CV info, full 7925 langs

app.py

@@ -6,20 +6,27 @@ from functools import lru_cache
 import csv
 from io import StringIO
 import re
+import json
 
 # Configuration
 LANGUAGE_CODES_FILE = "language-codes-full.csv"
 APP_CONTENT_FILE = "app_content.md"
 LANGUAGE_TAXONOMY_URL = "https://microsoft.github.io/linguisticdiversity/assets/lang2tax.txt"
+COMMON_VOICE_DATA_FILE = "cv-corpus-20.0-2024-12-06.json"
+COMMON_VOICE_VERSION = "20.0 (2024-12-06)"
 
 # Language list will be loaded from CSV
-# Structure: {alpha3_b: {"name": str, "alpha3_t": str, "alpha2": str}}
+# Structure: {iso_639_2: {"name": str, "iso_639_1": str, "french_name": str}}
 LANGUAGES = {}
 
 # Language taxonomy mapping (from Joshi et al.'s linguistic diversity paper)
 # Structure: {language_name_lowercase: level}
 LANGUAGE_TAXONOMY = {}
 
+# Common Voice dataset
+# Structure: {locale_code: {validHrs: float, totalHrs: float, splits: {gender: {...}}, ...}}
+COMMON_VOICE_DATA = {}
+
 # Taxonomy level descriptions
 TAXONOMY_LEVELS = {
     0: "The Left-Behinds",
@@ -89,32 +96,27 @@ def load_language_list(csv_path=None):
         with open(csv_path, 'r', encoding='utf-8') as f:
             reader = csv.DictReader(f)
             for row in reader:
-                # Use alpha3-b as primary key, fallback to alpha3-t if empty
-                code_b = row['alpha3-b'].strip()
-                code_t = row['alpha3-t'].strip()
-                code_2 = row['alpha2'].strip()
-                name = row['English'].strip()
-
-                primary_code = code_b if code_b else code_t
-
-                if primary_code and name:
-                    LANGUAGES[primary_code] = {
+                # New CSV structure: ISO 639-2, ISO 639-1, English name, French name
+                iso_639_2 = row['ISO 639-2'].strip()
+                iso_639_1 = row['ISO 639-1'].strip()
+                name = row['English name'].strip()
+                french_name = row['French name'].strip()
+
+                if iso_639_2 and name:
+                    LANGUAGES[iso_639_2] = {
                         "name": name,
-                        "alpha3_b": code_b,
-                        "alpha3_t": code_t,
-                        "alpha2": code_2
+                        "iso_639_1": iso_639_1,
+                        "french_name": french_name,
+                        # Keep legacy field names for backward compatibility
+                        "alpha2": iso_639_1,
+                        "alpha3_b": iso_639_2,
+                        "alpha3_t": ""  # Not used in new format
                     }
         print(f"Loaded {len(LANGUAGES)} languages from {csv_path}")
     except Exception as e:
-        print(f"Error loading language list: {e}")
-        # Fallback to a minimal set
-        LANGUAGES = {
-            "eng": {"name": "English", "alpha3_b": "eng", "alpha3_t": "", "alpha2": "en"},
-            "spa": {"name": "Spanish", "alpha3_b": "spa", "alpha3_t": "", "alpha2": "es"},
-            "fra": {"name": "French", "alpha3_b": "fra", "alpha3_t": "", "alpha2": "fr"},
-            "deu": {"name": "German", "alpha3_b": "ger", "alpha3_t": "deu", "alpha2": "de"},
-        }
-        print(f"Using fallback with {len(LANGUAGES)} languages")
+        print(f"ERROR: Failed to load language list from {csv_path}: {e}")
+        print("The application cannot run without the language codes CSV file.")
+        LANGUAGES = {}
 
 def load_language_taxonomy():
     """Load language taxonomy data from Microsoft's linguistic diversity project"""
@@ -138,6 +140,67 @@ def load_language_taxonomy():
         print(f"Warning: Could not load language taxonomy: {e}")
         print("Language classification will show as 'Unknown'")
 
+def load_common_voice_data():
+    """Load Common Voice dataset statistics"""
+    global COMMON_VOICE_DATA
+
+    try:
+        with open(COMMON_VOICE_DATA_FILE, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+            COMMON_VOICE_DATA = data.get('locales', {})
+
+        print(f"Loaded Common Voice data for {len(COMMON_VOICE_DATA)} languages")
+    except Exception as e:
+        print(f"Warning: Could not load Common Voice data: {e}")
+        print("Common Voice statistics will not be available")
+
+def get_common_voice_stats(language_code):
+    """
+    Get Common Voice statistics for a language
+    Returns dict with validHrs, totalHrs, gender balance, and locale code or None if not available
+    """
+    lang_info = LANGUAGES.get(language_code)
+    if not lang_info:
+        return None
+
+    # Try to find CV data using different code formats
+    # 1. Try ISO 639-2 (3-letter) code directly (e.g., "zgh", "kab")
+    if language_code in COMMON_VOICE_DATA:
+        cv_locale = language_code
+        cv_data = COMMON_VOICE_DATA[language_code]
+    # 2. Try ISO 639-1 (2-letter) code (e.g., "en", "fr")
+    elif lang_info.get('iso_639_1') and lang_info['iso_639_1'] in COMMON_VOICE_DATA:
+        cv_locale = lang_info['iso_639_1']
+        cv_data = COMMON_VOICE_DATA[lang_info['iso_639_1']]
+    # 3. Try to find any locale that starts with the 2-letter code (e.g., "fy-NL", "ga-IE")
+    elif lang_info.get('iso_639_1'):
+        iso_639_1 = lang_info['iso_639_1']
+        matching_locales = [loc for loc in COMMON_VOICE_DATA.keys() if loc.startswith(iso_639_1 + '-')]
+        if matching_locales:
+            cv_locale = matching_locales[0]  # Take the first match
+            cv_data = COMMON_VOICE_DATA[cv_locale]
+        else:
+            return None
+    else:
+        return None
+
+    # Extract statistics
+    valid_hrs = cv_data.get('validHrs', 0)
+    total_hrs = cv_data.get('totalHrs', 0)
+
+    # Extract gender balance
+    gender_splits = cv_data.get('splits', {}).get('gender', {})
+    male_pct = gender_splits.get('male_masculine', 0) * 100
+    female_pct = gender_splits.get('female_feminine', 0) * 100
+
+    return {
+        'locale': cv_locale,
+        'valid_hrs': valid_hrs,
+        'total_hrs': total_hrs,
+        'male_pct': male_pct,
+        'female_pct': female_pct
+    }
+
 def get_taxonomy_color(level):
     """
     Get color code for taxonomy level (red for left-behind, green for winners)
@@ -987,15 +1050,25 @@ def search_language_resources(language_code, deduplicate=False):
     all_logs = []
 
     if not language_code:
-        return None, None, None, 0, 0, None, None, 0, 0, ""
+        return None, None, None, None, 0, 0, None, None, 0, 0, ""
 
     lang_info = LANGUAGES.get(language_code)
     if not lang_info:
-        return None, None, None, 0, 0, None, None, 0, 0, ""
+        return None, None, None, None, 0, 0, None, None, 0, 0, ""
 
     language_name = lang_info['name']
     all_logs.append(f"=== Searching for {language_name} ({language_code}) ===")
-    all_logs.append(f"Language codes: alpha2={lang_info['alpha2']}, alpha3_b={lang_info['alpha3_b']}, alpha3_t={lang_info['alpha3_t']}")
+    all_logs.append(f"Language codes: ISO 639-1={lang_info['iso_639_1']}, ISO 639-2={language_code}")
+
+    # Check Common Voice data
+    all_logs.append("\n[Common Voice Dataset]")
+    cv_stats = get_common_voice_stats(language_code)
+    if cv_stats:
+        all_logs.append(f"  ✅ Available in Common Voice (locale: {cv_stats['locale']})")
+        all_logs.append(f"  Valid hours: {cv_stats['valid_hrs']:.1f}h, Total hours: {cv_stats['total_hrs']:.1f}h")
+        all_logs.append(f"  Gender balance: {cv_stats['male_pct']:.1f}% male, {cv_stats['female_pct']:.1f}% female")
+    else:
+        all_logs.append(f"  ❌ Not available in Common Voice")
 
     # Fetch Azure data
     all_logs.append("\n[Azure Speech Services]")
@@ -1250,14 +1323,15 @@ def search_language_resources(language_code, deduplicate=False):
     # Combine all logs
     log_text = "\n".join(all_logs)
 
-    # Return separate ASR and TTS dataframes, plus counts for tab labels, plus logs
-    return commercial_df, asr_models_df, tts_models_df, len(asr_models), len(tts_models), asr_datasets_df, tts_datasets_df, len(asr_datasets), len(tts_datasets), log_text
+    # Return CV stats, commercial services, models, datasets, and logs
+    return cv_stats, commercial_df, asr_models_df, tts_models_df, len(asr_models), len(tts_models), asr_datasets_df, tts_datasets_df, len(asr_datasets), len(tts_datasets), log_text
 
 # Initialize - load language list and app content
 print("Initializing Speech Resource Finder...")
 load_app_content()
 load_language_list()
 load_language_taxonomy()
+load_common_voice_data()
 
 # Create language choices for dropdown (code: name format for easy searching)
 language_choices = [f"{code}: {info['name']}" for code, info in sorted(LANGUAGES.items(), key=lambda x: x[1]['name'])]
@@ -1268,7 +1342,7 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
     gr.Markdown(APP_CONTENT["description"])
 
     with gr.Row(equal_height=True):
-        with gr.Column(scale=2):
+        with gr.Column(scale=70):
             language_dropdown = gr.Dropdown(
                 choices=language_choices,
                 label="Select Language",
@@ -1276,7 +1350,7 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
                 allow_custom_value=False,
                 filterable=True,
             )
-        with gr.Column(scale=1):
+        with gr.Column(scale=30):
             language_metadata = gr.HTML(
                 """<div style='padding: 15px; border: 2px solid #e0e0e0; border-radius: 4px; background-color: #fafafa; height: 100%; display: flex; align-items: center; justify-content: center; box-sizing: border-box;'>
                 <p style='margin: 0; color: #333; font-size: 14px;'>Select a language to see resource classification</p>
@@ -1284,12 +1358,23 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
                 elem_id="language-metadata"
             )
 
-    gr.Markdown("## Commercial Services")
-    commercial_table = gr.Dataframe(
-        headers=["Service", "ASR", "TTS"],
-        interactive=False,
-        wrap=True,
-    )
+    with gr.Row():
+        with gr.Column(scale=70):
+            gr.Markdown("## Commercial Services")
+            commercial_table = gr.Dataframe(
+                headers=["Service", "ASR", "TTS"],
+                interactive=False,
+                wrap=True,
+            )
+
+        with gr.Column(scale=30):
+            gr.Markdown("## Common Voice")
+            cv_info = gr.HTML(
+                """<div style='padding: 15px; border: 2px solid #e0e0e0; border-radius: 4px; background-color: #fafafa;'>
+                <p style='margin: 0; color: #666; font-size: 13px;'>Select a language</p>
+                </div>""",
+                elem_id="cv-info"
+            )
 
     gr.Markdown("## HuggingFace Models")
 
@@ -1361,7 +1446,10 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
             default_html = """<div style='padding: 15px; border: 2px solid #e0e0e0; border-radius: 4px; background-color: #fafafa; height: 100%; display: flex; align-items: center; justify-content: center; box-sizing: border-box;'>
             <p style='margin: 0; color: #333; font-size: 14px;'>Select a language to see resource classification</p>
             </div>"""
-            return default_html, None, "", None, "", None, "", None, "", None, ""
+            cv_default_html = """<div style='padding: 15px; border: 2px solid #e0e0e0; border-radius: 4px; background-color: #fafafa;'>
+            <p style='margin: 0; color: #666; font-size: 13px;'>Select a language</p>
+            </div>"""
+            return default_html, cv_default_html, None, "", None, "", None, "", None, "", None, ""
         # Extract the language code from "code: name" format
         language_code = language_selection.split(":")[0].strip()
 
@@ -1390,7 +1478,46 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
             <p style='margin: 8px 0 0 0; font-size: 11px; color: #555;'>Source: <a href='https://microsoft.github.io/linguisticdiversity/' target='_blank' style='color: #1976d2; text-decoration: none;'>Joshi et al.</a></p>
             </div>"""
 
-        commercial_df, asr_models_df, tts_models_df, asr_models_count, tts_models_count, asr_datasets_df, tts_datasets_df, asr_datasets_count, tts_datasets_count, logs = search_language_resources(language_code, deduplicate=deduplicate)
+        cv_stats, commercial_df, asr_models_df, tts_models_df, asr_models_count, tts_models_count, asr_datasets_df, tts_datasets_df, asr_datasets_count, tts_datasets_count, logs = search_language_resources(language_code, deduplicate=deduplicate)
+
+        # Create Common Voice info HTML
+        if cv_stats:
+            cv_info_html = f"""<div style='padding: 15px; border: 2px solid #4caf50; border-radius: 4px; background-color: #ffffff;'>
+            <div style='margin-bottom: 12px;'>
+                <span style='font-size: 18px;'>✅</span>
+                <span style='font-weight: bold; color: #2e7d32; font-size: 14px; margin-left: 4px;'>Available</span>
+            </div>
+            <table style='width: 100%; border-collapse: collapse; font-size: 13px;'>
+                <tr>
+                    <td style='padding: 3px 8px 3px 0; color: #666; width: 45%;'>Locale</td>
+                    <td style='padding: 3px 0; color: #000; font-weight: 500;'>{cv_stats['locale']}</td>
+                </tr>
+                <tr>
+                    <td style='padding: 3px 8px 3px 0; color: #666;'>Valid Hours</td>
+                    <td style='padding: 3px 0; color: #000; font-weight: 500;'>{cv_stats['valid_hrs']:.1f}h</td>
+                </tr>
+                <tr>
+                    <td style='padding: 3px 8px 3px 0; color: #666;'>Total Hours</td>
+                    <td style='padding: 3px 0; color: #000; font-weight: 500;'>{cv_stats['total_hrs']:.1f}h</td>
+                </tr>
+                <tr>
+                    <td style='padding: 3px 8px 3px 0; color: #666;'>Gender</td>
+                    <td style='padding: 3px 0; color: #000; font-weight: 500;'>{cv_stats['male_pct']:.0f}% M / {cv_stats['female_pct']:.0f}% F</td>
+                </tr>
+                <tr>
+                    <td style='padding: 3px 8px 3px 0; color: #666;'>Version</td>
+                    <td style='padding: 3px 0; color: #000; font-weight: 500;'>{COMMON_VOICE_VERSION}</td>
+                </tr>
+            </table>
+            </div>"""
+        else:
+            cv_info_html = """<div style='padding: 15px; border: 2px solid #e0e0e0; border-radius: 4px; background-color: #fafafa;'>
+            <div style='margin-bottom: 8px;'>
+                <span style='font-size: 18px;'>❌</span>
+                <span style='font-weight: bold; color: #666; font-size: 14px; margin-left: 4px;'>Not Available</span>
+            </div>
+            <p style='margin: 0; color: #999; font-size: 12px;'>Not in Common Voice dataset</p>
+            </div>"""
 
         # Create count labels
         asr_models_label = f"**Found {asr_models_count} ASR model(s)**"
@@ -1398,20 +1525,20 @@ with gr.Blocks(title=APP_CONTENT["title"]) as demo:
         asr_datasets_label = f"**Found {asr_datasets_count} ASR dataset(s)**"
         tts_datasets_label = f"**Found {tts_datasets_count} TTS dataset(s)**"
 
-        return metadata_html, commercial_df, asr_models_label, asr_models_df, tts_models_label, tts_models_df, asr_datasets_label, asr_datasets_df, tts_datasets_label, tts_datasets_df, logs
+        return metadata_html, cv_info_html, commercial_df, asr_models_label, asr_models_df, tts_models_label, tts_models_df, asr_datasets_label, asr_datasets_df, tts_datasets_label, tts_datasets_df, logs
 
     # Trigger search when language is selected
     language_dropdown.change(
         fn=on_search,
         inputs=[language_dropdown, deduplicate_checkbox],
-        outputs=[language_metadata, commercial_table, asr_count_label, asr_models_table, tts_count_label, tts_models_table, asr_datasets_count_label, asr_datasets_table, tts_datasets_count_label, tts_datasets_table, log_textbox],
+        outputs=[language_metadata, cv_info, commercial_table, asr_count_label, asr_models_table, tts_count_label, tts_models_table, asr_datasets_count_label, asr_datasets_table, tts_datasets_count_label, tts_datasets_table, log_textbox],
     )
 
     # Trigger search when deduplicate checkbox is changed
     deduplicate_checkbox.change(
         fn=on_search,
         inputs=[language_dropdown, deduplicate_checkbox],
-        outputs=[language_metadata, commercial_table, asr_count_label, asr_models_table, tts_count_label, tts_models_table, asr_datasets_count_label, asr_datasets_table, tts_datasets_count_label, tts_datasets_table, log_textbox],
+        outputs=[language_metadata, cv_info, commercial_table, asr_count_label, asr_models_table, tts_count_label, tts_models_table, asr_datasets_count_label, asr_datasets_table, tts_datasets_count_label, tts_datasets_table, log_textbox],
     )
 
 if __name__ == "__main__":

app_content.md

@@ -14,6 +14,12 @@ Built by CLEAR Global to support language inclusion and help close the digital l
 
 ## Data Sources
 
+### Common Voice
+
+[Common Voice](https://commonvoice.mozilla.org/) is Mozilla's crowdsourced, open-source speech dataset. For languages listed here, anyone can contribute voice recordings and anyone can download the data to build speech technology. Datasets are available through the [Mozilla Data Collective](https://datacollective.mozillafoundation.org/datasets).
+
+**Dataset Version:** 20.0 (2024-12-06)
+
 ### Commercial Speech Services
 
 Commercial service support is automatically pulled from the language support page of each service provider.
@@ -44,10 +50,17 @@ The resource classification shown for each language is based on [Joshi et al.'s
 
 **Note:** This classification is from 2020 research and may not reflect the current state of resources for all languages. The landscape of speech technology is rapidly evolving, and some languages have surely gained more resources since this study was conducted.
 
+## Language Coverage
+
+This tool supports **7,925 languages** based on the comprehensive [ISO 639-3 standard](https://iso639-3.sil.org/code_tables/639/data) maintained by SIL International. ISO 639-3 provides unique three-letter codes for all known human languages, including living, extinct, ancient, and constructed languages.
+
+The language codes are mapped to both:
+- **ISO 639-1** (2-letter codes) - Used by most modern services
+- **ISO 639-2** (3-letter codes) - Used for broader language coverage
+
 ## Disclaimer
 
-- The language list only contains 487 languages and is taken from this [Github repository](https://github.com/datasets/language-codes).
-- This is not an exhaustive list of speech and language technology resources. There are other commercial voice technology providers and dataset/model resources that this app doesn't cover. 
+- This is not an exhaustive list of speech and language technology resources. There are other commercial voice technology providers and dataset/model resources that this app doesn't cover.
 - Data fetched in real-time and can change.
 - Model deduplication discards models with same name uploaded by others and keeps only the most downloaded version in the list.
 - A maximum of 100 dataset and model entries from Hugging Face are shown.