Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
Normalize transcription language codes from display names and locale codes
Murtaza Nasir06dd2d74e56a
diff --git a/src/api/auth.py b/src/api/auth.pyindex 033474d1..5328a7a5 100644--- a/src/api/auth.py+++ b/src/api/auth.py@@ -577,7 +577,10 @@ def account():current_user.job_title = user_job_title if user_job_title else Nonecurrent_user.company = user_company if user_company else Nonecurrent_user.ui_language = ui_lang if ui_lang else 'en'- current_user.transcription_language = transcription_lang if transcription_lang else None+ # Normalize transcription_language so legacy display-name values+ # ("Français", "Deutsch", ...) can't reach the ASR call (issue #256).+ from src.utils.language import normalize_language_code+ current_user.transcription_language = normalize_language_code(transcription_lang)current_user.output_language = output_lang if output_lang else None# Check if this is the custom prompts form (has summary_prompt field)diff --git a/src/init_db.py b/src/init_db.pyindex 800fa246..d8b6508d 100644--- a/src/init_db.py+++ b/src/init_db.py@@ -14,7 +14,7 @@ import tempfilefrom sqlalchemy import text, inspectfrom src.database import db-from src.models import Recording, TranscriptChunk, SystemSetting+from src.models import Recording, TranscriptChunk, SystemSetting, Userfrom src.services.embeddings import process_recording_chunksfrom src.utils import add_column_if_not_exists, migrate_column_type, create_index_if_not_exists@@ -651,6 +651,31 @@ def initialize_database(app):)app.logger.info("Initialized enable_folders setting")+ # One-shot migration: clean up legacy User.transcription_language values+ # that were stored as display names ("Français", "English") before the+ # account-settings input was a dropdown. Issue #256.+ try:+ from src.utils.language import normalize_language_code+ from sqlalchemy import or_+ # Touch only rows where the value isn't already a valid 2-letter code,+ # to keep this idempotent across restarts.+ stale_users = User.query.filter(User.transcription_language.isnot(None)).all()+ cleaned = 0+ for u in stale_users:+ normalized = normalize_language_code(u.transcription_language)+ if normalized != u.transcription_language:+ app.logger.info(+ f"Migrating user {u.id} transcription_language: {u.transcription_language!r} -> {normalized!r}"+ )+ u.transcription_language = normalized+ cleaned += 1+ if cleaned:+ db.session.commit()+ app.logger.info(f"Normalized transcription_language for {cleaned} user(s)")+ except Exception as e:+ db.session.rollback()+ app.logger.warning(f"transcription_language normalization migration skipped: {e}")+# Process existing recordings for inquire mode (chunk and embed them)# Only run if inquire mode is enabledif ENABLE_INQUIRE_MODE:diff --git a/src/tasks/processing.py b/src/tasks/processing.pyindex d57f1673..2f4d7d70 100644--- a/src/tasks/processing.py+++ b/src/tasks/processing.py@@ -1464,6 +1464,11 @@ def transcribe_with_connector(app_context, recording_id, filepath, original_filefrom src.services.transcription import (get_connector, TranscriptionRequest, TranscriptionCapability)+ from src.utils.language import normalize_language_code++ # Defensive normalization at the connector boundary — guards legacy DB+ # values like "français" that pre-date the dropdown migration (issue #256).+ language = normalize_language_code(language)with app_context:recording = db.session.get(Recording, recording_id)@@ -2077,6 +2082,11 @@ def transcribe_incognito(filepath, original_filename, language=None, min_speakerif language is None and user:language = user.transcription_language+ # Normalize at the boundary — legacy values like "français" must+ # become "fr" before the connector receives them (issue #256).+ from src.utils.language import normalize_language_code+ language = normalize_language_code(language)+# Check if chunking is neededif video_passthrough_active:should_chunk = Falsediff --git a/src/utils/language.py b/src/utils/language.pynew file mode 100644index 00000000..30de9c76--- /dev/null+++ b/src/utils/language.py@@ -0,0 +1,114 @@+"""Language code normalization for ASR requests.++Older user records were saved when the transcription-language input was a+free-text field, so values like "français", "English", or "fr-FR" are present+in the wild. WhisperX and other ASR backends expect ISO 639-1 codes (`fr`,+`en`, ...) and reject anything else with a hard 500. This helper coerces+arbitrary input to a clean ISO code or returns None to mean "auto-detect".++Whitelist below matches the languages WhisperX accepts and the dropdown+offered in account settings.+"""++# ISO 639-1 codes accepted by WhisperX. Source: faster-whisper tokenizer.+# Keep this in sync with the dropdown in templates/account.html.+SUPPORTED_CODES = frozenset({+ "af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs",+ "ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi",+ "fo", "fr", "gl", "gu", "ha", "haw", "he", "hi", "hr", "ht", "hu", "hy",+ "id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb",+ "ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt",+ "my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", "ro", "ru",+ "sa", "sd", "si", "sk", "sl", "sn", "so", "sq", "sr", "su", "sv", "sw",+ "ta", "te", "tg", "th", "tk", "tl", "tr", "tt", "uk", "ur", "uz", "vi",+ "yi", "yo", "zh", "yue",+})++# Display names → ISO code. Lowercased keys; we lowercase input before lookup.+# Covers: English names, native names, common alternates.+_NAME_TO_CODE = {+ # English names+ "english": "en", "spanish": "es", "french": "fr", "german": "de",+ "italian": "it", "portuguese": "pt", "dutch": "nl", "russian": "ru",+ "chinese": "zh", "japanese": "ja", "korean": "ko", "arabic": "ar",+ "hindi": "hi", "polish": "pl", "ukrainian": "uk", "vietnamese": "vi",+ "thai": "th", "turkish": "tr", "indonesian": "id", "swedish": "sv",+ "norwegian": "no", "danish": "da", "finnish": "fi", "czech": "cs",+ "greek": "el", "hebrew": "he", "hungarian": "hu", "romanian": "ro",+ "bulgarian": "bg", "croatian": "hr", "serbian": "sr", "slovak": "sk",+ "slovenian": "sl", "estonian": "et", "latvian": "lv", "lithuanian": "lt",+ "persian": "fa", "farsi": "fa", "urdu": "ur", "bengali": "bn",+ "tamil": "ta", "telugu": "te", "marathi": "mr", "gujarati": "gu",+ "kannada": "kn", "malayalam": "ml", "punjabi": "pa", "burmese": "my",+ "khmer": "km", "lao": "lo", "mongolian": "mn", "nepali": "ne",+ "malay": "ms", "filipino": "tl", "tagalog": "tl", "swahili": "sw",+ "afrikaans": "af", "albanian": "sq", "armenian": "hy", "azerbaijani": "az",+ "basque": "eu", "belarusian": "be", "bosnian": "bs", "catalan": "ca",+ "welsh": "cy", "galician": "gl", "georgian": "ka", "icelandic": "is",+ "kazakh": "kk", "macedonian": "mk", "maltese": "mt", "yiddish": "yi",+ "yoruba": "yo", "cantonese": "yue",+ # Native names (selected — only those known to be misstored)+ "français": "fr", "francais": "fr",+ "deutsch": "de",+ "español": "es", "espanol": "es",+ "italiano": "it",+ "português": "pt", "portugues": "pt",+ "русский": "ru",+ "中文": "zh", "汉语": "zh", "中國話": "zh", "普通话": "zh",+ "日本語": "ja",+ "한국어": "ko",+ "العربية": "ar",+ "हिन्दी": "hi",+ "polski": "pl",+ "українська": "uk",+ "tiếng việt": "vi",+ "ไทย": "th",+ "türkçe": "tr",+ "bahasa indonesia": "id",+ "svenska": "sv",+ "norsk": "no",+ "dansk": "da",+ "suomi": "fi",+ "čeština": "cs", "cestina": "cs",+ "ελληνικά": "el",+ "עברית": "he",+ "magyar": "hu",+ "română": "ro", "romana": "ro",+}+++def normalize_language_code(value):+ """Coerce a user-supplied language string to an ISO 639-1 code or None.++ Empty / None / 'auto' / 'auto-detect' → None (signals auto-detect).+ Already-valid 2-letter codes pass through (lowercased).+ Locale codes like 'en-US' are stripped to 'en'.+ Display names like 'English', 'Français' map via _NAME_TO_CODE.+ Anything else returns None — better to auto-detect than crash the ASR call.+ """+ if value is None:+ return None+ s = str(value).strip().lower()+ if not s or s in ("auto", "auto-detect", "auto detect"):+ return None++ # Locale codes like 'en-US', 'fr_FR' → take the language portion.+ for sep in ("-", "_"):+ if sep in s:+ head = s.split(sep, 1)[0]+ if head in SUPPORTED_CODES:+ return head+ # Fall through; might still match by display name below.++ # Already a valid 2-letter (or 3-letter for 'haw'/'yue') code.+ if s in SUPPORTED_CODES:+ return s++ # Display name lookup.+ if s in _NAME_TO_CODE:+ code = _NAME_TO_CODE[s]+ return code if code in SUPPORTED_CODES else None++ # Last resort: first 2 chars happen to be a valid code (e.g. 'english' → 'en').+ # Skip — too aggressive and would map garbage to wrong languages.+ return Nonediff --git a/tests/test_language_normalization.py b/tests/test_language_normalization.pynew file mode 100644index 00000000..81c32f86--- /dev/null+++ b/tests/test_language_normalization.py@@ -0,0 +1,85 @@+#!/usr/bin/env python3+"""Tests for src.utils.language.normalize_language_code.++Run with: docker exec speakr-dev python /app/tests/test_language_normalization.py+"""++import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).parent.parent))++from src.utils.language import normalize_language_code, SUPPORTED_CODES+++PASSED = 0+FAILED = 0+++def expect(value, expected):+ global PASSED, FAILED+ got = normalize_language_code(value)+ if got == expected:+ print(f" ✓ normalize({value!r}) → {got!r}")+ PASSED += 1+ else:+ print(f" ✗ normalize({value!r}) → {got!r} (expected {expected!r})")+ FAILED += 1+++def main():+ print("=== normalize_language_code ===\n")++ # Empty / auto-detect+ expect(None, None)+ expect("", None)+ expect(" ", None)+ expect("auto", None)+ expect("Auto-Detect", None)++ # Already valid ISO codes+ expect("en", "en")+ expect("FR", "fr")+ expect("zh", "zh")+ expect("yue", "yue") # 3-letter cantonese code++ # Display names — English+ expect("English", "en")+ expect("french", "fr")+ expect("German", "de")+ expect("Chinese", "zh")+ expect("Cantonese", "yue")++ # Native names — the actual issue #256 case+ expect("Français", "fr")+ expect("français", "fr")+ expect("Francais", "fr")+ expect("Deutsch", "de")+ expect("Español", "es")+ expect("Português", "pt")+ expect("русский", "ru")+ expect("日本語", "ja")+ expect("中文", "zh")++ # Locale codes+ expect("en-US", "en")+ expect("fr-FR", "fr")+ expect("zh_CN", "zh")+ expect("pt-BR", "pt")++ # Invalid → None (don't crash the ASR call, let auto-detect kick in)+ expect("Klingon", None)+ expect("xyz", None)+ expect("123", None)+ expect("???", None)++ # Sanity check the supported set+ assert "fr" in SUPPORTED_CODES+ assert "français" not in SUPPORTED_CODES, "supported set must hold codes only"++ print(f"\nResults: {PASSED} passed, {FAILED} failed")+ return 0 if FAILED == 0 else 1+++if __name__ == "__main__":+ sys.exit(main())