Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Normalize transcription language codes from display names and locale codes

Murtaza Nasir06dd2d74e56a
diff --git a/src/api/auth.py b/src/api/auth.py
index 033474d1..5328a7a5 100644
--- a/src/api/auth.py
+++ b/src/api/auth.py
@@ -577,7 +577,10 @@ def account():
current_user.job_title = user_job_title if user_job_title else None
current_user.company = user_company if user_company else None
current_user.ui_language = ui_lang if ui_lang else 'en'
- current_user.transcription_language = transcription_lang if transcription_lang else None
+ # Normalize transcription_language so legacy display-name values
+ # ("Français", "Deutsch", ...) can't reach the ASR call (issue #256).
+ from src.utils.language import normalize_language_code
+ current_user.transcription_language = normalize_language_code(transcription_lang)
current_user.output_language = output_lang if output_lang else None
# Check if this is the custom prompts form (has summary_prompt field)
diff --git a/src/init_db.py b/src/init_db.py
index 800fa246..d8b6508d 100644
--- a/src/init_db.py
+++ b/src/init_db.py
@@ -14,7 +14,7 @@ import tempfile
from sqlalchemy import text, inspect
from src.database import db
-from src.models import Recording, TranscriptChunk, SystemSetting
+from src.models import Recording, TranscriptChunk, SystemSetting, User
from src.services.embeddings import process_recording_chunks
from src.utils import add_column_if_not_exists, migrate_column_type, create_index_if_not_exists
@@ -651,6 +651,31 @@ def initialize_database(app):
)
app.logger.info("Initialized enable_folders setting")
+ # One-shot migration: clean up legacy User.transcription_language values
+ # that were stored as display names ("Français", "English") before the
+ # account-settings input was a dropdown. Issue #256.
+ try:
+ from src.utils.language import normalize_language_code
+ from sqlalchemy import or_
+ # Touch only rows where the value isn't already a valid 2-letter code,
+ # to keep this idempotent across restarts.
+ stale_users = User.query.filter(User.transcription_language.isnot(None)).all()
+ cleaned = 0
+ for u in stale_users:
+ normalized = normalize_language_code(u.transcription_language)
+ if normalized != u.transcription_language:
+ app.logger.info(
+ f"Migrating user {u.id} transcription_language: {u.transcription_language!r} -> {normalized!r}"
+ )
+ u.transcription_language = normalized
+ cleaned += 1
+ if cleaned:
+ db.session.commit()
+ app.logger.info(f"Normalized transcription_language for {cleaned} user(s)")
+ except Exception as e:
+ db.session.rollback()
+ app.logger.warning(f"transcription_language normalization migration skipped: {e}")
+
# Process existing recordings for inquire mode (chunk and embed them)
# Only run if inquire mode is enabled
if ENABLE_INQUIRE_MODE:
diff --git a/src/tasks/processing.py b/src/tasks/processing.py
index d57f1673..2f4d7d70 100644
--- a/src/tasks/processing.py
+++ b/src/tasks/processing.py
@@ -1464,6 +1464,11 @@ def transcribe_with_connector(app_context, recording_id, filepath, original_file
from src.services.transcription import (
get_connector, TranscriptionRequest, TranscriptionCapability
)
+ from src.utils.language import normalize_language_code
+
+ # Defensive normalization at the connector boundary — guards legacy DB
+ # values like "français" that pre-date the dropdown migration (issue #256).
+ language = normalize_language_code(language)
with app_context:
recording = db.session.get(Recording, recording_id)
@@ -2077,6 +2082,11 @@ def transcribe_incognito(filepath, original_filename, language=None, min_speaker
if language is None and user:
language = user.transcription_language
+ # Normalize at the boundary — legacy values like "français" must
+ # become "fr" before the connector receives them (issue #256).
+ from src.utils.language import normalize_language_code
+ language = normalize_language_code(language)
+
# Check if chunking is needed
if video_passthrough_active:
should_chunk = False
diff --git a/src/utils/language.py b/src/utils/language.py
new file mode 100644
index 00000000..30de9c76
--- /dev/null
+++ b/src/utils/language.py
@@ -0,0 +1,114 @@
+"""Language code normalization for ASR requests.
+
+Older user records were saved when the transcription-language input was a
+free-text field, so values like "français", "English", or "fr-FR" are present
+in the wild. WhisperX and other ASR backends expect ISO 639-1 codes (`fr`,
+`en`, ...) and reject anything else with a hard 500. This helper coerces
+arbitrary input to a clean ISO code or returns None to mean "auto-detect".
+
+Whitelist below matches the languages WhisperX accepts and the dropdown
+offered in account settings.
+"""
+
+# ISO 639-1 codes accepted by WhisperX. Source: faster-whisper tokenizer.
+# Keep this in sync with the dropdown in templates/account.html.
+SUPPORTED_CODES = frozenset({
+ "af", "am", "ar", "as", "az", "ba", "be", "bg", "bn", "bo", "br", "bs",
+ "ca", "cs", "cy", "da", "de", "el", "en", "es", "et", "eu", "fa", "fi",
+ "fo", "fr", "gl", "gu", "ha", "haw", "he", "hi", "hr", "ht", "hu", "hy",
+ "id", "is", "it", "ja", "jw", "ka", "kk", "km", "kn", "ko", "la", "lb",
+ "ln", "lo", "lt", "lv", "mg", "mi", "mk", "ml", "mn", "mr", "ms", "mt",
+ "my", "ne", "nl", "nn", "no", "oc", "pa", "pl", "ps", "pt", "ro", "ru",
+ "sa", "sd", "si", "sk", "sl", "sn", "so", "sq", "sr", "su", "sv", "sw",
+ "ta", "te", "tg", "th", "tk", "tl", "tr", "tt", "uk", "ur", "uz", "vi",
+ "yi", "yo", "zh", "yue",
+})
+
+# Display names → ISO code. Lowercased keys; we lowercase input before lookup.
+# Covers: English names, native names, common alternates.
+_NAME_TO_CODE = {
+ # English names
+ "english": "en", "spanish": "es", "french": "fr", "german": "de",
+ "italian": "it", "portuguese": "pt", "dutch": "nl", "russian": "ru",
+ "chinese": "zh", "japanese": "ja", "korean": "ko", "arabic": "ar",
+ "hindi": "hi", "polish": "pl", "ukrainian": "uk", "vietnamese": "vi",
+ "thai": "th", "turkish": "tr", "indonesian": "id", "swedish": "sv",
+ "norwegian": "no", "danish": "da", "finnish": "fi", "czech": "cs",
+ "greek": "el", "hebrew": "he", "hungarian": "hu", "romanian": "ro",
+ "bulgarian": "bg", "croatian": "hr", "serbian": "sr", "slovak": "sk",
+ "slovenian": "sl", "estonian": "et", "latvian": "lv", "lithuanian": "lt",
+ "persian": "fa", "farsi": "fa", "urdu": "ur", "bengali": "bn",
+ "tamil": "ta", "telugu": "te", "marathi": "mr", "gujarati": "gu",
+ "kannada": "kn", "malayalam": "ml", "punjabi": "pa", "burmese": "my",
+ "khmer": "km", "lao": "lo", "mongolian": "mn", "nepali": "ne",
+ "malay": "ms", "filipino": "tl", "tagalog": "tl", "swahili": "sw",
+ "afrikaans": "af", "albanian": "sq", "armenian": "hy", "azerbaijani": "az",
+ "basque": "eu", "belarusian": "be", "bosnian": "bs", "catalan": "ca",
+ "welsh": "cy", "galician": "gl", "georgian": "ka", "icelandic": "is",
+ "kazakh": "kk", "macedonian": "mk", "maltese": "mt", "yiddish": "yi",
+ "yoruba": "yo", "cantonese": "yue",
+ # Native names (selected — only those known to be misstored)
+ "français": "fr", "francais": "fr",
+ "deutsch": "de",
+ "español": "es", "espanol": "es",
+ "italiano": "it",
+ "português": "pt", "portugues": "pt",
+ "русский": "ru",
+ "中文": "zh", "汉语": "zh", "中國話": "zh", "普通话": "zh",
+ "日本語": "ja",
+ "한국어": "ko",
+ "العربية": "ar",
+ "हिन्दी": "hi",
+ "polski": "pl",
+ "українська": "uk",
+ "tiếng việt": "vi",
+ "ไทย": "th",
+ "türkçe": "tr",
+ "bahasa indonesia": "id",
+ "svenska": "sv",
+ "norsk": "no",
+ "dansk": "da",
+ "suomi": "fi",
+ "čeština": "cs", "cestina": "cs",
+ "ελληνικά": "el",
+ "עברית": "he",
+ "magyar": "hu",
+ "română": "ro", "romana": "ro",
+}
+
+
+def normalize_language_code(value):
+ """Coerce a user-supplied language string to an ISO 639-1 code or None.
+
+ Empty / None / 'auto' / 'auto-detect' → None (signals auto-detect).
+ Already-valid 2-letter codes pass through (lowercased).
+ Locale codes like 'en-US' are stripped to 'en'.
+ Display names like 'English', 'Français' map via _NAME_TO_CODE.
+ Anything else returns None — better to auto-detect than crash the ASR call.
+ """
+ if value is None:
+ return None
+ s = str(value).strip().lower()
+ if not s or s in ("auto", "auto-detect", "auto detect"):
+ return None
+
+ # Locale codes like 'en-US', 'fr_FR' → take the language portion.
+ for sep in ("-", "_"):
+ if sep in s:
+ head = s.split(sep, 1)[0]
+ if head in SUPPORTED_CODES:
+ return head
+ # Fall through; might still match by display name below.
+
+ # Already a valid 2-letter (or 3-letter for 'haw'/'yue') code.
+ if s in SUPPORTED_CODES:
+ return s
+
+ # Display name lookup.
+ if s in _NAME_TO_CODE:
+ code = _NAME_TO_CODE[s]
+ return code if code in SUPPORTED_CODES else None
+
+ # Last resort: first 2 chars happen to be a valid code (e.g. 'english' → 'en').
+ # Skip — too aggressive and would map garbage to wrong languages.
+ return None
diff --git a/tests/test_language_normalization.py b/tests/test_language_normalization.py
new file mode 100644
index 00000000..81c32f86
--- /dev/null
+++ b/tests/test_language_normalization.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Tests for src.utils.language.normalize_language_code.
+
+Run with: docker exec speakr-dev python /app/tests/test_language_normalization.py
+"""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.utils.language import normalize_language_code, SUPPORTED_CODES
+
+
+PASSED = 0
+FAILED = 0
+
+
+def expect(value, expected):
+ global PASSED, FAILED
+ got = normalize_language_code(value)
+ if got == expected:
+ print(f" ✓ normalize({value!r}) → {got!r}")
+ PASSED += 1
+ else:
+ print(f" ✗ normalize({value!r}) → {got!r} (expected {expected!r})")
+ FAILED += 1
+
+
+def main():
+ print("=== normalize_language_code ===\n")
+
+ # Empty / auto-detect
+ expect(None, None)
+ expect("", None)
+ expect(" ", None)
+ expect("auto", None)
+ expect("Auto-Detect", None)
+
+ # Already valid ISO codes
+ expect("en", "en")
+ expect("FR", "fr")
+ expect("zh", "zh")
+ expect("yue", "yue") # 3-letter cantonese code
+
+ # Display names — English
+ expect("English", "en")
+ expect("french", "fr")
+ expect("German", "de")
+ expect("Chinese", "zh")
+ expect("Cantonese", "yue")
+
+ # Native names — the actual issue #256 case
+ expect("Français", "fr")
+ expect("français", "fr")
+ expect("Francais", "fr")
+ expect("Deutsch", "de")
+ expect("Español", "es")
+ expect("Português", "pt")
+ expect("русский", "ru")
+ expect("日本語", "ja")
+ expect("中文", "zh")
+
+ # Locale codes
+ expect("en-US", "en")
+ expect("fr-FR", "fr")
+ expect("zh_CN", "zh")
+ expect("pt-BR", "pt")
+
+ # Invalid → None (don't crash the ASR call, let auto-detect kick in)
+ expect("Klingon", None)
+ expect("xyz", None)
+ expect("123", None)
+ expect("???", None)
+
+ # Sanity check the supported set
+ assert "fr" in SUPPORTED_CODES
+ assert "français" not in SUPPORTED_CODES, "supported set must hold codes only"
+
+ print(f"\nResults: {PASSED} passed, {FAILED} failed")
+ return 0 if FAILED == 0 else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())