Personal AI, On Personal Devices
Python
fix(speech): honor language and isolate the Kokoro fallback (#1086)
AgenciaAlliera8d1ef909f56
diff --git a/src/openjarvis/cli/_voice_chat.py b/src/openjarvis/cli/_voice_chat.pyindex b4623080..73f5bb7f 100644--- a/src/openjarvis/cli/_voice_chat.py+++ b/src/openjarvis/cli/_voice_chat.py@@ -58,6 +58,13 @@ class VoiceSession:self._stt_resolved = Truereturn self._stt_backend+ def get_stt_language(self) -> Optional[str]:+ """Return the configured ``speech.language``, or None to auto-detect."""+ from openjarvis.core.config import load_config++ config = self._config if self._config is not None else load_config()+ return (getattr(config.speech, "language", "") or "").strip() or None+def get_tts_backend(self) -> Any:"""Return a cached healthy TTS backend, falling through once per key."""if self._tts_backend is not None:@@ -152,7 +159,9 @@ def record_voice(console.print("[dim]Transcribing…[/dim]")try:- result = backend.transcribe(audio_bytes, format="wav")+ result = backend.transcribe(+ audio_bytes, format="wav", language=active_session.get_stt_language()+ )text = result.text.strip()if text:console.print(f"[bold]You (voice):[/bold] {_terminal_safe_text(text)}")diff --git a/src/openjarvis/speech/kokoro_tts.py b/src/openjarvis/speech/kokoro_tts.pyindex b222c66e..6c553f61 100644--- a/src/openjarvis/speech/kokoro_tts.py+++ b/src/openjarvis/speech/kokoro_tts.py@@ -13,14 +13,20 @@ female (prefix ``z`` → ``lang_code="z"``).from __future__ import annotations+import builtinsimport io+import logging+import sysimport threading+import typesfrom collections import OrderedDictfrom typing import Any, Dict, Listfrom openjarvis.core.registry import TTSRegistryfrom openjarvis.speech.tts import TTSBackend, TTSResult+logger = logging.getLogger(__name__)+# Kokoro's voice-prefix → ``lang_code`` mapping. Keep this in sync with# ``kokoro.pipeline.LANG_CODES``: Kokoro 0.9.x does not expose a Korean# pipeline, even though Korean voice files have appeared in some catalogs.@@ -38,6 +44,65 @@ _VOICE_PREFIX_TO_LANG: Dict[str, str] = {_DEFAULT_LANG_CODE = "a"_DEFAULT_VOICE_ID = "af_heart"+_IMPORT_LOCK = threading.RLock()+++def _import_kokoro() -> Any:+ """Import ``kokoro``, tolerating a spaCy install that cannot be loaded.++ ``kokoro.pipeline`` imports ``misaki.en`` -- and with it spaCy -- for every+ language, but only the English G2P (voices ``a*``/``b*``) uses spaCy.+ Windows Smart App Control, for one, can refuse to load spaCy's unsigned+ extension DLLs, which would otherwise take every Kokoro voice down with+ it. When spaCy is the culprit, stub it so espeak-based voices (e.g.+ ``pf_dora``) keep working; English pipelines then fail on first use.+ """+ # Serialize hook installation across backend instances. The replacement+ # module is visible only to this thread, never through sys.modules.+ with _IMPORT_LOCK:+ try:+ import kokoro+ except ImportError as exc:+ if exc.name == "kokoro" or sys.modules.get("spacy") is not None:+ raise+ try:+ import spacy # noqa: F401+ except ImportError as spacy_exc:+ spacy_error = spacy_exc+ else:+ raise exc from None+ shim = types.ModuleType("spacy")+ owner = threading.get_ident()+ original_import = builtins.__import__+ active = True++ def fallback_import(name, globals=None, locals=None, fromlist=(), level=0):+ if (+ active+ and threading.get_ident() == owner+ and name == "spacy"+ and level == 0+ ):+ return shim+ return original_import(name, globals, locals, fromlist, level)++ builtins.__import__ = fallback_import+ try:+ import kokoro+ finally:+ # misaki.en may retain the private shim for non-English use,+ # while unrelated threads always use the original importer.+ # Deactivate even if another hook retained this wrapper.+ active = False+ if builtins.__import__ is fallback_import:+ builtins.__import__ = original_import+ kokoro._openjarvis_spacy_error = str(spacy_error)+ logger.warning(+ "spaCy unavailable (%s); English Kokoro voices disabled, "+ "other languages still work",+ spacy_error,+ )+ return kokoro@TTSRegistry.register("kokoro")@@ -86,11 +151,19 @@ class KokoroTTSBackend(TTSBackend):self._pipelines.move_to_end(lang_code)return pipelinetry:- from kokoro import KPipeline+ kokoro = _import_kokoro()+ KPipeline = kokoro.KPipelineexcept ImportError as exc:raise RuntimeError("kokoro package not installed. Install with: pip install kokoro") from exc+ if lang_code in {"a", "b"} and getattr(+ kokoro, "_openjarvis_spacy_error", None+ ):+ raise RuntimeError(+ "English Kokoro voices require a working spaCy installation: "+ f"{kokoro._openjarvis_spacy_error}"+ )try:# KModel is language-blind and is by far the heaviest part of# Kokoro. Reuse one model across every language pipeline.@@ -128,7 +201,7 @@ class KokoroTTSBackend(TTSBackend):return self._modeltry:- from kokoro import KModel+ KModel = _import_kokoro().KModelexcept ImportError as exc:raise RuntimeError("kokoro package not installed. Install with: pip install kokoro"@@ -255,8 +328,13 @@ class KokoroTTSBackend(TTSBackend):]def health(self) -> bool:- try:- self._ensure_pipeline(_DEFAULT_LANG_CODE)- return True- except RuntimeError:- return False+ # Check the shared model, not the English pipeline: language-specific+ # G2P problems surface at synthesis time for the voice actually used.+ # Use the same lock as synthesis so concurrent health probes cannot+ # allocate duplicate model weights or race with close().+ with self._pipeline_lock:+ try:+ self._ensure_model()+ return True+ except RuntimeError:+ return Falsediff --git a/tests/cli/test_chat_cmd.py b/tests/cli/test_chat_cmd.pyindex 458d6803..8c6d7509 100644--- a/tests/cli/test_chat_cmd.py+++ b/tests/cli/test_chat_cmd.py@@ -199,6 +199,35 @@ class TestVoiceInput:discover.assert_called_once()assert backend.transcribe.call_count == 2+ @pytest.mark.parametrize(+ ("language", "expected"),+ [("pt", "pt"), (" pt ", "pt"), ("", None), (" ", None)],+ )+ def test_configured_speech_language_is_passed_to_stt(+ self, language: str, expected: str | None+ ) -> None:+ backend = MagicMock()+ backend.transcribe.return_value = SimpleNamespace(text="olá")+ config = JarvisConfig()+ config.speech.language = language+ session = VoiceSession(config)++ with (+ patch(+ "openjarvis.speech._discovery.get_speech_backend",+ return_value=backend,+ ),+ patch(+ "openjarvis.speech.voice_io.record_until_silence",+ return_value=b"wav",+ ),+ ):+ assert record_voice(MagicMock(), session) == "olá"++ backend.transcribe.assert_called_once_with(+ b"wav", format="wav", language=expected+ )+def test_voice_transcript_cannot_inject_rich_hyperlink(self) -> None:backend = MagicMock()backend.transcribe.return_value = SimpleNamespace(diff --git a/tests/speech/test_tts_backends.py b/tests/speech/test_tts_backends.pyindex fc9c217c..11f9b511 100644--- a/tests/speech/test_tts_backends.py+++ b/tests/speech/test_tts_backends.py@@ -74,16 +74,16 @@ def test_kokoro_registered():def test_kokoro_health_false_without_package():+ import importlib.util+from openjarvis.speech.kokoro_tts import KokoroTTSBackend# The assertion only holds when kokoro is genuinely absent. With the# optional `voice` extra installed, health() legitimately returns True- # (and warming the pipeline would hit the HF Hub), so skip instead.- try:- import kokoro # noqa: F401- except ImportError:- pass- else:+ # (and loading the model would hit the HF Hub), so skip instead. Check+ # for the package without importing it: an installed kokoro whose spaCy+ # cannot load still counts as installed.+ if importlib.util.find_spec("kokoro") is not None:pytest.skip("kokoro installed (openjarvis[voice]); health() is True")backend = KokoroTTSBackend()@@ -401,6 +401,321 @@ def test_kokoro_reuses_model_and_honors_path_and_device(monkeypatch):assert pipeline_models[0][1] is pipeline_models[1][1]+class _FakeKModel:+ def to(self, _device):+ return self++ def eval(self):+ return self+++def _serve_kokoro_importing_spacy(+ monkeypatch,+ kpipeline,+ *,+ missing_dep=None,+ import_error=None,+ block_spacy=False,+ after_spacy=None,+):+ """Serve a fake ``kokoro`` that imports spaCy on load, like the real one+ (``kokoro.pipeline`` -> ``misaki.en`` -> ``spacy``)."""+ import importlib.abc+ import importlib.util+ import sys++ class Finder(importlib.abc.MetaPathFinder, importlib.abc.Loader):+ def find_spec(self, name, path=None, target=None):+ if name != "kokoro" and not (block_spacy and name == "spacy"):+ return None+ return importlib.util.spec_from_loader(name, self)++ def create_module(self, spec):+ return None++ def exec_module(self, module):+ if module.__name__ == "spacy":+ raise ImportError("spaCy native dependency blocked", name="spacy")+ import spacy # noqa: F401++ if after_spacy:+ after_spacy(spacy)+ if import_error:+ raise import_error+ if missing_dep:+ raise ModuleNotFoundError(+ f"No module named {missing_dep!r}", name=missing_dep+ )+ module.KModel = _FakeKModel+ module.KPipeline = kpipeline++ monkeypatch.setattr(sys, "meta_path", [Finder(), *sys.meta_path])+ # setitem records the original entry so it is restored after the test;+ # delitem then makes ``import kokoro`` go through the finder above.+ monkeypatch.setitem(sys.modules, "kokoro", None)+ monkeypatch.delitem(sys.modules, "kokoro")+++def test_kokoro_tolerates_unloadable_spacy_for_non_english_voices(monkeypatch):+ """A spaCy that cannot load (e.g. DLLs blocked by Windows Smart App+ Control) must not take down voices whose G2P never uses it."""+ import sys++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ pipelines = []++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ pipelines.append(lang_code)++ _serve_kokoro_importing_spacy(monkeypatch, FakeKPipeline)+ monkeypatch.setitem(sys.modules, "spacy", None) # import spacy -> ImportError++ KokoroTTSBackend(device="cpu")._ensure_pipeline("p")++ assert pipelines == ["p"]+ assert sys.modules["spacy"] is None # No fake global module after success.+ with pytest.raises(ModuleNotFoundError):+ __import__("spacy")+++def test_kokoro_degraded_import_gives_actionable_english_error(monkeypatch):+ import sys++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ _serve_kokoro_importing_spacy(monkeypatch, object)+ monkeypatch.setitem(sys.modules, "spacy", None)+ backend = KokoroTTSBackend(device="cpu")+ assert backend.health() is True+ with pytest.raises(RuntimeError, match="English Kokoro voices require.*spaCy"):+ backend._ensure_pipeline("b")+++def test_kokoro_restores_spacy_after_non_import_error(monkeypatch):+ import builtins+ import sys++ from openjarvis.speech.kokoro_tts import _import_kokoro++ _serve_kokoro_importing_spacy(+ monkeypatch, object, import_error=ValueError("bad model config")+ )+ monkeypatch.setitem(sys.modules, "spacy", None)+ original_import = builtins.__import__+ with pytest.raises(ValueError, match="bad model config"):+ _import_kokoro()+ assert sys.modules["spacy"] is None+ assert builtins.__import__ is original_import+++def test_kokoro_fallback_does_not_publish_spacy_to_other_threads(monkeypatch):+ """A plain import outside the fallback must see the real failed import."""+ import builtins+ import sys+ import threading++ from openjarvis.speech.kokoro_tts import _import_kokoro++ entered = threading.Event()+ release = threading.Event()+ loaded_shims = []+ retained_importers = []+ retained_import_errors = []+ errors = []++ def pause_fallback(spacy):+ loaded_shims.append(spacy)+ retained_importers.append(builtins.__import__)+ entered.set()+ if not release.wait(5):+ raise TimeoutError("test did not release Kokoro import")++ _serve_kokoro_importing_spacy(+ monkeypatch, object, block_spacy=True, after_spacy=pause_fallback+ )+ monkeypatch.setitem(sys.modules, "spacy", None)+ monkeypatch.delitem(sys.modules, "spacy")+ original_import = builtins.__import__++ def load():+ try:+ _import_kokoro()+ # Even a hook captured by third-party code must stop substituting+ # once the fallback ends, including on the original owner thread.+ try:+ retained_importers[0]("spacy")+ except ImportError as exc:+ retained_import_errors.append(exc)+ except Exception as exc:+ errors.append(exc)++ worker = threading.Thread(target=load)+ worker.start()+ try:+ assert entered.wait(5)+ # This thread does not acquire our Kokoro-only import lock.+ with pytest.raises(ImportError, match="spaCy native dependency blocked"):+ __import__("spacy")+ assert "spacy" not in sys.modules+ finally:+ release.set()+ worker.join(timeout=5)+ assert not worker.is_alive()+ assert not errors+ assert len(loaded_shims) == 1+ assert len(retained_import_errors) == 1+ assert builtins.__import__ is original_import+++def test_concurrent_kokoro_imports_do_not_leave_a_fake_spacy_module(monkeypatch):+ import sys+ import threading++ from openjarvis.speech.kokoro_tts import _import_kokoro++ _serve_kokoro_importing_spacy(monkeypatch, object, block_spacy=True)+ monkeypatch.setitem(sys.modules, "spacy", None)+ monkeypatch.delitem(sys.modules, "spacy")+ results = []+ errors = []++ def load():+ try:+ results.append(_import_kokoro())+ except Exception as exc:+ errors.append(exc)++ workers = [threading.Thread(target=load) for _ in range(8)]+ for worker in workers:+ worker.start()+ for worker in workers:+ worker.join(timeout=2)+ assert not worker.is_alive()+ assert not errors+ assert len(results) == 8+ assert len({id(result) for result in results}) == 1+ assert "spacy" not in sys.modules+ with pytest.raises(ImportError, match="spaCy native dependency blocked"):+ __import__("spacy")+++def test_kokoro_preserves_healthy_spacy_when_another_dependency_is_missing(+ monkeypatch,+):+ import sys+ import types++ from openjarvis.speech.kokoro_tts import _import_kokoro++ real_spacy = types.ModuleType("spacy")+ monkeypatch.setitem(sys.modules, "spacy", real_spacy)+ _serve_kokoro_importing_spacy(monkeypatch, object, missing_dep="torch")+ with pytest.raises(ModuleNotFoundError, match="torch"):+ _import_kokoro()+ assert sys.modules["spacy"] is real_spacy+++def test_kokoro_health_and_pipeline_share_one_model(monkeypatch):+ import sys+ import threading+ import time+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ models = []+ errors = []++ class SlowModel(_FakeKModel):+ def __init__(self):+ models.append(self)+ time.sleep(0.02)++ monkeypatch.setitem(+ sys.modules,+ "kokoro",+ types.SimpleNamespace(+ KModel=SlowModel,+ KPipeline=lambda **kwargs: kwargs["model"],+ ),+ )+ backend = KokoroTTSBackend(device="cpu")++ def load(index):+ try:+ if index % 2:+ assert backend.health()+ else:+ backend._ensure_pipeline("p")+ except Exception as exc:+ errors.append(exc)++ workers = [threading.Thread(target=load, args=(i,)) for i in range(8)]+ for worker in workers:+ worker.start()+ for worker in workers:+ worker.join(timeout=2)+ assert not worker.is_alive()+ assert not errors+ assert len(models) == 1+ assert backend._pipelines["p"] is models[0]+++def test_kokoro_unloadable_spacy_stub_is_removed_if_kokoro_still_fails(+ monkeypatch,+):+ """When something besides spaCy breaks the import, the stub must not+ linger in ``sys.modules`` and the backend stays unhealthy."""+ import sys++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ _serve_kokoro_importing_spacy(monkeypatch, object, missing_dep="torch")+ monkeypatch.setitem(sys.modules, "spacy", None)++ assert KokoroTTSBackend(device="cpu").health() is False+ assert sys.modules.get("spacy") is None+++def test_kokoro_missing_package_leaves_spacy_alone(monkeypatch):+ """No kokoro at all is a plain 'not installed' -- spaCy is not touched."""+ import sys++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ monkeypatch.setitem(sys.modules, "kokoro", None)+ monkeypatch.setitem(sys.modules, "spacy", None)++ assert KokoroTTSBackend().health() is False+ assert sys.modules["spacy"] is None+++def test_kokoro_health_does_not_build_english_pipeline(monkeypatch):+ """health() checks the shared model only: a broken English G2P (spaCy ++ en_core_web_sm) must not mark every other language unhealthy."""+ import sys+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ pipelines = []++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ pipelines.append(lang_code)++ monkeypatch.setitem(+ sys.modules,+ "kokoro",+ types.SimpleNamespace(KModel=_FakeKModel, KPipeline=FakeKPipeline),+ )++ assert KokoroTTSBackend(device="cpu").health() is True+ assert pipelines == []++# ---------------------------------------------------------------------------# OpenAI TTS backend tests# ---------------------------------------------------------------------------