Skip to content

Personal AI, On Personal Devices

Python

fix(speech): honor language and isolate the Kokoro fallback (#1086)

AgenciaAlliera8d1ef909f56
diff --git a/src/openjarvis/cli/_voice_chat.py b/src/openjarvis/cli/_voice_chat.py
index b4623080..73f5bb7f 100644
--- a/src/openjarvis/cli/_voice_chat.py
+++ b/src/openjarvis/cli/_voice_chat.py
@@ -58,6 +58,13 @@ class VoiceSession:
self._stt_resolved = True
return self._stt_backend
+ def get_stt_language(self) -> Optional[str]:
+ """Return the configured ``speech.language``, or None to auto-detect."""
+ from openjarvis.core.config import load_config
+
+ config = self._config if self._config is not None else load_config()
+ return (getattr(config.speech, "language", "") or "").strip() or None
+
def get_tts_backend(self) -> Any:
"""Return a cached healthy TTS backend, falling through once per key."""
if self._tts_backend is not None:
@@ -152,7 +159,9 @@ def record_voice(
console.print("[dim]Transcribing…[/dim]")
try:
- result = backend.transcribe(audio_bytes, format="wav")
+ result = backend.transcribe(
+ audio_bytes, format="wav", language=active_session.get_stt_language()
+ )
text = result.text.strip()
if text:
console.print(f"[bold]You (voice):[/bold] {_terminal_safe_text(text)}")
diff --git a/src/openjarvis/speech/kokoro_tts.py b/src/openjarvis/speech/kokoro_tts.py
index b222c66e..6c553f61 100644
--- a/src/openjarvis/speech/kokoro_tts.py
+++ b/src/openjarvis/speech/kokoro_tts.py
@@ -13,14 +13,20 @@ female (prefix ``z`` → ``lang_code="z"``).
from __future__ import annotations
+import builtins
import io
+import logging
+import sys
import threading
+import types
from collections import OrderedDict
from typing import Any, Dict, List
from openjarvis.core.registry import TTSRegistry
from openjarvis.speech.tts import TTSBackend, TTSResult
+logger = logging.getLogger(__name__)
+
# Kokoro's voice-prefix → ``lang_code`` mapping. Keep this in sync with
# ``kokoro.pipeline.LANG_CODES``: Kokoro 0.9.x does not expose a Korean
# pipeline, even though Korean voice files have appeared in some catalogs.
@@ -38,6 +44,65 @@ _VOICE_PREFIX_TO_LANG: Dict[str, str] = {
_DEFAULT_LANG_CODE = "a"
_DEFAULT_VOICE_ID = "af_heart"
+_IMPORT_LOCK = threading.RLock()
+
+
+def _import_kokoro() -> Any:
+ """Import ``kokoro``, tolerating a spaCy install that cannot be loaded.
+
+ ``kokoro.pipeline`` imports ``misaki.en`` -- and with it spaCy -- for every
+ language, but only the English G2P (voices ``a*``/``b*``) uses spaCy.
+ Windows Smart App Control, for one, can refuse to load spaCy's unsigned
+ extension DLLs, which would otherwise take every Kokoro voice down with
+ it. When spaCy is the culprit, stub it so espeak-based voices (e.g.
+ ``pf_dora``) keep working; English pipelines then fail on first use.
+ """
+ # Serialize hook installation across backend instances. The replacement
+ # module is visible only to this thread, never through sys.modules.
+ with _IMPORT_LOCK:
+ try:
+ import kokoro
+ except ImportError as exc:
+ if exc.name == "kokoro" or sys.modules.get("spacy") is not None:
+ raise
+ try:
+ import spacy # noqa: F401
+ except ImportError as spacy_exc:
+ spacy_error = spacy_exc
+ else:
+ raise exc from None
+ shim = types.ModuleType("spacy")
+ owner = threading.get_ident()
+ original_import = builtins.__import__
+ active = True
+
+ def fallback_import(name, globals=None, locals=None, fromlist=(), level=0):
+ if (
+ active
+ and threading.get_ident() == owner
+ and name == "spacy"
+ and level == 0
+ ):
+ return shim
+ return original_import(name, globals, locals, fromlist, level)
+
+ builtins.__import__ = fallback_import
+ try:
+ import kokoro
+ finally:
+ # misaki.en may retain the private shim for non-English use,
+ # while unrelated threads always use the original importer.
+ # Deactivate even if another hook retained this wrapper.
+ active = False
+ if builtins.__import__ is fallback_import:
+ builtins.__import__ = original_import
+ kokoro._openjarvis_spacy_error = str(spacy_error)
+ logger.warning(
+ "spaCy unavailable (%s); English Kokoro voices disabled, "
+ "other languages still work",
+ spacy_error,
+ )
+ return kokoro
@TTSRegistry.register("kokoro")
@@ -86,11 +151,19 @@ class KokoroTTSBackend(TTSBackend):
self._pipelines.move_to_end(lang_code)
return pipeline
try:
- from kokoro import KPipeline
+ kokoro = _import_kokoro()
+ KPipeline = kokoro.KPipeline
except ImportError as exc:
raise RuntimeError(
"kokoro package not installed. Install with: pip install kokoro"
) from exc
+ if lang_code in {"a", "b"} and getattr(
+ kokoro, "_openjarvis_spacy_error", None
+ ):
+ raise RuntimeError(
+ "English Kokoro voices require a working spaCy installation: "
+ f"{kokoro._openjarvis_spacy_error}"
+ )
try:
# KModel is language-blind and is by far the heaviest part of
# Kokoro. Reuse one model across every language pipeline.
@@ -128,7 +201,7 @@ class KokoroTTSBackend(TTSBackend):
return self._model
try:
- from kokoro import KModel
+ KModel = _import_kokoro().KModel
except ImportError as exc:
raise RuntimeError(
"kokoro package not installed. Install with: pip install kokoro"
@@ -255,8 +328,13 @@ class KokoroTTSBackend(TTSBackend):
]
def health(self) -> bool:
- try:
- self._ensure_pipeline(_DEFAULT_LANG_CODE)
- return True
- except RuntimeError:
- return False
+ # Check the shared model, not the English pipeline: language-specific
+ # G2P problems surface at synthesis time for the voice actually used.
+ # Use the same lock as synthesis so concurrent health probes cannot
+ # allocate duplicate model weights or race with close().
+ with self._pipeline_lock:
+ try:
+ self._ensure_model()
+ return True
+ except RuntimeError:
+ return False
diff --git a/tests/cli/test_chat_cmd.py b/tests/cli/test_chat_cmd.py
index 458d6803..8c6d7509 100644
--- a/tests/cli/test_chat_cmd.py
+++ b/tests/cli/test_chat_cmd.py
@@ -199,6 +199,35 @@ class TestVoiceInput:
discover.assert_called_once()
assert backend.transcribe.call_count == 2
+ @pytest.mark.parametrize(
+ ("language", "expected"),
+ [("pt", "pt"), (" pt ", "pt"), ("", None), (" ", None)],
+ )
+ def test_configured_speech_language_is_passed_to_stt(
+ self, language: str, expected: str | None
+ ) -> None:
+ backend = MagicMock()
+ backend.transcribe.return_value = SimpleNamespace(text="olá")
+ config = JarvisConfig()
+ config.speech.language = language
+ session = VoiceSession(config)
+
+ with (
+ patch(
+ "openjarvis.speech._discovery.get_speech_backend",
+ return_value=backend,
+ ),
+ patch(
+ "openjarvis.speech.voice_io.record_until_silence",
+ return_value=b"wav",
+ ),
+ ):
+ assert record_voice(MagicMock(), session) == "olá"
+
+ backend.transcribe.assert_called_once_with(
+ b"wav", format="wav", language=expected
+ )
+
def test_voice_transcript_cannot_inject_rich_hyperlink(self) -> None:
backend = MagicMock()
backend.transcribe.return_value = SimpleNamespace(
diff --git a/tests/speech/test_tts_backends.py b/tests/speech/test_tts_backends.py
index fc9c217c..11f9b511 100644
--- a/tests/speech/test_tts_backends.py
+++ b/tests/speech/test_tts_backends.py
@@ -74,16 +74,16 @@ def test_kokoro_registered():
def test_kokoro_health_false_without_package():
+ import importlib.util
+
from openjarvis.speech.kokoro_tts import KokoroTTSBackend
# The assertion only holds when kokoro is genuinely absent. With the
# optional `voice` extra installed, health() legitimately returns True
- # (and warming the pipeline would hit the HF Hub), so skip instead.
- try:
- import kokoro # noqa: F401
- except ImportError:
- pass
- else:
+ # (and loading the model would hit the HF Hub), so skip instead. Check
+ # for the package without importing it: an installed kokoro whose spaCy
+ # cannot load still counts as installed.
+ if importlib.util.find_spec("kokoro") is not None:
pytest.skip("kokoro installed (openjarvis[voice]); health() is True")
backend = KokoroTTSBackend()
@@ -401,6 +401,321 @@ def test_kokoro_reuses_model_and_honors_path_and_device(monkeypatch):
assert pipeline_models[0][1] is pipeline_models[1][1]
+class _FakeKModel:
+ def to(self, _device):
+ return self
+
+ def eval(self):
+ return self
+
+
+def _serve_kokoro_importing_spacy(
+ monkeypatch,
+ kpipeline,
+ *,
+ missing_dep=None,
+ import_error=None,
+ block_spacy=False,
+ after_spacy=None,
+):
+ """Serve a fake ``kokoro`` that imports spaCy on load, like the real one
+ (``kokoro.pipeline`` -> ``misaki.en`` -> ``spacy``)."""
+ import importlib.abc
+ import importlib.util
+ import sys
+
+ class Finder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
+ def find_spec(self, name, path=None, target=None):
+ if name != "kokoro" and not (block_spacy and name == "spacy"):
+ return None
+ return importlib.util.spec_from_loader(name, self)
+
+ def create_module(self, spec):
+ return None
+
+ def exec_module(self, module):
+ if module.__name__ == "spacy":
+ raise ImportError("spaCy native dependency blocked", name="spacy")
+ import spacy # noqa: F401
+
+ if after_spacy:
+ after_spacy(spacy)
+ if import_error:
+ raise import_error
+ if missing_dep:
+ raise ModuleNotFoundError(
+ f"No module named {missing_dep!r}", name=missing_dep
+ )
+ module.KModel = _FakeKModel
+ module.KPipeline = kpipeline
+
+ monkeypatch.setattr(sys, "meta_path", [Finder(), *sys.meta_path])
+ # setitem records the original entry so it is restored after the test;
+ # delitem then makes ``import kokoro`` go through the finder above.
+ monkeypatch.setitem(sys.modules, "kokoro", None)
+ monkeypatch.delitem(sys.modules, "kokoro")
+
+
+def test_kokoro_tolerates_unloadable_spacy_for_non_english_voices(monkeypatch):
+ """A spaCy that cannot load (e.g. DLLs blocked by Windows Smart App
+ Control) must not take down voices whose G2P never uses it."""
+ import sys
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ pipelines = []
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ pipelines.append(lang_code)
+
+ _serve_kokoro_importing_spacy(monkeypatch, FakeKPipeline)
+ monkeypatch.setitem(sys.modules, "spacy", None) # import spacy -> ImportError
+
+ KokoroTTSBackend(device="cpu")._ensure_pipeline("p")
+
+ assert pipelines == ["p"]
+ assert sys.modules["spacy"] is None # No fake global module after success.
+ with pytest.raises(ModuleNotFoundError):
+ __import__("spacy")
+
+
+def test_kokoro_degraded_import_gives_actionable_english_error(monkeypatch):
+ import sys
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ _serve_kokoro_importing_spacy(monkeypatch, object)
+ monkeypatch.setitem(sys.modules, "spacy", None)
+ backend = KokoroTTSBackend(device="cpu")
+ assert backend.health() is True
+ with pytest.raises(RuntimeError, match="English Kokoro voices require.*spaCy"):
+ backend._ensure_pipeline("b")
+
+
+def test_kokoro_restores_spacy_after_non_import_error(monkeypatch):
+ import builtins
+ import sys
+
+ from openjarvis.speech.kokoro_tts import _import_kokoro
+
+ _serve_kokoro_importing_spacy(
+ monkeypatch, object, import_error=ValueError("bad model config")
+ )
+ monkeypatch.setitem(sys.modules, "spacy", None)
+ original_import = builtins.__import__
+ with pytest.raises(ValueError, match="bad model config"):
+ _import_kokoro()
+ assert sys.modules["spacy"] is None
+ assert builtins.__import__ is original_import
+
+
+def test_kokoro_fallback_does_not_publish_spacy_to_other_threads(monkeypatch):
+ """A plain import outside the fallback must see the real failed import."""
+ import builtins
+ import sys
+ import threading
+
+ from openjarvis.speech.kokoro_tts import _import_kokoro
+
+ entered = threading.Event()
+ release = threading.Event()
+ loaded_shims = []
+ retained_importers = []
+ retained_import_errors = []
+ errors = []
+
+ def pause_fallback(spacy):
+ loaded_shims.append(spacy)
+ retained_importers.append(builtins.__import__)
+ entered.set()
+ if not release.wait(5):
+ raise TimeoutError("test did not release Kokoro import")
+
+ _serve_kokoro_importing_spacy(
+ monkeypatch, object, block_spacy=True, after_spacy=pause_fallback
+ )
+ monkeypatch.setitem(sys.modules, "spacy", None)
+ monkeypatch.delitem(sys.modules, "spacy")
+ original_import = builtins.__import__
+
+ def load():
+ try:
+ _import_kokoro()
+ # Even a hook captured by third-party code must stop substituting
+ # once the fallback ends, including on the original owner thread.
+ try:
+ retained_importers[0]("spacy")
+ except ImportError as exc:
+ retained_import_errors.append(exc)
+ except Exception as exc:
+ errors.append(exc)
+
+ worker = threading.Thread(target=load)
+ worker.start()
+ try:
+ assert entered.wait(5)
+ # This thread does not acquire our Kokoro-only import lock.
+ with pytest.raises(ImportError, match="spaCy native dependency blocked"):
+ __import__("spacy")
+ assert "spacy" not in sys.modules
+ finally:
+ release.set()
+ worker.join(timeout=5)
+ assert not worker.is_alive()
+ assert not errors
+ assert len(loaded_shims) == 1
+ assert len(retained_import_errors) == 1
+ assert builtins.__import__ is original_import
+
+
+def test_concurrent_kokoro_imports_do_not_leave_a_fake_spacy_module(monkeypatch):
+ import sys
+ import threading
+
+ from openjarvis.speech.kokoro_tts import _import_kokoro
+
+ _serve_kokoro_importing_spacy(monkeypatch, object, block_spacy=True)
+ monkeypatch.setitem(sys.modules, "spacy", None)
+ monkeypatch.delitem(sys.modules, "spacy")
+ results = []
+ errors = []
+
+ def load():
+ try:
+ results.append(_import_kokoro())
+ except Exception as exc:
+ errors.append(exc)
+
+ workers = [threading.Thread(target=load) for _ in range(8)]
+ for worker in workers:
+ worker.start()
+ for worker in workers:
+ worker.join(timeout=2)
+ assert not worker.is_alive()
+ assert not errors
+ assert len(results) == 8
+ assert len({id(result) for result in results}) == 1
+ assert "spacy" not in sys.modules
+ with pytest.raises(ImportError, match="spaCy native dependency blocked"):
+ __import__("spacy")
+
+
+def test_kokoro_preserves_healthy_spacy_when_another_dependency_is_missing(
+ monkeypatch,
+):
+ import sys
+ import types
+
+ from openjarvis.speech.kokoro_tts import _import_kokoro
+
+ real_spacy = types.ModuleType("spacy")
+ monkeypatch.setitem(sys.modules, "spacy", real_spacy)
+ _serve_kokoro_importing_spacy(monkeypatch, object, missing_dep="torch")
+ with pytest.raises(ModuleNotFoundError, match="torch"):
+ _import_kokoro()
+ assert sys.modules["spacy"] is real_spacy
+
+
+def test_kokoro_health_and_pipeline_share_one_model(monkeypatch):
+ import sys
+ import threading
+ import time
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ models = []
+ errors = []
+
+ class SlowModel(_FakeKModel):
+ def __init__(self):
+ models.append(self)
+ time.sleep(0.02)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "kokoro",
+ types.SimpleNamespace(
+ KModel=SlowModel,
+ KPipeline=lambda **kwargs: kwargs["model"],
+ ),
+ )
+ backend = KokoroTTSBackend(device="cpu")
+
+ def load(index):
+ try:
+ if index % 2:
+ assert backend.health()
+ else:
+ backend._ensure_pipeline("p")
+ except Exception as exc:
+ errors.append(exc)
+
+ workers = [threading.Thread(target=load, args=(i,)) for i in range(8)]
+ for worker in workers:
+ worker.start()
+ for worker in workers:
+ worker.join(timeout=2)
+ assert not worker.is_alive()
+ assert not errors
+ assert len(models) == 1
+ assert backend._pipelines["p"] is models[0]
+
+
+def test_kokoro_unloadable_spacy_stub_is_removed_if_kokoro_still_fails(
+ monkeypatch,
+):
+ """When something besides spaCy breaks the import, the stub must not
+ linger in ``sys.modules`` and the backend stays unhealthy."""
+ import sys
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ _serve_kokoro_importing_spacy(monkeypatch, object, missing_dep="torch")
+ monkeypatch.setitem(sys.modules, "spacy", None)
+
+ assert KokoroTTSBackend(device="cpu").health() is False
+ assert sys.modules.get("spacy") is None
+
+
+def test_kokoro_missing_package_leaves_spacy_alone(monkeypatch):
+ """No kokoro at all is a plain 'not installed' -- spaCy is not touched."""
+ import sys
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ monkeypatch.setitem(sys.modules, "kokoro", None)
+ monkeypatch.setitem(sys.modules, "spacy", None)
+
+ assert KokoroTTSBackend().health() is False
+ assert sys.modules["spacy"] is None
+
+
+def test_kokoro_health_does_not_build_english_pipeline(monkeypatch):
+ """health() checks the shared model only: a broken English G2P (spaCy +
+ en_core_web_sm) must not mark every other language unhealthy."""
+ import sys
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ pipelines = []
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ pipelines.append(lang_code)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "kokoro",
+ types.SimpleNamespace(KModel=_FakeKModel, KPipeline=FakeKPipeline),
+ )
+
+ assert KokoroTTSBackend(device="cpu").health() is True
+ assert pipelines == []
+
+
# ---------------------------------------------------------------------------
# OpenAI TTS backend tests
# ---------------------------------------------------------------------------