Personal AI, On Personal Devices
Python
feat(speech): multilingual Kokoro TTS — Mandarin / British / Japanese voices (#296)
TX-Huang1039a72f8adb
diff --git a/src/openjarvis/speech/kokoro_tts.py b/src/openjarvis/speech/kokoro_tts.pyindex 23c1028f..b222c66e 100644--- a/src/openjarvis/speech/kokoro_tts.py+++ b/src/openjarvis/speech/kokoro_tts.py@@ -2,58 +2,209 @@Requires the kokoro package: pip install kokoroFalls back gracefully if not installed.++Kokoro v1.x supports multiple languages. Each language has its own+``KPipeline`` (the model loads language-specific G2P resources at init).+This backend lazily creates one pipeline per language code, derived from+the voice ID prefix per Kokoro's naming convention+``{lang_prefix}{gender}_{name}`` — e.g. ``zf_xiaoxiao`` is Mandarin+female (prefix ``z`` → ``lang_code="z"``)."""from __future__ import annotationsimport io-from typing import List+import threading+from collections import OrderedDict+from typing import Any, Dict, Listfrom openjarvis.core.registry import TTSRegistryfrom openjarvis.speech.tts import TTSBackend, TTSResult+# Kokoro's voice-prefix → ``lang_code`` mapping. Keep this in sync with+# ``kokoro.pipeline.LANG_CODES``: Kokoro 0.9.x does not expose a Korean+# pipeline, even though Korean voice files have appeared in some catalogs.+_VOICE_PREFIX_TO_LANG: Dict[str, str] = {+ "a": "a", # American English+ "b": "b", # British English+ "z": "z", # Mandarin Chinese+ "j": "j", # Japanese+ "f": "f", # French+ "i": "i", # Italian+ "p": "p", # Brazilian Portuguese+ "h": "h", # Hindi+ "e": "e", # Spanish+}++_DEFAULT_LANG_CODE = "a"+_DEFAULT_VOICE_ID = "af_heart"+@TTSRegistry.register("kokoro")class KokoroTTSBackend(TTSBackend):- """Kokoro TTS — local open-source voice synthesis."""+ """Kokoro TTS — local open-source voice synthesis, multilingual."""backend_id = "kokoro"- def __init__(self, *, model_path: str = "", device: str = "auto") -> None:+ def __init__(+ self,+ *,+ model_path: str = "",+ device: str = "auto",+ max_cached_pipelines: int = 4,+ ) -> None:+ if max_cached_pipelines < 1:+ raise ValueError("max_cached_pipelines must be at least 1")self._model_path = model_pathself._device = device- self._pipeline = None+ self._max_cached_pipelines = max_cached_pipelines+ self._model: Any | None = None+ self._pipelines: OrderedDict[str, Any] = OrderedDict()+ # Pipeline creation and inference are both guarded: KPipeline is not+ # documented as thread-safe, and an LRU entry must not be evicted and+ # closed while another thread is still synthesizing with it.+ self._pipeline_lock = threading.RLock()- def _ensure_pipeline(self) -> None:- if self._pipeline is not None:- return- try:- from kokoro import KPipeline+ @staticmethod+ def _lang_for_voice(voice_id: str) -> str:+ """Return the Kokoro ``lang_code`` for a voice ID.++ Voice IDs follow the convention ``{lang}{gender}_{name}`` where+ ``lang`` is a single letter prefix (e.g. ``z`` for Mandarin in+ ``zf_xiaoxiao``). Falls back to American English if the prefix is+ unknown.+ """+ if not voice_id:+ return _DEFAULT_LANG_CODE+ return _VOICE_PREFIX_TO_LANG.get(voice_id[:1], _DEFAULT_LANG_CODE)++ def _ensure_pipeline(self, lang_code: str) -> Any:+ """Lazily create and retain a bounded LRU of language pipelines."""+ with self._pipeline_lock:+ pipeline = self._pipelines.get(lang_code)+ if pipeline is not None:+ self._pipelines.move_to_end(lang_code)+ return pipeline+ try:+ from kokoro import KPipeline+ except ImportError as exc:+ raise RuntimeError(+ "kokoro package not installed. Install with: pip install kokoro"+ ) from exc+ try:+ # KModel is language-blind and is by far the heaviest part of+ # Kokoro. Reuse one model across every language pipeline.+ pipeline = KPipeline(lang_code=lang_code, model=self._ensure_model())+ except ImportError as exc:+ # Kokoro loads language-specific G2P resources at init time and+ # several languages need extra dependencies that aren't pulled in+ # by the base ``kokoro`` package. The known cases:+ # z (Mandarin) → misaki.zh (jieba / pypinyin / ordered-set)+ # j (Japanese) → misaki.ja (fugashi / unidic-lite)+ # Translate the cryptic ``ordered_set``/``fugashi`` ImportError+ # into a one-line install hint so the user isn't left guessing.+ hints = {+ "z": 'pip install "misaki[zh]"',+ "j": 'pip install "misaki[ja]"',+ }+ hint = hints.get(+ lang_code,+ f'extra phoneme deps for lang_code="{lang_code}" (see Kokoro docs)',+ )+ raise RuntimeError(+ f"Kokoro lang_code={lang_code!r} requires extra dependencies. "+ f"Try: {hint}. Original error: {exc}"+ ) from exc++ self._pipelines[lang_code] = pipeline+ if len(self._pipelines) > self._max_cached_pipelines:+ _, evicted = self._pipelines.popitem(last=False)+ self._cleanup_pipeline(evicted)+ return pipeline++ def _ensure_model(self) -> Any:+ """Load the configured Kokoro model once and move it to the device."""+ if self._model is not None:+ return self._model- self._pipeline = KPipeline(lang_code="a")- except ImportError:+ try:+ from kokoro import KModel+ except ImportError as exc:raise RuntimeError("kokoro package not installed. Install with: pip install kokoro"- )+ ) from exc++ device = self._device+ if device == "auto":+ # A real Kokoro install always brings torch. Keeping this fallback+ # makes dependency probes and lightweight mocked tests graceful.+ try:+ import torch+ except ImportError:+ device = "cpu"+ else:+ if torch.cuda.is_available():+ device = "cuda"+ elif getattr(torch.backends, "mps", None) is not None and (+ torch.backends.mps.is_available()+ ):+ device = "mps"+ else:+ device = "cpu"++ model_kwargs = {"model": self._model_path} if self._model_path else {}+ try:+ self._model = KModel(**model_kwargs).to(device).eval()+ except RuntimeError as exc:+ raise RuntimeError(+ f"Failed to initialize Kokoro on device {device!r}: {exc}"+ ) from exc+ return self._model++ @staticmethod+ def _cleanup_pipeline(pipeline: Any) -> None:+ """Release resources exposed by a pipeline implementation, if any."""+ close = getattr(pipeline, "close", None)+ if callable(close):+ try:+ close()+ except Exception: # pragma: no cover - third-party cleanup is best effort+ pass++ def close(self) -> None:+ """Release every cached pipeline and clear the cache."""+ with self._pipeline_lock:+ pipelines = list(self._pipelines.values())+ self._pipelines.clear()+ for pipeline in pipelines:+ self._cleanup_pipeline(pipeline)+ self._model = Nonedef synthesize(self,text: str,*,- voice_id: str = "af_heart",+ voice_id: str = _DEFAULT_VOICE_ID,speed: float = 1.0,output_format: str = "wav",) -> TTSResult:- self._ensure_pipeline()import numpy as npimport soundfile as sf- samples = []- for _, _, audio in self._pipeline(text, voice=voice_id, speed=speed):- samples.append(audio)+ lang_code = self._lang_for_voice(voice_id)+ with self._pipeline_lock:+ pipeline = self._ensure_pipeline(lang_code)+ samples = []+ for _, _, audio in pipeline(text, voice=voice_id, speed=speed):+ samples.append(audio)if not samples:- return TTSResult(audio=b"", format=output_format, voice_id=voice_id)+ return TTSResult(+ audio=b"",+ format=output_format,+ voice_id=voice_id,+ metadata={"backend": "kokoro", "lang_code": lang_code},+ )combined = np.concatenate(samples)buf = io.BytesIO()@@ -66,15 +217,46 @@ class KokoroTTSBackend(TTSBackend):voice_id=voice_id,sample_rate=24000,duration_seconds=len(combined) / 24000,- metadata={"backend": "kokoro"},+ metadata={"backend": "kokoro", "lang_code": lang_code},)def available_voices(self) -> List[str]:- return ["af_heart", "af_bella", "am_adam", "am_michael"]+ # Curated subset of Kokoro v1.x voices. The full catalog is larger;+ # the list here covers the languages OpenJarvis users most commonly+ # ask for and avoids voice IDs that have changed across Kokoro+ # releases.+ return [+ # American English+ "af_heart",+ "af_bella",+ "af_nicole",+ "af_sarah",+ "af_sky",+ "am_adam",+ "am_michael",+ # British English+ "bf_emma",+ "bf_isabella",+ "bm_george",+ "bm_lewis",+ # Mandarin Chinese+ "zf_xiaobei",+ "zf_xiaoni",+ "zf_xiaoxiao",+ "zf_xiaoyi",+ "zm_yunjian",+ "zm_yunxi",+ "zm_yunxia",+ "zm_yunyang",+ # Japanese+ "jf_alpha",+ "jf_gongitsune",+ "jm_kumo",+ ]def health(self) -> bool:try:- self._ensure_pipeline()+ self._ensure_pipeline(_DEFAULT_LANG_CODE)return Trueexcept RuntimeError:return Falsediff --git a/tests/speech/test_tts_backends.py b/tests/speech/test_tts_backends.pyindex e1fd56c9..b5819174 100644--- a/tests/speech/test_tts_backends.py+++ b/tests/speech/test_tts_backends.py@@ -79,6 +79,316 @@ def test_kokoro_health_false_without_package():assert backend.health() is False+def test_kokoro_lang_for_voice_mandarin():+ """Mandarin voice IDs (zf_*, zm_*) map to lang_code='z'."""+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ assert KokoroTTSBackend._lang_for_voice("zf_xiaoxiao") == "z"+ assert KokoroTTSBackend._lang_for_voice("zm_yunxi") == "z"+++def test_kokoro_lang_for_voice_english_and_other_languages():+ """Voice prefix detection covers English variants and other languages."""+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ assert KokoroTTSBackend._lang_for_voice("af_heart") == "a"+ assert KokoroTTSBackend._lang_for_voice("am_adam") == "a"+ assert KokoroTTSBackend._lang_for_voice("bf_emma") == "b"+ assert KokoroTTSBackend._lang_for_voice("jf_alpha") == "j"+ # Kokoro 0.9.x has no Korean KPipeline language code. Do not route a+ # catalog-style Korean voice ID to an unsupported ``lang_code='k'``.+ assert KokoroTTSBackend._lang_for_voice("kf_example") == "a"+++def test_kokoro_lang_for_voice_unknown_falls_back_to_english():+ """Unknown prefixes fall back to American English (Kokoro's most-stocked+ language) rather than crashing — keeps backward-compat for any+ user-supplied voice ID we haven't catalogued."""+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ assert KokoroTTSBackend._lang_for_voice("") == "a"+ assert KokoroTTSBackend._lang_for_voice("xx_unknown") == "a"+++def test_kokoro_available_voices_includes_mandarin_and_english():+ """Voice catalog must list at least one Mandarin voice and preserve+ the original English voices for backward compatibility."""+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ voices = KokoroTTSBackend().available_voices()+ # New Mandarin coverage (the focus of this change)+ for v in ("zf_xiaoxiao", "zf_xiaoyi", "zm_yunxi", "zm_yunjian"):+ assert v in voices, f"Mandarin voice {v} missing from catalog"+ # Backward compat — voices the previous implementation listed+ for v in ("af_heart", "af_bella", "am_adam", "am_michael"):+ assert v in voices, f"Existing voice {v} dropped from catalog"+++def test_kokoro_missing_chinese_deps_gives_actionable_error(monkeypatch):+ """When Mandarin G2P deps are missing, the user must see an install hint+ pointing at ``misaki[zh]`` rather than a raw ``ordered_set`` ImportError."""+ import sys+ import types++ import pytest++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ # Mimic what happens inside Kokoro when misaki.zh can't import.+ raise ImportError("No module named 'ordered_set'")++ class FakeKModel:+ def to(self, _device):+ return self++ def eval(self):+ return self++ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)++ backend = KokoroTTSBackend()+ with pytest.raises(RuntimeError) as exc_info:+ backend._ensure_pipeline("z")+ msg = str(exc_info.value)+ assert "misaki[zh]" in msg+ assert "lang_code='z'" in msg or 'lang_code="z"' in msg+++def test_kokoro_pipeline_cached_per_language(monkeypatch):+ """Each lang_code instantiates one pipeline; same lang reuses the cache."""+ import sys+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ init_calls = []++ class FakeKModel:+ def __init__(self, **kwargs):+ self.kwargs = kwargs++ def to(self, _device):+ return self++ def eval(self):+ return self++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ init_calls.append(lang_code)+ self.model = model++ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)++ backend = KokoroTTSBackend()+ backend._ensure_pipeline("a")+ backend._ensure_pipeline("z")+ backend._ensure_pipeline("a") # cache hit — must NOT re-instantiate+ backend._ensure_pipeline("z") # cache hit++ assert init_calls == ["a", "z"]+++def test_kokoro_pipeline_cache_is_thread_safe(monkeypatch):+ """Concurrent requests for one language construct exactly one pipeline."""+ import sys+ import threading+ import time+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ init_calls = []++ class FakeKModel:+ def to(self, _device):+ return self++ def eval(self):+ return self++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ init_calls.append(lang_code)+ time.sleep(0.02)++ monkeypatch.setitem(+ sys.modules,+ "kokoro",+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),+ )+ backend = KokoroTTSBackend()+ pipelines = []++ def load_pipeline():+ pipelines.append(backend._ensure_pipeline("z"))++ threads = [threading.Thread(target=load_pipeline) for _ in range(8)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(timeout=2)+ assert not thread.is_alive()++ assert init_calls == ["z"]+ assert len({id(pipeline) for pipeline in pipelines}) == 1+++def test_kokoro_pipeline_cache_evicts_lru_and_cleans_up(monkeypatch):+ """The cache is bounded and closes evicted and explicitly released entries."""+ import sys+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ instances = {}++ class FakeKModel:+ def to(self, _device):+ return self++ def eval(self):+ return self++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ self.closed = False+ instances[lang_code] = self++ def close(self):+ self.closed = True++ monkeypatch.setitem(+ sys.modules,+ "kokoro",+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),+ )+ backend = KokoroTTSBackend(max_cached_pipelines=2)++ backend._ensure_pipeline("a")+ backend._ensure_pipeline("z")+ backend._ensure_pipeline("a") # make American English most recently used+ backend._ensure_pipeline("j")++ assert list(backend._pipelines) == ["a", "j"]+ assert instances["z"].closed is True+ assert instances["a"].closed is False+ assert instances["j"].closed is False++ backend.close()+ assert not backend._pipelines+ assert instances["a"].closed is True+ assert instances["j"].closed is True+++def test_kokoro_pipeline_cache_size_must_be_positive():+ import pytest++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ with pytest.raises(ValueError, match="at least 1"):+ KokoroTTSBackend(max_cached_pipelines=0)+++def test_kokoro_synthesize_routes_voice_to_correct_language(monkeypatch):+ """synthesize() with a Mandarin voice initializes a Mandarin pipeline+ and tags the result metadata with the resolved lang_code."""+ import sys+ import types++ import numpy as np++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ init_args = []+ call_args = []++ class FakeKModel:+ def to(self, device):+ self.device = device+ return self++ def eval(self):+ return self++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ init_args.append(lang_code)+ self.model = model++ def __call__(self, text, voice, speed):+ call_args.append({"text": text, "voice": voice, "speed": speed})+ yield (None, None, np.zeros(2400, dtype=np.float32))++ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)++ # ``soundfile`` is a kokoro-runtime dep; stub it so we can test the+ # routing logic without requiring it in the OpenJarvis test env.+ def _fake_sf_write(buf, _samples, _sr, format=None): # noqa: A002+ buf.write(b"FAKE_AUDIO_BYTES")++ fake_soundfile = types.SimpleNamespace(write=_fake_sf_write)+ monkeypatch.setitem(sys.modules, "soundfile", fake_soundfile)++ backend = KokoroTTSBackend()+ result = backend.synthesize("你好世界", voice_id="zf_xiaoxiao")++ assert init_args == ["z"]+ assert call_args[0]["voice"] == "zf_xiaoxiao"+ assert call_args[0]["text"] == "你好世界"+ assert result.voice_id == "zf_xiaoxiao"+ assert result.metadata["lang_code"] == "z"+ assert result.metadata["backend"] == "kokoro"+ assert result.format == "wav"+ assert len(result.audio) > 0+++def test_kokoro_reuses_model_and_honors_path_and_device(monkeypatch):+ """All language pipelines share the configured model instance."""+ import sys+ import types++ from openjarvis.speech.kokoro_tts import KokoroTTSBackend++ model_inits = []+ model_devices = []+ pipeline_models = []++ class FakeKModel:+ def __init__(self, **kwargs):+ model_inits.append(kwargs)++ def to(self, device):+ model_devices.append(device)+ return self++ def eval(self):+ return self++ class FakeKPipeline:+ def __init__(self, lang_code, model):+ pipeline_models.append((lang_code, model))++ monkeypatch.setitem(+ sys.modules,+ "kokoro",+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),+ )++ backend = KokoroTTSBackend(model_path="/models/kokoro.pth", device="cpu")+ backend._ensure_pipeline("a")+ backend._ensure_pipeline("z")++ assert model_inits == [{"model": "/models/kokoro.pth"}]+ assert model_devices == ["cpu"]+ assert pipeline_models[0][1] is pipeline_models[1][1]++# ---------------------------------------------------------------------------# OpenAI TTS backend tests# ---------------------------------------------------------------------------