Skip to content

Personal AI, On Personal Devices

Python

feat(speech): multilingual Kokoro TTS — Mandarin / British / Japanese voices (#296)

TX-Huang1039a72f8adb
diff --git a/src/openjarvis/speech/kokoro_tts.py b/src/openjarvis/speech/kokoro_tts.py
index 23c1028f..b222c66e 100644
--- a/src/openjarvis/speech/kokoro_tts.py
+++ b/src/openjarvis/speech/kokoro_tts.py
@@ -2,58 +2,209 @@
Requires the kokoro package: pip install kokoro
Falls back gracefully if not installed.
+
+Kokoro v1.x supports multiple languages. Each language has its own
+``KPipeline`` (the model loads language-specific G2P resources at init).
+This backend lazily creates one pipeline per language code, derived from
+the voice ID prefix per Kokoro's naming convention
+``{lang_prefix}{gender}_{name}`` — e.g. ``zf_xiaoxiao`` is Mandarin
+female (prefix ``z`` → ``lang_code="z"``).
"""
from __future__ import annotations
import io
-from typing import List
+import threading
+from collections import OrderedDict
+from typing import Any, Dict, List
from openjarvis.core.registry import TTSRegistry
from openjarvis.speech.tts import TTSBackend, TTSResult
+# Kokoro's voice-prefix → ``lang_code`` mapping. Keep this in sync with
+# ``kokoro.pipeline.LANG_CODES``: Kokoro 0.9.x does not expose a Korean
+# pipeline, even though Korean voice files have appeared in some catalogs.
+_VOICE_PREFIX_TO_LANG: Dict[str, str] = {
+ "a": "a", # American English
+ "b": "b", # British English
+ "z": "z", # Mandarin Chinese
+ "j": "j", # Japanese
+ "f": "f", # French
+ "i": "i", # Italian
+ "p": "p", # Brazilian Portuguese
+ "h": "h", # Hindi
+ "e": "e", # Spanish
+}
+
+_DEFAULT_LANG_CODE = "a"
+_DEFAULT_VOICE_ID = "af_heart"
+
@TTSRegistry.register("kokoro")
class KokoroTTSBackend(TTSBackend):
- """Kokoro TTS — local open-source voice synthesis."""
+ """Kokoro TTS — local open-source voice synthesis, multilingual."""
backend_id = "kokoro"
- def __init__(self, *, model_path: str = "", device: str = "auto") -> None:
+ def __init__(
+ self,
+ *,
+ model_path: str = "",
+ device: str = "auto",
+ max_cached_pipelines: int = 4,
+ ) -> None:
+ if max_cached_pipelines < 1:
+ raise ValueError("max_cached_pipelines must be at least 1")
self._model_path = model_path
self._device = device
- self._pipeline = None
+ self._max_cached_pipelines = max_cached_pipelines
+ self._model: Any | None = None
+ self._pipelines: OrderedDict[str, Any] = OrderedDict()
+ # Pipeline creation and inference are both guarded: KPipeline is not
+ # documented as thread-safe, and an LRU entry must not be evicted and
+ # closed while another thread is still synthesizing with it.
+ self._pipeline_lock = threading.RLock()
- def _ensure_pipeline(self) -> None:
- if self._pipeline is not None:
- return
- try:
- from kokoro import KPipeline
+ @staticmethod
+ def _lang_for_voice(voice_id: str) -> str:
+ """Return the Kokoro ``lang_code`` for a voice ID.
+
+ Voice IDs follow the convention ``{lang}{gender}_{name}`` where
+ ``lang`` is a single letter prefix (e.g. ``z`` for Mandarin in
+ ``zf_xiaoxiao``). Falls back to American English if the prefix is
+ unknown.
+ """
+ if not voice_id:
+ return _DEFAULT_LANG_CODE
+ return _VOICE_PREFIX_TO_LANG.get(voice_id[:1], _DEFAULT_LANG_CODE)
+
+ def _ensure_pipeline(self, lang_code: str) -> Any:
+ """Lazily create and retain a bounded LRU of language pipelines."""
+ with self._pipeline_lock:
+ pipeline = self._pipelines.get(lang_code)
+ if pipeline is not None:
+ self._pipelines.move_to_end(lang_code)
+ return pipeline
+ try:
+ from kokoro import KPipeline
+ except ImportError as exc:
+ raise RuntimeError(
+ "kokoro package not installed. Install with: pip install kokoro"
+ ) from exc
+ try:
+ # KModel is language-blind and is by far the heaviest part of
+ # Kokoro. Reuse one model across every language pipeline.
+ pipeline = KPipeline(lang_code=lang_code, model=self._ensure_model())
+ except ImportError as exc:
+ # Kokoro loads language-specific G2P resources at init time and
+ # several languages need extra dependencies that aren't pulled in
+ # by the base ``kokoro`` package. The known cases:
+ # z (Mandarin) → misaki.zh (jieba / pypinyin / ordered-set)
+ # j (Japanese) → misaki.ja (fugashi / unidic-lite)
+ # Translate the cryptic ``ordered_set``/``fugashi`` ImportError
+ # into a one-line install hint so the user isn't left guessing.
+ hints = {
+ "z": 'pip install "misaki[zh]"',
+ "j": 'pip install "misaki[ja]"',
+ }
+ hint = hints.get(
+ lang_code,
+ f'extra phoneme deps for lang_code="{lang_code}" (see Kokoro docs)',
+ )
+ raise RuntimeError(
+ f"Kokoro lang_code={lang_code!r} requires extra dependencies. "
+ f"Try: {hint}. Original error: {exc}"
+ ) from exc
+
+ self._pipelines[lang_code] = pipeline
+ if len(self._pipelines) > self._max_cached_pipelines:
+ _, evicted = self._pipelines.popitem(last=False)
+ self._cleanup_pipeline(evicted)
+ return pipeline
+
+ def _ensure_model(self) -> Any:
+ """Load the configured Kokoro model once and move it to the device."""
+ if self._model is not None:
+ return self._model
- self._pipeline = KPipeline(lang_code="a")
- except ImportError:
+ try:
+ from kokoro import KModel
+ except ImportError as exc:
raise RuntimeError(
"kokoro package not installed. Install with: pip install kokoro"
- )
+ ) from exc
+
+ device = self._device
+ if device == "auto":
+ # A real Kokoro install always brings torch. Keeping this fallback
+ # makes dependency probes and lightweight mocked tests graceful.
+ try:
+ import torch
+ except ImportError:
+ device = "cpu"
+ else:
+ if torch.cuda.is_available():
+ device = "cuda"
+ elif getattr(torch.backends, "mps", None) is not None and (
+ torch.backends.mps.is_available()
+ ):
+ device = "mps"
+ else:
+ device = "cpu"
+
+ model_kwargs = {"model": self._model_path} if self._model_path else {}
+ try:
+ self._model = KModel(**model_kwargs).to(device).eval()
+ except RuntimeError as exc:
+ raise RuntimeError(
+ f"Failed to initialize Kokoro on device {device!r}: {exc}"
+ ) from exc
+ return self._model
+
+ @staticmethod
+ def _cleanup_pipeline(pipeline: Any) -> None:
+ """Release resources exposed by a pipeline implementation, if any."""
+ close = getattr(pipeline, "close", None)
+ if callable(close):
+ try:
+ close()
+ except Exception: # pragma: no cover - third-party cleanup is best effort
+ pass
+
+ def close(self) -> None:
+ """Release every cached pipeline and clear the cache."""
+ with self._pipeline_lock:
+ pipelines = list(self._pipelines.values())
+ self._pipelines.clear()
+ for pipeline in pipelines:
+ self._cleanup_pipeline(pipeline)
+ self._model = None
def synthesize(
self,
text: str,
*,
- voice_id: str = "af_heart",
+ voice_id: str = _DEFAULT_VOICE_ID,
speed: float = 1.0,
output_format: str = "wav",
) -> TTSResult:
- self._ensure_pipeline()
import numpy as np
import soundfile as sf
- samples = []
- for _, _, audio in self._pipeline(text, voice=voice_id, speed=speed):
- samples.append(audio)
+ lang_code = self._lang_for_voice(voice_id)
+ with self._pipeline_lock:
+ pipeline = self._ensure_pipeline(lang_code)
+ samples = []
+ for _, _, audio in pipeline(text, voice=voice_id, speed=speed):
+ samples.append(audio)
if not samples:
- return TTSResult(audio=b"", format=output_format, voice_id=voice_id)
+ return TTSResult(
+ audio=b"",
+ format=output_format,
+ voice_id=voice_id,
+ metadata={"backend": "kokoro", "lang_code": lang_code},
+ )
combined = np.concatenate(samples)
buf = io.BytesIO()
@@ -66,15 +217,46 @@ class KokoroTTSBackend(TTSBackend):
voice_id=voice_id,
sample_rate=24000,
duration_seconds=len(combined) / 24000,
- metadata={"backend": "kokoro"},
+ metadata={"backend": "kokoro", "lang_code": lang_code},
)
def available_voices(self) -> List[str]:
- return ["af_heart", "af_bella", "am_adam", "am_michael"]
+ # Curated subset of Kokoro v1.x voices. The full catalog is larger;
+ # the list here covers the languages OpenJarvis users most commonly
+ # ask for and avoids voice IDs that have changed across Kokoro
+ # releases.
+ return [
+ # American English
+ "af_heart",
+ "af_bella",
+ "af_nicole",
+ "af_sarah",
+ "af_sky",
+ "am_adam",
+ "am_michael",
+ # British English
+ "bf_emma",
+ "bf_isabella",
+ "bm_george",
+ "bm_lewis",
+ # Mandarin Chinese
+ "zf_xiaobei",
+ "zf_xiaoni",
+ "zf_xiaoxiao",
+ "zf_xiaoyi",
+ "zm_yunjian",
+ "zm_yunxi",
+ "zm_yunxia",
+ "zm_yunyang",
+ # Japanese
+ "jf_alpha",
+ "jf_gongitsune",
+ "jm_kumo",
+ ]
def health(self) -> bool:
try:
- self._ensure_pipeline()
+ self._ensure_pipeline(_DEFAULT_LANG_CODE)
return True
except RuntimeError:
return False
diff --git a/tests/speech/test_tts_backends.py b/tests/speech/test_tts_backends.py
index e1fd56c9..b5819174 100644
--- a/tests/speech/test_tts_backends.py
+++ b/tests/speech/test_tts_backends.py
@@ -79,6 +79,316 @@ def test_kokoro_health_false_without_package():
assert backend.health() is False
+def test_kokoro_lang_for_voice_mandarin():
+ """Mandarin voice IDs (zf_*, zm_*) map to lang_code='z'."""
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ assert KokoroTTSBackend._lang_for_voice("zf_xiaoxiao") == "z"
+ assert KokoroTTSBackend._lang_for_voice("zm_yunxi") == "z"
+
+
+def test_kokoro_lang_for_voice_english_and_other_languages():
+ """Voice prefix detection covers English variants and other languages."""
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ assert KokoroTTSBackend._lang_for_voice("af_heart") == "a"
+ assert KokoroTTSBackend._lang_for_voice("am_adam") == "a"
+ assert KokoroTTSBackend._lang_for_voice("bf_emma") == "b"
+ assert KokoroTTSBackend._lang_for_voice("jf_alpha") == "j"
+ # Kokoro 0.9.x has no Korean KPipeline language code. Do not route a
+ # catalog-style Korean voice ID to an unsupported ``lang_code='k'``.
+ assert KokoroTTSBackend._lang_for_voice("kf_example") == "a"
+
+
+def test_kokoro_lang_for_voice_unknown_falls_back_to_english():
+ """Unknown prefixes fall back to American English (Kokoro's most-stocked
+ language) rather than crashing — keeps backward-compat for any
+ user-supplied voice ID we haven't catalogued."""
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ assert KokoroTTSBackend._lang_for_voice("") == "a"
+ assert KokoroTTSBackend._lang_for_voice("xx_unknown") == "a"
+
+
+def test_kokoro_available_voices_includes_mandarin_and_english():
+ """Voice catalog must list at least one Mandarin voice and preserve
+ the original English voices for backward compatibility."""
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ voices = KokoroTTSBackend().available_voices()
+ # New Mandarin coverage (the focus of this change)
+ for v in ("zf_xiaoxiao", "zf_xiaoyi", "zm_yunxi", "zm_yunjian"):
+ assert v in voices, f"Mandarin voice {v} missing from catalog"
+ # Backward compat — voices the previous implementation listed
+ for v in ("af_heart", "af_bella", "am_adam", "am_michael"):
+ assert v in voices, f"Existing voice {v} dropped from catalog"
+
+
+def test_kokoro_missing_chinese_deps_gives_actionable_error(monkeypatch):
+ """When Mandarin G2P deps are missing, the user must see an install hint
+ pointing at ``misaki[zh]`` rather than a raw ``ordered_set`` ImportError."""
+ import sys
+ import types
+
+ import pytest
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ # Mimic what happens inside Kokoro when misaki.zh can't import.
+ raise ImportError("No module named 'ordered_set'")
+
+ class FakeKModel:
+ def to(self, _device):
+ return self
+
+ def eval(self):
+ return self
+
+ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)
+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)
+
+ backend = KokoroTTSBackend()
+ with pytest.raises(RuntimeError) as exc_info:
+ backend._ensure_pipeline("z")
+ msg = str(exc_info.value)
+ assert "misaki[zh]" in msg
+ assert "lang_code='z'" in msg or 'lang_code="z"' in msg
+
+
+def test_kokoro_pipeline_cached_per_language(monkeypatch):
+ """Each lang_code instantiates one pipeline; same lang reuses the cache."""
+ import sys
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ init_calls = []
+
+ class FakeKModel:
+ def __init__(self, **kwargs):
+ self.kwargs = kwargs
+
+ def to(self, _device):
+ return self
+
+ def eval(self):
+ return self
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ init_calls.append(lang_code)
+ self.model = model
+
+ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)
+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)
+
+ backend = KokoroTTSBackend()
+ backend._ensure_pipeline("a")
+ backend._ensure_pipeline("z")
+ backend._ensure_pipeline("a") # cache hit — must NOT re-instantiate
+ backend._ensure_pipeline("z") # cache hit
+
+ assert init_calls == ["a", "z"]
+
+
+def test_kokoro_pipeline_cache_is_thread_safe(monkeypatch):
+ """Concurrent requests for one language construct exactly one pipeline."""
+ import sys
+ import threading
+ import time
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ init_calls = []
+
+ class FakeKModel:
+ def to(self, _device):
+ return self
+
+ def eval(self):
+ return self
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ init_calls.append(lang_code)
+ time.sleep(0.02)
+
+ monkeypatch.setitem(
+ sys.modules,
+ "kokoro",
+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),
+ )
+ backend = KokoroTTSBackend()
+ pipelines = []
+
+ def load_pipeline():
+ pipelines.append(backend._ensure_pipeline("z"))
+
+ threads = [threading.Thread(target=load_pipeline) for _ in range(8)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(timeout=2)
+ assert not thread.is_alive()
+
+ assert init_calls == ["z"]
+ assert len({id(pipeline) for pipeline in pipelines}) == 1
+
+
+def test_kokoro_pipeline_cache_evicts_lru_and_cleans_up(monkeypatch):
+ """The cache is bounded and closes evicted and explicitly released entries."""
+ import sys
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ instances = {}
+
+ class FakeKModel:
+ def to(self, _device):
+ return self
+
+ def eval(self):
+ return self
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ self.closed = False
+ instances[lang_code] = self
+
+ def close(self):
+ self.closed = True
+
+ monkeypatch.setitem(
+ sys.modules,
+ "kokoro",
+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),
+ )
+ backend = KokoroTTSBackend(max_cached_pipelines=2)
+
+ backend._ensure_pipeline("a")
+ backend._ensure_pipeline("z")
+ backend._ensure_pipeline("a") # make American English most recently used
+ backend._ensure_pipeline("j")
+
+ assert list(backend._pipelines) == ["a", "j"]
+ assert instances["z"].closed is True
+ assert instances["a"].closed is False
+ assert instances["j"].closed is False
+
+ backend.close()
+ assert not backend._pipelines
+ assert instances["a"].closed is True
+ assert instances["j"].closed is True
+
+
+def test_kokoro_pipeline_cache_size_must_be_positive():
+ import pytest
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ with pytest.raises(ValueError, match="at least 1"):
+ KokoroTTSBackend(max_cached_pipelines=0)
+
+
+def test_kokoro_synthesize_routes_voice_to_correct_language(monkeypatch):
+ """synthesize() with a Mandarin voice initializes a Mandarin pipeline
+ and tags the result metadata with the resolved lang_code."""
+ import sys
+ import types
+
+ import numpy as np
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ init_args = []
+ call_args = []
+
+ class FakeKModel:
+ def to(self, device):
+ self.device = device
+ return self
+
+ def eval(self):
+ return self
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ init_args.append(lang_code)
+ self.model = model
+
+ def __call__(self, text, voice, speed):
+ call_args.append({"text": text, "voice": voice, "speed": speed})
+ yield (None, None, np.zeros(2400, dtype=np.float32))
+
+ fake_kokoro = types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline)
+ monkeypatch.setitem(sys.modules, "kokoro", fake_kokoro)
+
+ # ``soundfile`` is a kokoro-runtime dep; stub it so we can test the
+ # routing logic without requiring it in the OpenJarvis test env.
+ def _fake_sf_write(buf, _samples, _sr, format=None): # noqa: A002
+ buf.write(b"FAKE_AUDIO_BYTES")
+
+ fake_soundfile = types.SimpleNamespace(write=_fake_sf_write)
+ monkeypatch.setitem(sys.modules, "soundfile", fake_soundfile)
+
+ backend = KokoroTTSBackend()
+ result = backend.synthesize("你好世界", voice_id="zf_xiaoxiao")
+
+ assert init_args == ["z"]
+ assert call_args[0]["voice"] == "zf_xiaoxiao"
+ assert call_args[0]["text"] == "你好世界"
+ assert result.voice_id == "zf_xiaoxiao"
+ assert result.metadata["lang_code"] == "z"
+ assert result.metadata["backend"] == "kokoro"
+ assert result.format == "wav"
+ assert len(result.audio) > 0
+
+
+def test_kokoro_reuses_model_and_honors_path_and_device(monkeypatch):
+ """All language pipelines share the configured model instance."""
+ import sys
+ import types
+
+ from openjarvis.speech.kokoro_tts import KokoroTTSBackend
+
+ model_inits = []
+ model_devices = []
+ pipeline_models = []
+
+ class FakeKModel:
+ def __init__(self, **kwargs):
+ model_inits.append(kwargs)
+
+ def to(self, device):
+ model_devices.append(device)
+ return self
+
+ def eval(self):
+ return self
+
+ class FakeKPipeline:
+ def __init__(self, lang_code, model):
+ pipeline_models.append((lang_code, model))
+
+ monkeypatch.setitem(
+ sys.modules,
+ "kokoro",
+ types.SimpleNamespace(KModel=FakeKModel, KPipeline=FakeKPipeline),
+ )
+
+ backend = KokoroTTSBackend(model_path="/models/kokoro.pth", device="cpu")
+ backend._ensure_pipeline("a")
+ backend._ensure_pipeline("z")
+
+ assert model_inits == [{"model": "/models/kokoro.pth"}]
+ assert model_devices == ["cpu"]
+ assert pipeline_models[0][1] is pipeline_models[1][1]
+
+
# ---------------------------------------------------------------------------
# OpenAI TTS backend tests
# ---------------------------------------------------------------------------