Agent Zero AI framework
Enhance native Kokoro voice blending
Alessandro14b5a03e7f8f
diff --git a/plugins/_kokoro_tts/AGENTS.md b/plugins/_kokoro_tts/AGENTS.mdindex 8e7ffc6bf..d9a6f1464 100644--- a/plugins/_kokoro_tts/AGENTS.md+++ b/plugins/_kokoro_tts/AGENTS.md@@ -17,6 +17,7 @@- Keep Kokoro dependencies on Docker/bootstrap paths, not opportunistic runtime installs.- Preserve browser-native fallback when the plugin is disabled or unavailable.- Do not expose generated speech artifacts outside intended response paths.+- Preserve `voice` and `speed` configuration compatibility; weighted blends stay in memory and use positive finite weights.## Work Guidancediff --git a/plugins/_kokoro_tts/README.md b/plugins/_kokoro_tts/README.mdindex 28b3e61b9..e86092f56 100644--- a/plugins/_kokoro_tts/README.md+++ b/plugins/_kokoro_tts/README.md@@ -5,12 +5,15 @@ Built-in speech synthesis plugin backed by Kokoro.## Behavior- Registers Kokoro as the active TTS provider when the plugin is enabled.+- Supports single voices, comma-separated equal blends, and optional in-memory weighted blends.- Keeps browser-native `speechSynthesis` as the fallback path when disabled.- Keeps Python dependencies on the core Docker/bootstrap path. This plugin does not install packages or binaries on demand.+- Uses Kokoro's existing voice-pack cache and does not create persistent blend files.## Config-- `voice`: Kokoro voice identifier+- `voice`: Kokoro voice identifier or comma-separated equal blend+- `voice_weights`: optional mapping of voice identifiers to positive weights; when present, it defines the active blend- `speed`: Kokoro playback speed multiplier## Routesdiff --git a/plugins/_kokoro_tts/default_config.yaml b/plugins/_kokoro_tts/default_config.yamlindex 85be3ad69..bd1c4ec63 100644--- a/plugins/_kokoro_tts/default_config.yaml+++ b/plugins/_kokoro_tts/default_config.yaml@@ -1,2 +1,3 @@voice: am_puck,am_onyx+voice_weights: {}speed: 1.1diff --git a/plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.html b/plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.htmlindex ff4a2ebb0..84560a9c3 100644--- a/plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.html+++ b/plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.html@@ -13,7 +13,7 @@<div class="voice-plugin-meta"><div class="voice-plugin-meta-row"><span>Voice</span>- <code x-text="$store.kokoroTts?.config?.voice || ''"></code>+ <code x-text="$store.kokoroTts?.voiceSummary || ''"></code></div><div class="voice-plugin-meta-row"><span>Speed</span>diff --git a/plugins/_kokoro_tts/helpers/runtime.py b/plugins/_kokoro_tts/helpers/runtime.pyindex 365886a2e..c8237c6ae 100644--- a/plugins/_kokoro_tts/helpers/runtime.py+++ b/plugins/_kokoro_tts/helpers/runtime.py@@ -3,6 +3,8 @@ from __future__ import annotationsimport asyncioimport base64import io+import math+import reimport warningsfrom typing import Any@@ -25,15 +27,17 @@ warnings.filterwarnings("ignore", category=UserWarning)PLUGIN_NAME = "_kokoro_tts"DEFAULT_CONFIG = {"voice": "am_puck,am_onyx",+ "voice_weights": {},"speed": 1.1,}+VOICE_ID_PATTERN = re.compile(r"^[a-z]{2}_[a-z0-9_]+$")_pipeline = Noneis_updating_model = Falsedef normalize_config(config: dict[str, Any] | None) -> dict[str, Any]:- normalized = dict(DEFAULT_CONFIG)+ normalized = {**DEFAULT_CONFIG, "voice_weights": {}}if not isinstance(config, dict):return normalized@@ -41,9 +45,25 @@ def normalize_config(config: dict[str, Any] | None) -> dict[str, Any]:if voice:normalized["voice"] = voice+ weights = config.get("voice_weights")+ if isinstance(weights, dict):+ for raw_voice, raw_weight in weights.items():+ voice_id = str(raw_voice or "").strip()+ if not VOICE_ID_PATTERN.fullmatch(voice_id):+ continue+ try:+ weight = float(raw_weight)+ except (TypeError, ValueError):+ continue+ if math.isfinite(weight) and weight > 0:+ normalized["voice_weights"][voice_id] = weight++ if normalized["voice_weights"]:+ normalized["voice"] = ",".join(normalized["voice_weights"])+try:speed = float(config.get("speed", normalized["speed"]))- if speed > 0:+ if math.isfinite(speed) and speed > 0:normalized["speed"] = speedexcept (TypeError, ValueError):pass@@ -113,23 +133,43 @@ async def synthesize_sentences(return await _synthesize_sentences(sentences,voice=str(cfg["voice"]),+ voice_weights=dict(cfg["voice_weights"]),speed=float(cfg["speed"]),)+def _resolve_voice(+ pipeline: Any, voice: str, voice_weights: dict[str, float]+) -> Any:+ if not voice_weights:+ return voice++ total = sum(voice_weights.values())+ if not math.isfinite(total) or total <= 0:+ return voice+ blend = None+ for voice_id, weight in voice_weights.items():+ weighted_pack = pipeline.load_single_voice(voice_id) * (weight / total)+ blend = weighted_pack if blend is None else blend + weighted_pack+ return blend++async def _synthesize_sentences(- sentences: list[str], *, voice: str, speed: float+ sentences: list[str], *, voice: str, voice_weights: dict[str, float], speed: float) -> str:await _preload()combined_audio: list[float] = []+ resolved_voice = _resolve_voice(_pipeline, voice, voice_weights)try:for sentence in sentences:if not sentence.strip():continue- segments = _pipeline(sentence.strip(), voice=voice, speed=speed) # type: ignore[misc]+ segments = _pipeline( # type: ignore[misc]+ sentence.strip(), voice=resolved_voice, speed=speed+ )for segment in list(segments):audio_tensor = segment.audioaudio_numpy = audio_tensor.detach().cpu().numpy() # type: ignore[union-attr]diff --git a/plugins/_kokoro_tts/plugin.yaml b/plugins/_kokoro_tts/plugin.yamlindex 8c810fafc..963e75404 100644--- a/plugins/_kokoro_tts/plugin.yaml+++ b/plugins/_kokoro_tts/plugin.yaml@@ -1,7 +1,7 @@name: _kokoro_ttstitle: Kokoro TTS-description: Built-in Kokoro text-to-speech plugin with browser TTS fallback when disabled.-version: 1.0.0+description: Built-in Kokoro text-to-speech plugin with weighted voice blending and browser fallback.+version: 1.1.0always_enabled: falsesettings_sections:- agentdiff --git a/plugins/_kokoro_tts/webui/config.html b/plugins/_kokoro_tts/webui/config.htmlindex 755258cc2..010baeffa 100644--- a/plugins/_kokoro_tts/webui/config.html+++ b/plugins/_kokoro_tts/webui/config.html@@ -4,36 +4,332 @@</head><body>- <div x-data>+ <div x-data="{+ catalog: [+ { id: 'af_alloy', label: 'Alloy', group: 'American Female' },+ { id: 'af_aoede', label: 'Aoede', group: 'American Female' },+ { id: 'af_bella', label: 'Bella', group: 'American Female' },+ { id: 'af_heart', label: 'Heart', group: 'American Female' },+ { id: 'af_jessica', label: 'Jessica', group: 'American Female' },+ { id: 'af_kore', label: 'Kore', group: 'American Female' },+ { id: 'af_nicole', label: 'Nicole', group: 'American Female' },+ { id: 'af_nova', label: 'Nova', group: 'American Female' },+ { id: 'af_river', label: 'River', group: 'American Female' },+ { id: 'af_sarah', label: 'Sarah', group: 'American Female' },+ { id: 'af_sky', label: 'Sky', group: 'American Female' },+ { id: 'am_adam', label: 'Adam', group: 'American Male' },+ { id: 'am_echo', label: 'Echo', group: 'American Male' },+ { id: 'am_eric', label: 'Eric', group: 'American Male' },+ { id: 'am_fenrir', label: 'Fenrir', group: 'American Male' },+ { id: 'am_liam', label: 'Liam', group: 'American Male' },+ { id: 'am_michael', label: 'Michael', group: 'American Male' },+ { id: 'am_onyx', label: 'Onyx', group: 'American Male' },+ { id: 'am_puck', label: 'Puck', group: 'American Male' },+ { id: 'am_santa', label: 'Santa', group: 'American Male' },+ ],+ initConfig() {+ if (!config.voice) config.voice = 'am_puck,am_onyx';+ const sanitized = {};+ const weights = config.voice_weights;+ if (weights && typeof weights === 'object' && !Array.isArray(weights)) {+ for (const [id, rawWeight] of Object.entries(weights)) {+ const weight = Number(rawWeight);+ if (/^[a-z]{2}_[a-z0-9_]+$/.test(id) && Number.isFinite(weight) && weight > 0) {+ sanitized[id] = weight;+ }+ }+ }+ config.voice_weights = sanitized;+ if (Object.keys(sanitized).length) this.syncVoice(Object.keys(sanitized));+ const speed = Number(config.speed);+ config.speed = Number.isFinite(speed) && speed > 0 ? speed : 1.1;+ },+ voiceIds() {+ const weighted = Object.keys(config.voice_weights || {});+ if (weighted.length) return weighted;+ return String(config.voice || '')+ .split(',')+ .map(id => id.trim())+ .filter(id => /^[a-z]{2}_[a-z0-9_]+$/.test(id));+ },+ hasWeights() {+ return Object.keys(config.voice_weights || {}).length > 0;+ },+ voiceInfo(id) {+ return this.catalog.find(voice => voice.id === id) || {+ id,+ label: id,+ group: 'Custom voice',+ };+ },+ availableVoices(group) {+ const selected = new Set(this.voiceIds());+ return this.catalog.filter(voice => voice.group === group && !selected.has(voice.id));+ },+ weightFor(id) {+ const weight = Number(config.voice_weights?.[id]);+ return Number.isFinite(weight) && weight > 0 ? weight : 1;+ },+ weightPercent(id) {+ const ids = this.voiceIds();+ const total = ids.reduce((sum, voiceId) => sum + this.weightFor(voiceId), 0);+ return total > 0 ? Math.round((this.weightFor(id) / total) * 100) : 0;+ },+ setWeight(id, rawWeight) {+ const next = {};+ for (const voiceId of this.voiceIds()) next[voiceId] = this.weightFor(voiceId);+ next[id] = Math.min(10, Math.max(0.1, Number(rawWeight) || 1));+ config.voice_weights = next;+ this.syncVoice(Object.keys(next));+ },+ useEqualWeights() {+ config.voice_weights = {};+ },+ addVoice(id) {+ if (!id || !/^[a-z]{2}_[a-z0-9_]+$/.test(id)) return;+ const ids = this.voiceIds();+ if (!ids.includes(id)) ids.push(id);+ if (this.hasWeights()) {+ const next = {};+ for (const voiceId of ids) next[voiceId] = this.weightFor(voiceId);+ config.voice_weights = next;+ }+ this.syncVoice(ids);+ },+ removeVoice(id) {+ const ids = this.voiceIds();+ if (ids.length <= 1) return;+ const remaining = ids.filter(voiceId => voiceId !== id);+ if (this.hasWeights()) {+ const next = {};+ for (const voiceId of remaining) next[voiceId] = this.weightFor(voiceId);+ config.voice_weights = next;+ }+ this.syncVoice(remaining);+ },+ syncVoice(ids) {+ config.voice = ids.join(',');+ },+ }"><template x-if="config">- <div class="plugin-config-page">+ <div class="plugin-config-page" x-init="initConfig()"><div class="section-title">Kokoro TTS</div><div class="section-description">- Configure the built-in Kokoro voice provider. When this plugin is disabled,- spoken output falls back to the browser speech API.+ Choose a voice or blend several voices in memory. Disabling this plugin+ returns spoken output to the browser speech API.</div><div class="field"><div class="field-label">- <div class="field-title">Voice</div>- <div class="field-description">Kokoro voice identifier passed to the backend pipeline.</div>+ <div class="field-title">Voice expression</div>+ <div class="field-description">+ Pick a voice below or enter Kokoro voice identifiers separated by commas for an equal blend.+ </div></div><div class="field-control">- <input type="text" x-model="config.voice" />+ <input+ type="text"+ list="kokoro-voice-catalog"+ x-model.trim="config.voice"+ @input="config.voice_weights = {}"+ placeholder="am_puck,am_onyx"+ />+ <datalist id="kokoro-voice-catalog">+ <template x-for="voice in catalog" :key="voice.id">+ <option :value="voice.id" :label="`${voice.label} — ${voice.group}`"></option>+ </template>+ </datalist>+ </div>+ </div>++ <div class="field">+ <div class="field-label">+ <div class="field-title">Voice blend</div>+ <div class="field-description">+ Adjusting a weight enables a weighted blend. Voice packs remain in Kokoro's normal cache; no blend files are created.+ </div>+ </div>+ <div class="field-control kokoro-blend-editor">+ <div class="kokoro-blend-toolbar">+ <span class="kokoro-mode" x-text="hasWeights() ? 'Weighted' : 'Equal weights'"></span>+ <button+ type="button"+ class="button"+ x-show="hasWeights()"+ @click="useEqualWeights()"+ >+ Use equal weights+ </button>+ </div>++ <template x-if="voiceIds().length === 0">+ <div class="kokoro-empty">+ Enter a standard Kokoro voice identifier to use the blend editor.+ </div>+ </template>++ <div class="kokoro-voice-list">+ <template x-for="id in voiceIds()" :key="id">+ <div class="kokoro-voice-row">+ <div class="kokoro-voice-copy">+ <strong x-text="voiceInfo(id).label"></strong>+ <span x-text="`${id} · ${voiceInfo(id).group}`"></span>+ </div>+ <label class="kokoro-weight">+ <span x-text="`${weightPercent(id)}%`"></span>+ <input+ type="range"+ min="0.1"+ max="10"+ step="0.1"+ :value="weightFor(id)"+ :aria-label="`Weight for ${voiceInfo(id).label}`"+ @input="setWeight(id, $event.target.value)"+ />+ </label>+ <button+ type="button"+ class="button icon-button"+ :disabled="voiceIds().length <= 1"+ :title="`Remove ${voiceInfo(id).label}`"+ :aria-label="`Remove ${voiceInfo(id).label}`"+ @click="removeVoice(id)"+ >+ <x-icon name="close"></x-icon>+ </button>+ </div>+ </template>+ </div>++ <select @change="addVoice($event.target.value); $event.target.value = ''">+ <option value="">Add a voice...</option>+ <optgroup label="American Female">+ <template x-for="voice in availableVoices('American Female')" :key="voice.id">+ <option :value="voice.id" x-text="`${voice.label} (${voice.id})`"></option>+ </template>+ </optgroup>+ <optgroup label="American Male">+ <template x-for="voice in availableVoices('American Male')" :key="voice.id">+ <option :value="voice.id" x-text="`${voice.label} (${voice.id})`"></option>+ </template>+ </optgroup>+ </select></div></div><div class="field"><div class="field-label"><div class="field-title">Speed</div>- <div class="field-description">Playback speed multiplier for Kokoro synthesis.</div>+ <div class="field-description">Speech rate from 0.5× to 3.0×.</div></div>- <div class="field-control">- <input type="number" min="0.1" step="0.1" x-model.number="config.speed" />+ <div class="field-control kokoro-speed">+ <input+ type="range"+ min="0.5"+ max="3"+ step="0.1"+ x-model.number="config.speed"+ aria-label="Kokoro speech speed"+ />+ <output x-text="`${Number(config.speed || 1.1).toFixed(1)}×`"></output></div></div></div></template></div>++ <style>+ .kokoro-blend-editor {+ display: flex;+ flex-direction: column;+ gap: 0.65rem;+ }++ .kokoro-blend-toolbar,+ .kokoro-voice-row,+ .kokoro-speed {+ display: flex;+ align-items: center;+ gap: 0.75rem;+ }++ .kokoro-blend-toolbar {+ justify-content: space-between;+ }++ .kokoro-mode {+ padding: 0.2rem 0.55rem;+ border: 1px solid var(--color-border);+ border-radius: 999px;+ color: var(--color-text-secondary);+ font-size: var(--font-size-small);+ }++ .kokoro-voice-list {+ display: flex;+ flex-direction: column;+ gap: 0.45rem;+ }++ .kokoro-voice-row,+ .kokoro-empty {+ padding: 0.65rem;+ border: 1px solid var(--color-border);+ border-radius: 0.5rem;+ background: var(--color-input);+ }++ .kokoro-voice-copy {+ display: flex;+ flex: 1 1 11rem;+ min-width: 0;+ flex-direction: column;+ gap: 0.15rem;+ }++ .kokoro-voice-copy span,+ .kokoro-empty {+ color: var(--color-text-secondary);+ font-size: var(--font-size-small);+ }++ .kokoro-weight {+ display: flex;+ flex: 1 1 10rem;+ align-items: center;+ gap: 0.55rem;+ }++ .kokoro-weight span {+ width: 2.7rem;+ text-align: right;+ font-variant-numeric: tabular-nums;+ }++ .kokoro-weight input,+ .kokoro-speed input {+ min-width: 7rem;+ flex: 1;+ }++ .kokoro-speed output {+ min-width: 3rem;+ font-weight: 600;+ font-variant-numeric: tabular-nums;+ text-align: right;+ }++ @media (max-width: 620px) {+ .kokoro-voice-row {+ align-items: stretch;+ flex-wrap: wrap;+ }++ .kokoro-weight {+ order: 3;+ flex-basis: 100%;+ }+ }+ </style></body></html>diff --git a/plugins/_kokoro_tts/webui/kokoro-tts-store.js b/plugins/_kokoro_tts/webui/kokoro-tts-store.jsindex 715836ae4..1e32d81fc 100644--- a/plugins/_kokoro_tts/webui/kokoro-tts-store.js+++ b/plugins/_kokoro_tts/webui/kokoro-tts-store.js@@ -13,6 +13,7 @@ const model = {enabled: false,config: {voice: "",+ voice_weights: {},speed: 1.1,},modelReady: false,@@ -41,6 +42,7 @@ const model = {this.enabled = !!status?.enabled;this.config = {voice: status?.config?.voice || "",+ voice_weights: status?.config?.voice_weights || {},speed: Number(status?.config?.speed || 1.1),};this.modelReady = !!status?.model?.ready;@@ -103,6 +105,17 @@ const model = {return "warn";},+ get voiceSummary() {+ const entries = Object.entries(this.config.voice_weights || {});+ if (!entries.length) return this.config.voice || "";++ const total = entries.reduce((sum, [, weight]) => sum + Number(weight || 0), 0);+ if (total <= 0) return this.config.voice || "";+ return entries+ .map(([voice, weight]) => `${voice} ${Math.round((Number(weight) / total) * 100)}%`)+ .join(" · ");+ },+async openConfig() {const { store } = await import("/components/plugins/plugin-settings-store.js");await store.openConfig(PLUGIN_NAME);diff --git a/plugins/_kokoro_tts/webui/main.html b/plugins/_kokoro_tts/webui/main.htmlindex 58595b3df..d9aa54dc2 100644--- a/plugins/_kokoro_tts/webui/main.html+++ b/plugins/_kokoro_tts/webui/main.html@@ -42,7 +42,7 @@<div class="field-title">Resolved Config</div><div class="status-row"><span class="status-key">Voice</span>- <span class="status-value mono" x-text="$store.kokoroTts.config.voice"></span>+ <span class="status-value mono" x-text="$store.kokoroTts.voiceSummary"></span></div><div class="status-row"><span class="status-key">Speed</span>diff --git a/tests/test_kokoro_tts.py b/tests/test_kokoro_tts.pynew file mode 100644index 000000000..68addf37c--- /dev/null+++ b/tests/test_kokoro_tts.py@@ -0,0 +1,114 @@+from __future__ import annotations++import asyncio+from pathlib import Path++from plugins._kokoro_tts.helpers import runtime+++PROJECT_ROOT = Path(__file__).resolve().parents[1]+++def test_config_keeps_legacy_voice_and_normalizes_weighted_blends() -> None:+ legacy = runtime.normalize_config(+ {"voice": "custom/voice.pt", "speed": 2.2}+ )+ assert legacy == {+ "voice": "custom/voice.pt",+ "voice_weights": {},+ "speed": 2.2,+ }++ weighted = runtime.normalize_config(+ {+ "voice": "ignored_when_weights_are_present",+ "voice_weights": {+ "af_heart": "3",+ "am_puck": 1,+ "../bad.pt": 5,+ "am_onyx": float("nan"),+ "am_echo": 0,+ },+ "speed": float("inf"),+ }+ )+ assert weighted == {+ "voice": "af_heart,am_puck",+ "voice_weights": {"af_heart": 3.0, "am_puck": 1.0},+ "speed": 1.1,+ }+++def test_weighted_blend_reuses_the_existing_pipeline() -> None:+ class FakePipeline:+ packs = {"af_heart": 2.0, "am_puck": 10.0}++ def __init__(self) -> None:+ self.loaded: list[str] = []++ def load_single_voice(self, voice: str) -> float:+ self.loaded.append(voice)+ return self.packs[voice]++ pipeline = FakePipeline()+ blend = runtime._resolve_voice(+ pipeline,+ "legacy",+ {"af_heart": 3.0, "am_puck": 1.0},+ )++ assert blend == 4.0+ assert pipeline.loaded == ["af_heart", "am_puck"]+ assert runtime._resolve_voice(pipeline, "am_puck,am_onyx", {}) == (+ "am_puck,am_onyx"+ )+++def test_synthesis_forwards_normalized_weights(monkeypatch) -> None:+ captured: dict = {}++ async def fake_synthesize(sentences, **kwargs):+ captured.update({"sentences": sentences, **kwargs})+ return "audio"++ monkeypatch.setattr(runtime, "_synthesize_sentences", fake_synthesize)++ result = asyncio.run(+ runtime.synthesize_sentences(+ ["Hello"],+ {+ "voice": "legacy",+ "voice_weights": {"af_heart": 2, "am_puck": 1},+ "speed": 1.4,+ },+ )+ )++ assert result == "audio"+ assert captured == {+ "sentences": ["Hello"],+ "voice": "af_heart,am_puck",+ "voice_weights": {"af_heart": 2.0, "am_puck": 1.0},+ "speed": 1.4,+ }+++def test_settings_expose_catalog_weights_and_speed_without_disk_blends() -> None:+ config_ui = (+ PROJECT_ROOT / "plugins/_kokoro_tts/webui/config.html"+ ).read_text(encoding="utf-8")+ store = (+ PROJECT_ROOT / "plugins/_kokoro_tts/webui/kokoro-tts-store.js"+ ).read_text(encoding="utf-8")++ assert "af_heart" in config_ui+ assert "am_puck" in config_ui+ assert "am_onyx" in config_ui+ assert "config.voice_weights" in config_ui+ assert "Use equal weights" in config_ui+ assert 'type="range"' in config_ui+ assert 'min="0.5"' in config_ui+ assert 'max="3"' in config_ui+ assert "create_blend" not in config_ui+ assert "lang_code" not in config_ui+ assert "voiceSummary" in store