Skip to content

The open-source AI voice studio. Clone, dictate, create.

fix(omnivoice): address review feedback

SKAL40587c7c40cd
diff --git a/app/src/components/Generation/FloatingGenerateBox.tsx b/app/src/components/Generation/FloatingGenerateBox.tsx
index d09d489f..51ea701f 100644
--- a/app/src/components/Generation/FloatingGenerateBox.tsx
+++ b/app/src/components/Generation/FloatingGenerateBox.tsx
@@ -16,6 +16,7 @@ import {
import { Textarea } from '@/components/ui/textarea';
import { useToast } from '@/components/ui/use-toast';
import { apiClient } from '@/lib/api/client';
+import { engineSupportsInstruct } from '@/lib/constants/engines';
import { getLanguageOptionsForEngine, type LanguageCode } from '@/lib/constants/languages';
import { useGenerationForm } from '@/lib/hooks/useGenerationForm';
import { useProfile, useProfiles } from '@/lib/hooks/useProfiles';
@@ -438,7 +439,7 @@ export function FloatingGenerateBox({
{/* Instruct toggle — only for Qwen CustomVoice, which actually honors the kwarg */}
<AnimatePresence>
- {isExpanded && form.watch('engine') === 'qwen_custom_voice' && (
+ {isExpanded && engineSupportsInstruct(form.watch('engine')) && (
<motion.div
initial={{ opacity: 0, scale: 0.8 }}
animate={{ opacity: 1, scale: 1 }}
@@ -507,7 +508,7 @@ export function FloatingGenerateBox({
{/* Additive instruct textarea — shown below main text when toggle is on and engine supports it */}
<AnimatePresence>
- {isInstructExpanded && form.watch('engine') === 'qwen_custom_voice' && (
+ {isInstructExpanded && engineSupportsInstruct(form.watch('engine')) && (
<motion.div
initial={{ opacity: 0, height: 0 }}
animate={{ opacity: 1, height: 'auto' }}
diff --git a/app/src/lib/constants/engines.ts b/app/src/lib/constants/engines.ts
new file mode 100644
index 00000000..36b3219a
--- /dev/null
+++ b/app/src/lib/constants/engines.ts
@@ -0,0 +1,18 @@
+/** Engine capability sets shared across the generation UI and its form hook. */
+
+/**
+ * Engines whose model actually honours the `instruct` field.
+ *
+ * Base Qwen3-TTS accepts the kwarg and silently ignores it, so it stays out.
+ *
+ * Note the two members read the field very differently: Qwen CustomVoice takes
+ * free-form prose ("speak in an angry tone"), while OmniVoice takes a closed
+ * vocabulary of attributes ("female, low pitch, british accent") and raises on
+ * anything outside it.
+ */
+export const INSTRUCT_ENGINES: ReadonlySet<string> = new Set(['qwen_custom_voice', 'omnivoice']);
+
+/** Whether the given engine should show and forward the instruct field. */
+export function engineSupportsInstruct(engine: string | undefined): boolean {
+ return engine !== undefined && INSTRUCT_ENGINES.has(engine);
+}
diff --git a/app/src/lib/hooks/useGenerationForm.ts b/app/src/lib/hooks/useGenerationForm.ts
index 38118df9..110451ed 100644
--- a/app/src/lib/hooks/useGenerationForm.ts
+++ b/app/src/lib/hooks/useGenerationForm.ts
@@ -5,6 +5,7 @@ import * as z from 'zod';
import { useToast } from '@/components/ui/use-toast';
import { apiClient } from '@/lib/api/client';
import type { EffectConfig } from '@/lib/api/types';
+import { engineSupportsInstruct } from '@/lib/constants/engines';
import { LANGUAGE_CODES, type LanguageCode } from '@/lib/constants/languages';
import { useGeneration } from '@/lib/hooks/useGeneration';
import { useModelDownloadToast } from '@/lib/hooks/useModelDownloadToast';
@@ -144,9 +145,7 @@ export function useGenerationForm(options: UseGenerationFormOptions = {}) {
const hasModelSizes =
engine === 'qwen' || engine === 'qwen_custom_voice' || engine === 'tada';
- // Only Qwen CustomVoice actually honors the instruct kwarg at model level.
- // Base Qwen3-TTS accepts the kwarg but ignores it.
- const supportsInstruct = engine === 'qwen_custom_voice';
+ const supportsInstruct = engineSupportsInstruct(engine);
const effectsChain = options.getEffectsChain?.();
// This now returns immediately with status="generating"
const result = await generation.mutateAsync({
diff --git a/backend/backends/omnivoice_backend.py b/backend/backends/omnivoice_backend.py
index b4904f0a..36bf6443 100644
--- a/backend/backends/omnivoice_backend.py
+++ b/backend/backends/omnivoice_backend.py
@@ -97,6 +97,7 @@ class OmniVoiceBackend:
self.model_size = "default" # OmniVoice ships a single checkpoint
self._device = None
self._asr_loaded = False
+ self._model_load_lock = asyncio.Lock()
def _get_device(self) -> str:
# MPS is allowed: from_pretrained keeps the audio tokenizer on CPU
@@ -123,7 +124,13 @@ class OmniVoiceBackend:
if self.model is not None:
return
- await asyncio.to_thread(self._load_model_sync)
+ # Both create_voice_prompt() and generate() call this, so two requests
+ # arriving together can clear the check above before either assigns
+ # self.model. At 3.3 GB a duplicate load is an OOM, not just waste.
+ async with self._model_load_lock:
+ if self.model is not None:
+ return
+ await asyncio.to_thread(self._load_model_sync)
def _load_model_sync(self) -> None:
model_name = "omnivoice"
@@ -212,7 +219,14 @@ class OmniVoiceBackend:
if cache_key:
cached = get_cached_voice_prompt(cache_key)
- if isinstance(cached, dict) and "ref_audio_tokens" in cached:
+ # Check the version here rather than at generation time: a stale
+ # entry should be a cache miss and get re-encoded, not a hard
+ # failure every time the profile is used.
+ if (
+ isinstance(cached, dict)
+ and "ref_audio_tokens" in cached
+ and cached.get("format_version") == _VOICE_CLONE_PROMPT_FORMAT_VERSION
+ ):
return cached, True
def _encode_sync() -> dict:
diff --git a/backend/build_binary.py b/backend/build_binary.py
index 88e3a93c..625357a5 100644
--- a/backend/build_binary.py
+++ b/backend/build_binary.py
@@ -298,10 +298,12 @@ def build_server(cuda=False, rocm=False):
"omnivoice",
# The vendored codec builds its two sub-models through
# AutoModel.from_config(), which transformers resolves at runtime —
- # static analysis cannot see either of them.
- "--hidden-import",
+ # static analysis cannot see either of them. The parent package is
+ # not enough: the weights live in modeling_dac / modeling_hubert,
+ # which transformers itself imports lazily.
+ "--collect-submodules",
"transformers.models.dac",
- "--hidden-import",
+ "--collect-submodules",
"transformers.models.hubert",
# Reference-audio preprocessing (silence trimming) in
# omnivoice.utils.audio.
diff --git a/backend/requirements.txt b/backend/requirements.txt
index 8c274bd2..c8cbdaa2 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -8,7 +8,11 @@ sqlalchemy>=2.0.0
alembic>=1.13.0
# ML models
-torch>=2.2.0
+# 2.5 is the first release with torch.nn.Buffer, which the vendored Higgs
+# Audio V2 codec uses (backend/vendor/higgs_audio_v2_tokenizer). The old >=2.2
+# floor was already well below what the CUDA/ROCm install paths and the
+# transformers 4.57 workarounds actually run against.
+torch>=2.5.0
transformers>=4.36.0,<=4.57.6
accelerate>=0.26.0
huggingface_hub>=0.20.0
diff --git a/backend/tests/test_omnivoice_compat.py b/backend/tests/test_omnivoice_compat.py
index ed959ff2..cefb8616 100644
--- a/backend/tests/test_omnivoice_compat.py
+++ b/backend/tests/test_omnivoice_compat.py
@@ -146,9 +146,28 @@ def test_strict_returns_target_unchanged():
@pytest.fixture
def restore_special_tokens_hook():
- saved = PreTrainedTokenizerBase.__dict__.get("_set_model_specific_special_tokens")
+ """Undo the test double, including when the class never owned the attribute.
+
+ ``_set_model_specific_special_tokens`` is defined on ``SpecialTokensMixin``,
+ not on ``PreTrainedTokenizerBase``, so it is absent from the latter's own
+ ``__dict__``. Restoring only when a previous value existed would leave the
+ stub installed for the rest of the session and every later tokenizer test
+ would run against it.
+ """
+ sentinel = object()
+ saved = PreTrainedTokenizerBase.__dict__.get(
+ "_set_model_specific_special_tokens", sentinel
+ )
yield
- if saved is not None:
+ if saved is sentinel:
+ # Nothing was shadowed -- drop ours so the inherited one is found
+ # again. A class __dict__ is a read-only mappingproxy, so this has to
+ # go through del on the class itself.
+ try:
+ del PreTrainedTokenizerBase._set_model_specific_special_tokens
+ except AttributeError:
+ pass
+ else:
PreTrainedTokenizerBase._set_model_specific_special_tokens = saved
@@ -191,6 +210,27 @@ def test_patch_is_idempotent(restore_special_tokens_hook):
assert compat.patch_extra_special_tokens() is False
+def test_teardown_leaves_no_stub_behind(restore_special_tokens_hook):
+ """The fixture must not leak the double into the rest of the session.
+
+ Regression: the original teardown only restored when a previous value had
+ been shadowed. ``_set_model_specific_special_tokens`` lives on
+ ``SpecialTokensMixin``, so ``PreTrainedTokenizerBase`` never owns it and the
+ stub survived every test that used this fixture.
+ """
+ assert "_set_model_specific_special_tokens" not in PreTrainedTokenizerBase.__dict__
+
+ PreTrainedTokenizerBase._set_model_specific_special_tokens = lambda self, tokens: None
+ compat.patch_extra_special_tokens()
+ assert "_set_model_specific_special_tokens" in PreTrainedTokenizerBase.__dict__
+
+
+def test_inherited_method_is_reachable_after_teardown():
+ """Runs after the fixture above has torn down; the real method must be back."""
+ assert "_set_model_specific_special_tokens" not in PreTrainedTokenizerBase.__dict__
+ assert callable(PreTrainedTokenizerBase._set_model_specific_special_tokens)
+
+
@pytest.mark.parametrize(
"token,expected",
[