Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fix Base vs LoRA compare for voice messages on fine-tuned Whisper (#12381)

Nilaya6bf9b3ac613
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 3fa64bfae8b..f88709fe1ec 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -2157,6 +2157,7 @@ class InferenceBackend:
def generate_whisper_response(
self,
audio_array,
+ use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
extra_audio_arrays: Optional[list] = None,
) -> Generator[str, None, None]:
@@ -2178,7 +2179,13 @@ class InferenceBackend:
if cancel_event is not None and cancel_event.is_set():
return
with self._generation_lock:
- result = whisper_pipe({"raw": clip, "sampling_rate": 16000})
+ self._apply_adapter_state(use_adapter)
+ try:
+ result = whisper_pipe({"raw": clip, "sampling_rate": 16000})
+ finally:
+ # Plain requests keep the adapter state, so turn the LoRA back on.
+ if use_adapter is False and isinstance(model_info.get("model"), PeftModel):
+ model_info["model"].base_model.enable_adapter_layers()
text = result.get("text", "") if isinstance(result, dict) else str(result)
if text:
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index e94437d90f5..d4ae76ef0d5 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -3317,6 +3317,7 @@ class InferenceOrchestrator:
def generate_whisper_response(
self,
audio_array,
+ use_adapter: Optional[Union[bool, str]] = None,
cancel_event = None,
stats_holder: Optional[dict] = None,
extra_audio_arrays: Optional[list] = None,
@@ -3327,6 +3328,7 @@ class InferenceOrchestrator:
audio_type = "whisper",
messages = [],
system_prompt = "",
+ use_adapter = use_adapter,
cancel_event = cancel_event,
stats_holder = stats_holder,
extra_audio_arrays = extra_audio_arrays,
diff --git a/studio/backend/core/inference/worker.py b/studio/backend/core/inference/worker.py
index 6475449246f..881ec60b431 100644
--- a/studio/backend/core/inference/worker.py
+++ b/studio/backend/core/inference/worker.py
@@ -1653,6 +1653,7 @@ def _handle_generate_audio_input(backend, cmd: dict, resp_queue: Any, cancel_eve
raise RuntimeError("Whisper transcription is not supported on the MLX backend yet.")
generator = backend.generate_whisper_response(
audio_array = audio_array,
+ use_adapter = cmd.get("use_adapter"),
cancel_event = cancel_event,
**extra_audio_kwargs,
)
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index db04ee9e43e..a952d15a16e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -27008,6 +27008,7 @@ async def produce_openai_chat_completions(
if model_info.get("audio_type") == "whisper":
return backend.generate_whisper_response(
audio_array = audio_array,
+ use_adapter = payload.use_adapter,
cancel_event = cancel_event,
stats_holder = _audio_stats_holder,
**extra_audio_kwargs,
diff --git a/studio/backend/tests/test_whisper_compare_adapter.py b/studio/backend/tests/test_whisper_compare_adapter.py
new file mode 100644
index 00000000000..edd77092767
--- /dev/null
+++ b/studio/backend/tests/test_whisper_compare_adapter.py
@@ -0,0 +1,219 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import importlib
+import sys
+import threading
+import types
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(name, attrs):
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception:
+ pass
+ _STUBBED.append(name)
+ mod = types.ModuleType(name)
+ mod.__spec__ = None
+ for attr in attrs:
+ setattr(mod, attr, MagicMock())
+ sys.modules[name] = mod
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, mod)
+
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+try:
+ import core.inference.inference
+except ImportError:
+ pass
+
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+import routes.inference as inference_route
+from core.inference import worker
+from core.inference.orchestrator import InferenceOrchestrator
+from models.inference import ChatCompletionRequest
+
+
+class _ChatRequest:
+ method = "POST"
+ url = SimpleNamespace(path = "/v1/chat/completions")
+ state = SimpleNamespace(skip_api_monitor = True)
+ scope: dict = {}
+
+ async def is_disconnected(self):
+ return False
+
+
+def test_the_chat_route_passes_each_compare_panes_adapter_state_to_whisper(monkeypatch):
+ seen = []
+
+ class _WhisperBackend:
+ active_model_name = "org/whisper-lora"
+ models = {"org/whisper-lora": {"has_audio_input": True, "audio_type": "whisper"}}
+
+ def generate_whisper_response(
+ self,
+ use_adapter = None,
+ **_kwargs,
+ ):
+ seen.append(use_adapter)
+ yield "hello"
+
+ def reset_generation_state(self, caller_cancel_event = None):
+ pass
+
+ monkeypatch.setattr(
+ inference_route,
+ "get_llama_cpp_backend",
+ lambda: SimpleNamespace(
+ is_loaded = False, supports_tools = False, is_vision = False, context_length = None
+ ),
+ )
+ monkeypatch.setattr(inference_route, "get_inference_backend", lambda: _WhisperBackend())
+ monkeypatch.setattr(inference_route, "_automatic_model_load_may_run", lambda: False)
+ monkeypatch.setattr(
+ inference_route, "_decode_audio_clips", lambda _clips: [np.zeros(16, np.float32)]
+ )
+
+ async def _no_auto_switch(*_args, **_kwargs):
+ return None
+
+ monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _no_auto_switch)
+
+ for use_adapter in (False, True):
+ payload = ChatCompletionRequest(
+ model = "org/whisper-lora",
+ messages = [{"role": "user", "content": ""}],
+ audio_base64 = "ZmFrZQ==",
+ use_adapter = use_adapter,
+ )
+ response = asyncio.run(
+ inference_route.openai_chat_completions(
+ payload, _ChatRequest(), current_subject = "tester"
+ )
+ )
+ assert response.status_code == 200
+
+ assert seen == [False, True]
+
+
+def test_the_orchestrator_sends_the_adapter_state_with_a_whisper_request(monkeypatch):
+ orchestrator = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ sent = {}
+
+ def _inner(**kwargs):
+ sent.update(kwargs)
+ yield from ()
+
+ monkeypatch.setattr(orchestrator, "_generate_audio_input_inner", _inner)
+
+ list(orchestrator.generate_whisper_response(audio_array = [0.0], use_adapter = False))
+
+ assert sent["audio_type"] == "whisper"
+ assert sent["use_adapter"] is False
+
+
+def test_the_worker_hands_the_adapter_state_to_whisper():
+ calls = []
+
+ class _Backend:
+ def generate_whisper_response(self, **kwargs):
+ calls.append(kwargs)
+ return iter(())
+
+ worker._handle_generate_audio_input(
+ _Backend(),
+ {
+ "request_id": "r",
+ "audio_clips": [np.zeros(16, np.float32).tobytes()],
+ "audio_type": "whisper",
+ "use_adapter": False,
+ },
+ SimpleNamespace(put = lambda *_args, **_kwargs: None),
+ SimpleNamespace(is_set = lambda: False),
+ )
+
+ assert calls[0]["use_adapter"] is False
+
+
+def test_whisper_applies_the_adapter_state_under_the_lock_before_transcribing(monkeypatch):
+ pytest.importorskip("peft")
+ from core.inference.inference import InferenceBackend
+
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend.active_model_name = "whisper-lora"
+ backend._generation_lock = threading.Lock()
+ events = []
+
+ def _pipe(_inputs):
+ events.append(("transcribe", backend._generation_lock.locked()))
+ return {"text": "hello"}
+
+ backend.models = {"whisper-lora": {"model": object(), "whisper_pipeline": _pipe}}
+ monkeypatch.setattr(
+ backend,
+ "_apply_adapter_state",
+ lambda state: events.append((state, backend._generation_lock.locked())),
+ )
+ audio = np.zeros(16, np.float32)
+
+ for use_adapter in (False, True):
+ assert list(backend.generate_whisper_response(audio, use_adapter = use_adapter)) == ["hello"]
+
+ assert events == [(False, True), ("transcribe", True), (True, True), ("transcribe", True)]
+
+
+def test_a_base_pane_transcription_turns_the_lora_back_on_afterwards():
+ peft = pytest.importorskip("peft")
+ import torch
+ from core.inference.inference import InferenceBackend
+
+ class _Lora:
+ disabled = False
+
+ def disable_adapter_layers(self):
+ self.disabled = True
+
+ def enable_adapter_layers(self):
+ self.disabled = False
+
+ class _PeftWhisper(peft.PeftModel):
+ def __init__(self):
+ torch.nn.Module.__init__(self)
+ self.base_model = _Lora()
+
+ model = _PeftWhisper()
+ backend = InferenceBackend.__new__(InferenceBackend)
+ backend.active_model_name = "whisper-lora"
+ backend._generation_lock = threading.Lock()
+ seen = []
+
+ def _pipe(_inputs):
+ seen.append(model.base_model.disabled)
+ return {"text": "hello"}
+
+ backend.models = {"whisper-lora": {"model": model, "whisper_pipeline": _pipe}}
+ audio = np.zeros(16, np.float32)
+
+ list(backend.generate_whisper_response(audio, use_adapter = False))
+ list(backend.generate_whisper_response(audio))
+
+ assert seen == [True, False]