Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: run llama-server with a per-launch API key by default (#13011)
Daniel Hane0fa1a49ce7d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 41777084028..558c4aa92aa 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -7565,13 +7565,32 @@ def _expanded_user_path(value) -> Path:return Path(os.path.expanduser(str(value)))-def _write_direct_stream_key(key: str) -> "Path":- """Store the direct-streaming key where only the server user can read it."""+def _llama_server_api_key_enabled() -> bool:+ """UNSLOTH_LLAMA_SERVER_API_KEY=0 opts out, unless UNSLOTH_DIRECT_STREAM=1 needs the key."""+ return (+ os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1"+ or os.getenv("UNSLOTH_LLAMA_SERVER_API_KEY", "1") != "0"+ )+++def _write_direct_stream_key(key: str, previous: "Optional[Path]" = None) -> "Path":+ """0600 key file, one per backend so concurrent launches never swap keys; ``previous`` is rewritten on relaunch."""+ import secrets as _secrets+from utils.paths.storage_roots import auth_rootdirectory = auth_root()directory.mkdir(parents = True, exist_ok = True)- path = directory / "llama_api_key"+ if previous is None:+ # Left by a killed Studio; llama-server reads its key only at startup.+ stale_before = time.time() - 600+ for stale in directory.glob("llama_api_key_*"):+ with contextlib.suppress(OSError):+ if stale.stat().st_mtime < stale_before:+ stale.unlink()+ path = (+ previous if previous is not None else directory / f"llama_api_key_{_secrets.token_hex(8)}"+ )fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)with os.fdopen(fd, "w", encoding = "utf-8") as handle:handle.write(key)@@ -7920,6 +7939,7 @@ class LlamaCppBackend:self._llama_log_path: Optional[Path] = Noneself._cancel_event = threading.Event()self._api_key: Optional[str] = None+ self._api_key_file: Optional[Path] = Noneself._slot_save_dir: Optional[str] = Noneself._slot_save_binary: Optional[tuple[str, int]] = None# (gguf_identity, launch_fingerprint) snapshotted at load, so a later slot@@ -8045,8 +8065,7 @@ class LlamaCppBackend:@propertydef _auth_headers(self) -> "Optional[dict[str, str]]":- """Bearer header matching the --api-key direct-stream mode uses, else- None (so unauthenticated llama-server calls don't get a spurious 401)."""+ """Bearer header for the child's per-launch --api-key, else None (key disabled)."""return {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None@property@@ -9563,6 +9582,7 @@ class LlamaCppBackend:# only a build whose --help positively lacks one drops it."supports_no_context_shift": True,"supports_jinja": True,+ "supports_api_key_file": True,"supports_flash_attn": True,"flash_attn_takes_value": True,"supports_fit_ctx": False,@@ -9619,6 +9639,7 @@ class LlamaCppBackend:# See the fallback dict: these fail open.supports_no_context_shift = Truesupports_jinja = True+ supports_api_key_file = Truesupports_flash_attn = Trueflash_attn_takes_value = Truesupports_fit_ctx = False@@ -9845,6 +9866,7 @@ class LlamaCppBackend:if probe_ok and blocks:supports_no_context_shift = _is_real("--no-context-shift")supports_jinja = _is_real("--jinja")+ supports_api_key_file = _is_real("--api-key-file")# Two answers, not one: whether the flag exists, and whether its# declaration takes a value. A build predating flash attention# has neither, and emitting the flag there is an immediate exit.@@ -9958,6 +9980,7 @@ class LlamaCppBackend:"supports_kv_unified": supports_kv_unified,"supports_no_context_shift": supports_no_context_shift,"supports_jinja": supports_jinja,+ "supports_api_key_file": supports_api_key_file,"supports_flash_attn": supports_flash_attn,"flash_attn_takes_value": flash_attn_takes_value,"supports_fit_ctx": supports_fit_ctx,@@ -28428,14 +28451,21 @@ class LlamaCppBackend:"device is virtualised.")- # Option C: --api-key for direct client access when enabled+ # llama-server sends permissive CORS, so without a key any open web page could drive it.import secrets as _secrets- if os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1":+ _key_on = _llama_server_api_key_enabled()+ if _key_on and not server_caps.get("supports_api_key_file", True):+ # A custom build predating --api-key-file would exit on the flag; run it keyless as before.+ logger.warning(+ "This llama-server build has no --api-key-file; starting it without an API key."+ )+ _key_on = False+ if _key_on:self._api_key = _secrets.token_urlsafe(32)# Through a file, not argv: a command line is readable by every process of this Unix user, and the auth directory is not.- cmd.extend(["--api-key-file", str(_write_direct_stream_key(self._api_key))])- logger.info("llama-server started with --api-key-file for direct streaming")+ self._api_key_file = _write_direct_stream_key(self._api_key, self._api_key_file)+ cmd.extend(["--api-key-file", str(self._api_key_file)])else:self._api_key = None@@ -31884,7 +31914,9 @@ class LlamaCppBackend:from core.inference.llama_stats import maybe_start_stats_loggerif self._stats_logger is not None:self._stats_logger.stop()- self._stats_logger = maybe_start_stats_logger(self.base_url, logger)+ self._stats_logger = maybe_start_stats_logger(+ self.base_url, logger, headers = self._auth_headers+ )except Exception as e:logger.debug(f"engine-stats logger not started: {e}")else:@@ -32741,6 +32773,10 @@ class LlamaCppBackend:_was_resident = self._process is not Noneself._kill_process()self._cleanup_cpu_fallback_runtime()+ if self._api_key_file is not None:+ with contextlib.suppress(OSError):+ os.unlink(self._api_key_file)+ self._api_key_file = None# The one unload line: routes/inference.py logged a second, differently named.if _was_resident:logger.info(f"Unloaded GGUF model: {self._model_identifier}")@@ -35083,9 +35119,8 @@ class LlamaCppBackend:"""llama-server's ``/props``, or None when it cannot be read."""url = f"{self.base_url}/props"try:- # /props is not one of llama-server's public endpoints, so under- # UNSLOTH_DIRECT_STREAM=1 (which launches the child with --api-key)- # an unauthenticated read 401s: the context readback silently keeps+ # /props is not one of llama-server's public endpoints, so with the+ # child's --api-key an unauthenticated read 401s: the context readback silently keeps# the requested -c, and video reads as unsupported on a model that# supports it. None when there is no child key, which is httpx's# default and what every other call site here relies on.diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.pyindex 74a6be38085..6acd4e9b493 100644--- a/studio/backend/core/inference/llama_stats.py+++ b/studio/backend/core/inference/llama_stats.py@@ -34,8 +34,19 @@ class LlamaServerStatsLogger:logger,interval_s = 10.0,stall_timeout_s = 600.0,+ headers = None,):self._url = f"{base_url.rstrip('/')}/metrics"+ self._headers = dict(headers or {})+ # No HTTPSHandler: it builds an SSL context this loopback scrape never needs.+ self._opener = urllib.request.OpenerDirector()+ for handler in (+ urllib.request.ProxyHandler({}),+ urllib.request.HTTPHandler(),+ urllib.request.HTTPDefaultErrorHandler(),+ urllib.request.HTTPErrorProcessor(),+ ):+ self._opener.add_handler(handler)self._log = loggerself._interval = max(1.0, float(interval_s))self._stop = threading.Event()@@ -57,7 +68,10 @@ class LlamaServerStatsLogger:def _scrape(self):try:- with urllib.request.urlopen(self._url, timeout = 3) as r:+ # No proxy: the bearer must only ever go to the loopback server.+ with self._opener.open(+ urllib.request.Request(self._url, headers = self._headers), timeout = 3+ ) as r:if r.status != 200:return Nonebody = r.read().decode("utf-8", "replace")@@ -252,7 +266,11 @@ def _env_float(name, default, logger):return value-def maybe_start_stats_logger(base_url, logger):+def maybe_start_stats_logger(+ base_url,+ logger,+ headers = None,+):"""Start a stats logger unless UNSLOTH_STUDIO_ENGINE_STATS disables it."""if (os.environ.get("UNSLOTH_STUDIO_ENGINE_STATS", "1") or "").strip().lower() in _OFF:return None@@ -265,6 +283,7 @@ def maybe_start_stats_logger(base_url, logger):logger,interval,stall_timeout_s = stall_timeout,+ headers = headers,)sl.start()return sldiff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.pyindex 4bea1c0ac27..a1d7e5deedf 100644--- a/studio/backend/core/inference/stt_mtmd_sidecar.py+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py@@ -17,13 +17,14 @@ from __future__ import annotationsimport base64import http.clientimport json+import secretsimport socketimport subprocessimport threadingimport timeimport urllib.requestimport uuid-from contextlib import contextmanager+from contextlib import contextmanager, suppressfrom dataclasses import dataclassfrom pathlib import Pathfrom typing import Iterator, Optional@@ -591,6 +592,7 @@ class MtmdSttSidecar:self._start_lock = threading.Lock()self._process: Optional[subprocess.Popen] = Noneself._port: Optional[int] = None+ self._api_key: Optional[str] = Noneself._model_id: Optional[str] = Noneself._binary_path_revision: Optional[int] = Noneself._loading = False@@ -667,6 +669,7 @@ class MtmdSttSidecar:finally:self._process = Noneself._port = None+ self._api_key = Noneself._model_id = Noneself._binary_path_revision = None@@ -970,6 +973,22 @@ class MtmdSttSidecar:raise SttLoadCancelledError("Unsloth is shutting down; not starting the MTMD server.")+ from core.inference.llama_cpp import (+ LlamaCppBackend,+ _llama_server_api_key_enabled,+ _write_direct_stream_key,+ )++ api_key = key_file = None+ # A custom build without --api-key-file would exit on the flag; it runs keyless as before.+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(+ binary+ ).get("supports_api_key_file", True):+ # Per-launch key against permissive CORS; read once at startup, so deleted once ready.+ api_key = secrets.token_urlsafe(32)+ key_file = _write_direct_stream_key(api_key)+ # Right after the binary: --no-mmproj-offload has to stay last.+ cmd[1:1] = ["--api-key-file", str(key_file)]process = subprocess.Popen(cmd,# nothing reads these, and an undrained pipe blocks llama-server mid-startup once its logs fill the@@ -995,7 +1014,13 @@ class MtmdSttSidecar:raise SttLoadCancelledError("Unsloth is shutting down; not starting the MTMD server.")- if not self._wait_for_server(process, port, cancel_event):+ try:+ ready = self._wait_for_server(process, port, cancel_event)+ finally:+ if key_file is not None:+ with suppress(OSError):+ key_file.unlink()+ if not ready:# Reap it here: _process was never assigned, so unload() cannot reach a child that ignores SIGTERM and# keeps port and VRAM._reap(process)@@ -1009,6 +1034,7 @@ class MtmdSttSidecar:with self._lock:self._process = processself._port = port+ self._api_key = api_keyself._model_id = model_idself._binary_path_revision = path_revisionself._gpu_disabled = training@@ -1076,6 +1102,7 @@ class MtmdSttSidecar:self.load(model_id, request_cancel_event = cancel_event)with self._lock:port = self._port+ api_key = self._api_keyif port is None or not self._process_alive():raise SttUnavailableError("The dictation server is not running.")# Another client can switch models in the gap between that load returning and this lock, and the port read@@ -1098,6 +1125,7 @@ class MtmdSttSidecar:wav_bytes,audio_seconds,cancel_event = cancel_event,+ **({"api_key": api_key} if api_key else {}),**({"on_progress": on_progress} if on_progress is not None else {}),)if cancel_event is not None and cancel_event.is_set():@@ -1131,6 +1159,7 @@ class MtmdSttSidecar:*,cancel_event: Optional[threading.Event] = None,on_progress = None,+ api_key: Optional[str] = None,) -> str:spec = MTMD_STT_MODELS[model_id]payload = {@@ -1172,7 +1201,10 @@ class MtmdSttSidecar:"POST","/v1/chat/completions",body = json.dumps(payload).encode("utf-8"),- headers = {"Content-Type": "application/json"},+ headers = {+ "Content-Type": "application/json",+ **({"Authorization": f"Bearer {api_key}"} if api_key else {}),+ },)with connection.getresponse() as response:if on_progress is not None and 200 <= response.status < 300:diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 38b8407e7a8..089b108afb7 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -2991,9 +2991,12 @@ _STUDIO_CREDENTIAL_BASENAME_RE = re.compile(# Dotted names nothing else spells, so they match bare too.r"(?:^|[/\\\s'\"=])(?:\.cli_api_key_[^/\\\s'\";&|)(<>`]*|\.bootstrap_password|\.desktop_secret)"+ _WORD_END+ # The per-launch key file, bare or as a glob (find / -name 'llama_api_key_*').+ + r"|(?:^|[/\\\s'\"=])llama_api_key_[^/\\\s'\";&|)(<>`]*"+ + _WORD_END# Path form only: the bare name is an ordinary identifier. auth.db is absent for the same reason,# and Studio's copy is covered by the auth-directory patterns below.- + r"|[/\\]llama_api_key"+ + r"|[/\\]llama_api_key(?:_\w+)?"+ _WORD_END# `unsloth start` keeps the coding-agent keys here. Path form only: matching the bare name# refused `print('agent_api_key.json')`.diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex c992bd5b82b..b924d96de9a 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -210,6 +210,7 @@ class LlamaServerBackend:self._force_cpu = False# trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)+ self._api_key: str | None = Noneatexit.register(self._shutdown)@contextmanager@@ -236,6 +237,9 @@ class LlamaServerBackend:def _base_url(self) -> str:return f"http://{config.EMBED_HOST}:{self._port}"+ def _auth_headers(self) -> dict[str, str]:+ return {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}+def _resolve_binary(self) -> str:"""Find llama-server, verify embeddings support, cache it. Raises ifmissing/unsupported."""@@ -942,6 +946,22 @@ class LlamaServerBackend:"gpu" if use_gpu else "cpu"," ".join(cmd),)+ from core.inference.llama_cpp import (+ LlamaCppBackend,+ _llama_server_api_key_enabled,+ _write_direct_stream_key,+ )++ self._api_key = key_file = None+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(+ binary+ ).get("supports_api_key_file", True):+ # Per-spawn key against permissive CORS; read once at startup, so deleted once healthy.+ import secrets++ self._api_key = secrets.token_urlsafe(32)+ key_file = _write_direct_stream_key(self._api_key)+ cmd[1:1] = ["--api-key-file", str(key_file)]self._stdout_lines = []# One flag at every spawn. No _graceful_shutdown step stops this backend, so an# encode still resolving or downloading its model as the app quits would@@ -979,7 +999,15 @@ class LlamaServerBackend:name = "llama-embed-stdout",)self._stdout_thread.start()- if not self._wait_for_health(config.EMBED_STARTUP_TIMEOUT_S):+ try:+ healthy = self._wait_for_health(config.EMBED_STARTUP_TIMEOUT_S)+ finally:+ if key_file is not None:+ try:+ key_file.unlink()+ except OSError:+ pass+ if not healthy:tail = "\n".join(self._stdout_lines[-30:])self._kill_process()raise RuntimeError(@@ -1133,7 +1161,9 @@ class LlamaServerBackend:# Held across readiness AND the request, so a swap can only land between whole requests.self._ensure_ready(model_name)try:- resp = self._client.post(f"{self._base_url}{path}", json = payload)+ resp = self._client.post(+ f"{self._base_url}{path}", json = payload, headers = self._auth_headers()+ )resp.raise_for_status()return resp.json()except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as e:@@ -1229,7 +1259,12 @@ class LlamaServerBackend:malformed base URL, which is what an un-started server has."""try:- data = httpx.get(f"{self._base_url}/props", timeout = 2.0, trust_env = False).json()+ data = httpx.get(+ f"{self._base_url}/props",+ headers = self._auth_headers(),+ timeout = 2.0,+ trust_env = False,+ ).json()except Exception: # noqa: BLE001 - see docstringreturn Nonereturn data if isinstance(data, dict) else Nonediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 7dfe12b02fd..c16d5cfe3de 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -36428,7 +36428,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge_direct_llama_request_started()try:req = client.build_request(- "POST", target_url, json = upstream_body, headers = {"Connection": "close"}+ "POST",+ target_url,+ json = upstream_body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),)first_token_deadline = time.monotonic() + _first_token_timeout_s()# Same event the relay loop polls, so a forced swap ends the request during prefill@@ -36575,6 +36578,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(geresp = await _client.post(target_url,json = body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),timeout = _llama_non_streaming_generation_timeout(),)except httpx.RequestError:@@ -37201,6 +37205,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(getresp = await _client.post(target_url,json = body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,)except httpx.RequestError:@@ -38844,7 +38849,10 @@ async def _responses_stream(disconnect_event = cancel_eventtry:req = client.build_request(- "POST", target_url, json = body, headers = {"Connection": "close"}+ "POST",+ target_url,+ json = body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),)first_token_deadline = time.monotonic() + _first_token_timeout_s()try:@@ -43146,7 +43154,12 @@ async def _anthropic_passthrough_stream(try:url = target_urltry:- req = client.build_request("POST", url, json = body, headers = {"Connection": "close"})+ req = client.build_request(+ "POST",+ url,+ json = body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),+ )first_token_deadline = time.monotonic() + _first_token_timeout_s()resp = await _send_stream_with_preheader_cancel(client, req, cancel_event, request = request@@ -43157,7 +43170,12 @@ async def _anthropic_passthrough_stream(url = await _passthrough_retry_url(llama_backend, exc)if url is None:raise- req = client.build_request("POST", url, json = body, headers = {"Connection": "close"})+ req = client.build_request(+ "POST",+ url,+ json = body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),+ )first_token_deadline = time.monotonic() + _first_token_timeout_s()resp = await _send_stream_with_preheader_cancel(client, req, cancel_event, request = request@@ -43377,6 +43395,7 @@ async def _anthropic_passthrough_non_streaming(response = await _client.post(target_url,json = payload_body,+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),timeout = _llama_non_streaming_generation_timeout(),)except httpx.RequestError:diff --git a/studio/backend/tests/test_engine_stats_live_rate.py b/studio/backend/tests/test_engine_stats_live_rate.pyindex f61509b3113..f8beb92395e 100644--- a/studio/backend/tests/test_engine_stats_live_rate.py+++ b/studio/backend/tests/test_engine_stats_live_rate.py@@ -264,7 +264,7 @@ def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):"llamacpp:prompt_tokens_total 10\n""llamacpp:prompt_seconds_total 1\n").encode()- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp(body))+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(body))m = LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()diff --git a/studio/backend/tests/test_llama_extra_args_end_to_end.py b/studio/backend/tests/test_llama_extra_args_end_to_end.pyindex 9e4234be9b6..4b103711910 100644--- a/studio/backend/tests/test_llama_extra_args_end_to_end.py+++ b/studio/backend/tests/test_llama_extra_args_end_to_end.py@@ -36,10 +36,10 @@ def _cmd(tmp_path, **load_kwargs):def _stable(cmd: list[str]) -> list[str]:- """The command with the two values that differ per launch masked.+ """The command with the values that differ per launch masked.- The port is picked from whatever is free and the GGUF lives under a per-test- tmp dir, so a literal comparison could never hold. Everything else is the+ The port is picked from whatever is free, the GGUF lives under a per-test+ tmp dir and the API key file is per launch, so a literal comparison could never hold. Everything else is thepart this feature must not move."""masked = list(cmd)@@ -48,6 +48,8 @@ def _stable(cmd: list[str]) -> list[str]:masked[index] = "<port>"elif index and masked[index - 1] in {"-m", "--model"}:masked[index] = "<model>"+ elif index and masked[index - 1] == "--api-key-file":+ masked[index] = "<api-key-file>"return maskeddiff --git a/studio/backend/tests/test_llama_extra_args_platforms.py b/studio/backend/tests/test_llama_extra_args_platforms.pyindex 3d9068efb6c..258b6802d27 100644--- a/studio/backend/tests/test_llama_extra_args_platforms.py+++ b/studio/backend/tests/test_llama_extra_args_platforms.py@@ -74,13 +74,15 @@ def _apply_platform(monkeypatch, platform) -> None:def _stable(cmd: list[str]) -> list[str]:- """The command with the two per-launch values masked (port, model path)."""+ """The command with the per-launch values masked (port, model path, API key file)."""masked = list(cmd)for index, token in enumerate(masked):if index and masked[index - 1] == "--port":masked[index] = "<port>"elif index and masked[index - 1] in {"-m", "--model"}:masked[index] = "<model>"+ elif index and masked[index - 1] == "--api-key-file":+ masked[index] = "<api-key-file>"return maskeddiff --git a/studio/backend/tests/test_llama_server_api_key_default.py b/studio/backend/tests/test_llama_server_api_key_default.pynew file mode 100644index 00000000000..28340d753c3--- /dev/null+++ b/studio/backend/tests/test_llama_server_api_key_default.py@@ -0,0 +1,142 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""llama-server runs with a per-launch API key by default, and every Studio request to it sends the key."""++import ast+import os+import stat+import sys+from pathlib import Path++import pytest++BACKEND = Path(__file__).resolve().parents[1]+if str(BACKEND) not in sys.path:+ sys.path.insert(0, str(BACKEND))++from core.inference import llama_cpp # noqa: E402+from core.inference import llama_stats # noqa: E402+++ "env,expected",+ [+ ({}, True),+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "0"}, False),+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "1"}, True),+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "0", "UNSLOTH_DIRECT_STREAM": "1"}, True),+ ({"UNSLOTH_DIRECT_STREAM": "1"}, True),+ ],+)+def test_key_is_on_by_default(monkeypatch, env, expected):+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)+ monkeypatch.delenv("UNSLOTH_DIRECT_STREAM", raising = False)+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ assert llama_cpp._llama_server_api_key_enabled() is expected+++def test_key_file_is_per_backend_private_and_rewritten_on_relaunch(monkeypatch, tmp_path):+ import utils.paths.storage_roots as roots++ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)+ first = llama_cpp._write_direct_stream_key("one")+ second = llama_cpp._write_direct_stream_key("two", first)+ other = llama_cpp._write_direct_stream_key("three")+ assert second == first and other != first+ assert second.read_text() == "two" and other.read_text() == "three"+ assert second.name.startswith("llama_api_key_")+ if os.name == "posix":+ assert stat.S_IMODE(second.stat().st_mode) == 0o600+++def test_stale_key_files_are_swept_on_a_new_launch(monkeypatch, tmp_path):+ import utils.paths.storage_roots as roots++ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)+ stale = tmp_path / "llama_api_key_dead"+ stale.write_text("old")+ os.utime(stale, (1, 1))+ recent = llama_cpp._write_direct_stream_key("live")+ current = llama_cpp._write_direct_stream_key("new")+ assert not stale.exists()+ assert recent.read_text() == "live" and current.read_text() == "new"+++def test_tool_guard_still_refuses_the_per_launch_key_file():+ from core.inference import tools++ assert tools._references_studio_credential("cat /tmp/x/llama_api_key_0123abcd")+ assert tools._references_studio_credential("cat /tmp/x/llama_api_key")+ # A basename or glob search finds the live file anywhere, so it is refused bare too.+ assert tools._references_studio_credential("find / -name 'llama_api_key_*' -exec cat {} +")+ # The unsuffixed name stays an ordinary identifier.+ assert not tools._references_studio_credential("print(llama_api_key)")+++def test_stats_scrape_sends_the_key(monkeypatch):+ seen = {}++ class _Resp:+ status = 200++ def __enter__(self):+ return self++ def __exit__(self, *a):+ return False++ def read(self):+ return b""++ def _open(request, timeout = None):+ seen["auth"] = request.get_header("Authorization")+ return _Resp()++ logger = llama_stats.LlamaServerStatsLogger(+ "http://127.0.0.1:1", None, headers = {"Authorization": "Bearer k"}+ )+ # The scrape never goes through an ambient proxy, which would receive the bearer.+ assert not any(+ isinstance(h, llama_stats.urllib.request.ProxyHandler) and h.proxies+ for h in logger._opener.handlers+ )+ monkeypatch.setattr(logger._opener, "open", _open)+ logger._scrape()+ assert seen["auth"] == "Bearer k"+++def test_capability_probe_reports_api_key_file_and_fails_open():+ src = Path(llama_cpp.__file__).read_text(encoding = "utf-8")+ assert '"supports_api_key_file": True' in src+ assert 'supports_api_key_file = _is_real("--api-key-file")' in src+ assert 'server_caps.get("supports_api_key_file", True)' in src+++def _llama_requests(tree):+ """Every build_request / post in routes/inference.py whose URL is a llama-server target."""+ for node in ast.walk(tree):+ if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):+ continue+ if node.func.attr not in ("build_request", "post"):+ continue+ names = {a.id for a in node.args if isinstance(a, ast.Name)}+ if names & {"target_url", "url"} and not names & {"provider_url"}:+ yield node+++def test_every_llama_request_in_inference_routes_sends_auth():+ src = (BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8")+ tree = ast.parse(src)+ calls = list(_llama_requests(tree))+ assert len(calls) >= 10, "the call-site scan matched too few requests to mean anything"+ missing = []+ for call in calls:+ headers = next((kw.value for kw in call.keywords if kw.arg == "headers"), None)+ text = ast.unparse(headers) if headers is not None else ""+ if "upstream_headers" not in text and "_openai_passthrough_upstream_headers" not in text:+ missing.append(f"line {call.lineno}: {ast.unparse(call)[:120]}")+ assert not missing, "llama-server requests without the backend auth header:\n" + "\n".join(+ missing+ )diff --git a/studio/backend/tests/test_llama_stats.py b/studio/backend/tests/test_llama_stats.pyindex c46ac40ecf8..26aea440154 100644--- a/studio/backend/tests/test_llama_stats.py+++ b/studio/backend/tests/test_llama_stats.py@@ -110,7 +110,7 @@ def test_scrape_parses_labelled_and_bare_metrics(monkeypatch):def __exit__(self, *a):return False- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp())+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp())m = ls.LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()assert m["tokens_predicted_total"] == 20.0assert m["prompt_tokens_total"] == 5.0diff --git a/studio/backend/tests/test_llama_stats_platform_matrix.py b/studio/backend/tests/test_llama_stats_platform_matrix.pyindex c257d51381d..9a909ddcddd 100644--- a/studio/backend/tests/test_llama_stats_platform_matrix.py+++ b/studio/backend/tests/test_llama_stats_platform_matrix.py@@ -146,7 +146,7 @@ def _scrape_body(body, monkeypatch):def __exit__(self, *a):return False- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp())+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp())return LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()@@ -184,7 +184,7 @@ def test_scrape_failure_returns_none(monkeypatch):def _boom(*a, **k):raise OSError("connection refused")- monkeypatch.setattr(ls.urllib.request, "urlopen", _boom)+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", _boom)assert LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape() is Nonediff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.pyindex bd45b70e43f..5df7a4636df 100644--- a/studio/backend/tests/test_rag_embed_llama_server.py+++ b/studio/backend/tests/test_rag_embed_llama_server.py@@ -662,7 +662,12 @@ def test_post_restarts_once_on_connect_error(monkeypatch):attempts = {"n": 0}class _Client:- def post(self, url, json):+ def post(+ self,+ url,+ json,+ headers = None,+ ):attempts["n"] += 1if attempts["n"] == 1:raise httpx.ConnectError("boom")@@ -697,7 +702,12 @@ def test_post_restarts_once_on_read_timeout(monkeypatch):attempts = {"n": 0}class _Client:- def post(self, url, json):+ def post(+ self,+ url,+ json,+ headers = None,+ ):attempts["n"] += 1if attempts["n"] == 1:raise httpx.ReadTimeout("timed out")@@ -1564,7 +1574,11 @@ def test_a_swap_cannot_land_between_readiness_and_the_request(monkeypatch):def fake_ensure_ready(model_name = None):order.append(f"ready:{model_name}")- def fake_post(url, json = None): # noqa: A002 - httpx's parameter name+ def fake_post(+ url,+ json = None,+ headers = None,+ ): # noqa: A002 - httpx's parameter nameorder.append("post")# A competing swap tries to run right here, where the old code let it.swapper = threading.Thread(target = _swap)@@ -2090,3 +2104,27 @@ def test_llama_identity_only_predicts_a_stand_in_quant_while_its_download_is_penlegacy = config.embedding_identity("llama-server", model, gguf_repo = repo)assert embeddings._identity(True, model) == legacy + suffix+++def test_spawn_uses_a_per_spawn_key_that_requests_send(monkeypatch, tmp_path):+ import utils.paths.storage_roots as roots++ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)+ monkeypatch.setattr(config, "EMBED_PORT", 0)+ b = LlamaServerBackend()+ proc = _FakeProc(alive = True)+ captured, keys = {}, []+ _patch_spawn_deps(monkeypatch, proc)+ _intercept_server_popen(monkeypatch, proc, captured)++ def healthy(*_a, **_k):+ cmd = captured["cmd"]+ keys.append(Path(cmd[cmd.index("--api-key-file") + 1]).read_text())+ return True++ monkeypatch.setattr(b, "_wait_for_health", healthy)+ b._spawn()+ assert keys == [b._api_key] and b._auth_headers() == {"Authorization": f"Bearer {b._api_key}"}+ # llama-server read it at startup; nothing is left on disk.+ assert not list(tmp_path.glob("llama_api_key_*"))diff --git a/studio/backend/tests/test_stt_mtmd_sidecar.py b/studio/backend/tests/test_stt_mtmd_sidecar.pyindex 367832ead16..b94d1844fb4 100644--- a/studio/backend/tests/test_stt_mtmd_sidecar.py+++ b/studio/backend/tests/test_stt_mtmd_sidecar.py@@ -10,6 +10,7 @@ when the tests were written rather than what they cover. Same tests, one subjectimport subprocessimport threadingimport time+from pathlib import Pathimport pytestfrom core.inference import stt_mtmd_sidecar as mtmd_modfrom core.inference.stt_mtmd_sidecar import MtmdSttSidecar@@ -1205,3 +1206,85 @@ def test_catalogue_probes_stay_answerable(model_id):mtmd_mod._forget_downloaded_probe()assert isinstance(mtmd_mod.is_model_downloaded(model_id), bool)mtmd_mod._forget_downloaded_probe()+++def test_the_dictation_server_gets_a_per_launch_key_that_requests_send(+ spawned, monkeypatch, tmp_path+):+ import utils.paths.storage_roots as roots++ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)+ commands, sent = [], []++ def capture(cmd, **kwargs):+ key_file = Path(cmd[cmd.index("--api-key-file") + 1])+ commands.append((list(cmd), key_file.read_text()))+ return _FakeProcess()++ class _Response:+ status = 200++ def __enter__(self):+ return self++ def __exit__(self, *_args):+ return False++ def read(self):+ return b'{"choices":[{"message":{"content":"hi"}}]}'++ class _Connection:+ sock = None++ def __init__(self, *args, **kwargs):+ pass++ def request(+ self,+ method,+ path,+ body = None,+ headers = None,+ ):+ sent.append(headers)++ def getresponse(self):+ return _Response()++ def close(self):+ pass++ monkeypatch.setattr(mtmd_mod.subprocess, "Popen", capture)+ monkeypatch.setattr(MtmdSttSidecar, "_wait_for_server", staticmethod(lambda *a, **k: True))+ monkeypatch.setattr(mtmd_mod.http.client, "HTTPConnection", _Connection)+ sidecar = MtmdSttSidecar(keep_alive_seconds = 0)+ sidecar.load("qwen3-asr-0.6b")+ ((cmd, key),) = commands+ assert cmd[-1] != "--api-key-file" and len(key) > 20+ # llama-server read it at startup; nothing is left on disk.+ assert not list(tmp_path.glob("llama_api_key_*"))+ sidecar._post_transcribe(65000, "qwen3-asr-0.6b", b"RIFFwav", api_key = sidecar._api_key)+ assert sent[0]["Authorization"] == f"Bearer {key}"+ sidecar.unload()+ assert sidecar._api_key is None+++def test_a_build_without_api_key_file_still_starts_keyless(spawned, monkeypatch):+ from core.inference.llama_cpp import LlamaCppBackend++ commands = []+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)+ monkeypatch.setattr(+ LlamaCppBackend,+ "probe_server_capabilities",+ classmethod(lambda cls, binary = None: {"supports_api_key_file": False}),+ )+ monkeypatch.setattr(+ mtmd_mod.subprocess, "Popen", lambda cmd, **k: commands.append(list(cmd)) or _FakeProcess()+ )+ monkeypatch.setattr(MtmdSttSidecar, "_wait_for_server", staticmethod(lambda *a, **k: True))+ sidecar = MtmdSttSidecar(keep_alive_seconds = 0)+ sidecar.load("qwen3-asr-0.6b")+ assert "--api-key-file" not in commands[0] and sidecar._api_key is None+ sidecar.unload()