Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: run llama-server with a per-launch API key by default (#13011)

Daniel Hane0fa1a49ce7d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 41777084028..558c4aa92aa 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -7565,13 +7565,32 @@ def _expanded_user_path(value) -> Path:
return Path(os.path.expanduser(str(value)))
-def _write_direct_stream_key(key: str) -> "Path":
- """Store the direct-streaming key where only the server user can read it."""
+def _llama_server_api_key_enabled() -> bool:
+ """UNSLOTH_LLAMA_SERVER_API_KEY=0 opts out, unless UNSLOTH_DIRECT_STREAM=1 needs the key."""
+ return (
+ os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1"
+ or os.getenv("UNSLOTH_LLAMA_SERVER_API_KEY", "1") != "0"
+ )
+
+
+def _write_direct_stream_key(key: str, previous: "Optional[Path]" = None) -> "Path":
+ """0600 key file, one per backend so concurrent launches never swap keys; ``previous`` is rewritten on relaunch."""
+ import secrets as _secrets
+
from utils.paths.storage_roots import auth_root
directory = auth_root()
directory.mkdir(parents = True, exist_ok = True)
- path = directory / "llama_api_key"
+ if previous is None:
+ # Left by a killed Studio; llama-server reads its key only at startup.
+ stale_before = time.time() - 600
+ for stale in directory.glob("llama_api_key_*"):
+ with contextlib.suppress(OSError):
+ if stale.stat().st_mtime < stale_before:
+ stale.unlink()
+ path = (
+ previous if previous is not None else directory / f"llama_api_key_{_secrets.token_hex(8)}"
+ )
fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "w", encoding = "utf-8") as handle:
handle.write(key)
@@ -7920,6 +7939,7 @@ class LlamaCppBackend:
self._llama_log_path: Optional[Path] = None
self._cancel_event = threading.Event()
self._api_key: Optional[str] = None
+ self._api_key_file: Optional[Path] = None
self._slot_save_dir: Optional[str] = None
self._slot_save_binary: Optional[tuple[str, int]] = None
# (gguf_identity, launch_fingerprint) snapshotted at load, so a later slot
@@ -8045,8 +8065,7 @@ class LlamaCppBackend:
@property
def _auth_headers(self) -> "Optional[dict[str, str]]":
- """Bearer header matching the --api-key direct-stream mode uses, else
- None (so unauthenticated llama-server calls don't get a spurious 401)."""
+ """Bearer header for the child's per-launch --api-key, else None (key disabled)."""
return {"Authorization": f"Bearer {self._api_key}"} if self._api_key else None
@property
@@ -9563,6 +9582,7 @@ class LlamaCppBackend:
# only a build whose --help positively lacks one drops it.
"supports_no_context_shift": True,
"supports_jinja": True,
+ "supports_api_key_file": True,
"supports_flash_attn": True,
"flash_attn_takes_value": True,
"supports_fit_ctx": False,
@@ -9619,6 +9639,7 @@ class LlamaCppBackend:
# See the fallback dict: these fail open.
supports_no_context_shift = True
supports_jinja = True
+ supports_api_key_file = True
supports_flash_attn = True
flash_attn_takes_value = True
supports_fit_ctx = False
@@ -9845,6 +9866,7 @@ class LlamaCppBackend:
if probe_ok and blocks:
supports_no_context_shift = _is_real("--no-context-shift")
supports_jinja = _is_real("--jinja")
+ supports_api_key_file = _is_real("--api-key-file")
# Two answers, not one: whether the flag exists, and whether its
# declaration takes a value. A build predating flash attention
# has neither, and emitting the flag there is an immediate exit.
@@ -9958,6 +9980,7 @@ class LlamaCppBackend:
"supports_kv_unified": supports_kv_unified,
"supports_no_context_shift": supports_no_context_shift,
"supports_jinja": supports_jinja,
+ "supports_api_key_file": supports_api_key_file,
"supports_flash_attn": supports_flash_attn,
"flash_attn_takes_value": flash_attn_takes_value,
"supports_fit_ctx": supports_fit_ctx,
@@ -28428,14 +28451,21 @@ class LlamaCppBackend:
"device is virtualised."
)
- # Option C: --api-key for direct client access when enabled
+ # llama-server sends permissive CORS, so without a key any open web page could drive it.
import secrets as _secrets
- if os.getenv("UNSLOTH_DIRECT_STREAM", "0") == "1":
+ _key_on = _llama_server_api_key_enabled()
+ if _key_on and not server_caps.get("supports_api_key_file", True):
+ # A custom build predating --api-key-file would exit on the flag; run it keyless as before.
+ logger.warning(
+ "This llama-server build has no --api-key-file; starting it without an API key."
+ )
+ _key_on = False
+ if _key_on:
self._api_key = _secrets.token_urlsafe(32)
# Through a file, not argv: a command line is readable by every process of this Unix user, and the auth directory is not.
- cmd.extend(["--api-key-file", str(_write_direct_stream_key(self._api_key))])
- logger.info("llama-server started with --api-key-file for direct streaming")
+ self._api_key_file = _write_direct_stream_key(self._api_key, self._api_key_file)
+ cmd.extend(["--api-key-file", str(self._api_key_file)])
else:
self._api_key = None
@@ -31884,7 +31914,9 @@ class LlamaCppBackend:
from core.inference.llama_stats import maybe_start_stats_logger
if self._stats_logger is not None:
self._stats_logger.stop()
- self._stats_logger = maybe_start_stats_logger(self.base_url, logger)
+ self._stats_logger = maybe_start_stats_logger(
+ self.base_url, logger, headers = self._auth_headers
+ )
except Exception as e:
logger.debug(f"engine-stats logger not started: {e}")
else:
@@ -32741,6 +32773,10 @@ class LlamaCppBackend:
_was_resident = self._process is not None
self._kill_process()
self._cleanup_cpu_fallback_runtime()
+ if self._api_key_file is not None:
+ with contextlib.suppress(OSError):
+ os.unlink(self._api_key_file)
+ self._api_key_file = None
# The one unload line: routes/inference.py logged a second, differently named.
if _was_resident:
logger.info(f"Unloaded GGUF model: {self._model_identifier}")
@@ -35083,9 +35119,8 @@ class LlamaCppBackend:
"""llama-server's ``/props``, or None when it cannot be read."""
url = f"{self.base_url}/props"
try:
- # /props is not one of llama-server's public endpoints, so under
- # UNSLOTH_DIRECT_STREAM=1 (which launches the child with --api-key)
- # an unauthenticated read 401s: the context readback silently keeps
+ # /props is not one of llama-server's public endpoints, so with the
+ # child's --api-key an unauthenticated read 401s: the context readback silently keeps
# the requested -c, and video reads as unsupported on a model that
# supports it. None when there is no child key, which is httpx's
# default and what every other call site here relies on.
diff --git a/studio/backend/core/inference/llama_stats.py b/studio/backend/core/inference/llama_stats.py
index 74a6be38085..6acd4e9b493 100644
--- a/studio/backend/core/inference/llama_stats.py
+++ b/studio/backend/core/inference/llama_stats.py
@@ -34,8 +34,19 @@ class LlamaServerStatsLogger:
logger,
interval_s = 10.0,
stall_timeout_s = 600.0,
+ headers = None,
):
self._url = f"{base_url.rstrip('/')}/metrics"
+ self._headers = dict(headers or {})
+ # No HTTPSHandler: it builds an SSL context this loopback scrape never needs.
+ self._opener = urllib.request.OpenerDirector()
+ for handler in (
+ urllib.request.ProxyHandler({}),
+ urllib.request.HTTPHandler(),
+ urllib.request.HTTPDefaultErrorHandler(),
+ urllib.request.HTTPErrorProcessor(),
+ ):
+ self._opener.add_handler(handler)
self._log = logger
self._interval = max(1.0, float(interval_s))
self._stop = threading.Event()
@@ -57,7 +68,10 @@ class LlamaServerStatsLogger:
def _scrape(self):
try:
- with urllib.request.urlopen(self._url, timeout = 3) as r:
+ # No proxy: the bearer must only ever go to the loopback server.
+ with self._opener.open(
+ urllib.request.Request(self._url, headers = self._headers), timeout = 3
+ ) as r:
if r.status != 200:
return None
body = r.read().decode("utf-8", "replace")
@@ -252,7 +266,11 @@ def _env_float(name, default, logger):
return value
-def maybe_start_stats_logger(base_url, logger):
+def maybe_start_stats_logger(
+ base_url,
+ logger,
+ headers = None,
+):
"""Start a stats logger unless UNSLOTH_STUDIO_ENGINE_STATS disables it."""
if (os.environ.get("UNSLOTH_STUDIO_ENGINE_STATS", "1") or "").strip().lower() in _OFF:
return None
@@ -265,6 +283,7 @@ def maybe_start_stats_logger(base_url, logger):
logger,
interval,
stall_timeout_s = stall_timeout,
+ headers = headers,
)
sl.start()
return sl
diff --git a/studio/backend/core/inference/stt_mtmd_sidecar.py b/studio/backend/core/inference/stt_mtmd_sidecar.py
index 4bea1c0ac27..a1d7e5deedf 100644
--- a/studio/backend/core/inference/stt_mtmd_sidecar.py
+++ b/studio/backend/core/inference/stt_mtmd_sidecar.py
@@ -17,13 +17,14 @@ from __future__ import annotations
import base64
import http.client
import json
+import secrets
import socket
import subprocess
import threading
import time
import urllib.request
import uuid
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
from dataclasses import dataclass
from pathlib import Path
from typing import Iterator, Optional
@@ -591,6 +592,7 @@ class MtmdSttSidecar:
self._start_lock = threading.Lock()
self._process: Optional[subprocess.Popen] = None
self._port: Optional[int] = None
+ self._api_key: Optional[str] = None
self._model_id: Optional[str] = None
self._binary_path_revision: Optional[int] = None
self._loading = False
@@ -667,6 +669,7 @@ class MtmdSttSidecar:
finally:
self._process = None
self._port = None
+ self._api_key = None
self._model_id = None
self._binary_path_revision = None
@@ -970,6 +973,22 @@ class MtmdSttSidecar:
raise SttLoadCancelledError(
"Unsloth is shutting down; not starting the MTMD server."
)
+ from core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _llama_server_api_key_enabled,
+ _write_direct_stream_key,
+ )
+
+ api_key = key_file = None
+ # A custom build without --api-key-file would exit on the flag; it runs keyless as before.
+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(
+ binary
+ ).get("supports_api_key_file", True):
+ # Per-launch key against permissive CORS; read once at startup, so deleted once ready.
+ api_key = secrets.token_urlsafe(32)
+ key_file = _write_direct_stream_key(api_key)
+ # Right after the binary: --no-mmproj-offload has to stay last.
+ cmd[1:1] = ["--api-key-file", str(key_file)]
process = subprocess.Popen(
cmd,
# nothing reads these, and an undrained pipe blocks llama-server mid-startup once its logs fill the
@@ -995,7 +1014,13 @@ class MtmdSttSidecar:
raise SttLoadCancelledError(
"Unsloth is shutting down; not starting the MTMD server."
)
- if not self._wait_for_server(process, port, cancel_event):
+ try:
+ ready = self._wait_for_server(process, port, cancel_event)
+ finally:
+ if key_file is not None:
+ with suppress(OSError):
+ key_file.unlink()
+ if not ready:
# Reap it here: _process was never assigned, so unload() cannot reach a child that ignores SIGTERM and
# keeps port and VRAM.
_reap(process)
@@ -1009,6 +1034,7 @@ class MtmdSttSidecar:
with self._lock:
self._process = process
self._port = port
+ self._api_key = api_key
self._model_id = model_id
self._binary_path_revision = path_revision
self._gpu_disabled = training
@@ -1076,6 +1102,7 @@ class MtmdSttSidecar:
self.load(model_id, request_cancel_event = cancel_event)
with self._lock:
port = self._port
+ api_key = self._api_key
if port is None or not self._process_alive():
raise SttUnavailableError("The dictation server is not running.")
# Another client can switch models in the gap between that load returning and this lock, and the port read
@@ -1098,6 +1125,7 @@ class MtmdSttSidecar:
wav_bytes,
audio_seconds,
cancel_event = cancel_event,
+ **({"api_key": api_key} if api_key else {}),
**({"on_progress": on_progress} if on_progress is not None else {}),
)
if cancel_event is not None and cancel_event.is_set():
@@ -1131,6 +1159,7 @@ class MtmdSttSidecar:
*,
cancel_event: Optional[threading.Event] = None,
on_progress = None,
+ api_key: Optional[str] = None,
) -> str:
spec = MTMD_STT_MODELS[model_id]
payload = {
@@ -1172,7 +1201,10 @@ class MtmdSttSidecar:
"POST",
"/v1/chat/completions",
body = json.dumps(payload).encode("utf-8"),
- headers = {"Content-Type": "application/json"},
+ headers = {
+ "Content-Type": "application/json",
+ **({"Authorization": f"Bearer {api_key}"} if api_key else {}),
+ },
)
with connection.getresponse() as response:
if on_progress is not None and 200 <= response.status < 300:
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 38b8407e7a8..089b108afb7 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -2991,9 +2991,12 @@ _STUDIO_CREDENTIAL_BASENAME_RE = re.compile(
# Dotted names nothing else spells, so they match bare too.
r"(?:^|[/\\\s'\"=])(?:\.cli_api_key_[^/\\\s'\";&|)(<>`]*|\.bootstrap_password|\.desktop_secret)"
+ _WORD_END
+ # The per-launch key file, bare or as a glob (find / -name 'llama_api_key_*').
+ + r"|(?:^|[/\\\s'\"=])llama_api_key_[^/\\\s'\";&|)(<>`]*"
+ + _WORD_END
# Path form only: the bare name is an ordinary identifier. auth.db is absent for the same reason,
# and Studio's copy is covered by the auth-directory patterns below.
- + r"|[/\\]llama_api_key"
+ + r"|[/\\]llama_api_key(?:_\w+)?"
+ _WORD_END
# `unsloth start` keeps the coding-agent keys here. Path form only: matching the bare name
# refused `print('agent_api_key.json')`.
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index c992bd5b82b..b924d96de9a 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -210,6 +210,7 @@ class LlamaServerBackend:
self._force_cpu = False
# trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.
self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)
+ self._api_key: str | None = None
atexit.register(self._shutdown)
@contextmanager
@@ -236,6 +237,9 @@ class LlamaServerBackend:
def _base_url(self) -> str:
return f"http://{config.EMBED_HOST}:{self._port}"
+ def _auth_headers(self) -> dict[str, str]:
+ return {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}
+
def _resolve_binary(self) -> str:
"""Find llama-server, verify embeddings support, cache it. Raises if
missing/unsupported."""
@@ -942,6 +946,22 @@ class LlamaServerBackend:
"gpu" if use_gpu else "cpu",
" ".join(cmd),
)
+ from core.inference.llama_cpp import (
+ LlamaCppBackend,
+ _llama_server_api_key_enabled,
+ _write_direct_stream_key,
+ )
+
+ self._api_key = key_file = None
+ if _llama_server_api_key_enabled() and LlamaCppBackend.probe_server_capabilities(
+ binary
+ ).get("supports_api_key_file", True):
+ # Per-spawn key against permissive CORS; read once at startup, so deleted once healthy.
+ import secrets
+
+ self._api_key = secrets.token_urlsafe(32)
+ key_file = _write_direct_stream_key(self._api_key)
+ cmd[1:1] = ["--api-key-file", str(key_file)]
self._stdout_lines = []
# One flag at every spawn. No _graceful_shutdown step stops this backend, so an
# encode still resolving or downloading its model as the app quits would
@@ -979,7 +999,15 @@ class LlamaServerBackend:
name = "llama-embed-stdout",
)
self._stdout_thread.start()
- if not self._wait_for_health(config.EMBED_STARTUP_TIMEOUT_S):
+ try:
+ healthy = self._wait_for_health(config.EMBED_STARTUP_TIMEOUT_S)
+ finally:
+ if key_file is not None:
+ try:
+ key_file.unlink()
+ except OSError:
+ pass
+ if not healthy:
tail = "\n".join(self._stdout_lines[-30:])
self._kill_process()
raise RuntimeError(
@@ -1133,7 +1161,9 @@ class LlamaServerBackend:
# Held across readiness AND the request, so a swap can only land between whole requests.
self._ensure_ready(model_name)
try:
- resp = self._client.post(f"{self._base_url}{path}", json = payload)
+ resp = self._client.post(
+ f"{self._base_url}{path}", json = payload, headers = self._auth_headers()
+ )
resp.raise_for_status()
return resp.json()
except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as e:
@@ -1229,7 +1259,12 @@ class LlamaServerBackend:
malformed base URL, which is what an un-started server has.
"""
try:
- data = httpx.get(f"{self._base_url}/props", timeout = 2.0, trust_env = False).json()
+ data = httpx.get(
+ f"{self._base_url}/props",
+ headers = self._auth_headers(),
+ timeout = 2.0,
+ trust_env = False,
+ ).json()
except Exception: # noqa: BLE001 - see docstring
return None
return data if isinstance(data, dict) else None
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7dfe12b02fd..c16d5cfe3de 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -36428,7 +36428,10 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
_direct_llama_request_started()
try:
req = client.build_request(
- "POST", target_url, json = upstream_body, headers = {"Connection": "close"}
+ "POST",
+ target_url,
+ json = upstream_body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
)
first_token_deadline = time.monotonic() + _first_token_timeout_s()
# Same event the relay loop polls, so a forced swap ends the request during prefill
@@ -36575,6 +36578,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
resp = await _client.post(
target_url,
json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
timeout = _llama_non_streaming_generation_timeout(),
)
except httpx.RequestError:
@@ -37201,6 +37205,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
resp = await _client.post(
target_url,
json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S,
)
except httpx.RequestError:
@@ -38844,7 +38849,10 @@ async def _responses_stream(
disconnect_event = cancel_event
try:
req = client.build_request(
- "POST", target_url, json = body, headers = {"Connection": "close"}
+ "POST",
+ target_url,
+ json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
)
first_token_deadline = time.monotonic() + _first_token_timeout_s()
try:
@@ -43146,7 +43154,12 @@ async def _anthropic_passthrough_stream(
try:
url = target_url
try:
- req = client.build_request("POST", url, json = body, headers = {"Connection": "close"})
+ req = client.build_request(
+ "POST",
+ url,
+ json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
+ )
first_token_deadline = time.monotonic() + _first_token_timeout_s()
resp = await _send_stream_with_preheader_cancel(
client, req, cancel_event, request = request
@@ -43157,7 +43170,12 @@ async def _anthropic_passthrough_stream(
url = await _passthrough_retry_url(llama_backend, exc)
if url is None:
raise
- req = client.build_request("POST", url, json = body, headers = {"Connection": "close"})
+ req = client.build_request(
+ "POST",
+ url,
+ json = body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
+ )
first_token_deadline = time.monotonic() + _first_token_timeout_s()
resp = await _send_stream_with_preheader_cancel(
client, req, cancel_event, request = request
@@ -43377,6 +43395,7 @@ async def _anthropic_passthrough_non_streaming(
response = await _client.post(
target_url,
json = payload_body,
+ headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend),
timeout = _llama_non_streaming_generation_timeout(),
)
except httpx.RequestError:
diff --git a/studio/backend/tests/test_engine_stats_live_rate.py b/studio/backend/tests/test_engine_stats_live_rate.py
index f61509b3113..f8beb92395e 100644
--- a/studio/backend/tests/test_engine_stats_live_rate.py
+++ b/studio/backend/tests/test_engine_stats_live_rate.py
@@ -264,7 +264,7 @@ def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):
"llamacpp:prompt_tokens_total 10\n"
"llamacpp:prompt_seconds_total 1\n"
).encode()
- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp(body))
+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(body))
m = LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()
diff --git a/studio/backend/tests/test_llama_extra_args_end_to_end.py b/studio/backend/tests/test_llama_extra_args_end_to_end.py
index 9e4234be9b6..4b103711910 100644
--- a/studio/backend/tests/test_llama_extra_args_end_to_end.py
+++ b/studio/backend/tests/test_llama_extra_args_end_to_end.py
@@ -36,10 +36,10 @@ def _cmd(tmp_path, **load_kwargs):
def _stable(cmd: list[str]) -> list[str]:
- """The command with the two values that differ per launch masked.
+ """The command with the values that differ per launch masked.
- The port is picked from whatever is free and the GGUF lives under a per-test
- tmp dir, so a literal comparison could never hold. Everything else is the
+ The port is picked from whatever is free, the GGUF lives under a per-test
+ tmp dir and the API key file is per launch, so a literal comparison could never hold. Everything else is the
part this feature must not move.
"""
masked = list(cmd)
@@ -48,6 +48,8 @@ def _stable(cmd: list[str]) -> list[str]:
masked[index] = "<port>"
elif index and masked[index - 1] in {"-m", "--model"}:
masked[index] = "<model>"
+ elif index and masked[index - 1] == "--api-key-file":
+ masked[index] = "<api-key-file>"
return masked
diff --git a/studio/backend/tests/test_llama_extra_args_platforms.py b/studio/backend/tests/test_llama_extra_args_platforms.py
index 3d9068efb6c..258b6802d27 100644
--- a/studio/backend/tests/test_llama_extra_args_platforms.py
+++ b/studio/backend/tests/test_llama_extra_args_platforms.py
@@ -74,13 +74,15 @@ def _apply_platform(monkeypatch, platform) -> None:
def _stable(cmd: list[str]) -> list[str]:
- """The command with the two per-launch values masked (port, model path)."""
+ """The command with the per-launch values masked (port, model path, API key file)."""
masked = list(cmd)
for index, token in enumerate(masked):
if index and masked[index - 1] == "--port":
masked[index] = "<port>"
elif index and masked[index - 1] in {"-m", "--model"}:
masked[index] = "<model>"
+ elif index and masked[index - 1] == "--api-key-file":
+ masked[index] = "<api-key-file>"
return masked
diff --git a/studio/backend/tests/test_llama_server_api_key_default.py b/studio/backend/tests/test_llama_server_api_key_default.py
new file mode 100644
index 00000000000..28340d753c3
--- /dev/null
+++ b/studio/backend/tests/test_llama_server_api_key_default.py
@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""llama-server runs with a per-launch API key by default, and every Studio request to it sends the key."""
+
+import ast
+import os
+import stat
+import sys
+from pathlib import Path
+
+import pytest
+
+BACKEND = Path(__file__).resolve().parents[1]
+if str(BACKEND) not in sys.path:
+ sys.path.insert(0, str(BACKEND))
+
+from core.inference import llama_cpp # noqa: E402
+from core.inference import llama_stats # noqa: E402
+
+
+ "env,expected",
+ [
+ ({}, True),
+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "0"}, False),
+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "1"}, True),
+ ({"UNSLOTH_LLAMA_SERVER_API_KEY": "0", "UNSLOTH_DIRECT_STREAM": "1"}, True),
+ ({"UNSLOTH_DIRECT_STREAM": "1"}, True),
+ ],
+)
+def test_key_is_on_by_default(monkeypatch, env, expected):
+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
+ monkeypatch.delenv("UNSLOTH_DIRECT_STREAM", raising = False)
+ for key, value in env.items():
+ monkeypatch.setenv(key, value)
+ assert llama_cpp._llama_server_api_key_enabled() is expected
+
+
+def test_key_file_is_per_backend_private_and_rewritten_on_relaunch(monkeypatch, tmp_path):
+ import utils.paths.storage_roots as roots
+
+ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
+ first = llama_cpp._write_direct_stream_key("one")
+ second = llama_cpp._write_direct_stream_key("two", first)
+ other = llama_cpp._write_direct_stream_key("three")
+ assert second == first and other != first
+ assert second.read_text() == "two" and other.read_text() == "three"
+ assert second.name.startswith("llama_api_key_")
+ if os.name == "posix":
+ assert stat.S_IMODE(second.stat().st_mode) == 0o600
+
+
+def test_stale_key_files_are_swept_on_a_new_launch(monkeypatch, tmp_path):
+ import utils.paths.storage_roots as roots
+
+ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
+ stale = tmp_path / "llama_api_key_dead"
+ stale.write_text("old")
+ os.utime(stale, (1, 1))
+ recent = llama_cpp._write_direct_stream_key("live")
+ current = llama_cpp._write_direct_stream_key("new")
+ assert not stale.exists()
+ assert recent.read_text() == "live" and current.read_text() == "new"
+
+
+def test_tool_guard_still_refuses_the_per_launch_key_file():
+ from core.inference import tools
+
+ assert tools._references_studio_credential("cat /tmp/x/llama_api_key_0123abcd")
+ assert tools._references_studio_credential("cat /tmp/x/llama_api_key")
+ # A basename or glob search finds the live file anywhere, so it is refused bare too.
+ assert tools._references_studio_credential("find / -name 'llama_api_key_*' -exec cat {} +")
+ # The unsuffixed name stays an ordinary identifier.
+ assert not tools._references_studio_credential("print(llama_api_key)")
+
+
+def test_stats_scrape_sends_the_key(monkeypatch):
+ seen = {}
+
+ class _Resp:
+ status = 200
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *a):
+ return False
+
+ def read(self):
+ return b""
+
+ def _open(request, timeout = None):
+ seen["auth"] = request.get_header("Authorization")
+ return _Resp()
+
+ logger = llama_stats.LlamaServerStatsLogger(
+ "http://127.0.0.1:1", None, headers = {"Authorization": "Bearer k"}
+ )
+ # The scrape never goes through an ambient proxy, which would receive the bearer.
+ assert not any(
+ isinstance(h, llama_stats.urllib.request.ProxyHandler) and h.proxies
+ for h in logger._opener.handlers
+ )
+ monkeypatch.setattr(logger._opener, "open", _open)
+ logger._scrape()
+ assert seen["auth"] == "Bearer k"
+
+
+def test_capability_probe_reports_api_key_file_and_fails_open():
+ src = Path(llama_cpp.__file__).read_text(encoding = "utf-8")
+ assert '"supports_api_key_file": True' in src
+ assert 'supports_api_key_file = _is_real("--api-key-file")' in src
+ assert 'server_caps.get("supports_api_key_file", True)' in src
+
+
+def _llama_requests(tree):
+ """Every build_request / post in routes/inference.py whose URL is a llama-server target."""
+ for node in ast.walk(tree):
+ if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)):
+ continue
+ if node.func.attr not in ("build_request", "post"):
+ continue
+ names = {a.id for a in node.args if isinstance(a, ast.Name)}
+ if names & {"target_url", "url"} and not names & {"provider_url"}:
+ yield node
+
+
+def test_every_llama_request_in_inference_routes_sends_auth():
+ src = (BACKEND / "routes" / "inference.py").read_text(encoding = "utf-8")
+ tree = ast.parse(src)
+ calls = list(_llama_requests(tree))
+ assert len(calls) >= 10, "the call-site scan matched too few requests to mean anything"
+ missing = []
+ for call in calls:
+ headers = next((kw.value for kw in call.keywords if kw.arg == "headers"), None)
+ text = ast.unparse(headers) if headers is not None else ""
+ if "upstream_headers" not in text and "_openai_passthrough_upstream_headers" not in text:
+ missing.append(f"line {call.lineno}: {ast.unparse(call)[:120]}")
+ assert not missing, "llama-server requests without the backend auth header:\n" + "\n".join(
+ missing
+ )
diff --git a/studio/backend/tests/test_llama_stats.py b/studio/backend/tests/test_llama_stats.py
index c46ac40ecf8..26aea440154 100644
--- a/studio/backend/tests/test_llama_stats.py
+++ b/studio/backend/tests/test_llama_stats.py
@@ -110,7 +110,7 @@ def test_scrape_parses_labelled_and_bare_metrics(monkeypatch):
def __exit__(self, *a):
return False
- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp())
+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp())
m = ls.LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()
assert m["tokens_predicted_total"] == 20.0
assert m["prompt_tokens_total"] == 5.0
diff --git a/studio/backend/tests/test_llama_stats_platform_matrix.py b/studio/backend/tests/test_llama_stats_platform_matrix.py
index c257d51381d..9a909ddcddd 100644
--- a/studio/backend/tests/test_llama_stats_platform_matrix.py
+++ b/studio/backend/tests/test_llama_stats_platform_matrix.py
@@ -146,7 +146,7 @@ def _scrape_body(body, monkeypatch):
def __exit__(self, *a):
return False
- monkeypatch.setattr(ls.urllib.request, "urlopen", lambda *a, **k: _Resp())
+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp())
return LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()
@@ -184,7 +184,7 @@ def test_scrape_failure_returns_none(monkeypatch):
def _boom(*a, **k):
raise OSError("connection refused")
- monkeypatch.setattr(ls.urllib.request, "urlopen", _boom)
+ monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", _boom)
assert LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape() is None
diff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.py
index bd45b70e43f..5df7a4636df 100644
--- a/studio/backend/tests/test_rag_embed_llama_server.py
+++ b/studio/backend/tests/test_rag_embed_llama_server.py
@@ -662,7 +662,12 @@ def test_post_restarts_once_on_connect_error(monkeypatch):
attempts = {"n": 0}
class _Client:
- def post(self, url, json):
+ def post(
+ self,
+ url,
+ json,
+ headers = None,
+ ):
attempts["n"] += 1
if attempts["n"] == 1:
raise httpx.ConnectError("boom")
@@ -697,7 +702,12 @@ def test_post_restarts_once_on_read_timeout(monkeypatch):
attempts = {"n": 0}
class _Client:
- def post(self, url, json):
+ def post(
+ self,
+ url,
+ json,
+ headers = None,
+ ):
attempts["n"] += 1
if attempts["n"] == 1:
raise httpx.ReadTimeout("timed out")
@@ -1564,7 +1574,11 @@ def test_a_swap_cannot_land_between_readiness_and_the_request(monkeypatch):
def fake_ensure_ready(model_name = None):
order.append(f"ready:{model_name}")
- def fake_post(url, json = None): # noqa: A002 - httpx's parameter name
+ def fake_post(
+ url,
+ json = None,
+ headers = None,
+ ): # noqa: A002 - httpx's parameter name
order.append("post")
# A competing swap tries to run right here, where the old code let it.
swapper = threading.Thread(target = _swap)
@@ -2090,3 +2104,27 @@ def test_llama_identity_only_predicts_a_stand_in_quant_while_its_download_is_pen
legacy = config.embedding_identity("llama-server", model, gguf_repo = repo)
assert embeddings._identity(True, model) == legacy + suffix
+
+
+def test_spawn_uses_a_per_spawn_key_that_requests_send(monkeypatch, tmp_path):
+ import utils.paths.storage_roots as roots
+
+ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
+ monkeypatch.setattr(config, "EMBED_PORT", 0)
+ b = LlamaServerBackend()
+ proc = _FakeProc(alive = True)
+ captured, keys = {}, []
+ _patch_spawn_deps(monkeypatch, proc)
+ _intercept_server_popen(monkeypatch, proc, captured)
+
+ def healthy(*_a, **_k):
+ cmd = captured["cmd"]
+ keys.append(Path(cmd[cmd.index("--api-key-file") + 1]).read_text())
+ return True
+
+ monkeypatch.setattr(b, "_wait_for_health", healthy)
+ b._spawn()
+ assert keys == [b._api_key] and b._auth_headers() == {"Authorization": f"Bearer {b._api_key}"}
+ # llama-server read it at startup; nothing is left on disk.
+ assert not list(tmp_path.glob("llama_api_key_*"))
diff --git a/studio/backend/tests/test_stt_mtmd_sidecar.py b/studio/backend/tests/test_stt_mtmd_sidecar.py
index 367832ead16..b94d1844fb4 100644
--- a/studio/backend/tests/test_stt_mtmd_sidecar.py
+++ b/studio/backend/tests/test_stt_mtmd_sidecar.py
@@ -10,6 +10,7 @@ when the tests were written rather than what they cover. Same tests, one subject
import subprocess
import threading
import time
+from pathlib import Path
import pytest
from core.inference import stt_mtmd_sidecar as mtmd_mod
from core.inference.stt_mtmd_sidecar import MtmdSttSidecar
@@ -1205,3 +1206,85 @@ def test_catalogue_probes_stay_answerable(model_id):
mtmd_mod._forget_downloaded_probe()
assert isinstance(mtmd_mod.is_model_downloaded(model_id), bool)
mtmd_mod._forget_downloaded_probe()
+
+
+def test_the_dictation_server_gets_a_per_launch_key_that_requests_send(
+ spawned, monkeypatch, tmp_path
+):
+ import utils.paths.storage_roots as roots
+
+ monkeypatch.setattr(roots, "auth_root", lambda: tmp_path)
+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
+ commands, sent = [], []
+
+ def capture(cmd, **kwargs):
+ key_file = Path(cmd[cmd.index("--api-key-file") + 1])
+ commands.append((list(cmd), key_file.read_text()))
+ return _FakeProcess()
+
+ class _Response:
+ status = 200
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def read(self):
+ return b'{"choices":[{"message":{"content":"hi"}}]}'
+
+ class _Connection:
+ sock = None
+
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def request(
+ self,
+ method,
+ path,
+ body = None,
+ headers = None,
+ ):
+ sent.append(headers)
+
+ def getresponse(self):
+ return _Response()
+
+ def close(self):
+ pass
+
+ monkeypatch.setattr(mtmd_mod.subprocess, "Popen", capture)
+ monkeypatch.setattr(MtmdSttSidecar, "_wait_for_server", staticmethod(lambda *a, **k: True))
+ monkeypatch.setattr(mtmd_mod.http.client, "HTTPConnection", _Connection)
+ sidecar = MtmdSttSidecar(keep_alive_seconds = 0)
+ sidecar.load("qwen3-asr-0.6b")
+ ((cmd, key),) = commands
+ assert cmd[-1] != "--api-key-file" and len(key) > 20
+ # llama-server read it at startup; nothing is left on disk.
+ assert not list(tmp_path.glob("llama_api_key_*"))
+ sidecar._post_transcribe(65000, "qwen3-asr-0.6b", b"RIFFwav", api_key = sidecar._api_key)
+ assert sent[0]["Authorization"] == f"Bearer {key}"
+ sidecar.unload()
+ assert sidecar._api_key is None
+
+
+def test_a_build_without_api_key_file_still_starts_keyless(spawned, monkeypatch):
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ commands = []
+ monkeypatch.delenv("UNSLOTH_LLAMA_SERVER_API_KEY", raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "probe_server_capabilities",
+ classmethod(lambda cls, binary = None: {"supports_api_key_file": False}),
+ )
+ monkeypatch.setattr(
+ mtmd_mod.subprocess, "Popen", lambda cmd, **k: commands.append(list(cmd)) or _FakeProcess()
+ )
+ monkeypatch.setattr(MtmdSttSidecar, "_wait_for_server", staticmethod(lambda *a, **k: True))
+ sidecar = MtmdSttSidecar(keep_alive_seconds = 0)
+ sidecar.load("qwen3-asr-0.6b")
+ assert "--api-key-file" not in commands[0] and sidecar._api_key is None
+ sidecar.unload()