Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: put llama-server warnings and errors in the default session log (#10848)

Sourav Rajvi8947133de68a
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 4287fbd788a..1c6d9fb8ff9 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -752,13 +752,22 @@ _LOG_VERBOSITY_FLAGS = frozenset(
)
# Past this, the active llama-server log keeps only warnings and errors.
_LLAMA_LOG_FULL_BYTES = 64 * 1024 * 1024
-_LOG_LEVEL_TOKEN_RE = re.compile(r"^\S+ ([A-Z]) ")
+# llama.cpp's prefix (stamp optional, colours allowed); a mid-line "x W " is record text.
+_ANSI = r"(?:\x1b\[[0-9;]*m)*"
+_LOG_LEVEL_TOKEN_RE = re.compile(rf"^{_ANSI}(?:\d+\.\d+\.\d+\.\d+{_ANSI} )?{_ANSI}([A-Z]) ")
# The startup head is never trimmed: its load lines are read for the server's whole life.
_STDOUT_TRIM_AT = 20000
_STDOUT_TAIL_KEEP = 5000
_STDOUT_HEAD_MAX = 5000
+def _llama_line_is_warning_or_error(line: str, line_lower: str) -> bool:
+ level = _LOG_LEVEL_TOKEN_RE.match(line)
+ if level is not None:
+ return level.group(1) in "WE"
+ return "error" in line_lower or "fail" in line_lower or "warn" in line_lower
+
+
def _trace_verbosity_args(caps: dict, extra_args: Optional[Iterable[str]], environ) -> list[str]:
"""``--verbosity 4`` when the build hides load lines at its default and the user chose none."""
if not caps.get("supports_trace_verbosity"):
@@ -17814,6 +17823,7 @@ class LlamaCppBackend:
"""
startup_len: Optional[int] = None
log_bytes = 0
+ levelled = to_info = False
try:
for line in self._process.stdout:
line = line.rstrip()
@@ -17828,23 +17838,33 @@ class LlamaCppBackend:
# Two forms across llama.cpp builds. Only readiness lines wake the
# probe: waking on every tensor-load log spins startup.
line_lower = line.lower()
- if "server is listening" in line_lower or "model loaded" in line_lower:
+ ready = "server is listening" in line_lower or "model loaded" in line_lower
+ if ready:
if startup_len is None:
startup_len = len(self._stdout_lines)
health_probe_event = getattr(self, "_health_probe_event", None)
if health_probe_event is not None:
health_probe_event.set()
- logger.debug(f"[llama-server] {line}")
+ # Unprefixed lines continue the previous record (e.g. a request dump).
+ level = _LOG_LEVEL_TOKEN_RE.match(line)
+ if level is not None:
+ levelled = True
+ to_info = level.group(1) in "WE" or (ready and level.group(1) == "I")
+ elif not levelled:
+ to_info = ready or _llama_line_is_warning_or_error(line, line_lower)
+ try:
+ if to_info:
+ logger.info(f"[llama-server] {line}")
+ else:
+ logger.debug(f"[llama-server] {line}")
+ except Exception:
+ # A closed stdout must not stop the drain and stall the server.
+ pass
fh = getattr(self, "_llama_log_fh", None)
if fh is not None:
try:
if log_bytes >= _LLAMA_LOG_FULL_BYTES:
- level = _LOG_LEVEL_TOKEN_RE.match(line)
- if not (
- level.group(1) in "WE"
- if level is not None
- else ("error" in line_lower or "fail" in line_lower)
- ):
+ if not _llama_line_is_warning_or_error(line, line_lower):
continue
elif log_bytes + len(line) + 1 >= _LLAMA_LOG_FULL_BYTES:
fh.write(
@@ -17853,9 +17873,19 @@ class LlamaCppBackend:
log_bytes += len(line) + 1
fh.write(line + "\n")
fh.flush()
- except (ValueError, OSError):
- # Log file closed under us; tee silently.
+ except ValueError:
+ # Closed under us by the kill path.
pass
+ except OSError as e:
+ if not getattr(self, "_llama_log_tee_failed", False):
+ self._llama_log_tee_failed = True
+ try:
+ logger.warning(
+ f"Could not write llama-server log "
+ f"{getattr(self, '_llama_log_path', None)}: {e}"
+ )
+ except Exception:
+ pass
except Exception:
# Never let the drain thread die: a full stdout pipe can deadlock
# llama-server (Windows). Pipe-closed on exit is the common case.
@@ -23232,7 +23262,7 @@ class LlamaCppBackend:
logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}")
except (OSError, UnicodeDecodeError) as e:
# Best-effort; never block the load on logging.
- logger.debug(f"Could not open llama-server log file: {e}")
+ logger.warning(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
# Log the argv per attempt (the text-only mmproj retry re-enters here
@@ -29790,7 +29820,7 @@ class LlamaCppBackend:
logger.info(f"llama-server stdout/stderr -> {self._llama_log_path}")
except (OSError, UnicodeDecodeError) as e:
# Best-effort; never block the load on logging.
- logger.debug(f"Could not open llama-server log file: {e}")
+ logger.warning(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
_last_spawn_cmd = list(run_cmd)
# Read off the argv actually spawned rather than the intent, so
@@ -32803,20 +32833,44 @@ class LlamaCppBackend:
self._healthy = True
return True
- def _close_attempt_log(self) -> None:
+ def _close_attempt_log(
+ self,
+ *,
+ reason: str = "closed",
+ exit_code: "Optional[int]" = None,
+ ) -> None:
"""Close the per-attempt tee log opened just before a spawn.
A refusal publishes no process and _kill_process returns early when there is
none, so nothing else closes it: the next attempt leaks the descriptor and,
on Windows, holds the file lock an update needs.
"""
+ self._llama_log_tee_failed = False
fh = getattr(self, "_llama_log_fh", None)
- if fh is not None:
+ if fh is None:
+ return
+ if exit_code is None:
try:
- fh.close()
+ exit_code = self._process.poll()
except Exception:
pass
- self._llama_log_fh = None
+ try:
+ fh.write(f"[studio] attempt end reason={reason} exit_code={exit_code}\n")
+ except Exception:
+ pass
+ try:
+ fh.close()
+ except Exception:
+ pass
+ self._llama_log_fh = None
+ try:
+ logger.info(
+ f"attempt log closed reason={reason} exit_code={exit_code} "
+ f"path={getattr(self, '_llama_log_path', None)}"
+ )
+ except Exception:
+ # The logger raises once stdout is closed; teardown must not.
+ pass
def _begin_server_lifecycle(self) -> None:
"""Clear shutdown state so a restarted server can launch again.
@@ -32908,6 +32962,7 @@ class LlamaCppBackend:
_pid = getattr(self._process, "pid", None)
_pgid = self._leading_process_group(_pid)
_descendants, _descendants_known = self._collect_descendants(_pid)
+ _exited_first = getattr(self._process, "poll", lambda: None)() is not None
if teardown:
# Before the signal, and as the process itself: the reference stays set
# across the waits below, and only identity says which child a teardown
@@ -32945,7 +33000,8 @@ class LlamaCppBackend:
# next startup sweep cannot reap it. The record stores a start-time
# identity, so a recycled pid is never signalled either way.
_killed_pid = getattr(self._process, "pid", None)
- _exited = getattr(self._process, "poll", lambda: None)() is not None
+ _exit_code = getattr(self._process, "poll", lambda: None)()
+ _exited = _exit_code is not None
# The tree kill below tells the reaper the owner is known, which waives the
# "cannot prove this pid is still our child" refusal. That waiver is only
# true while we hold the handle that spawned the pid, so it is spent on a
@@ -33030,13 +33086,9 @@ class LlamaCppBackend:
if stdout_thread is not None:
stdout_thread.join(timeout = 2)
self._stdout_thread = None
- fh = getattr(self, "_llama_log_fh", None)
- if fh is not None:
- try:
- fh.close()
- except Exception:
- pass
- self._llama_log_fh = None
+ self._close_attempt_log(
+ reason = "exited" if _exited_first else "killed", exit_code = _exit_code
+ )
@staticmethod
def _server_pidfile_path() -> Optional[Path]:
diff --git a/studio/backend/run.py b/studio/backend/run.py
index c422cc07a13..f9b8b1b66da 100644
--- a/studio/backend/run.py
+++ b/studio/backend/run.py
@@ -2424,7 +2424,17 @@ def run_server(
env = os.getenv("ENVIRONMENT_TYPE", "production"),
)
- logger.info("run_server startup begin api_only=%s host=%s port=%s", api_only, host, port)
+ logger.info(
+ "run_server startup begin api_only=%s host=%s port=%s log_level=%s studio_home=%s "
+ "python=%s pid=%s",
+ api_only,
+ host,
+ port,
+ os.getenv("LOG_LEVEL", "INFO"),
+ _studio_root(),
+ sys.version.split()[0],
+ os.getpid(),
+ )
cloudflare_intent = _consume_cloudflare_intent(cloudflare, secure)
# Reap every child if the parent dies abnormally (terminal close, Task Manager kill, SIGKILL); must
diff --git a/studio/backend/tests/test_llama_cpp_atexit_quiet.py b/studio/backend/tests/test_llama_cpp_atexit_quiet.py
index 48ce8fb35ee..bfcb41ac58d 100644
--- a/studio/backend/tests/test_llama_cpp_atexit_quiet.py
+++ b/studio/backend/tests/test_llama_cpp_atexit_quiet.py
@@ -241,3 +241,30 @@ def test_a_failing_kill_does_not_escape_the_atexit_handler(monkeypatch):
# Same trap: a TypeError from a stale signature is swallowed too, and then
# the RuntimeError this test is about is never raised at all.
assert raised, "the failing kill never ran; the handler swallowed the wrong error"
+
+
+class _Exited:
+ pid = None
+
+ def terminate(self):
+ pass
+
+ def wait(self, timeout = None):
+ return 3
+
+ def poll(self):
+ return 3
+
+
+def test_a_raising_logger_still_closes_the_attempt_log_and_names_a_crash(monkeypatch, tmp_path):
+ monkeypatch.setattr(mod, "logger", _RaisingLogger())
+ backend = _stub()
+ backend._process = _Exited()
+ path = tmp_path / "llama.log"
+ backend._llama_log_fh = open(path, "w", encoding = "utf-8")
+ backend._llama_log_path = path
+
+ backend._kill_process()
+
+ assert backend._llama_log_fh is None
+ assert path.read_text(encoding = "utf-8").endswith("reason=exited exit_code=3\n")
diff --git a/studio/backend/tests/test_llama_server_log_lines.py b/studio/backend/tests/test_llama_server_log_lines.py
new file mode 100644
index 00000000000..ec6f0624056
--- /dev/null
+++ b/studio/backend/tests/test_llama_server_log_lines.py
@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import sys
+import types as _types
+from pathlib import Path
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+_loggers_stub = _types.ModuleType("loggers")
+_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
+sys.modules.setdefault("loggers", _loggers_stub)
+_structlog_stub = _types.ModuleType("structlog")
+_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("structlog")
+sys.modules.setdefault("structlog", _structlog_stub)
+if not hasattr(sys.modules["structlog"], "get_logger"):
+ sys.modules["structlog"].get_logger = _structlog_stub.get_logger
+
+import core.inference.llama_cpp as llama_cpp # noqa: E402
+from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
+
+# Verbatim from llama.cpp b11317 (load of a missing GGUF, then a normal start).
+_NOISE = (
+ "0.00.003.868 I srv llama_server: initializing ...",
+ "0.13.635.789 I srv load_model: loading model '/nonexistent.gguf'",
+ "0.34.561.161 I cmn init: llama threadpool init, n_threads = 96",
+ "0.13.636.121 I srv operator(): operator(): cleaning up before exit...",
+)
+_NOTABLE = (
+ "0.13.616.121 W srv llama_server: security: no API key is set and CORS allows all origins",
+ "0.13.635.929 E gguf_init_from_file: failed to open GGUF file '/nonexistent.gguf'",
+ "0.13.650.267 E srv llama_server: exiting due to model loading error",
+)
+
+
+class _Proc:
+ def __init__(
+ self,
+ lines,
+ returncode = None,
+ ):
+ self.stdout = iter(line + "\n" for line in lines)
+ self.returncode = returncode
+
+ def poll(self):
+ return self.returncode
+
+
+def _backend(lines):
+ b = LlamaCppBackend.__new__(LlamaCppBackend)
+ b._process = _Proc(lines)
+ b._stdout_lines = []
+ return b
+
+
+class _Recorder:
+ def __init__(self):
+ self.records = []
+
+ def _at(level):
+ def log(self, msg, *args, **kwargs):
+ self.records.append((level, msg % args if args else msg))
+
+ return log
+
+ debug, info, warning, error = _at(10), _at(20), _at(30), _at(40)
+
+
+def _logger(monkeypatch):
+ rec = _Recorder()
+ monkeypatch.setattr(llama_cpp, "logger", rec)
+ return rec
+
+
+def _info_lines(rec, floor = 20):
+ return [m for level, m in rec.records if level >= floor]
+
+
+def test_only_warnings_errors_and_readiness_reach_info(monkeypatch):
+ ready = "0.40.000.000 I srv llama_server: server is listening on http://127.0.0.1:8080"
+ log = _logger(monkeypatch)
+ _backend(_NOISE + _NOTABLE + (ready,))._drain_stdout()
+ info = _info_lines(log)
+ assert info == [f"[llama-server] {line}" for line in _NOTABLE + (ready,)]
+
+
+def test_builds_without_level_letters_fall_back_to_keywords(monkeypatch):
+ log = _logger(monkeypatch)
+ _backend(
+ [
+ "llama_model_loader: - kv 2: general.name str = test",
+ "error: failed to load model",
+ "warning: GPU backend unavailable",
+ ]
+ )._drain_stdout()
+ assert _info_lines(log) == [
+ "[llama-server] error: failed to load model",
+ "[llama-server] warning: GPU backend unavailable",
+ ]
+
+
+def test_user_text_in_a_trace_dump_stays_debug(monkeypatch):
+ log = _logger(monkeypatch)
+ warning = "0.13.616.121 W srv llama_server: security: no API key is set"
+ _backend(
+ [
+ "0.29.142.020 D srv params_from_: request: model loaded, it failed with an error",
+ "and then model loaded",
+ "an error on a continuation line",
+ "hello W private prompt",
+ warning,
+ "the warning's own continuation",
+ ]
+ )._drain_stdout()
+ assert _info_lines(log) == [
+ f"[llama-server] {warning}",
+ "[llama-server] the warning's own continuation",
+ ]
+
+
+def test_a_failing_tee_warns_once_and_a_closed_one_stays_quiet(monkeypatch):
+ class _Broken:
+ def __init__(self, exc):
+ self.exc = exc
+
+ def write(self, _):
+ raise self.exc
+
+ log = _logger(monkeypatch)
+ b = _backend(_NOISE)
+ b._llama_log_fh = _Broken(OSError("disk full"))
+ b._drain_stdout()
+ assert sum("disk full" in m for m in _info_lines(log)) == 1
+
+ log.records.clear()
+ b = _backend(_NOISE)
+ b._llama_log_fh = _Broken(ValueError("I/O operation on closed file"))
+ b._drain_stdout()
+ assert not _info_lines(log, 30)
+
+
+def test_close_writes_reason_and_exit_code_and_rearms_the_tee_warning(tmp_path, monkeypatch):
+ log = _logger(monkeypatch)
+ b = _backend([])
+ b._process.returncode = 1
+ b._llama_log_path = tmp_path / "llama.log"
+ b._llama_log_fh = open(b._llama_log_path, "w", encoding = "utf-8")
+ b._llama_log_tee_failed = True
+ b._close_attempt_log(reason = "killed")
+ assert b._llama_log_fh is None
+ assert not b._llama_log_tee_failed
+ assert b._llama_log_path.read_text(encoding = "utf-8").endswith(
+ "attempt end reason=killed exit_code=1\n"
+ )
+ assert any("exit_code=1" in m for m in _info_lines(log))
+
+ log.records.clear()
+ b._close_attempt_log()
+ assert not _info_lines(log)
+
+
+def test_a_raising_logger_does_not_stop_the_drain():
+ class _Raising:
+ def __getattr__(self, name):
+ def boom(*a, **k):
+ raise ValueError("I/O operation on closed file")
+
+ return boom
+
+ class _Full:
+ def write(self, _):
+ raise OSError("disk full")
+
+ lines = list(_NOTABLE) + list(_NOISE)
+ b = _backend(lines)
+ b._llama_log_fh = _Full()
+ llama_cpp_logger = llama_cpp.logger
+ llama_cpp.logger = _Raising()
+ try:
+ b._drain_stdout()
+ finally:
+ llama_cpp.logger = llama_cpp_logger
+ assert b._stdout_lines == lines
+
+
+def test_coloured_and_timestamp_free_prefixes_keep_their_level(monkeypatch):
+ log = _logger(monkeypatch)
+ # Verbatim from llama.cpp b11317 with --log-colors on / --no-log-timestamps.
+ coloured = "\x1b[34m0.08.894.798\x1b[0m \x1b[35mW srv llama_server: no API key is set"
+ bare = "E gguf_init_from_file: cannot open GGUF file '/x.gguf'"
+ _backend(
+ [
+ "\x1b[34m0.00.002.742\x1b[0m \x1b[32mI \x1b[0msrv llama_server: initializing",
+ coloured,
+ bare,
+ ]
+ )._drain_stdout()
+ assert _info_lines(log) == [f"[llama-server] {coloured}", f"[llama-server] {bare}"]
diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py
index 8e2dbdddc6c..979c23143ad 100644
--- a/unsloth_cli/commands/studio.py
+++ b/unsloth_cli/commands/studio.py
@@ -38,6 +38,7 @@ studio_app = typer.Typer(help = "Unsloth Studio commands.")
def _enable_verbose_access_logs() -> None:
os.environ["UNSLOTH_STUDIO_ACCESS_LOG_DEDUP_MS"] = "0"
os.environ["UNSLOTH_STUDIO_ACCESS_LOG_POLL_DEDUP_MS"] = "0"
+ os.environ["LOG_LEVEL"] = "DEBUG"
# Root order: UNSLOTH_STUDIO_HOME, STUDIO_HOME, UNSLOTH_HOME/studio, sys.prefix,