Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: show prompt progress, live generation details and where a GGUF loaded (#14909)

oobaboogae139d53d2fb2
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 63c2ebd01ce..7f804da5f6b 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -1049,6 +1049,165 @@ _GPU_DEVICE_PREFIXES = (
)
+# Any backend's "model buffer size" line (CUDA0, CPU_Mapped, RPC0[host:port], ...).
+_MODEL_BUFFER_RE = re.compile(
+ r"\b([A-Za-z][^\s=]*)\s+model buffer size\s*=\s*([0-9]+(?:\.[0-9]+)?)\s+MiB\b"
+)
+# --fit's verdict per device, e.g. "- CUDA0 (NVIDIA GeForce RTX 3090): 31 layers ( 8 overflowing), 22815 MiB used".
+# An overflowing layer keeps its MoE experts (or part of a layer) out of that device's memory.
+_FIT_DEVICE_RE = re.compile(
+ r"common_params_fit_impl:\s+-\s+(?P<device>[A-Za-z]+\d+)\s+\(.*?\):\s+(?P<layers>\d+)\s+layers?"
+ r"\s*(?:\(\s*(?P<overflow>\d+)\s+overflowing\s*\))?\s*,\s*\d+\s+MiB used"
+)
+# The MoE-only fill: printed for a model with experts, whose GPU layers then hold only their dense weights.
+_FIT_DENSE_ONLY_MARK = "filling dense-only layers back-to-front"
+# From here the per-device lines carry an explicit "(N overflowing)" count.
+_FIT_FULL_LAYERS_MARK = "converting dense-only layers to full layers"
+# One extra partial layer squeezed onto device i; on any device but the last it overflows to the NEXT GPU, not RAM.
+_FIT_EXTRA_LAYER_RE = re.compile(
+ r"set ngl_per_device\[(?P<index>\d+)\]\.\(n_layer, n_part, overflow_type\)=\(\d+, \d+, (?:UP|GATE|ATTN)\)"
+)
+# --moe-cache-mib auto: every routed expert stays in host RAM and a cache of the busiest ones goes on the GPU. --fit
+# returns before any per-device line on this path.
+_FIT_MOE_CACHE_RE = re.compile(
+ r"moe cache auto: all routed experts in host RAM, cache (?P<mib>\d+) MiB"
+)
+
+
+def _host_buffer(name: str) -> bool:
+ """CPU* buffers, AMX and a GPU backend's pinned host buffer (CUDA_Host) live in system memory."""
+ upper = name.upper()
+ return "CPU" in upper or upper.endswith("_HOST") or upper == "AMX"
+
+
+def parse_weight_placement(
+ lines: "list[str]", model_bytes: Optional[int] = None
+) -> "Optional[dict]":
+ """Where llama.cpp put the main model's weights, from its startup log.
+
+ ``gpu_mib`` / ``host_mib`` sum the main model's buffer block, the first allocated one (dry runs log 0.00 MiB).
+ "offloaded 31/31" hides an expert spill, so ``overflow_layers`` comes from --fit's final per-device
+ lines; the last device's count includes the output layer, which spills nothing (a 3090 gemma-4-26B
+ load printing "8 overflowing" holds exactly seven layers' worth in CUDA_Host). None without --fit.
+ A ``*_Mapped`` buffer logs its mmap span, first to last host tensor, which covers GPU tensors in
+ between when experts spill; ``model_bytes`` (the main GGUF set) caps host at the file minus the GPU side."""
+ blocks: list[list] = []
+ current: Optional[list] = None
+ # A build without GPU offload logs no "offloaded N/M layers" line, only the buffers: each run of them is a block.
+ unmarked = False
+ moe = False
+ full_phase = False
+ # device -> (layers, overflow or None for a dense-only line), in the order of the final verdict
+ verdict: dict[str, tuple[int, Optional[int]]] = {}
+ dense_only_order: list[str] = []
+ extra_layer_on: set[int] = set()
+ expert_cache_mib: Optional[float] = None
+ for line in lines:
+ if _FIT_DENSE_ONLY_MARK in line:
+ moe = True
+ continue
+ if _FIT_FULL_LAYERS_MARK in line:
+ full_phase = True
+ verdict = {}
+ continue
+ cache = _FIT_MOE_CACHE_RE.search(line)
+ if cache is not None:
+ expert_cache_mib = float(cache.group("mib"))
+ continue
+ extra = _FIT_EXTRA_LAYER_RE.search(line)
+ if extra is not None and full_phase:
+ extra_layer_on.add(int(extra.group("index")))
+ continue
+ fit = _FIT_DEVICE_RE.search(line)
+ if fit is not None:
+ device = fit.group("device")
+ overflow = fit.group("overflow")
+ verdict[device] = (
+ int(fit.group("layers")),
+ int(overflow) if overflow is not None else None,
+ )
+ if not full_phase and device not in dense_only_order:
+ dense_only_order.append(device)
+ continue
+ counts = _OFFLOADED_LAYERS_RE.search(line)
+ if counts is not None:
+ current, unmarked = [int(counts.group(2)), 0.0, 0.0, False], False
+ blocks.append(current)
+ continue
+ buffer = _MODEL_BUFFER_RE.search(line)
+ if buffer is None:
+ if unmarked:
+ current, unmarked = None, False
+ continue
+ if current is None:
+ current, unmarked = [0, 0.0, 0.0, False], True
+ blocks.append(current)
+ mib = float(buffer.group(2))
+ if _host_buffer(buffer.group(1)):
+ current[2] += mib
+ current[3] = current[3] or (mib > 0 and buffer.group(1).upper().endswith("_MAPPED"))
+ else:
+ current[1] += mib
+ # A no-alloc dry run logs the same block with every buffer at 0.00 MiB.
+ loaded = [block for block in blocks if block[1] + block[2] > 0]
+ if not loaded:
+ return None
+ # llama-server loads the main model before any drafter, which can have MORE layers (Gemma 4 E2B: 35 vs 26B A4B: 30).
+ main = loaded[0]
+ host_mib = main[2]
+ if main[3] and main[1] > 0 and model_bytes:
+ host_mib = min(host_mib, max(0.0, model_bytes / (1024 * 1024) - main[1]))
+
+ overflow: Optional[int] = None
+ if verdict:
+ devices = list(verdict)
+ overflow = 0
+ for index, device in enumerate(devices):
+ layers, counted = verdict[device]
+ if counted is None:
+ # Dense-only fill (filled back to front, so the first device it printed is the last one). A dense
+ # model's layers are full. The last device keeps one layer full and its output slot spills nothing.
+ if not moe:
+ continue
+ last = bool(dense_only_order) and device == dense_only_order[0]
+ counted = max(0, layers - 2) if last else layers
+ elif index < len(devices) - 1:
+ if index in extra_layer_on and counted > 0:
+ counted -= 1
+ elif counted > 0:
+ counted -= 1
+ overflow += counted
+ return {
+ "gpu_mib": round(main[1], 1),
+ "host_mib": round(host_mib, 1),
+ "overflow_layers": overflow,
+ "all_experts_in_ram": expert_cache_mib is not None,
+ "expert_cache_mib": expert_cache_mib,
+ }
+
+
+def _extra_weight_file_bytes(argv, env: Optional[Mapping[str, str]] = None) -> int:
+ """GGUF sets of the projector, local drafter and adapters a llama-server launch loads. arg.cpp reads
+ the ``LLAMA_ARG_*`` env first and lets argv override it, so ``env`` is the one the child got."""
+ tokens = [str(token) for token in argv] if isinstance(argv, (list, tuple)) else []
+ env_map = env or {}
+ projector: Optional[str] = None
+ for index, token in enumerate(tokens):
+ if _flag_name(token) not in ("--mmproj", "-mm"):
+ continue
+ _, eq, value = token.partition("=")
+ if eq:
+ projector = value
+ elif index + 1 < len(tokens):
+ projector = tokens[index + 1]
+ if projector is None:
+ projector = (env_map.get("LLAMA_ARG_MMPROJ") or "").strip() or None
+ drafter, remote = _extra_args_mtp_draft_source(tokens, env_map)
+ paths = [path for path in (projector, None if remote else drafter) if path]
+ paths += _sidecar_adapter_paths(tokens)
+ return sum(_gguf_set_bytes(Path(path)) for path in dict.fromkeys(paths))
+
+
def parse_gpu_offload_counts(lines: "list[str]") -> "Optional[tuple[int, int]]":
"""``(offloaded, total)`` for the largest-M line, or None; ties keep the first
line since llama.cpp logs the main model before its drafter."""
@@ -2758,6 +2917,39 @@ _SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)
_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$", re.IGNORECASE)
+def _gguf_set_bytes(path: Path) -> int:
+ """Size of a GGUF plus, when it is a split one, its sibling shards: llama-server opens the whole set."""
+ total = 0
+ try:
+ if path.is_file():
+ total += path.stat().st_size
+ m = _SHARD_FULL_RE.match(path.name)
+ if m is None or not path.parent.is_dir():
+ return total
+ # llama.cpp names a shard "<prefix>-<i>-of-<count>.gguf" from the first one's, so both must match: a
+ # drafter split as "<main prefix>-MTP-..." or a stale set with another count is not this set.
+ prefix, count = m.group(1).lower(), int(m.group(3))
+ for sibling in path.parent.iterdir():
+ if sibling.name == path.name:
+ continue
+ sibling_match = _SHARD_FULL_RE.match(sibling.name)
+ if (
+ sibling_match is None
+ or sibling_match.group(1).lower() != prefix
+ or int(sibling_match.group(3)) != count
+ or not 1 <= int(sibling_match.group(2)) <= count
+ ):
+ continue
+ try:
+ if sibling.is_file():
+ total += sibling.stat().st_size
+ except OSError:
+ pass
+ except OSError:
+ pass
+ return total
+
+
# ── Sliding-window-pattern resolver ───────────────────────────
# Resolves the per-layer SWA mask when a GGUF reports a sliding window but
# no `sliding_window_pattern` field. Tier order in `_resolve_swa_pattern`:
@@ -8585,6 +8777,8 @@ class LlamaCppBackend:
self._gpu_offload_active: Optional[bool] = None
# (offloaded, total) layers llama.cpp reported for this load, when it said.
self._gpu_offload_layers: Optional[tuple[int, int]] = None
+ # parse_weight_placement's result for this load.
+ self._weight_placement: Optional[dict] = None
# Whether the user's own extras pinned the layer split for this load.
self._offload_overridden: bool = False
# Studio saw a GPU for this load but llama.cpp's device table did not.
@@ -9112,6 +9306,29 @@ class LlamaCppBackend:
def offload_total_layers(self) -> Optional[int]:
return self._gpu_offload_layers[1] if self._gpu_offload_layers else None
+ def _placement_value(self, key: str):
+ placement = getattr(self, "_weight_placement", None)
+ return placement.get(key) if placement else None
+
+ @property
+ def gpu_weights_mib(self) -> Optional[float]:
+ return self._placement_value("gpu_mib")
+
+ @property
+ def host_weights_mib(self) -> Optional[float]:
+ return self._placement_value("host_mib")
+
+ @property
+ def overflow_layers(self) -> Optional[int]:
+ """Under --moe-cache-mib auto every routed expert is in RAM, so this is every MoE layer."""
+ if self._placement_value("all_experts_in_ram"):
+ return self.n_moe_layers or None
+ return self._placement_value("overflow_layers")
+
+ @property
+ def expert_cache_mib(self) -> Optional[float]:
+ return self._placement_value("expert_cache_mib")
+
@property
def gpu_backend_unavailable(self) -> bool:
"""Studio found a GPU for this load but llama.cpp reported none."""
@@ -9228,38 +9445,104 @@ class LlamaCppBackend:
@staticmethod
def _read_rss_bytes(pid: int) -> Optional[int]:
- """Resident set size of ``pid`` in bytes, from /proc/<pid>/status (Linux).
- 0 when the status has no VmRSS line (zombie / kernel thread); None where
- /proc is unavailable (macOS/Windows) or the value is unreadable."""
+ """``RssFile`` of ``pid`` (Linux), ``VmRSS`` before kernel 4.5; 0 when neither, None without /proc.
+ Not ``RssAnon``: CUDA pins the host weight buffer (``CUDA_Host``) before a byte of it is read."""
+ vm_rss = None
try:
with open(f"/proc/{pid}/status", "r", encoding = "utf-8") as f:
for line in f:
- if line.startswith("VmRSS:"):
- # IndexError guards a "VmRSS:" line with no value column.
+ # IndexError guards a line with no value column.
+ if line.startswith("RssFile:"):
return int(line.split()[1]) * 1024 # kB -> bytes
+ if line.startswith("VmRSS:"):
+ vm_rss = int(line.split()[1]) * 1024
except (FileNotFoundError, PermissionError, ValueError, IndexError, OSError):
return None
- return 0 # readable but no VmRSS line
+ return 0 if vm_rss is None else vm_rss
+
+ @staticmethod
+ def _read_load_bytes(pid: int) -> Optional[int]:
+ """Bytes ``pid`` has taken in, for a load that streams weights to VRAM and keeps little RSS.
+
+ Linux: ``rchar`` (page cache hits included). Windows: psutil read_bytes. macOS: the larger of
+ disk bytes read (a cold mmap Metal load) and physical footprint (a copy into Metal or CPU
+ buffers)."""
+ if os.path.isdir("/proc"):
+ try:
+ with open(f"/proc/{pid}/io", "r", encoding = "utf-8") as f:
+ for line in f:
+ if line.startswith("rchar:"):
+ return int(line.split()[1])
+ except (FileNotFoundError, PermissionError, ValueError, IndexError, OSError):
+ return None
+ return None
+ if sys.platform == "darwin":
+ usage = LlamaCppBackend._darwin_rusage(pid)
+ return max(usage) if usage else None
+ if sys.platform != "win32":
+ return None
+ try:
+ import psutil
+ return int(psutil.Process(pid).io_counters().read_bytes)
+ except Exception:
+ return None
+
+ @staticmethod
+ def _darwin_rusage(pid: int) -> Optional[tuple]:
+ """``(ri_diskio_bytesread, ri_phys_footprint)`` of ``pid`` from proc_pid_rusage, or None."""
+ import ctypes
+
+ class _RusageInfoV2(ctypes.Structure):
+ _fields_ = [("ri_uuid", ctypes.c_uint8 * 16)] + [
+ (name, ctypes.c_uint64)
+ for name in (
+ "ri_user_time",
+ "ri_system_time",
+ "ri_pkg_idle_wkups",
+ "ri_interrupt_wkups",
+ "ri_pageins",
+ "ri_wired_size",
+ "ri_resident_size",
+ "ri_phys_footprint",
+ "ri_proc_start_abstime",
+ "ri_proc_exit_abstime",
+ "ri_child_user_time",
+ "ri_child_system_time",
+ "ri_child_pkg_idle_wkups",
+ "ri_child_interrupt_wkups",
+ "ri_child_pageins",
+ "ri_child_elapsed_abstime",
+ "ri_diskio_bytesread",
+ "ri_diskio_byteswritten",
+ )
+ ]
+
+ try:
+ info = _RusageInfoV2()
+ # RUSAGE_INFO_V2 = 2
+ if (
+ ctypes.CDLL("/usr/lib/libproc.dylib").proc_pid_rusage(pid, 2, ctypes.byref(info))
+ != 0
+ ):
+ return None
+ return int(info.ri_diskio_bytesread), int(info.ri_phys_footprint)
+ except (OSError, AttributeError, ValueError):
+ return None
def load_progress(self) -> Optional[dict]:
"""Return live model-load progress, or None if not loading.
- During warm-up llama-server mmaps weight shards into page cache before
- pushing layers to VRAM, a window where status only reports ``loading``
- and the UI spinner looks stuck for minutes on large MoEs. Samples
- ``/proc/<pid>/status VmRSS`` against the sum of GGUF shard sizes for a
- real progress bar. Returns ``None`` when no load is in flight.
+ ``bytes_loaded`` is the larger of the bytes read and Linux file-backed RSS; never a working set,
+ which counts buffers allocated before any read. The total includes projector, drafter and adapters.
Shape::
{
"phase": "mmap" | "ready",
- "bytes_loaded": int, # VmRSS of the llama-server
- "bytes_total": int, # sum of shard file sizes
+ "bytes_loaded": int, # as above, never decreasing
+ "bytes_total": int, # weight files the server loads
"fraction": float, # bytes_loaded / bytes_total, 0..1
}
-
- Linux-only; returns ``None`` where ``/proc/<pid>/status`` is unavailable.
"""
proc = self._process
if proc is None:
@@ -9268,42 +9551,19 @@ class LlamaCppBackend:
if pid is None:
return None
- # Sum shard sizes (primary + any extras alongside).
- bytes_total = 0
gguf_path = self._gguf_path
- if gguf_path:
- primary = Path(gguf_path)
- try:
- if primary.is_file():
- bytes_total += primary.stat().st_size
- except OSError:
- pass
- # Extra shards share the primary's prefix before the shard index.
- try:
- parent = primary.parent
- stem = primary.name
- m = _SHARD_RE.match(stem)
- prefix = m.group(1) if m else None
- if prefix and parent.is_dir():
- prefix_lower = prefix.lower()
- for sibling in parent.iterdir():
- if (
- sibling.is_file()
- and sibling.name.lower().startswith(prefix_lower)
- and sibling.name != stem
- and sibling.suffix.lower() == ".gguf"
- ):
- try:
- bytes_total += sibling.stat().st_size
- except OSError:
- pass
- except OSError:
- pass
+ bytes_total = _gguf_set_bytes(Path(gguf_path)) if gguf_path else 0
+ launched = getattr(self, "_process_env", None)
+ bytes_total += _extra_weight_file_bytes(
+ getattr(proc, "args", None),
+ launched[1] if launched is not None and launched[0] is proc else None,
+ )
- # VmRSS of the llama-server; None where /proc is unavailable.
- bytes_loaded = LlamaCppBackend._read_rss_bytes(pid)
- if bytes_loaded is None:
+ rss = LlamaCppBackend._read_rss_bytes(pid)
+ read = LlamaCppBackend._read_load_bytes(pid)
+ if rss is None and read is None:
return None
+ bytes_loaded = max(rss or 0, read or 0)
# RSS climbs as weights page in, then drops once -ngl offloads them to
# VRAM and the mmap pages are freed. Hold a per-process high-water mark
@@ -9316,6 +9576,8 @@ class LlamaCppBackend:
phase = "ready" if self._healthy else "mmap"
fraction = 0.0
if bytes_total > 0:
+ # Reads past the weights (headers, a second pass) must not push the figure past the total.
+ bytes_loaded = min(bytes_loaded, bytes_total)
fraction = min(1.0, bytes_loaded / bytes_total)
# Once llama-server is healthy the load is complete by definition. With
# layers offloaded to VRAM (-ngl) the process releases the mmap'd weight
@@ -25791,6 +26053,8 @@ class LlamaCppBackend:
)
)
self._process = _spawned
+ # The env picks companions too (LLAMA_ARG_MMPROJ, ...): load_progress sizes them from it.
+ self._process_env = (_spawned, env)
_invalidate_gpu_memory("llama-server spawned")
# Cross-session backstop for when parent-death cleanup did not run.
# Under the lock: see _spawn_and_wait.
@@ -33548,6 +33812,7 @@ class LlamaCppBackend:
)
)
self._process = _spawned
+ self._process_env = (_spawned, env)
_invalidate_gpu_memory("llama-server spawned")
# Inside the lock: written after a sweep reaped the child,
# _pid_start_identity yields no start time, and the bare pid
@@ -35560,6 +35825,10 @@ class LlamaCppBackend:
):
_offload_counts = None
self._gpu_offload_layers = _offload_counts
+ self._weight_placement = parse_weight_placement(
+ self._stdout_lines,
+ _gguf_set_bytes(Path(self._gguf_path)) if self._gguf_path else None,
+ )
self._gpu_backend_unavailable = bool(_detected_gpus or _metal_capable_host()) and (
llama_saw_gpu_device(self._stdout_lines) is False
)
@@ -37141,6 +37410,7 @@ class LlamaCppBackend:
# server's stale zero-offload flag until the health probe reclassifies.
self._gpu_offload_active = None
self._gpu_offload_layers = None
+ self._weight_placement = None
self._offload_overridden = False
self._gpu_backend_unavailable = False
# Drives _wait_for_vram_settle in the next load_model; set in finally
diff --git a/studio/backend/core/inference/sse_control_frames.py b/studio/backend/core/inference/sse_control_frames.py
index 7777083896d..5cf9a6f7408 100644
--- a/studio/backend/core/inference/sse_control_frames.py
+++ b/studio/backend/core/inference/sse_control_frames.py
@@ -44,6 +44,7 @@ _CONTROL_TYPES = frozenset(
"diffusion_frame",
"reasoning_summary",
"context_usage",
+ "generation_progress",
}
)
@@ -54,6 +55,7 @@ _CONTROL_KEYS = (
"_diffusionFrame",
"_reasoningDurationMs",
"_contextUsage",
+ "_generationProgress",
"_mcp_provenance",
"quote_cut",
)
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 7e725aef751..fdf7b443a81 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -1938,6 +1938,33 @@ class _InferenceRuntimeFields(BaseModel):
None,
description = "Total layers in the model, alongside offloaded_layers.",
)
+ overflow_layers: Optional[int] = Field(
+ None,
+ description = (
+ "Layers with MoE expert weights that llama.cpp's --fit left in system memory: their "
+ "routed experts, or part of the FFN for the first partial layer on a device. "
+ "offloaded_layers still counts them as on the GPU, so a spill shows only here. Every "
+ "MoE layer when expert_cache_mib is set. None when --fit did not run."
+ ),
+ )
+ expert_cache_mib: Optional[float] = Field(
+ None,
+ description = (
+ "--moe-cache-mib auto: every routed expert is in system memory and this many MiB of "
+ "the most used ones are cached on the GPU. None when no MoE cache is in use."
+ ),
+ )
+ gpu_weights_mib: Optional[float] = Field(
+ None,
+ description = "MiB of model weights llama.cpp placed in GPU buffers, from its load log.",
+ )
+ host_weights_mib: Optional[float] = Field(
+ None,
+ description = (
+ "MiB of model weights in system memory (CPU, memory-mapped or pinned host buffers). "
+ "Token embeddings stay here even when every layer is on the GPU."
+ ),
+ )
gpu_backend_unavailable: bool = Field(
False,
description = (
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index aa38294dc07..013593afad3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2004,6 +2004,10 @@ _TEARDOWN_TASK_RECANCEL_S = 0.01
# (e.g. prompt prefill between tool iterations). A second layer atop the
# tool_stream_exec heartbeats, keeping proxies (Cloudflare drops idle at ~100s).
_LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0
+# How often a UI stream looks for new prefill progress while the generator is silent.
+_LIVE_PROGRESS_POLL_S = 0.25
+# Minimum gap between two live decode frames: the client only redraws one line of text.
+_LIVE_DECODE_FRAME_INTERVAL_S = 1.0
_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25
@@ -7697,22 +7701,62 @@ def _monitor_perf_callback(monitor_id: Optional[str], context_length):
return _callback
+def _live_count(value: Any) -> Optional[int]:
+ if isinstance(value, bool):
+ return None
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ return None
+ if not math.isfinite(number) or number < 0:
+ return None
+ return int(number)
+
+
+def _live_rate(value: Any) -> Optional[float]:
+ if isinstance(value, bool):
+ return None
+ try:
+ number = float(value)
+ except (TypeError, ValueError):
+ return None
+ if not math.isfinite(number) or number <= 0:
+ return None
+ return round(number, 1)
+
+
class _PrefillProgressSignal:
"""Counts advancing ``prompt_progress`` so the stall loop can send ``: prefill-progress``.
- Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease."""
+ Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease.
+
+ With ``live`` it also feeds ``live_frame``, the chat UI's ``generation_progress`` frames."""
- def __init__(self, inner):
+ def __init__(
+ self,
+ inner,
+ live: bool = False,
+ ):
self._inner = inner
self._last_processed = None
self._advances = 0
self._seen = 0
- self.wants_timings = inner is not None
+ self._live = live
+ # Written from the generator thread, read on the event loop: each is replaced whole, never mutated.
+ self._prefill: Optional[dict] = None
+ self._prefill_sent: Optional[dict] = None
+ self._decode: Optional[dict] = None
+ self._decode_sent_at = 0.0
+ self.wants_timings = inner is not None or live
@property
def needs_phase(self) -> bool:
return self._inner is not None and getattr(self._inner, "needs_phase", True)
+ @property
+ def live(self) -> bool:
+ return self._live
+
def __call__(self, sample: dict) -> None:
progress = sample.get("prompt_progress")
if isinstance(progress, dict):
@@ -7721,9 +7765,66 @@ class _PrefillProgressSignal:
if processed is not None and processed != self._last_processed:
self._last_processed = processed
self._advances += 1
+ if self._live:
+ self._record_prefill(progress, sample.get("prompt_per_second"))
+ elif self._live:
+ self._record_decode(sample)
if self._inner is not None:
self._inner(sample)
+ def _record_prefill(self, progress: dict, rate: Any) -> None:
+ total = _live_count(progress.get("total"))
+ processed = _live_count(progress.get("processed"))
+ if processed is None:
+ return
+ if total:
+ processed = min(processed, total)
+ cached = _live_count(progress.get("cache"))
+ # A new prompt pass (the next round of a tool loop): the last round's decode figures must not follow it.
+ self._decode = None
+ self._prefill = {
+ "type": "generation_progress",
+ "phase": "prefill",
+ "processed": processed,
+ "total": total,
+ "cached": min(cached, processed) if cached is not None else None,
+ "tok_per_s": _live_rate(rate),
+ }
+
+ def _record_decode(self, sample: dict) -> None:
+ tokens = _live_count(sample.get("predicted_n"))
+ if not tokens:
+ return
+ frame = {
+ "type": "generation_progress",
+ "phase": "decode",
+ "tokens": tokens,
+ "tok_per_s": _live_rate(sample.get("predicted_per_second")),
+ }
+ draft_n = _live_count(sample.get("draft_n"))
+ draft_accepted = _live_count(sample.get("draft_n_accepted"))
+ if draft_n and draft_accepted is not None:
+ frame["draft_n"] = draft_n
+ frame["draft_accepted"] = min(draft_accepted, draft_n)
+ self._decode = frame
+
+ def live_frame(self) -> Optional[str]:
+ """The next ``generation_progress`` line: the newest unsent prefill sample, else decode figures at most
+ once per ``_LIVE_DECODE_FRAME_INTERVAL_S``. None when nothing is due."""
+ if not self._live:
+ return None
+ prefill = self._prefill
+ if prefill is not None and prefill is not self._prefill_sent:
+ self._prefill_sent = prefill
+ return f"data: {json.dumps(prefill, separators = (',', ':'))}\n\n"
+ decode = self._decode
+ now = time.monotonic()
+ if decode is not None and now - self._decode_sent_at >= _LIVE_DECODE_FRAME_INTERVAL_S:
+ self._decode = None
+ self._decode_sent_at = now
+ return f"data: {json.dumps(decode, separators = (',', ':'))}\n\n"
+ return None
+
def advanced(self) -> bool:
advances = self._advances
if advances == self._seen:
@@ -7732,6 +7833,34 @@ class _PrefillProgressSignal:
return True
+async def _stall_wait_frames(next_task, signal: _PrefillProgressSignal, first_timeout: float):
+ """Wait for a GGUF generator step, yielding keep-alives on stall and, when live, progress frames.
+ Never cancels ``next_task``: the caller's finally drains it."""
+ timeout = first_timeout
+ waited = 0.0
+ while True:
+ poll = min(timeout - waited, _LIVE_PROGRESS_POLL_S) if signal.live else timeout
+ done_tasks, _ = await asyncio.wait({next_task}, timeout = poll)
+ if done_tasks:
+ # Steps faster than the poll get their pending prefill sample, then decode figures, sent here.
+ for _ in range(2):
+ frame = signal.live_frame()
+ if frame is not None:
+ yield frame
+ return
+ frame = signal.live_frame()
+ if frame is not None:
+ yield frame
+ waited += poll
+ if waited < timeout:
+ continue
+ yield (
+ _OPENAI_PREFILL_PROGRESS_SSE if signal.advanced() else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
+ )
+ waited = 0.0
+ timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S
+
+
def _monitor_call_text(name: Any, arguments: Any = None) -> str:
call_name = str(name or "tool")
if arguments is None or arguments == "":
@@ -34175,7 +34304,10 @@ async def produce_openai_chat_completions(
else None
)
# Only a stream has a stall loop to feed; elsewhere the monitor callback (or None) stands.
- _gguf_prefill_signal = _PrefillProgressSignal(_gguf_monitor_callback)
+ _gguf_prefill_signal = _PrefillProgressSignal(
+ _gguf_monitor_callback,
+ live = bool(payload.stream and _ui_events and not _wants_multiple_choices(payload)),
+ )
_gguf_perf_callback = _gguf_prefill_signal if payload.stream else _gguf_monitor_callback
def _gguf_chat_delta_line(delta: ChoiceDelta, finish_reason = None) -> str:
@@ -34704,20 +34836,10 @@ async def produce_openai_chat_completions(
if approval_flush_pending
else _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S
)
- while True:
- done_tasks, _ = await asyncio.wait(
- {next_task},
- timeout = wait_timeout,
- )
- if done_tasks:
- break
- yield (
- _OPENAI_PREFILL_PROGRESS_SSE
- if _gguf_prefill_signal.advanced()
- else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
- )
- approval_flush_pending = False
- wait_timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S
+ async for stall_line in _stall_wait_frames(
+ next_task, _gguf_prefill_signal, wait_timeout
+ ):
+ yield stall_line
event = next_task.result()
approval_flush_pending = False
finally:
@@ -35429,18 +35551,12 @@ async def produce_openai_chat_completions(
# Stall-timeout wait: keepalive while the generator stays
# silent (e.g. no-tool prefill). asyncio.wait never cancels
# next_task, matching the finally-drain shield (see GGUF stream).
- while True:
- done_tasks, _ = await asyncio.wait(
- {next_task},
- timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S,
- )
- if done_tasks:
- break
- yield (
- _OPENAI_PREFILL_PROGRESS_SSE
- if _gguf_prefill_signal.advanced()
- else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE
- )
+ async for stall_line in _stall_wait_frames(
+ next_task,
+ _gguf_prefill_signal,
+ _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S,
+ ):
+ yield stall_line
cumulative = next_task.result()
finally:
if next_task.done():
diff --git a/studio/backend/tests/test_live_generation_progress.py b/studio/backend/tests/test_live_generation_progress.py
new file mode 100644
index 00000000000..f1d31a8ac5b
--- /dev/null
+++ b/studio/backend/tests/test_live_generation_progress.py
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Live prefill / decode frames for the Unsloth chat UI (``generation_progress``)."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.sse_control_frames import ( # noqa: E402
+ is_ui_control_sse_line,
+ sanitize_provider_sse_line,
+)
+from routes import inference as route # noqa: E402
+
+
+def _frame(line: str) -> dict:
+ assert line.startswith("data: ") and line.endswith("\n\n")
+ return json.loads(line[len("data: ") :])
+
+
+def test_prefill_samples_become_frames_once_each():
+ signal = route._PrefillProgressSignal(None, live = True)
+ assert signal.wants_timings, "a live signal must ask llama-server for per-token timings"
+ assert signal.live_frame() is None
+
+ signal(
+ {
+ "prompt_progress": {"total": 5316, "processed": 2048, "cache": 0, "time_ms": 1500},
+ "prompt_per_second": 1365.33,
+ }
+ )
+ frame = _frame(signal.live_frame())
+ assert frame == {
+ "type": "generation_progress",
+ "phase": "prefill",
+ "processed": 2048,
+ "total": 5316,
+ "cached": 0,
+ "tok_per_s": 1365.3,
+ }
+ # Sent once; the stall loop polls far more often than llama-server reports a batch.
+ assert signal.live_frame() is None
+
+
+def test_prefill_counts_are_clamped_and_bad_rates_dropped():
+ signal = route._PrefillProgressSignal(None, live = True)
+ signal(
+ {
+ "prompt_progress": {"total": 100, "processed": 250, "cache": 400},
+ "prompt_per_second": float("inf"),
+ }
+ )
+ frame = _frame(signal.live_frame())
+ assert frame["processed"] == 100
+ assert frame["cached"] == 100
+ assert frame["tok_per_s"] is None
+
+
+def test_decode_frames_are_throttled(monkeypatch):
+ clock = {"now": 1000.0}
+ monkeypatch.setattr(route.time, "monotonic", lambda: clock["now"])
+ signal = route._PrefillProgressSignal(None, live = True)
+
+ signal({"predicted_n": 10, "predicted_per_second": 48.7})
+ assert _frame(signal.live_frame()) == {
+ "type": "generation_progress",
+ "phase": "decode",
+ "tokens": 10,
+ "tok_per_s": 48.7,
+ }
+ signal({"predicted_n": 11, "predicted_per_second": 49.0})
+ assert signal.live_frame() is None, "a second decode frame inside the interval"
+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S
+ assert _frame(signal.live_frame())["tokens"] == 11
+ assert signal.live_frame() is None, "the same sample twice"
+
+
+def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():
+ signal = route._PrefillProgressSignal(None, live = True)
+ signal({"predicted_n": 50, "predicted_per_second": 40.0})
+ signal({"prompt_progress": {"total": 3000, "processed": 2048}})
+ assert _frame(signal.live_frame())["phase"] == "prefill"
+ assert signal.live_frame() is None, "round 1's decode frame must not follow round 2's prefill"
+
+
+def test_decode_frame_carries_draft_acceptance():
+ signal = route._PrefillProgressSignal(None, live = True)
+ signal({"predicted_n": 40, "predicted_per_second": 90.0, "draft_n": 23, "draft_n_accepted": 14})
+ frame = _frame(signal.live_frame())
+ assert (frame["draft_n"], frame["draft_accepted"]) == (23, 14)
+
+
+def test_not_live_keeps_the_old_behaviour():
+ calls = []
+
+ def inner(sample):
+ calls.append(sample)
+
+ signal = route._PrefillProgressSignal(inner)
+ signal({"prompt_progress": {"total": 10, "processed": 5}})
+ signal({"predicted_n": 3, "predicted_per_second": 20.0})
+ assert signal.live_frame() is None
+ assert signal.advanced() is True
+ assert len(calls) == 2, "the monitor callback still sees every sample"
+ assert route._PrefillProgressSignal(None).wants_timings is False
+
+
+def test_stall_wait_sends_progress_while_the_generator_is_silent(monkeypatch):
+ monkeypatch.setattr(route, "_LIVE_PROGRESS_POLL_S", 0.01)
+ signal = route._PrefillProgressSignal(None, live = True)
+
+ async def scenario():
+ release = asyncio.Event()
+
+ async def step():
+ await release.wait()
+ return "token"
+
+ task = asyncio.create_task(step())
+ lines = []
+
+ async def collect():
+ async for line in route._stall_wait_frames(task, signal, first_timeout = 15.0):
+ lines.append(line)
+
+ collector = asyncio.create_task(collect())
+ await asyncio.sleep(0.03)
+ signal({"prompt_progress": {"total": 4000, "processed": 2048}})
+ await asyncio.sleep(0.05)
+ release.set()
+ await collector
+ return lines, task.result()
+
+ lines, result = asyncio.run(scenario())
+ assert result == "token"
+ assert [_frame(line)["processed"] for line in lines] == [2048]
+
+
+def test_a_step_faster_than_the_poll_still_sends_what_it_recorded():
+ # Fast tokens (or tool-call markup stripped to no text) never reach the poll.
+ signal = route._PrefillProgressSignal(None, live = True)
+
+ async def scenario():
+ async def step():
+ signal({"prompt_progress": {"total": 4000, "processed": 4000}})
+ signal({"predicted_n": 3, "predicted_per_second": 20.0})
+ return "token"
+
+ task = asyncio.create_task(step())
+ await task
+ return [line async for line in route._stall_wait_frames(task, signal, first_timeout = 15.0)]
+
+ lines = asyncio.run(scenario())
+ assert [_frame(line)["phase"] for line in lines] == ["prefill", "decode"]
+ assert _frame(lines[0])["processed"] == 4000 and _frame(lines[1])["tokens"] == 3
+
+
+def test_stall_wait_still_sends_the_comment_after_the_stall_interval(monkeypatch):
+ monkeypatch.setattr(route, "_LIVE_PROGRESS_POLL_S", 0.01)
+
+ async def scenario(live: bool):
+ signal = route._PrefillProgressSignal(None, live = live)
+ signal({"prompt_progress": {"total": 4000, "processed": 1}})
+ signal.live_frame()
+ release = asyncio.Event()
+
+ async def step():
+ await release.wait()
+
+ task = asyncio.create_task(step())
+ lines = []
+
+ async def collect():
+ async for line in route._stall_wait_frames(task, signal, first_timeout = 0.05):
+ lines.append(line)
+
+ collector = asyncio.create_task(collect())
+ await asyncio.sleep(0.12)
+ release.set()
+ await collector
+ return lines
+
+ for live in (False, True):
+ lines = asyncio.run(scenario(live))
+ assert lines and lines[0] == route._OPENAI_PREFILL_PROGRESS_SSE, (live, lines)
+ assert all(not line.startswith("data:") for line in lines)
+
+
+ "line",
+ [
+ 'data: {"type":"generation_progress","phase":"prefill","processed":1,"total":2}',
+ ],
+)
+def test_providers_cannot_forge_the_frame(line):
+ assert sanitize_provider_sse_line(line) is None
+ assert is_ui_control_sse_line(line)
diff --git a/studio/backend/tests/test_llama_cpp_load_progress_live.py b/studio/backend/tests/test_llama_cpp_load_progress_live.py
index dff8334314c..21f4b4abfca 100644
--- a/studio/backend/tests/test_llama_cpp_load_progress_live.py
+++ b/studio/backend/tests/test_llama_cpp_load_progress_live.py
@@ -71,14 +71,19 @@ def _make_backend(
return inst
-def test_live_rss_matches_kernel_vmrss(tmp_path):
- """Spawn a real child, let it allocate real bytes, confirm ``bytes_loaded``
- tracks the kernel's VmRSS within a sane tolerance."""
+def test_live_mapped_pages_count_and_allocations_do_not(tmp_path):
+ """Spawn a real child that pages in a mapped file and fills an allocation: ``bytes_loaded``
+ follows the mapped pages, not the allocation (a pinned upload buffer is one)."""
+ data = tmp_path / "weights.bin"
+ data.write_bytes(b"\1" * (100 * 1024 * 1024))
script = tmp_path / "burn.py"
script.write_text(
- "import time, sys\n"
- "buf = bytearray(100 * 1024 * 1024)\n"
- "# touch every page so RSS actually grows\n"
+ "import mmap, sys, time\n"
+ "f = open(sys.argv[1], 'rb')\n"
+ "m = mmap.mmap(f.fileno(), 0, access = mmap.ACCESS_READ)\n"
+ "# touch every page of the mapping and of the allocation so both are resident\n"
+ "seen = sum(m[i] for i in range(0, len(m), 4096))\n"
+ "buf = bytearray(200 * 1024 * 1024)\n"
"for i in range(0, len(buf), 4096):\n"
" buf[i] = 1\n"
"sys.stdout.write('ready\\n')\n"
@@ -86,7 +91,7 @@ def test_live_rss_matches_kernel_vmrss(tmp_path):
"time.sleep(10)\n"
)
proc = subprocess.Popen(
- [sys.executable, str(script)],
+ [sys.executable, str(script), str(data)],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
)
@@ -96,20 +101,22 @@ def test_live_rss_matches_kernel_vmrss(tmp_path):
gguf = tmp_path / "model.gguf"
with open(gguf, "wb") as f:
- f.truncate(200 * 1024 * 1024)
+ f.truncate(400 * 1024 * 1024)
inst = _make_backend(proc.pid, str(gguf), healthy = False)
out = inst.load_progress()
assert out is not None, "load_progress returned None for live pid"
assert out["phase"] == "mmap"
- assert out["bytes_total"] == 200 * 1024 * 1024
- # VmRSS includes the interpreter plus the 100MB buffer.
+ assert out["bytes_total"] == 400 * 1024 * 1024
+ # The 100 MB mapping plus the interpreter's own files; the 200 MB allocation is left out.
assert (
- out["bytes_loaded"] >= 50 * 1024 * 1024
+ out["bytes_loaded"] >= 90 * 1024 * 1024
), f"bytes_loaded unexpectedly low: {out['bytes_loaded']}"
- assert out["bytes_loaded"] <= 200 * 1024 * 1024
- assert 0.0 < out["fraction"] <= 1.0
+ assert (
+ out["bytes_loaded"] < 200 * 1024 * 1024
+ ), f"the allocation counted: {out['bytes_loaded']}"
+ assert 0.0 < out["fraction"] < 0.5
finally:
proc.terminate()
try:
diff --git a/studio/backend/tests/test_llama_weight_placement.py b/studio/backend/tests/test_llama_weight_placement.py
new file mode 100644
index 00000000000..a7439ad51a3
--- /dev/null
+++ b/studio/backend/tests/test_llama_weight_placement.py
@@ -0,0 +1,406 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Where llama.cpp put a model's weights, and load progress that sees weights headed for VRAM."""
+
+from __future__ import annotations
+
+import io
+import os
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
+if _BACKEND_DIR not in sys.path:
+ sys.path.insert(0, _BACKEND_DIR)
+
+from core.inference.llama_cpp import ( # noqa: E402
+ LlamaCppBackend,
+ _extra_weight_file_bytes,
+ _gguf_set_bytes,
+ parse_weight_placement,
+)
+
+# Gemma 4 26B A4B Q8_0 on a 24 GB RTX 3090 under --fit on (trimmed from a real llama-server log).
+_MOE_SPILL_LOG = """\
+0.00.997.437 I common_params_fit_impl: filling dense-only layers back-to-front:
+0.01.282.617 I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 3090): 31 layers, 5550 MiB used, 18347 MiB free
+0.02.623.039 I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 3090): 31 layers ( 8 overflowing), 22815 MiB used, 1082 MiB free
+0.02.623.041 I common_fit_params: successfully fit params to free device memory
+0.03.897.106 I load_tensors: offloading output layer to GPU
+0.03.897.109 I load_tensors: offloading 29 repeating layers to GPU
+0.03.897.109 I load_tensors: offloaded 31/31 layers to GPU
+0.03.897.113 I load_tensors: CUDA0 model buffer size = 20709.15 MiB
+0.03.897.113 I load_tensors: CUDA_Host model buffer size = 5639.40 MiB
+0.23.263.171 I llama_kv_cache: CUDA0 KV buffer size = 160.00 MiB
+""".splitlines()
+
+
+def test_moe_spill_is_read_from_fit_not_from_the_layer_count():
+ assert parse_weight_placement(_MOE_SPILL_LOG) == {
+ "gpu_mib": 20709.2,
+ "host_mib": 5639.4,
+ "overflow_layers": 7,
+ "all_experts_in_ram": False,
+ "expert_cache_mib": None,
+ }
+
+
+def test_dense_model_fully_on_gpu():
+ lines = [
+ "0.00.365.476 I load_tensors: offloaded 41/41 layers to GPU",
+ "0.00.365.484 I load_tensors: CUDA0 model buffer size = 8161.75 MiB",
+ "0.00.365.484 I load_tensors: CUDA_Host model buffer size = 417.30 MiB",
+ ]
+ assert parse_weight_placement(lines) == {
+ "gpu_mib": 8161.8,
+ "host_mib": 417.3,
+ "overflow_layers": None,
+ "all_experts_in_ram": False,
+ "expert_cache_mib": None,
+ }
+
+
+def test_dry_run_and_drafter_blocks_do_not_count():
+ lines = [
+ # --fit's no-alloc pass logs the same block with nothing allocated.
+ "I load_tensors: offloaded 31/31 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 0.00 MiB",
+ "I load_tensors: CUDA_Host model buffer size = 0.00 MiB",
+ "I load_tensors: offloaded 31/31 layers to GPU",
+ "I load_tensors: CPU_Mapped model buffer size = 13986.81 MiB",
+ "I load_tensors: CUDA0 model buffer size = 13258.54 MiB",
+ # A drafter logs its own, smaller block after the main model.
+ "I load_tensors: offloaded 3/3 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 600.00 MiB",
+ ]
+ # A drafter can also be DEEPER than its target (Gemma 4 E2B, 35 layers, for 26B A4B, 30): still the main model.
+ deeper = lines + [
+ "I load_tensors: offloaded 36/36 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 2000.00 MiB",
+ ]
+ assert parse_weight_placement(deeper)["gpu_mib"] == 13258.5
+ assert parse_weight_placement(lines) == {
+ "gpu_mib": 13258.5,
+ "host_mib": 13986.8,
+ "overflow_layers": None,
+ "all_experts_in_ram": False,
+ "expert_cache_mib": None,
+ }
+
+
+def test_multi_gpu_extra_layer_overflows_to_the_next_gpu_not_ram():
+ lines = [
+ "I common_params_fit_impl: filling dense-only layers back-to-front:",
+ "I common_params_fit_impl: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:",
+ # Device 0 took one extra partial layer; it overflows into CUDA1, not into RAM.
+ "I common_params_fit_impl: set ngl_per_device[0].(n_layer, n_part, overflow_type)=(20, 1, GATE), id_dense_start=1",
+ "I common_params_fit_impl: - CUDA0 (A): 20 layers ( 1 overflowing), 23000 MiB used, 900 MiB free",
+ "I common_params_fit_impl: - CUDA1 (B): 11 layers ( 4 overflowing), 15000 MiB used, 800 MiB free",
+ "I load_tensors: offloaded 31/31 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 20000.00 MiB",
+ "I load_tensors: CUDA1 model buffer size = 12000.00 MiB",
+ "I load_tensors: CPU model buffer size = 4000.00 MiB",
+ ]
+ out = parse_weight_placement(lines)
+ # CUDA1's 4 partial slots end at the output layer, which spills nothing.
+ assert (out["gpu_mib"], out["host_mib"], out["overflow_layers"]) == (32000.0, 4000.0, 3)
+
+
+def test_moe_fit_that_stops_after_the_dense_only_fill_has_every_gpu_layer_spilled():
+ """The dense-only lines carry no count, but every GPU layer there holds dense weights only."""
+ lines = [
+ "I common_params_fit_impl: filling dense-only layers back-to-front:",
+ "I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 5060 Ti): 60 layers, 15000 MiB used, 900 MiB free",
+ "I load_tensors: offloaded 60/95 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 14000.00 MiB",
+ "I load_tensors: CUDA_Host model buffer size = 120000.00 MiB",
+ ]
+ # One device, so it is also the last: its output layer carries no experts.
+ # 60 slots, 59 partial; the last partial slot is the output layer and the first layer stays full.
+ assert parse_weight_placement(lines)["overflow_layers"] == 58
+ dense = [line.replace("dense-only layers", "dense layers") for line in lines]
+ assert parse_weight_placement(dense)["overflow_layers"] == 0, "a dense model's layers are full"
+
+
+def test_moe_cache_auto_puts_every_expert_in_ram():
+ lines = [
+ "I common_params_fit_impl: moe cache auto: all routed experts in host RAM, cache 6144 MiB (margin 1024 MiB)",
+ "I common_fit_params: successfully fit params to free device memory",
+ "I load_tensors: offloaded 31/31 layers to GPU",
+ "I load_tensors: CUDA0 model buffer size = 3900.00 MiB",
+ "I load_tensors: CUDA_Host model buffer size = 22400.00 MiB",
+ ]
+ out = parse_weight_placement(lines)
+ assert out["all_experts_in_ram"] is True
+ assert out["expert_cache_mib"] == 6144.0
+ assert (
+ out["overflow_layers"] is None
+ ), "no per-device line on this path; the backend supplies the MoE layer count"
+
+
+def test_cpu_repack_buffers_count_as_ram_and_remote_buffers_parse():
+ lines = [
+ "I load_tensors: offloaded 0/49 layers to GPU",
+ "I load_tensors: AMX model buffer size = 9000.00 MiB",
+ "I load_tensors: CPU_Mapped model buffer size = 500.00 MiB",
+ "I load_tensors: RPC0[192.168.1.2:50052] model buffer size = 1000.00 MiB",
+ ]
+ out = parse_weight_placement(lines)
+ assert (out["gpu_mib"], out["host_mib"]) == (1000.0, 9500.0)
+
+
+def test_no_buffer_lines_means_nothing_to_report():
+ assert parse_weight_placement(["I srv load_model: loading model"]) is None
+
+
+def test_a_cpu_only_build_reports_its_ram_placement():
+ # llama.cpp prints "offloaded N/M layers" only when the build supports GPU offload.
+ lines = [
+ "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB",
+ "load_tensors: CPU model buffer size = 0.6 MiB",
+ "llama_context: n_ctx = 4096",
+ # A drafter's buffers after other lines are its own block, not the main model's.
+ "load_tensors: CPU_Mapped model buffer size = 500.0 MiB",
+ ]
+ placement = parse_weight_placement(lines)
+ assert placement is not None
+ assert (placement["gpu_mib"], placement["host_mib"]) == (0.0, 21000.6)
+ assert placement["overflow_layers"] is None
+
+
+def test_projector_and_drafter_on_the_command_line_count_toward_the_load(tmp_path):
+ projector = tmp_path / "mmproj-F16.gguf"
+ projector.write_bytes(b"x" * 1000)
+ drafter = tmp_path / "draft.gguf"
+ drafter.write_bytes(b"x" * 300)
+ argv = [
+ "llama-server",
+ "-m",
+ "model.gguf",
+ "--mmproj",
+ str(projector),
+ "-md",
+ str(drafter),
+ "--port",
+ "1",
+ ]
+ assert _extra_weight_file_bytes(argv) == 1300
+ assert _extra_weight_file_bytes(None) == 0
+ assert (
+ _extra_weight_file_bytes(["llama-server", "--mmproj", str(tmp_path / "missing.gguf")]) == 0
+ )
+
+
+def test_a_split_drafter_counts_every_shard(tmp_path):
+ # llama-server opens a split drafter's siblings from the named first shard.
+ for index in (1, 2, 3):
+ (tmp_path / f"Model-Q4_K_M-MTP-{index:05d}-of-00003.gguf").write_bytes(b"x" * 100)
+ # The main model's set in the same folder, whose prefix the drafter's starts with, is not the drafter's.
+ for index in (1, 2):
+ (tmp_path / f"Model-Q4_K_M-{index:05d}-of-00002.gguf").write_bytes(b"x" * 7)
+ # Nor is a stale set under the same prefix with another shard count.
+ for index in (1, 2, 3, 4):
+ (tmp_path / f"Model-Q4_K_M-MTP-{index:05d}-of-00004.gguf").write_bytes(b"x" * 1000)
+ argv = ["llama-server", "-md", str(tmp_path / "Model-Q4_K_M-MTP-00001-of-00003.gguf")]
+ assert _extra_weight_file_bytes(argv) == 300
+ assert _gguf_set_bytes(tmp_path / "Model-Q4_K_M-00001-of-00002.gguf") == 14
+
+
+def test_companions_picked_by_the_launch_env_count_too(tmp_path):
+ projector = tmp_path / "mmproj-F16.gguf"
+ projector.write_bytes(b"x" * 1000)
+ drafter = tmp_path / "draft.gguf"
+ drafter.write_bytes(b"x" * 300)
+ argv = ["llama-server", "-m", "model.gguf"]
+ env = {"LLAMA_ARG_MMPROJ": str(projector), "LLAMA_ARG_SPEC_DRAFT_MODEL": str(drafter)}
+ assert _extra_weight_file_bytes(argv, env) == 1300
+ # Pre-b8955 drafter spelling.
+ assert _extra_weight_file_bytes(argv, {"LLAMA_ARG_MODEL_DRAFT": str(drafter)}) == 300
+ # argv overrides the env, as in arg.cpp; a Hugging Face repo is not a local file.
+ other = tmp_path / "mmproj-BF16.gguf"
+ other.write_bytes(b"x" * 50)
+ assert _extra_weight_file_bytes(argv + ["--mmproj", str(other)], env) == 350
+ assert _extra_weight_file_bytes(argv, {"LLAMA_ARG_SPEC_DRAFT_HF_REPO": "org/draft"}) == 0
+
+
+def test_adapters_on_the_command_line_count_toward_the_load(tmp_path, monkeypatch):
+ lora = tmp_path / "lora.gguf"
+ lora.write_bytes(b"x" * 400)
+ (tmp_path / "vector.gguf").write_bytes(b"x" * 20)
+ # Relative: upstream splits FNAME:SCALE on every ':', so a drive-lettered path is its own error.
+ monkeypatch.chdir(tmp_path)
+ argv = ["llama-server", "-m", "model.gguf", "--lora", str(lora)]
+ assert _extra_weight_file_bytes(argv + ["--control-vector-scaled", "vector.gguf:0.5"]) == 420
+
+
+def test_load_progress_sizes_companions_from_the_env_its_process_got(tmp_path, monkeypatch):
+ inst = _fake_load(tmp_path, 1000)
+ projector = tmp_path / "mmproj-F16.gguf"
+ projector.write_bytes(b"x" * 1000)
+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))
+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: 500))
+ inst._process_env = (inst._process, {"LLAMA_ARG_MMPROJ": str(projector)})
+ assert inst.load_progress()["bytes_total"] == 2000
+ # An env recorded for an earlier process does not describe this one.
+ inst._process_env = (object(), {"LLAMA_ARG_MMPROJ": str(projector)})
+ assert inst.load_progress()["bytes_total"] == 1000
+
+
[email protected](not Path("/proc").exists(), reason = "live /proc test is Linux-only")
+def test_load_progress_follows_bytes_read_when_rss_stays_flat(tmp_path):
+ """A server that streams its weights to VRAM reads every byte but keeps little resident:
+ progress must follow the reads (it froze at the RSS share before)."""
+ weights = tmp_path / "model.gguf"
+ weights.write_bytes(os.urandom(64 * 1024 * 1024))
+ script = tmp_path / "reader.py"
+ script.write_text(
+ "import sys, time\n"
+ "with open(sys.argv[1], 'rb') as f:\n"
+ " while f.read(1 << 20):\n"
+ " pass\n"
+ "print('read', flush=True)\n"
+ "time.sleep(10)\n"
+ )
+ proc = subprocess.Popen([sys.executable, str(script), str(weights)], stdout = subprocess.PIPE)
+ try:
+ assert proc.stdout.readline().strip() == b"read"
+ inst = LlamaCppBackend.__new__(LlamaCppBackend)
+ inst._process = proc
+ inst._gguf_path = str(weights)
+ inst._healthy = False
+ rss = LlamaCppBackend._read_rss_bytes(proc.pid)
+ assert rss is not None and rss < 64 * 1024 * 1024, "the reader never holds the file"
+ out = inst.load_progress()
+ assert out is not None
+ assert out["bytes_total"] == 64 * 1024 * 1024
+ assert out["fraction"] == 1.0, out
+ finally:
+ proc.kill()
+ proc.wait(timeout = 5)
+
+
+def _fake_load(tmp_path, size):
+ weights = tmp_path / "model.gguf"
+ weights.write_bytes(b"\0" * size)
+ inst = LlamaCppBackend.__new__(LlamaCppBackend)
+ inst._process = type("P", (), {"pid": 4242, "args": []})()
+ inst._gguf_path = str(weights)
+ inst._healthy = False
+ return inst
+
+
+def test_windows_load_progress_follows_bytes_read_alone(tmp_path, monkeypatch):
+ """A Windows working set takes in the buffers llama.cpp allocates before reading a byte (DevLab:
+ 9.1 of 9.0 GB a second into the load, reads still at 0.01 GB), so only the read counter counts."""
+ inst = _fake_load(tmp_path, 1000)
+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))
+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: 250))
+ assert inst.load_progress()["fraction"] == 0.25
+
+
+def test_linux_rss_term_leaves_out_the_pinned_weight_buffer():
+ """With --load-mode none, CUDA pins the host buffer for the weights kept in RAM before a byte is
+ read (3090: VmRSS 6.39 GB, 5.99 GB of it RssShmem, 0.16 GB read), which put the bar at 23% and held
+ it there until the reads passed it. Only the file-backed share is what a mapped load pages in."""
+ status = "VmRSS:\t6390000 kB\nRssAnon:\t250000 kB\nRssFile:\t150000 kB\nRssShmem:\t5990000 kB\n"
+ with patch("builtins.open", side_effect = lambda *a, **kw: io.StringIO(status)):
+ assert LlamaCppBackend._read_rss_bytes(4242) == 150000 * 1024
+ # A kernel without the split (before 4.5) still gives VmRSS.
+ with patch("builtins.open", side_effect = lambda *a, **kw: io.StringIO("VmRSS:\t2048 kB\n")):
+ assert LlamaCppBackend._read_rss_bytes(4242) == 2048 * 1024
+
+
+def test_no_load_signal_reports_no_figure(tmp_path, monkeypatch):
+ inst = _fake_load(tmp_path, 1000)
+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))
+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: None))
+ assert inst.load_progress() is None
+
+
+def test_each_os_reads_its_own_load_counter(monkeypatch):
+ """Windows: psutil read bytes. macOS: the larger of disk bytes read (a cold memory-mapped Metal
+ load) and physical footprint (a copy into Metal or CPU buffers); RSS stays at ~20% of a Metal load."""
+ import core.inference.llama_cpp as llama_cpp
+
+ calls = []
+
+ class _Process:
+ def __init__(self, pid):
+ calls.append(pid)
+
+ def io_counters(self):
+ return type("C", (), {"read_bytes": 123})()
+
+ monkeypatch.setitem(sys.modules, "psutil", type("M", (), {"Process": _Process}))
+ monkeypatch.setattr(llama_cpp.os.path, "isdir", lambda path: False)
+ monkeypatch.setattr(LlamaCppBackend, "_darwin_rusage", staticmethod(lambda pid: (700, 300)))
+ monkeypatch.setattr(llama_cpp.sys, "platform", "win32")
+ assert LlamaCppBackend._read_load_bytes(7) == 123
+ monkeypatch.setattr(llama_cpp.sys, "platform", "darwin")
+ assert LlamaCppBackend._read_load_bytes(7) == 700
+ monkeypatch.setattr(LlamaCppBackend, "_darwin_rusage", staticmethod(lambda pid: None))
+ assert LlamaCppBackend._read_load_bytes(7) is None
+ monkeypatch.setattr(llama_cpp.sys, "platform", "freebsd14")
+ assert LlamaCppBackend._read_load_bytes(7) is None
+ assert calls == [7]
+
+
+def test_load_progress_follows_a_real_process_on_this_os(tmp_path):
+ """No mocks: a child reads a weight file into memory, and load_progress must see all of it.
+ Linux rchar and Windows read_bytes count the read; on macOS a page-cached file reads nothing from
+ disk, so the physical footprint of the held bytes carries it. Before, macOS and Windows got None."""
+ if not (os.path.isdir("/proc") or sys.platform in ("darwin", "win32")):
+ pytest.skip("no load counter on this platform")
+ size = 64 * 1024 * 1024
+ weights = tmp_path / "model.gguf"
+ weights.write_bytes(os.urandom(size))
+ child = subprocess.Popen(
+ [
+ sys.executable,
+ "-c",
+ "import sys, time\n"
+ "data = open(sys.argv[1], 'rb').read()\n"
+ "print(len(data), flush = True)\n"
+ "time.sleep(60)\n",
+ str(weights),
+ ],
+ stdout = subprocess.PIPE,
+ text = True,
+ )
+ try:
+ assert int(child.stdout.readline()) == size
+ backend = LlamaCppBackend.__new__(LlamaCppBackend)
+ backend._process = type("P", (), {"pid": child.pid, "args": None})()
+ backend._gguf_path = str(weights)
+ backend._healthy = False
+ progress = backend.load_progress()
+ assert progress is not None
+ assert progress["phase"] == "mmap"
+ assert progress["bytes_total"] == size
+ assert progress["fraction"] >= 0.99
+ finally:
+ child.kill()
+ child.wait(10)
+
+
+def test_an_mmap_span_over_gpu_tensors_is_capped_at_the_file_minus_the_gpu_side():
+ """With experts spilled under mmap, CPU_Mapped logs the span from the first to the last host tensor,
+ which covers the GPU tensors in between: 25 GiB "in RAM" for a 25.3 GiB file with 21 GiB on the GPU."""
+ lines = [
+ "load_tensors: offloaded 49/49 layers to GPU",
+ "load_tensors: CUDA0 model buffer size = 21504.00 MiB",
+ "load_tensors: CPU_Mapped model buffer size = 25600.00 MiB",
+ ]
+ model_bytes = 26419 * 1024 * 1024
+ out = parse_weight_placement(lines, model_bytes)
+ assert out["gpu_mib"] == 21504.0
+ assert out["host_mib"] == 26419 - 21504.0
+ # Without the file size, and for a buffer that is not a mapping, the logged size stands.
+ assert parse_weight_placement(lines)["host_mib"] == 25600.0
+ host = [line.replace("CPU_Mapped", "CUDA_Host") for line in lines]
+ assert parse_weight_placement(host, model_bytes)["host_mib"] == 25600.0
diff --git a/studio/backend/tests/test_provider_control_frame_spoofing.py b/studio/backend/tests/test_provider_control_frame_spoofing.py
index 235790b79ef..61e4c3d14e1 100644
--- a/studio/backend/tests/test_provider_control_frame_spoofing.py
+++ b/studio/backend/tests/test_provider_control_frame_spoofing.py
@@ -29,6 +29,7 @@ from core.inference.sse_control_frames import sanitize_provider_sse_line
"diffusion_frame",
"reasoning_summary",
"context_usage",
+ "generation_progress",
],
)
def test_every_control_type_is_dropped(frame_type):
@@ -45,6 +46,7 @@ def test_every_control_type_is_dropped(frame_type):
"_diffusionFrame",
"_reasoningDurationMs",
"_contextUsage",
+ "_generationProgress",
"_mcp_provenance",
"quote_cut",
],
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index dcff9a82f87..a310131adeb 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -416,8 +416,12 @@ export const BUDGET = {
// in the entry and settings chunks naming the example box, now a lazy chunk shared by the API page and Settings
// (6,932,829 -> 6,932,967 B raw, 2,088,616 -> 2,088,669 B transfer against cfc593f0bd, Node 26, 99 chunks).
// Per-account chat preferences (#15025): +338 B raw / +93 B transfer over the line above (Node 22.23.2, 99 chunks).
- transferBytes: 2_088_800,
- rawBytes: 6_933_450,
+ // Live prefill progress, the Generation details setting and the loaded placement readout (#14582): the
+ // stream frame reader, live store slice, reply footer line and timing tooltip rows in the eager chat
+ // chunk, +8,585 B raw / +2,268 B transfer over main (6,932,829 -> 6,941,414 B raw, 2,088,586 ->
+ // 2,090,854 B transfer against 1492952a09, Node 22, 99 chunks). Raised by that on top of #15084 and #15025.
+ transferBytes: 2_091_068,
+ rawBytes: 6_942_035,
};
// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering the
diff --git a/studio/frontend/src/components/assistant-ui/live-generation-line.tsx b/studio/frontend/src/components/assistant-ui/live-generation-line.tsx
new file mode 100644
index 00000000000..094a3e24ecb
--- /dev/null
+++ b/studio/frontend/src/components/assistant-ui/live-generation-line.tsx
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ formatLiveDecode,
+ useChatPreferencesStore,
+ useChatRuntimeStore,
+} from "@/features/chat";
+import { useLocale } from "@/i18n";
+import { useAuiState } from "@assistant-ui/react";
+import type { FC } from "react";
+
+/** Gated on the preference first so a reply mounts no assistant-ui selector while it is off
+ * (composer-keystroke-subscription-budget), like AssistantMessageTime. */
+export const LiveGenerationLine: FC = () => {
+ const enabled = useChatPreferencesStore((state) => state.showGenerationDetails);
+ return enabled ? <RunningGate /> : null;
+};
+
+const RunningGate: FC = () => {
+ const running = useAuiState(({ message }) => message.status?.type === "running");
+ return running ? <LiveDecodeFigures /> : null;
+};
+
+const LiveDecodeFigures: FC = () => {
+ const locale = useLocale();
+ // Keyed like the prefill indicator: a first turn reads "__default" until the thread persists.
+ const threadKey =
+ useAuiState(({ threadListItem }) => threadListItem.remoteId) ?? "__default";
+ const decode = useChatRuntimeStore(
+ (s) => s.liveGenerationByThreadId[threadKey]?.decode ?? null,
+ );
+ if (!decode) {
+ return null;
+ }
+ return (
+ <span
+ data-slot="live-generation-line"
+ className="aui-live-generation-line ml-[var(--icon-btn-inset)] mr-2 min-w-0 select-none font-mono text-ui-11p5 text-muted-foreground tabular-nums"
+ >
+ {formatLiveDecode(decode, locale)}
+ </span>
+ );
+};
diff --git a/studio/frontend/src/components/assistant-ui/message-timing.tsx b/studio/frontend/src/components/assistant-ui/message-timing.tsx
index 307a4d93d81..caffeb1c1ee 100644
--- a/studio/frontend/src/components/assistant-ui/message-timing.tsx
+++ b/studio/frontend/src/components/assistant-ui/message-timing.tsx
@@ -19,6 +19,27 @@ const formatNumber = (n: number): string => {
return n.toLocaleString();
};
+/** A user stop wins over the stream's last finish_reason. */
+function stopReasonLabel(
+ finishReason: unknown,
+ incompleteReason?: unknown,
+): string | null {
+ if (incompleteReason === "cancelled") return "Stopped by you";
+ switch (finishReason) {
+ case "stop":
+ return "End of turn";
+ case "length":
+ return "Length limit";
+ case "tool_calls":
+ case "function_call":
+ return "Tool call";
+ case "content_filter":
+ return "Content filter";
+ default:
+ return typeof finishReason === "string" && finishReason ? finishReason : null;
+ }
+}
+
const formatRate = (r: number | undefined): string => {
if (r === undefined || !Number.isFinite(r)) return "—";
return `${Math.round(r).toLocaleString()} tok/s`;
@@ -48,9 +69,18 @@ export const MessageTiming: FC<{
cachedTokens?: number;
cacheWriteTokens?: number;
};
+ stopReason?: unknown;
+ incomplete?: { reason?: unknown };
}
| undefined;
const st = custom?.serverTimings;
+ const stopReason = stopReasonLabel(custom?.stopReason, custom?.incomplete?.reason);
+ const draftN = st?.draft_n ?? 0;
+ const draftAccepted = st?.draft_n_accepted;
+ const draftText =
+ draftN > 0 && typeof draftAccepted === "number" && Number.isFinite(draftAccepted)
+ ? `${formatNumber(draftAccepted)} / ${formatNumber(draftN)} (${Math.round((draftAccepted / draftN) * 100)}%)`
+ : null;
// `??` (not `||`) so an explicit cache_n=0 isn't replaced by a stale
// contextUsage.cachedTokens from a prior turn.
const cacheHits =
@@ -239,6 +269,12 @@ export const MessageTiming: FC<{
</span>
</div>
)}
+ {draftText !== null && (
+ <div className="flex items-center justify-between gap-4">
+ <span className="text-muted-foreground">Draft accepted</span>
+ <span className="font-mono tabular-nums">{draftText}</span>
+ </div>
+ )}
{timing.firstTokenTime !== undefined && (
<div className="flex items-center justify-between gap-4">
<span className="text-muted-foreground">First token</span>
@@ -292,6 +328,12 @@ export const MessageTiming: FC<{
{timing.totalChunks}
</span>
</div>
+ {stopReason !== null && (
+ <div className="flex items-center justify-between gap-4">
+ <span className="text-muted-foreground">Stop reason</span>
+ <span>{stopReason}</span>
+ </div>
+ )}
</>
)
) : (
@@ -333,6 +375,12 @@ export const MessageTiming: FC<{
{timing.totalChunks}
</span>
</div>
+ {stopReason !== null && (
+ <div className="flex items-center justify-between gap-4">
+ <span className="text-muted-foreground">Stop reason</span>
+ <span>{stopReason}</span>
+ </div>
+ )}
</>
)}
</div>
diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx
index 49956beb174..4bc91960c21 100644
--- a/studio/frontend/src/components/assistant-ui/thread.tsx
+++ b/studio/frontend/src/components/assistant-ui/thread.tsx
@@ -53,6 +53,7 @@ import { QueueResumeIcon } from "@/components/assistant-ui/queue-resume-icon";
import { ProgressiveMessages } from "@/components/assistant-ui/progressive-messages";
import { MessageMenuTime } from "@/components/assistant-ui/message-menu-time";
import { AssistantMessageTime } from "@/components/assistant-ui/assistant-message-time";
+import { LiveGenerationLine } from "@/components/assistant-ui/live-generation-line";
import { UserMessageActionBar, UserMessageFooter } from "@/components/assistant-ui/user-message-actions";
import { useActionBarFocusReveal } from "@/components/assistant-ui/use-action-bar-focus-reveal";
import {
@@ -213,6 +214,7 @@ import {
providerModelSupportsStudioTools,
} from "@/features/chat/external-providers";
import { toolStatusKind } from "@/features/chat/utils/tool-status";
+import { formatPrefillDetails, formatPrefillTimeLeft, prefillPercent } from "@/features/chat";
import { replySourceMarkdown } from "@/features/chat/utils/reply-source-markdown";
import { toolResultModelText } from "@/features/chat/api/chat-adapter";
import {
@@ -260,7 +262,7 @@ import {
resolveBindings,
} from "@/features/settings";
import { FIND_SKIP_ATTRIBUTE } from "@/features/find-in-page";
-import { translate, useT } from "@/i18n";
+import { translate, useLocale, useT } from "@/i18n";
import type { TranslationKey } from "@/i18n";
import {
clampReasoningEffortToLevels,
@@ -7888,9 +7890,44 @@ const GeneratingIndicator: FC = () => {
if (!show) {
return null;
}
+ return <GeneratingStatus />;
+};
+
+// Mounted only under the empty running reply, so the thread's other messages subscribe to nothing new.
+const GeneratingStatus: FC = () => {
+ // Keyed like DiffusionCanvas: a first turn reads "__default" until the thread persists.
+ const threadKey =
+ useAuiState(({ threadListItem }) => threadListItem.remoteId) ?? "__default";
+ const prefill = useChatRuntimeStore(
+ (s) => s.liveGenerationByThreadId[threadKey]?.prefill ?? null,
+ );
+ const labelShown = useChatRuntimeStore(
+ (s) => s.liveGenerationByThreadId[threadKey]?.prefillShown === true,
+ );
+ // A reply can decode with nothing to show yet (a tool call being written): past the prompt, back to Working.
+ const decoding = useChatRuntimeStore(
+ (s) => s.liveGenerationByThreadId[threadKey]?.decode != null,
+ );
+ const showDetails = useChatPreferencesStore((s) => s.showGenerationDetails);
+ const locale = useLocale();
+ const percent = prefill ? prefillPercent(prefill) : null;
+ // Held at 100% until the first token: going back to Working... for that moment would flash too.
+ if (prefill === null || !labelShown || percent === null || decoding) {
+ return (
+ <span className="generating-glint text-sm text-muted-foreground">
+ Working...
+ </span>
+ );
+ }
+ const timeLeft = formatPrefillTimeLeft(prefill);
return (
- <span className="generating-glint text-sm text-muted-foreground">
- Working...
+ <span
+ data-slot="prefill-progress"
+ className="generating-glint text-sm text-muted-foreground tabular-nums whitespace-normal!"
+ >
+ Processing prompt {percent}%
+ {timeLeft ? ` · ${timeLeft}` : null}
+ {showDetails ? ` · ${formatPrefillDetails(prefill, locale)}` : null}
</span>
);
};
@@ -8550,6 +8587,8 @@ const AssistantMessage: FC = () => {
<div className="aui-assistant-message-footer mt-1.5 -ml-[var(--icon-btn-inset)] flex min-h-8 min-w-0 flex-wrap items-center gap-y-1">
{/* First, so the action bar mounting on hover never moves it. */}
<AssistantMessageTime />
+ {/* Only while streaming, where the action bar is hidden; the timing badge takes over after. */}
+ <LiveGenerationLine />
<BranchPicker className="mr-0.5" />
<AssistantActionBar />
</div>
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index d889771d1f1..1807e3ab74c 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -2,6 +2,7 @@
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { attachedMediaUnavailableReason } from "../lib/attached-media-gate";
+import { readGenerationProgress } from "../lib/generation-progress";
import { externalModelLabel } from "../lib/external-model-label";
import { mlxRuntimeStateFrom } from "../lib/mlx-runtime-state";
import { minPSamplingPayload } from "../lib/min-p-policy";
@@ -2792,6 +2793,7 @@ const VISIBLE_MODEL_RUNTIME_KEYS = [
"speculativeType",
"loadedSpeculativeType",
"specFallbackReason",
+ "loadedPlacement",
"specDraftNMax",
"loadedSpecDraftNMax",
"specDraftModel",
@@ -5884,6 +5886,7 @@ export function createOpenAIStreamAdapter(
let responseModelId = externalSelection?.modelId ?? params.checkpoint;
let firstTokenTime: number | undefined;
let totalChunks = 0;
+ let lastFinishReason: string | null = null;
let resolveFirstToken: (() => void) | null = null;
let rejectFirstToken: ((err: unknown) => void) | null = null;
const firstTokenPromise = new Promise<void>((resolve, reject) => {
@@ -7502,6 +7505,20 @@ export function createOpenAIStreamAdapter(
continue;
}
+ // Transient like the diffusion canvas: never transcript text, never a first token.
+ const generationProgress = (
+ chunk as unknown as { _generationProgress?: unknown }
+ )._generationProgress;
+ if (generationProgress !== undefined) {
+ const update = readGenerationProgress(generationProgress);
+ if (update?.phase === "prefill") {
+ runtime.setLivePrefill(liveThreadKey(serverCancel), update.prefill);
+ } else if (update?.phase === "decode") {
+ runtime.setLiveDecode(liveThreadKey(serverCancel), update.decode);
+ }
+ continue;
+ }
+
// Diffusion frame: a transient canvas snapshot routed to the transient store and skipped; it
// has no assistant text, so it never enters the transcript.
const diffusionFrame = (
@@ -8092,6 +8109,10 @@ export function createOpenAIStreamAdapter(
} else if (chunk.choices?.[0]?.finish_reason) {
incompleteReason = null;
}
+ const chunkFinishReason = chunk.choices?.[0]?.finish_reason;
+ if (typeof chunkFinishReason === "string" && chunkFinishReason) {
+ lastFinishReason = chunkFinishReason;
+ }
// Latch the chunk's `model` field so the openrouter/free chip shows the underlying model.
if (
isExternalRequest &&
@@ -9029,7 +9050,8 @@ export function createOpenAIStreamAdapter(
const finalTiming = buildTiming(
streamStartTime,
totalChunks,
- serverPromptEvalTime ?? firstTokenTime,
+ // What the user waited, as a resumed run records it. Server prompt eval has its own row.
+ firstTokenTime ?? serverPromptEvalTime,
finishedAt - streamStartTime,
finalTokenCount,
toolCallParts.length,
@@ -9078,6 +9100,7 @@ export function createOpenAIStreamAdapter(
// Persisted refusal flag driving the two-pass prune.
anthropicRefusal: anthropicRefusalSeen || undefined,
serverTimings: meta?.timings ?? undefined,
+ stopReason: lastFinishReason ?? undefined,
contextUsage: meta?.usage
? {
promptTokens: meta.usage.prompt_tokens,
@@ -9326,6 +9349,7 @@ export function createOpenAIStreamAdapter(
// Drop the transient denoising canvas so the finished bubble shows only the committed answer;
// scoped, since a global clear wiped another chat's frame.
runtime.clearActiveDiffusionCanvasForThread(cleanupKey);
+ runtime.clearLiveGenerationForThread(cleanupKey);
clearTimeout(warmupTimer);
if (waitingFirstChunk) {
if (firstTokenSettled) {
diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts
index 37adc248c81..b1e9a5d47a5 100644
--- a/studio/frontend/src/features/chat/api/chat-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-api.ts
@@ -2064,6 +2064,11 @@ export async function* streamChatCompletions(
separatorIndex = buffer.search(/\r?\n\r?\n/);
continue;
}
+ if ("type" in parsed && parsed.type === "generation_progress") {
+ yield { _generationProgress: parsed } as unknown as OpenAIChatChunk;
+ separatorIndex = buffer.search(/\r?\n\r?\n/);
+ continue;
+ }
const {
usage: parsedUsage,
timings: parsedTimings,
diff --git a/studio/frontend/src/features/chat/api/chat-generation-api.ts b/studio/frontend/src/features/chat/api/chat-generation-api.ts
index 79bad7f040d..476b81a7fc8 100644
--- a/studio/frontend/src/features/chat/api/chat-generation-api.ts
+++ b/studio/frontend/src/features/chat/api/chat-generation-api.ts
@@ -102,6 +102,9 @@ export function normalizeChatGenerationChunkPayload(
if (frameType === "context_usage") {
return { _contextUsage: payload } as unknown as OpenAIChatChunk;
}
+ if (frameType === "generation_progress") {
+ return { _generationProgress: payload } as unknown as OpenAIChatChunk;
+ }
}
return payload;
}
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index dab2197b65d..4432dcc89fd 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -3707,7 +3707,8 @@ export function useChatModelRuntime() {
const totalGb = prog.bytes_total / 1e9;
const pct = Math.min(99, Math.round(prog.fraction * 100));
const est = estimate(mmapSamples, prog.bytes_loaded, prog.bytes_total);
- const base = `${loadedGb.toFixed(1)} of ${totalGb.toFixed(1)} GB in memory`;
+ // Bytes the server has read or paged in, not bytes resident in RAM: weights headed for VRAM pass through.
+ const base = `${loadedGb.toFixed(1)} of ${totalGb.toFixed(1)} GB loaded`;
const label = est.stable
? `${base} • ${formatRate(est.rate)}${
formatEta(est.eta) !== "--" ? ` • ${formatEta(est.eta)} left` : ""
@@ -3728,7 +3729,7 @@ export function useChatModelRuntime() {
...modelLoadToastOptions(
renderLoadDescription(
"Starting model…",
- "Paging weights into memory.",
+ "Loading model weights.",
pct,
label,
),
diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts
index 797a8b79a94..8b4c68b0d85 100644
--- a/studio/frontend/src/features/chat/index.ts
+++ b/studio/frontend/src/features/chat/index.ts
@@ -90,7 +90,14 @@ export {
GPU_LAYERS_AUTO,
} from "./stores/chat-runtime-store";
export { resolveStagedDiffusionClassification } from "./lib/gpu-placement";
-export { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
+export { describePlacement, offloadCountsFrom, offloadWarning } from "./lib/partial-offload";
+export {
+ formatLiveDecode,
+ formatPrefillDetails,
+ formatPrefillTimeLeft,
+ prefillPercent,
+} from "./lib/generation-progress";
+export type { LiveDecodeProgress } from "./stores/chat-runtime-store";
export {
preferFullToolOutput,
preferSanitizedFullToolOutput,
diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
index 02e38f13655..0c188a65952 100644
--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts
@@ -9,6 +9,7 @@ import {
resolveResidentInitialConfig,
savedContextPin,
} from "@/features/model-picker";
+import { loadedPlacementFrom } from "./partial-offload";
// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.
import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";
import { modelDisplayName } from "@/features/hub/lib/model-identity";
@@ -502,6 +503,7 @@ export function applyActiveModelStatusToStore(
activeModelIsLocal: status.is_local_model ?? false,
specFallbackReason: status.spec_fallback_reason ?? null,
mmprojFallbackReason: status.mmproj_fallback_reason ?? null,
+ loadedPlacement: loadedPlacementFrom(status),
specDrafterKind: status.spec_drafter_kind ?? null,
// Controls follow the server while clean; loaded baselines always describe
// the settled resident, including same-model reloads from another client.
diff --git a/studio/frontend/src/features/chat/lib/generation-progress.ts b/studio/frontend/src/features/chat/lib/generation-progress.ts
new file mode 100644
index 00000000000..8ef980b2ddf
--- /dev/null
+++ b/studio/frontend/src/features/chat/lib/generation-progress.ts
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import type {
+ LiveDecodeProgress,
+ LiveGenerationProgress,
+ LivePrefillProgress,
+} from "../stores/chat-runtime-store";
+import { formatEta } from "../utils/format-transfer";
+
+export type GenerationProgressUpdate =
+ | { phase: "prefill"; prefill: LivePrefillProgress }
+ | { phase: "decode"; decode: LiveDecodeProgress };
+
+function count(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value >= 0
+ ? Math.floor(value)
+ : null;
+}
+
+function rate(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value > 0
+ ? value
+ : null;
+}
+
+/** Null for anything malformed, so a bad frame is dropped instead of drawing NaN. */
+export function readGenerationProgress(
+ frame: unknown,
+): GenerationProgressUpdate | null {
+ if (frame === null || typeof frame !== "object") return null;
+ const f = frame as Record<string, unknown>;
+ if (f.phase === "prefill") {
+ const processed = count(f.processed);
+ if (processed === null) return null;
+ const total = count(f.total);
+ return {
+ phase: "prefill",
+ prefill: {
+ processed: total ? Math.min(processed, total) : processed,
+ total: total || null,
+ cached: count(f.cached),
+ tokPerS: rate(f.tok_per_s),
+ },
+ };
+ }
+ if (f.phase === "decode") {
+ const tokens = count(f.tokens);
+ if (!tokens) return null;
+ const draftN = count(f.draft_n);
+ const draftAccepted = count(f.draft_accepted);
+ return {
+ phase: "decode",
+ decode: {
+ tokens,
+ tokPerS: rate(f.tok_per_s),
+ draftN: draftN || null,
+ draftAccepted:
+ draftN && draftAccepted !== null ? Math.min(draftAccepted, draftN) : null,
+ },
+ };
+ }
+ return null;
+}
+
+/** "9,216 / 23,700 tokens · 1,326 tok/s" for the Generation details setting, counted past the
+ * cached prefix like the percent. */
+export function formatPrefillDetails(
+ prefill: LivePrefillProgress,
+ locale?: string,
+): string {
+ const cached = prefill.cached ?? 0;
+ const done = Math.max(0, prefill.processed - cached).toLocaleString(locale);
+ const parts = [
+ prefill.total
+ ? `${done} / ${Math.max(0, prefill.total - cached).toLocaleString(locale)} tokens`
+ : `${done} tokens`,
+ ];
+ // The label shows only once prefillLabelShown has a rate it trusts, so the rate is past the first sample.
+ if (prefill.tokPerS !== null) {
+ parts.push(`${Math.round(prefill.tokPerS).toLocaleString(locale)} tok/s`);
+ }
+ return parts.join(" · ");
+}
+
+const MIN_TOKENS_FOR_PREFILL_RATE = 512;
+const PREFILL_LABEL_MIN_SECONDS = 2;
+
+/** Seconds left at the rate so far, or null while llama.cpp's first sample (a few tokens in, before
+ * the first batch) is all there is, since its rate is noise. */
+function prefillSecondsLeft(prefill: LivePrefillProgress): number | null {
+ if (!prefill.total || prefill.tokPerS === null) return null;
+ if (prefill.processed - (prefill.cached ?? 0) < MIN_TOKENS_FOR_PREFILL_RATE) return null;
+ return (prefill.total - prefill.processed) / prefill.tokPerS;
+}
+
+/** Whether "Processing prompt" shows after this sample. Only a pass expected to keep it up for
+ * PREFILL_LABEL_MIN_SECONDS turns it on, so a fast prompt stays on Working... instead of flashing it,
+ * and it then stays for the rest of that pass instead of dropping as the estimate shrinks. */
+export function prefillLabelShown(
+ previous: LiveGenerationProgress | undefined,
+ prefill: LivePrefillProgress,
+): boolean {
+ const samePass =
+ previous?.prefill != null &&
+ previous.decode === null &&
+ previous.prefill.total === prefill.total &&
+ previous.prefill.processed <= prefill.processed;
+ if (samePass && previous.prefillShown) return true;
+ const left = prefillSecondsLeft(prefill);
+ return left !== null && left >= PREFILL_LABEL_MIN_SECONDS;
+}
+
+/** "12s left" at the rate so far, like the load toast; null without a trusted rate or under a second. */
+export function formatPrefillTimeLeft(prefill: LivePrefillProgress): string | null {
+ const left = prefillSecondsLeft(prefill);
+ return left !== null && left >= 1 ? `${formatEta(left)} left` : null;
+}
+
+/** "4,306 tokens · 58.3 tok/s · draft 1,791/5,030 accepted (36%)", every number in the reader's locale. */
+export function formatLiveDecode(decode: LiveDecodeProgress, locale?: string): string {
+ const parts = [
+ `${decode.tokens.toLocaleString(locale)} ${decode.tokens === 1 ? "token" : "tokens"}`,
+ ];
+ if (decode.tokPerS !== null) {
+ const rate = decode.tokPerS.toLocaleString(locale, {
+ minimumFractionDigits: 1,
+ maximumFractionDigits: 1,
+ });
+ parts.push(`${rate} tok/s`);
+ }
+ if (decode.draftN && decode.draftAccepted !== null) {
+ const share = Math.round((decode.draftAccepted / decode.draftN) * 100);
+ parts.push(
+ `draft ${decode.draftAccepted.toLocaleString(locale)}/${decode.draftN.toLocaleString(locale)} accepted (${share}%)`,
+ );
+ }
+ return parts.join(" · ");
+}
+
+/** Percent of the part of the prompt the cache did not cover, llama.cpp's "timed progress": a
+ * follow-up in a long chat would otherwise start near 100% and crawl through the last few. */
+export function prefillPercent(prefill: LivePrefillProgress): number | null {
+ if (!prefill.total) return null;
+ const cached = prefill.cached ?? 0;
+ const todo = prefill.total - cached;
+ if (todo <= 0) return 100;
+ return Math.min(100, Math.max(0, Math.floor(((prefill.processed - cached) / todo) * 100)));
+}
diff --git a/studio/frontend/src/features/chat/lib/partial-offload.ts b/studio/frontend/src/features/chat/lib/partial-offload.ts
index ce5401809c1..7f91ab545d5 100644
--- a/studio/frontend/src/features/chat/lib/partial-offload.ts
+++ b/studio/frontend/src/features/chat/lib/partial-offload.ts
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { formatGiB } from "../../../lib/memory/format";
+import type { LoadedPlacement } from "../stores/chat-runtime-store";
import { CPU_FALLBACK_MESSAGE } from "../utils/mmproj-fallback";
// React-free so node:test can import it; shared by every model load path.
@@ -97,3 +99,98 @@ export function offloadWarning(counts: OffloadCounts): OffloadWarning | null {
description: `${offloaded} of ${total} layers are on the GPU. The rest run on CPU, so generation will be slower. A smaller quantization or a shorter context may let more of it fit on the GPU.`,
};
}
+
+function count(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value >= 0
+ ? value
+ : null;
+}
+
+/** GiB like the rest of the panel, or MiB under 0.1 GiB, which would otherwise print as "0.0 GiB". */
+function weightAmount(mib: number): string {
+ return mib < 102.4
+ ? `${Math.max(1, Math.round(mib))} MiB`
+ : formatGiB(mib / 1024);
+}
+
+function layerCount(n: number): string {
+ return `${n} ${n === 1 ? "layer" : "layers"}`;
+}
+
+/** --fit reports its own expert spill; a Manual load says it through n_cpu_moe instead. */
+export function loadedPlacementFrom(response: {
+ is_gguf?: boolean | null;
+ offloaded_layers?: number | null;
+ offload_total_layers?: number | null;
+ overflow_layers?: number | null;
+ gpu_weights_mib?: number | null;
+ host_weights_mib?: number | null;
+ expert_cache_mib?: number | null;
+ gpu_memory_mode?: string | null;
+ n_cpu_moe?: number | null;
+}): LoadedPlacement | null {
+ if (response.is_gguf === false) return null;
+ const manualExperts =
+ response.gpu_memory_mode === "manual" ? count(response.n_cpu_moe) : null;
+ const placement: LoadedPlacement = {
+ offloaded: count(response.offloaded_layers),
+ total: count(response.offload_total_layers),
+ overflowLayers:
+ count(response.overflow_layers) ?? (manualExperts ? manualExperts : null),
+ gpuWeightsMib: count(response.gpu_weights_mib),
+ hostWeightsMib: count(response.host_weights_mib),
+ expertCacheMib: count(response.expert_cache_mib) || null,
+ };
+ return placement.total === null &&
+ placement.gpuWeightsMib === null &&
+ placement.hostWeightsMib === null
+ ? null
+ : placement;
+}
+
+/** An overflowing layer has expert weights in RAM: all its routed experts, except --fit's first partial layer
+ * per device, which keeps only part of its FFN there, so the text says expert weights, not experts. */
+function describeLayers(p: LoadedPlacement): string | null {
+ const { offloaded, total, overflowLayers, expertCacheMib } = p;
+ if (offloaded === null || total === null || total <= 0) return null;
+ // A GPU build whose backend failed to start still logs "offloaded N/N"; its buffers are all host ones.
+ const noGpuBuffers = p.gpuWeightsMib === 0 && (p.hostWeightsMib ?? 0) > 0;
+ if (offloaded <= 0 || noGpuBuffers) return `All ${total} layers on the CPU`;
+ if (offloaded < total)
+ return `${offloaded} of ${total} layers on the GPU, ${total - offloaded} on the CPU`;
+ if (expertCacheMib !== null && expertCacheMib > 0) {
+ return `All ${total} layers on the GPU, all experts in RAM (${weightAmount(expertCacheMib)} expert cache on the GPU)`;
+ }
+ if (overflowLayers !== null && overflowLayers > 0) {
+ return `All ${total} layers on the GPU, expert weights of ${layerCount(overflowLayers)} in RAM`;
+ }
+ return `All ${total} layers on the GPU`;
+}
+
+/** Both figures are shown: token embeddings stay in RAM even when every layer is on the GPU. On a machine whose
+ * GPU and CPU share one pool, `pool` names it ("unified memory") and the weights are one figure. */
+function describeWeights(
+ gpuWeightsMib: number | null,
+ hostWeightsMib: number | null,
+ pool: string | null,
+): string | null {
+ const gpu = gpuWeightsMib !== null && gpuWeightsMib > 0 ? gpuWeightsMib : 0;
+ const host =
+ hostWeightsMib !== null && hostWeightsMib > 0 ? hostWeightsMib : 0;
+ if (pool !== null && gpu + host > 0)
+ return `${weightAmount(gpu + host)} in ${pool}`;
+ const parts: string[] = [];
+ if (gpu > 0) parts.push(`${weightAmount(gpu)} in VRAM`);
+ if (host > 0) parts.push(`${weightAmount(host)} in RAM`);
+ return parts.length > 0 ? parts.join(" · ") : null;
+}
+
+export function describePlacement(
+ p: LoadedPlacement,
+ pool: string | null = null,
+): { layers: string | null; memory: string | null } {
+ return {
+ layers: describeLayers(p),
+ memory: describeWeights(p.gpuWeightsMib, p.hostWeightsMib, pool),
+ };
+}
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index 055a8862899..81d4a0a6307 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+import { readGenerationProgress } from "./lib/generation-progress";
import { useAppShellReadySignal } from "@/components/app-readiness";
import { authFetch, getAuthSessionEpoch } from "@/features/auth";
import { isMcpToolOnly, mcpImageMappingsEnabled } from "./api/mcp-image";
@@ -1782,6 +1783,7 @@ function scheduleGenerationRecovery(
cursor = update.event.seq;
if (update.event.type === "chunk") {
const chunk = update.event.payload as {
+ _generationProgress?: unknown;
_reasoningDurationMs?: unknown;
usage?: {
prompt_tokens?: unknown;
@@ -1801,6 +1803,19 @@ function scheduleGenerationRecovery(
context_truncated?: OpenAIChatChunk["context_truncated"];
quote_cut?: boolean;
};
+ // Live figures only once caught up: a replayed backlog would flash every old sample.
+ if ("_generationProgress" in chunk) {
+ if (caughtUp(update.run)) {
+ const progress = readGenerationProgress(chunk._generationProgress);
+ const store = useChatRuntimeStore.getState();
+ if (progress?.phase === "prefill") {
+ store.setLivePrefill(threadId, progress.prefill);
+ } else if (progress?.phase === "decode") {
+ store.setLiveDecode(threadId, progress.decode);
+ }
+ }
+ continue;
+ }
if ("_reasoningDurationMs" in chunk) {
currentMetadata = recoveredReasoningSummaryMetadata(
currentMetadata,
@@ -1948,6 +1963,7 @@ function scheduleGenerationRecovery(
);
}
} finally {
+ useChatRuntimeStore.getState().clearLiveGenerationForThread(threadId);
// EVERY exit, not just the terminal one: a permanent follower error (another tab deletes the
// thread, the run row cascades, the request 404s) throws past the terminal branch into the
// outer catch, leaving a seeded card in the global store for the session. `soleRequest` counts
diff --git a/studio/frontend/src/features/chat/stores/chat-preferences-store.ts b/studio/frontend/src/features/chat/stores/chat-preferences-store.ts
index b5963ed9364..b59bff80b78 100644
--- a/studio/frontend/src/features/chat/stores/chat-preferences-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-preferences-store.ts
@@ -29,6 +29,8 @@ import {
// showInlineEditResponse: on keeps Edit response in each response's action bar instead of its
// More menu. notifyWhenReplyFinishes: on sends a system notification when a reply ends while the
// window is hidden or unfocused. showReplyTimes: on shows when each reply finished under it.
+// showGenerationDetails: on shows live prefill / decode figures while a reply streams and where
+// a local model's layers ended up after it loads.
// filesPanelCollapsed: on shrinks Chat with files to a header and
// a row of pills.
export interface ChatPreferencesState {
@@ -50,6 +52,8 @@ export interface ChatPreferencesState {
setShowResponseModel: (value: boolean) => void;
showReplyTimes: boolean;
setShowReplyTimes: (value: boolean) => void;
+ showGenerationDetails: boolean;
+ setShowGenerationDetails: (value: boolean) => void;
showInlineEditResponse: boolean;
setShowInlineEditResponse: (value: boolean) => void;
thinkingVisibility: DisplayVisibility;
@@ -103,6 +107,10 @@ export const useChatPreferencesStore = create<ChatPreferencesState>()(
// Off by default: it adds text under every reply, so it stays opt in.
showReplyTimes: false,
setShowReplyTimes: (showReplyTimes) => set({ showReplyTimes }),
+ // Off by default: it adds a line of numbers under streaming replies, so it stays opt in.
+ showGenerationDetails: false,
+ setShowGenerationDetails: (showGenerationDetails) =>
+ set({ showGenerationDetails }),
showInlineEditResponse: true,
setShowInlineEditResponse: (showInlineEditResponse) =>
set({ showInlineEditResponse }),
@@ -157,6 +165,7 @@ export const useChatPreferencesStore = create<ChatPreferencesState>()(
showModelDisclaimer: saved?.showModelDisclaimer ?? false,
showResponseModel: saved?.showResponseModel ?? false,
showReplyTimes: saved?.showReplyTimes === true,
+ showGenerationDetails: saved?.showGenerationDetails === true,
showInlineEditResponse: saved?.showInlineEditResponse ?? true,
thinkingVisibility: migrateVisibility(
saved?.thinkingVisibility,
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index 73679877ad0..3b84af1d85e 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -120,6 +120,7 @@ import {
type ModelLifecyclePhase,
} from "../utils/model-lifecycle-gate";
import { shouldAdvanceQueuedSettingsEpoch } from "../utils/queued-settings-epoch";
+import { prefillLabelShown } from "../lib/generation-progress";
import { adoptFirstTurnConfirmations } from "../utils/adopt-first-turn-confirmations";
import type { MmprojFallbackReason } from "../types/api";
import type {
@@ -427,6 +428,36 @@ export type DiffusionCanvasFrame = {
total: number;
text: string;
};
+/** Where llama.cpp placed a loaded GGUF's weights (counts from its log; MiB from its buffers). */
+export type LoadedPlacement = {
+ offloaded: number | null;
+ total: number | null;
+ /** Layers whose MoE experts --fit kept in system memory; null when --fit did not run. */
+ overflowLayers: number | null;
+ gpuWeightsMib: number | null;
+ hostWeightsMib: number | null;
+ /** Set when every routed expert is in RAM with a GPU cache of the busiest (--moe-cache-mib auto). */
+ expertCacheMib: number | null;
+};
+export type LivePrefillProgress = {
+ processed: number;
+ total: number | null;
+ cached: number | null;
+ tokPerS: number | null;
+};
+export type LiveDecodeProgress = {
+ tokens: number;
+ tokPerS: number | null;
+ draftN: number | null;
+ draftAccepted: number | null;
+};
+/** Transient, like the diffusion canvas: set from generation_progress frames, cleared when the run ends. */
+export type LiveGenerationProgress = {
+ prefill: LivePrefillProgress | null;
+ /** Whether "Processing prompt" is up for this pass; see prefillLabelShown. */
+ prefillShown: boolean;
+ decode: LiveDecodeProgress | null;
+};
export type PendingImageEditReference = {
threadId: string | null;
openaiImageGenerationCallId: string;
@@ -2683,6 +2714,8 @@ type ChatRuntimeStore = {
specFallbackReason: string | null;
/** Projector recovery outcome for the active GGUF, or null. */
mmprojFallbackReason: MmprojFallbackReason | null;
+ /** Where the loaded GGUF's weights went, from llama.cpp's load log; null for other backends. */
+ loadedPlacement: LoadedPlacement | null;
/** Which drafter the speculative resolution was about ("mtp", "dspark", "dflash").
* specFallbackReason alone cannot name the file to fix, since Auto resolves it server-side. */
specDrafterKind: string | null;
@@ -2777,6 +2810,8 @@ type ChatRuntimeStore = {
/** Live denoising frame per conversation ("__default" until the id exists). Transient and
* keyed, since two denoising chats overwrote each other's frame. */
activeDiffusionCanvasByThreadId: Record<string, DiffusionCanvasFrame>;
+ /** Live prefill / decode progress per conversation, keyed like the diffusion canvas. */
+ liveGenerationByThreadId: Record<string, LiveGenerationProgress>;
customContextLength: number | null;
/** The pinned context the loaded model used (null = Auto), so dirty-tracking and a later fit
* Apply can tell an explicit pin from Auto. */
@@ -2966,6 +3001,9 @@ type ChatRuntimeStore = {
) => void;
/** Drop only `threadId`'s canvas: a run ending in a background chat must not wipe another's. */
clearActiveDiffusionCanvasForThread: (threadId: string | null) => void;
+ setLivePrefill: (threadId: string | null, prefill: LivePrefillProgress) => void;
+ setLiveDecode: (threadId: string | null, decode: LiveDecodeProgress) => void;
+ clearLiveGenerationForThread: (threadId: string | null) => void;
setAutoHealToolCalls: (enabled: boolean) => void;
setNudgeToolCalls: (enabled: boolean) => void;
setDeduplicateToolCalls: (enabled: boolean) => void;
@@ -4622,6 +4660,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
toolFullOutput: {},
generatingStatus: null,
activeDiffusionCanvasByThreadId: {},
+ liveGenerationByThreadId: {},
autoHealToolCalls: true,
nudgeToolCalls: true,
deduplicateToolCalls: true,
@@ -4651,6 +4690,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedSpeculativeType: null,
specFallbackReason: null,
mmprojFallbackReason: null,
+ loadedPlacement: null,
specDrafterKind: null,
specDraftNMax: null,
loadedSpecDraftNMax: null,
@@ -5162,6 +5202,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
state.activeDiffusionCanvasByThreadId,
"activeDiffusionCanvasByThreadId",
);
+ move(state.liveGenerationByThreadId, "liveGenerationByThreadId");
// Its parked calls too, or the row never trades its spinner for the approval mark.
if (moved.runningByThreadId) {
const confirmations = adoptFirstTurnConfirmations(
@@ -5345,6 +5386,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
activeModelIsLocal: false,
specFallbackReason: null,
mmprojFallbackReason: null,
+ loadedPlacement: null,
specDrafterKind: null,
}
: {}),
@@ -5639,6 +5681,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
toolLiveOutput: {},
toolFullOutput: {},
activeDiffusionCanvasByThreadId: {},
+ liveGenerationByThreadId: {},
kvCacheDtype: null,
mlxKvQuant: null,
loadedMlxKvQuantRequested: null,
@@ -5661,6 +5704,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
loadedSpeculativeType: null,
specFallbackReason: null,
mmprojFallbackReason: null,
+ loadedPlacement: null,
specDrafterKind: null,
specDraftNMax: null,
loadedSpecDraftNMax: null,
@@ -6339,6 +6383,41 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({
delete next[key];
return { activeDiffusionCanvasByThreadId: next };
}),
+ setLivePrefill: (threadId, prefill) =>
+ set((state) => {
+ const key = threadId || "__default";
+ const prefillShown = prefillLabelShown(state.liveGenerationByThreadId[key], prefill);
+ // A new prompt pass (the next round of a tool loop) starts over, so its decode figures go too.
+ return {
+ liveGenerationByThreadId: {
+ ...state.liveGenerationByThreadId,
+ [key]: { prefill, prefillShown, decode: null },
+ },
+ };
+ }),
+ setLiveDecode: (threadId, decode) =>
+ set((state) => {
+ const key = threadId || "__default";
+ const current = state.liveGenerationByThreadId[key];
+ return {
+ liveGenerationByThreadId: {
+ ...state.liveGenerationByThreadId,
+ [key]: {
+ prefill: current?.prefill ?? null,
+ prefillShown: current?.prefillShown ?? false,
+ decode,
+ },
+ },
+ };
+ }),
+ clearLiveGenerationForThread: (threadId) =>
+ set((state) => {
+ const key = threadId || "__default";
+ if (state.liveGenerationByThreadId[key] === undefined) return state;
+ const next = { ...state.liveGenerationByThreadId };
+ delete next[key];
+ return { liveGenerationByThreadId: next };
+ }),
setGeneratingStatus: (generatingStatus) => set({ generatingStatus }),
setAutoHealToolCalls: (autoHealToolCalls) =>
set((state) => {
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index aa9dad0285f..ecbe754700c 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -493,6 +493,12 @@ export interface InferenceStatusResponse {
/** Set while the active model is a recovered CPU-only Vulkan load. */
offloaded_layers?: number | null;
offload_total_layers?: number | null;
+ /** Layers --fit left partly in system memory (MoE experts first); null without --fit. */
+ overflow_layers?: number | null;
+ gpu_weights_mib?: number | null;
+ host_weights_mib?: number | null;
+ /** --moe-cache-mib auto: every routed expert in RAM, this many MiB cached on the GPU. */
+ expert_cache_mib?: number | null;
offload_overridden?: boolean | null;
gpu_backend_unavailable?: boolean | null;
cpu_fallback_reason?: CpuFallbackReason | null;
diff --git a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
index 3503317f0ff..58878121faa 100644
--- a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
+++ b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
@@ -52,6 +52,7 @@ export function generationChunkCountsTowardTiming(payload: unknown): boolean {
| {
_reasoningDurationMs?: unknown;
_contextUsage?: unknown;
+ _generationProgress?: unknown;
context_truncated?: unknown;
quote_cut?: unknown;
usage?: unknown;
@@ -63,6 +64,7 @@ export function generationChunkCountsTowardTiming(payload: unknown): boolean {
if (
"_reasoningDurationMs" in chunk ||
"_contextUsage" in chunk ||
+ "_generationProgress" in chunk ||
chunk.context_truncated ||
chunk.quote_cut
) {
@@ -181,6 +183,8 @@ type RecoveryTimings = {
[key: string]: unknown;
};
+const RECOVERED_STOP_REASONS = new Set(["stop", "length", "tool_calls", "content_filter"]);
+
export function recoveredGenerationFinalMetadata(options: {
current: Record<string, unknown>;
run: {
@@ -189,6 +193,7 @@ export function recoveredGenerationFinalMetadata(options: {
createdAt: number;
startedAt: number | null;
completedAt: number | null;
+ finishReason?: string | null;
};
usage?: RecoveryUsage;
timings?: RecoveryTimings;
@@ -219,6 +224,10 @@ export function recoveredGenerationFinalMetadata(options: {
if (next.serverTimings === undefined && timings !== undefined) {
next.serverTimings = timings;
}
+ // Only the stream's own vocabulary: "cancelled" / "interrupted" are run states the reply already shows.
+ if (next.stopReason === undefined && RECOVERED_STOP_REASONS.has(run.finishReason ?? "")) {
+ next.stopReason = run.finishReason;
+ }
if (
next.contextUsage === undefined &&
typeof usage?.prompt_tokens === "number" &&
diff --git a/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsx b/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsx
new file mode 100644
index 00000000000..150970cc84f
--- /dev/null
+++ b/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsx
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import {
+ describePlacement,
+ useChatPreferencesStore,
+ useChatRuntimeStore,
+} from "@/features/chat";
+
+/** Where the resident GGUF's weights landed. Under Auto, --fit decides, and the layer count alone hides a
+ * MoE spill, so the experts and the RAM side are shown too. `pool` is the panel's single memory pool, if any. */
+export function LoadedPlacementRow({ pool }: { pool: string | null }) {
+ const enabled = useChatPreferencesStore((s) => s.showGenerationDetails);
+ const placement = useChatRuntimeStore((s) => s.loadedPlacement);
+ if (!enabled || placement === null) return null;
+ const { layers, memory } = describePlacement(placement, pool);
+ if (layers === null && memory === null) return null;
+ return (
+ <div
+ data-slot="loaded-placement"
+ className="flex flex-col gap-1 border-b border-border pb-5"
+ >
+ <span className="flex min-h-8 items-center text-ui-13 font-medium leading-[1.25] tracking-nav text-foreground">
+ Weight placement
+ </span>
+ {layers !== null && (
+ <span className="text-ui-12 leading-snug text-muted-foreground">
+ {layers}
+ </span>
+ )}
+ {memory !== null && (
+ <span className="text-ui-12 leading-snug text-muted-foreground tabular-nums">
+ {memory}
+ </span>
+ )}
+ </div>
+ );
+}
diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
index 0d4d154ad81..6e73837fe93 100644
--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx
+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx
@@ -229,6 +229,7 @@ import {
} from "../sharing";
import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog";
import { MemoryEstimateRow } from "./memory-estimate-row";
+import { LoadedPlacementRow } from "./loaded-placement-row";
import type { ModelPickTarget } from "./model-selector/types";
import type { ModelConfigFocus } from "../model-config/model-config-handoff";
import { reconcileTensorSplit } from "@/hooks/gpu-tensor-split";
@@ -4018,6 +4019,12 @@ export function ModelConfigPage({
onExpandedChange={setMemoryBreakdownOpen}
/>
)}
+ {isActiveModel && target.isGguf && !audioRuntimeGguf ? (
+ // The same pool the estimate above names: "Unified" on Apple Silicon, "Shared" on other one-pool machines.
+ <LoadedPlacementRow
+ pool={singleMemoryPool ? (isAppleUnifiedMemory ? "unified memory" : "shared memory") : null}
+ />
+ ) : null}
{target.isGguf && !audioRuntimeGguf && (
<>
<GgufContextLengthSetting
diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts
index 7826c1c03c9..fa11332e89b 100644
--- a/studio/frontend/src/features/settings/settings-search.ts
+++ b/studio/frontend/src/features/settings/settings-search.ts
@@ -147,6 +147,7 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = {
"settings.chat.modelDisclaimer",
"settings.chat.inlineEditResponse",
"settings.chat.showReplyTimes",
+ "settings.chat.showGenerationDetails",
"settings.chat.groups.menu.title",
],
browser: [
@@ -478,6 +479,7 @@ export const SETTINGS_SEARCH_KEYWORDS: Partial<
"settings.chat.scrollToBottomButton": "settings.chat.scrollToBottomButtonKeywords",
"settings.chat.replyNotification": "settings.chat.replyNotificationKeywords",
"settings.chat.showReplyTimes": "settings.chat.showReplyTimesKeywords",
+ "settings.chat.showGenerationDetails": "settings.chat.showGenerationDetailsKeywords",
// match protocol and agent names omitted from the title.
"settings.apiKeys.mcp.title": "settings.apiKeys.mcp.keywords",
// the intro names only some agents; every listed one has to find this page.
diff --git a/studio/frontend/src/features/settings/tabs/chat-tab.tsx b/studio/frontend/src/features/settings/tabs/chat-tab.tsx
index 1153c16f0c0..2bece356672 100644
--- a/studio/frontend/src/features/settings/tabs/chat-tab.tsx
+++ b/studio/frontend/src/features/settings/tabs/chat-tab.tsx
@@ -369,6 +369,12 @@ export function ChatTab() {
const setShowReplyTimes = useChatPreferencesStore(
(state) => state.setShowReplyTimes,
);
+ const showGenerationDetails = useChatPreferencesStore(
+ (state) => state.showGenerationDetails,
+ );
+ const setShowGenerationDetails = useChatPreferencesStore(
+ (state) => state.setShowGenerationDetails,
+ );
const autoScrollWhileGenerating = useChatPreferencesStore(
(state) => state.autoScrollWhileGenerating,
);
@@ -796,6 +802,16 @@ export function ChatTab() {
onCheckedChange={setShowReplyTimes}
/>
</SettingsRow>
+ <SettingsRow
+ label={t("settings.chat.showGenerationDetails")}
+ description={t("settings.chat.showGenerationDetailsDescription")}
+ >
+ <Switch
+ aria-label={t("settings.chat.showGenerationDetails")}
+ checked={showGenerationDetails}
+ onCheckedChange={setShowGenerationDetails}
+ />
+ </SettingsRow>
<SettingsRow
label={t("settings.chat.inlineEditResponse")}
description={t("settings.chat.inlineEditResponseDescription")}
diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts
index e269265cb47..1798315f4fe 100644
--- a/studio/frontend/src/i18n/locales/ar.ts
+++ b/studio/frontend/src/i18n/locales/ar.ts
@@ -2755,6 +2755,11 @@ export const ar = {
showReplyTimesDescription: "إظهار وقت انتهاء كل رد أسفله.",
showReplyTimesKeywords:
"طابع زمني وقت ساعة تاريخ متى انتهى رد رسالة timestamp time clock date reply",
+ showGenerationDetails: "تفاصيل التوليد",
+ showGenerationDetailsDescription:
+ "عرض السرعة وعدد الرموز مباشرةً أثناء بث الرد، ومكان وضع طبقات النموذج المحلي.",
+ showGenerationDetailsKeywords:
+ "تفاصيل سرعة رموز إحصاءات طبقات ذاكرة verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "تعديل الرد في الردود",
inlineEditResponseDescription: "إبقاء تعديل الرد على كل رد بدلًا من قائمة المزيد.",
modelDisclaimer: "إظهار إخلاء مسؤولية النموذج",
diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts
index 41ce024aea1..4737d6a6a73 100644
--- a/studio/frontend/src/i18n/locales/de.ts
+++ b/studio/frontend/src/i18n/locales/de.ts
@@ -2796,6 +2796,11 @@ export const de = {
showReplyTimesDescription: "Unter jeder Antwort anzeigen, wann sie fertig war.",
showReplyTimesKeywords:
"Zeitstempel Uhrzeit Datum wann fertig Antwort Nachricht timestamp time clock date reply",
+ showGenerationDetails: "Generierungsdetails",
+ showGenerationDetailsDescription:
+ "Tempo und Tokenzahl live anzeigen, während eine Antwort gestreamt wird, und wo die Schichten eines lokalen Modells liegen.",
+ showGenerationDetailsKeywords:
+ "Details Geschwindigkeit Token Statistik Schichten Auslagerung verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Antwort bearbeiten bei Antworten",
inlineEditResponseDescription: "Antwort bearbeiten bei jeder Antwort anzeigen statt im Menü „Mehr“.",
modelDisclaimer: "Modell-Hinweis anzeigen",
diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts
index f1a61a8e6e1..80930b9fcec 100644
--- a/studio/frontend/src/i18n/locales/en.ts
+++ b/studio/frontend/src/i18n/locales/en.ts
@@ -2813,6 +2813,11 @@ export const en = {
showReplyTimesDescription: "Show when each reply finished, below the reply.",
showReplyTimesKeywords:
"timestamp time clock date hour when finished arrived reply response message",
+ showGenerationDetails: "Generation details",
+ showGenerationDetailsDescription:
+ "Show live prompt and reply speeds with token counts, and where a model's layers loaded.",
+ showGenerationDetailsKeywords:
+ "verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Edit response on responses",
inlineEditResponseDescription:
"Keep Edit response on every response, instead of in the More menu.",
diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts
index ef5b0f5bb28..1c6322487ec 100644
--- a/studio/frontend/src/i18n/locales/es.ts
+++ b/studio/frontend/src/i18n/locales/es.ts
@@ -2787,6 +2787,11 @@ export const es = {
showReplyTimesDescription: "Mostrar debajo de cada respuesta cuándo terminó.",
showReplyTimesKeywords:
"marca de tiempo hora fecha cuándo terminó respuesta mensaje timestamp time clock date reply",
+ showGenerationDetails: "Detalles de generación",
+ showGenerationDetailsDescription:
+ "Muestra la velocidad y los tokens en vivo mientras se genera una respuesta, y dónde quedaron las capas de un modelo local.",
+ showGenerationDetailsKeywords:
+ "detalles velocidad tokens estadísticas capas descarga verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Editar respuesta en las respuestas",
inlineEditResponseDescription: "Mostrar Editar respuesta en cada respuesta, en lugar de en el menú Más.",
modelDisclaimer: "Mostrar aviso del modelo",
diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts
index ba38c337482..9c6d3b16217 100644
--- a/studio/frontend/src/i18n/locales/fr.ts
+++ b/studio/frontend/src/i18n/locales/fr.ts
@@ -2793,6 +2793,11 @@ export const fr = {
showReplyTimesDescription: "Afficher sous chaque réponse l'heure à laquelle elle s'est terminée.",
showReplyTimesKeywords:
"horodatage heure date quand terminé réponse message timestamp time clock date reply",
+ showGenerationDetails: "Détails de génération",
+ showGenerationDetailsDescription:
+ "Affiche la vitesse et le nombre de jetons en direct pendant une réponse, et où se trouvent les couches d'un modèle local.",
+ showGenerationDetailsKeywords:
+ "détails vitesse jetons statistiques couches déchargement verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Modifier la réponse sur les réponses",
inlineEditResponseDescription: "Afficher Modifier la réponse sur chaque réponse, plutôt que dans le menu Plus.",
modelDisclaimer: "Afficher l'avertissement du modèle",
diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.ts
index 647fea5fe42..9fe7695fab9 100644
--- a/studio/frontend/src/i18n/locales/he.ts
+++ b/studio/frontend/src/i18n/locales/he.ts
@@ -2871,6 +2871,11 @@ export const he = {
showReplyTimesDescription: "הצג מתחת לכל תשובה מתי היא הסתיימה.",
showReplyTimesKeywords:
"חותמת זמן שעה תאריך מתי הסתיים תשובה הודעה timestamp time clock date reply",
+ showGenerationDetails: "פרטי יצירה",
+ showGenerationDetailsDescription:
+ "הצגת מהירות ומספר אסימונים בזמן אמת בזמן שתשובה נוצרת, והיכן הוצבו השכבות של מודל מקומי.",
+ showGenerationDetailsKeywords:
+ "פרטים מהירות אסימונים סטטיסטיקה שכבות verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "עריכת תגובה על גבי התגובות",
inlineEditResponseDescription:
"הצג את 'עריכת תגובה' בכל תגובה, במקום בתפריט 'עוד'.",
diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts
index 9e8aeae3c58..b21def95af8 100644
--- a/studio/frontend/src/i18n/locales/hi.ts
+++ b/studio/frontend/src/i18n/locales/hi.ts
@@ -2764,6 +2764,11 @@ export const hi = {
showReplyTimesDescription: "हर जवाब के नीचे दिखाएँ कि वह कब पूरा हुआ।",
showReplyTimesKeywords:
"टाइमस्टैम्प समय घड़ी तारीख कब पूरा जवाब संदेश timestamp time clock date reply",
+ showGenerationDetails: "जनरेशन विवरण",
+ showGenerationDetailsDescription:
+ "जवाब बनते समय लाइव गति और टोकन गिनती दिखाएँ, और लोकल मॉडल की परतें कहाँ रखी गईं।",
+ showGenerationDetailsKeywords:
+ "विवरण गति टोकन आँकड़े परतें verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "जवाबों पर जवाब संपादित करें",
inlineEditResponseDescription: "जवाब संपादित करें को More मेन्यू के बजाय हर जवाब पर दिखाएँ।",
modelDisclaimer: "मॉडल अस्वीकरण दिखाएं",
diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.ts
index 7ffdea8431e..c696c90930e 100644
--- a/studio/frontend/src/i18n/locales/it.ts
+++ b/studio/frontend/src/i18n/locales/it.ts
@@ -2699,6 +2699,11 @@ export const it = {
showReplyTimesDescription: "Mostra sotto ogni risposta quando è terminata.",
showReplyTimesKeywords:
"orario ora data quando terminata risposta messaggio timestamp time clock date reply",
+ showGenerationDetails: "Dettagli generazione",
+ showGenerationDetailsDescription:
+ "Mostra velocità e token in tempo reale mentre una risposta viene generata, e dove sono stati collocati i livelli di un modello locale.",
+ showGenerationDetailsKeywords:
+ "dettagli velocità token statistiche livelli scaricamento verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Modifica risposta nelle risposte",
inlineEditResponseDescription: "Mostra Modifica risposta su ogni risposta, invece che nel menu Altro.",
modelDisclaimer: "Mostra l'avviso sul modello",
diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts
index f3ba6571bcb..e3e0d357d66 100644
--- a/studio/frontend/src/i18n/locales/ja.ts
+++ b/studio/frontend/src/i18n/locales/ja.ts
@@ -2741,6 +2741,11 @@ export const ja = {
showReplyTimesDescription: "各応答の下に、完了した時刻を表示します。",
showReplyTimesKeywords:
"タイムスタンプ 時刻 時間 日付 完了 応答 メッセージ timestamp time clock date reply",
+ showGenerationDetails: "生成の詳細",
+ showGenerationDetailsDescription:
+ "応答の生成中に速度とトークン数をリアルタイムで表示し、ローカルモデルのレイヤーの配置も表示します。",
+ showGenerationDetailsKeywords:
+ "詳細 速度 トークン 統計 レイヤー オフロード verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "応答に応答の編集を表示",
inlineEditResponseDescription: "応答の編集を「その他」メニューではなく、各応答に表示します。",
modelDisclaimer: "モデルの免責事項を表示",
diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts
index a4aeda644fa..48a8497073d 100644
--- a/studio/frontend/src/i18n/locales/ko.ts
+++ b/studio/frontend/src/i18n/locales/ko.ts
@@ -2746,6 +2746,11 @@ export const ko = {
showReplyTimesDescription: "각 응답 아래에 완료된 시간을 표시합니다.",
showReplyTimesKeywords:
"타임스탬프 시간 시각 날짜 완료 응답 메시지 timestamp time clock date reply",
+ showGenerationDetails: "생성 세부 정보",
+ showGenerationDetailsDescription:
+ "응답이 생성되는 동안 속도와 토큰 수를 실시간으로 표시하고, 로컬 모델의 레이어가 어디에 배치되었는지 보여줍니다.",
+ showGenerationDetailsKeywords:
+ "세부 속도 토큰 통계 레이어 오프로드 verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "응답에 응답 편집 표시",
inlineEditResponseDescription: "응답 편집을 더보기 메뉴 대신 모든 응답에 표시합니다.",
modelDisclaimer: "모델 고지 표시",
diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts
index 79c20db9b71..13f10b0881b 100644
--- a/studio/frontend/src/i18n/locales/pt-br.ts
+++ b/studio/frontend/src/i18n/locales/pt-br.ts
@@ -2774,6 +2774,11 @@ export const ptBR = {
showReplyTimesDescription: "Mostrar abaixo de cada resposta quando ela terminou.",
showReplyTimesKeywords:
"carimbo de data hora data quando terminou resposta mensagem timestamp time clock date reply",
+ showGenerationDetails: "Detalhes da geração",
+ showGenerationDetailsDescription:
+ "Mostrar ao vivo a velocidade do prompt e da resposta com a contagem de tokens, e onde as camadas de um modelo foram carregadas.",
+ showGenerationDetailsKeywords:
+ "detalhes geração velocidade tokens estatísticas camadas especialistas desenvolvedor verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Editar resposta nas respostas",
inlineEditResponseDescription: "Mostrar Editar resposta em cada resposta, em vez de no menu Mais.",
modelDisclaimer: "Mostrar aviso do modelo",
diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts
index 51261942772..71741fd1208 100644
--- a/studio/frontend/src/i18n/locales/ru.ts
+++ b/studio/frontend/src/i18n/locales/ru.ts
@@ -2772,6 +2772,11 @@ export const ru = {
showReplyTimesDescription: "Показывать под каждым ответом, когда он завершился.",
showReplyTimesKeywords:
"время метка дата когда завершено ответ сообщение timestamp time clock date reply",
+ showGenerationDetails: "Подробности генерации",
+ showGenerationDetailsDescription:
+ "Показывать скорость и число токенов в реальном времени во время ответа и где размещены слои локальной модели.",
+ showGenerationDetailsKeywords:
+ "подробности скорость токены статистика слои выгрузка verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Изменение ответа в ответах",
inlineEditResponseDescription: "Показывать «Изменить ответ» у каждого ответа, а не в меню «Ещё».",
modelDisclaimer: "Показывать предупреждение о модели",
diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.ts
index 1d89e28cac1..95a4ea693c6 100644
--- a/studio/frontend/src/i18n/locales/sv.ts
+++ b/studio/frontend/src/i18n/locales/sv.ts
@@ -2954,6 +2954,11 @@ export const sv = {
showReplyTimesDescription: "Visa under varje svar när det blev klart.",
showReplyTimesKeywords:
"tidsstämpel tid klocka datum när klar svar meddelande timestamp time clock date reply",
+ showGenerationDetails: "Genereringsdetaljer",
+ showGenerationDetailsDescription:
+ "Visa hastighet och antal token live medan ett svar strömmas, och var en lokal modells lager placerades.",
+ showGenerationDetailsKeywords:
+ "detaljer hastighet token statistik lager avlastning verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "Redigera svar på svaren",
inlineEditResponseDescription:
"Visa Redigera svar på varje svar i stället för i menyn Mer.",
diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts
index ad00ea959a9..a9db0f9d320 100644
--- a/studio/frontend/src/i18n/locales/zh-CN.ts
+++ b/studio/frontend/src/i18n/locales/zh-CN.ts
@@ -2712,6 +2712,11 @@ export const zhCN = {
showReplyTimesDescription: "在每条回复下方显示其完成的时间。",
showReplyTimesKeywords:
"时间戳 时间 时刻 日期 完成 回复 消息 timestamp time clock date reply",
+ showGenerationDetails: "生成详情",
+ showGenerationDetailsDescription:
+ "在回复生成时实时显示速度和 token 数,并显示本地模型各层的放置位置。",
+ showGenerationDetailsKeywords:
+ "详情 速度 统计 层 卸载 verbose developer stats speed tok/s prefill decode layers offload vram experts",
inlineEditResponse: "在回复上显示编辑回复",
inlineEditResponseDescription: "在每条回复上显示编辑回复,而不是放在“更多”菜单中。",
modelDisclaimer: "显示模型免责声明",
diff --git a/studio/frontend/tests/generation-details.test.ts b/studio/frontend/tests/generation-details.test.ts
new file mode 100644
index 00000000000..870d1ed25e9
--- /dev/null
+++ b/studio/frontend/tests/generation-details.test.ts
@@ -0,0 +1,279 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// Live prefill / decode figures, the Generation details setting, and the loaded placement readout.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import type { LiveDecodeProgress, LivePrefillProgress } from "../src/features/chat/stores/chat-runtime-store.ts";
+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";
+
+registerBundlerResolver();
+const {
+ formatLiveDecode,
+ formatPrefillDetails,
+ formatPrefillTimeLeft,
+ prefillLabelShown,
+ prefillPercent,
+ readGenerationProgress,
+} = await import("../src/features/chat/lib/generation-progress.ts");
+const { describePlacement, loadedPlacementFrom } = await import(
+ "../src/features/chat/lib/partial-offload.ts"
+);
+const { useChatPreferencesStore } = await import(
+ "../src/features/chat/stores/chat-preferences-store.ts"
+);
+const { normalizeChatGenerationChunkPayload } = await import(
+ "../src/features/chat/api/chat-generation-api.ts"
+);
+const { generationChunkCountsTowardTiming } = await import(
+ "../src/features/chat/utils/chat-generation-recovery.ts"
+);
+
+test("prefill frames read into store shape, clamped, and give a whole percent", () => {
+ const update = readGenerationProgress({
+ type: "generation_progress",
+ phase: "prefill",
+ processed: 6000,
+ total: 5316,
+ cached: 7000,
+ tok_per_s: 1326.4,
+ });
+ assert.equal(update?.phase, "prefill");
+ if (update?.phase !== "prefill") return;
+ assert.deepEqual(update.prefill, { processed: 5316, total: 5316, cached: 7000, tokPerS: 1326.4 });
+ assert.equal(prefillPercent({ processed: 2048, total: 5316, cached: 0, tokPerS: null }), 38);
+ assert.equal(prefillPercent({ processed: 10, total: null, cached: null, tokPerS: null }), null);
+});
+
+test("a follow-up in a long chat counts only the part the cache did not cover", () => {
+ // 29,000 of 30,000 tokens reused: halfway through the new 1,000, not 98% of the whole prompt.
+ const followUp = { processed: 29500, total: 30000, cached: 29000, tokPerS: 1000 };
+ assert.equal(prefillPercent(followUp), 50);
+ assert.equal(formatPrefillDetails(followUp, "en-US"), "500 / 1,000 tokens · 1,000 tok/s");
+ // llama.cpp evaluates at least one token, but a cache that covers it all is done, not NaN.
+ assert.equal(prefillPercent({ processed: 30000, total: 30000, cached: 30000, tokPerS: null }), 100);
+});
+
+test("the time left follows the rate so far and drops out under a second", () => {
+ assert.equal(formatPrefillTimeLeft({ processed: 4118, total: 18880, cached: 0, tokPerS: 1401.2 }), "11s left");
+ assert.equal(formatPrefillTimeLeft({ processed: 5000, total: 200000, cached: 0, tokPerS: 1500 }), "2m 10s left");
+ // The first sample's rate is noise, and at the end there is nothing worth counting down.
+ assert.equal(formatPrefillTimeLeft({ processed: 22, total: 18844, cached: 0, tokPerS: 242 }), null);
+ assert.equal(formatPrefillTimeLeft({ processed: 18500, total: 18880, cached: 0, tokPerS: 1400 }), null);
+});
+
+test("malformed frames are dropped rather than drawn as NaN", () => {
+ assert.equal(readGenerationProgress(null), null);
+ assert.equal(readGenerationProgress({ phase: "prefill", processed: "12" }), null);
+ assert.equal(readGenerationProgress({ phase: "decode", tokens: 0 }), null);
+ assert.equal(readGenerationProgress({ phase: "warmup" }), null);
+});
+
+test("decode frames keep draft acceptance only when both counts are there", () => {
+ const update = readGenerationProgress({ phase: "decode", tokens: 40, tok_per_s: 90, draft_n: 23, draft_accepted: 30 });
+ assert.equal(update?.phase, "decode");
+ if (update?.phase !== "decode") return;
+ assert.deepEqual(update.decode, { tokens: 40, tokPerS: 90, draftN: 23, draftAccepted: 23 });
+});
+
+test("the prefill details give the rate when there is one", () => {
+ assert.equal(
+ formatPrefillDetails({ processed: 4118, total: 18880, cached: 0, tokPerS: 1401.2 }, "en-US"),
+ "4,118 / 18,880 tokens · 1,401 tok/s",
+ );
+ assert.equal(formatPrefillDetails({ processed: 900, total: null, cached: null, tokPerS: null }, "en-US"), "900 tokens");
+});
+
+test("the prefill label shows only for a pass expected to keep it up, then stays for that pass", () => {
+ const sample = (processed: number, total: number, tokPerS: number | null, cached = 0) => ({
+ processed,
+ total,
+ cached,
+ tokPerS,
+ });
+ const after = (prefill: LivePrefillProgress, prefillShown: boolean, decode: LiveDecodeProgress | null = null) => ({
+ prefill,
+ prefillShown,
+ decode,
+ });
+ // llama.cpp's first sample, a few tokens in: its rate is noise, so no estimate yet.
+ assert.equal(prefillLabelShown(undefined, sample(22, 18844, 242)), false);
+ // One batch into a long prompt: about 10 s left.
+ const long = sample(4118, 18880, 1401.2);
+ assert.equal(prefillLabelShown(after(sample(22, 18844, 242), false), long), true);
+ // Stays up near the end, where the estimate alone would drop it.
+ assert.equal(prefillLabelShown(after(long, true), sample(18000, 18880, 1390)), true);
+ // A 5,000-token prompt at 2,600 tok/s has about 1 s left after its first batch, so it never shows.
+ const fast = sample(2048, 5000, 2600);
+ assert.equal(prefillLabelShown(after(sample(0, 5000, null), false), fast), false);
+ assert.equal(prefillLabelShown(after(fast, false), sample(4096, 5000, 2600)), false);
+ // A cache hit leaves too few fresh tokens for a rate.
+ assert.equal(prefillLabelShown(undefined, sample(29000, 30000, 0.5, 29000)), false);
+ // The next pass of a tool loop starts over: decode figures came in between.
+ const decode = { tokens: 40, tokPerS: 50, draftN: null, draftAccepted: null };
+ assert.equal(prefillLabelShown(after(long, true, decode), sample(2048, 5000, 2600)), false);
+});
+
+test("the live line writes every number in the reader's locale", () => {
+ const decode = { tokens: 3060, tokPerS: 58.25, draftN: 5030, draftAccepted: 1791 };
+ assert.equal(
+ formatLiveDecode(decode, "en-US"),
+ "3,060 tokens · 58.3 tok/s · draft 1,791/5,030 accepted (36%)",
+ );
+ // A locale that groups with "." must not get "." as the rate's decimal point too.
+ assert.equal(
+ formatLiveDecode(decode, "de-DE"),
+ "3.060 tokens · 58,3 tok/s · draft 1.791/5.030 accepted (36%)",
+ );
+});
+
+test("both live lines wrap instead of clipping on a narrow chat column", () => {
+ // .generating-glint is nowrap and overflow: hidden; at 390 px the prefill details and the decode line ran past the edge.
+ const thread = readSrc("components/assistant-ui/thread.tsx");
+ const prefill = thread.slice(thread.indexOf('data-slot="prefill-progress"'), thread.indexOf("Processing prompt"));
+ assert.match(prefill, /whitespace-normal!/);
+ const line = readSrc("components/assistant-ui/live-generation-line.tsx");
+ assert.doesNotMatch(line, /\bshrink-0\b/);
+ assert.match(line, /\bmin-w-0\b/);
+});
+
+test("Generation details is off by default and wired into Settings > Chat > Display", () => {
+ assert.equal(useChatPreferencesStore.getInitialState().showGenerationDetails, false);
+ const store = readSrc("features/chat/stores/chat-preferences-store.ts");
+ assert.match(store, /showGenerationDetails: saved\?\.showGenerationDetails === true/);
+ const tab = readSrc("features/settings/tabs/chat-tab.tsx");
+ const times = tab.indexOf('t("settings.chat.showReplyTimes")');
+ const details = tab.indexOf('t("settings.chat.showGenerationDetails")');
+ assert.ok(times !== -1 && details > times, "the switch sits after Reply times");
+ assert.match(readSrc("features/settings/settings-search.ts"), /"settings\.chat\.showGenerationDetailsKeywords"/);
+});
+
+test("the live line gates on the preference before any assistant-ui selector", () => {
+ const line = readSrc("components/assistant-ui/live-generation-line.tsx");
+ const gate = line.slice(line.indexOf("export const LiveGenerationLine"), line.indexOf("const RunningGate"));
+ assert.match(gate, /return enabled \? <RunningGate \/> : null;/);
+ assert.doesNotMatch(gate, /useAuiState/);
+});
+
+test("persisted runs re-tag generation_progress and keep it out of timing", () => {
+ const frame = { type: "generation_progress", phase: "prefill", processed: 1, total: 2 };
+ const normalized = normalizeChatGenerationChunkPayload(frame) as { _generationProgress?: unknown };
+ assert.deepEqual(normalized._generationProgress, frame);
+ assert.equal(generationChunkCountsTowardTiming(normalized), false);
+});
+
+// The runtime store pulls in the app shell, so like chat-model-residency.test.ts this reads its source.
+test("live progress is per thread, transient, and re-keyed with the thread's other run handles", () => {
+ const store = readSrc("features/chat/stores/chat-runtime-store.ts");
+ // A new prompt pass (the next round of a tool loop) drops the previous round's decode figures.
+ assert.match(store, /\[key\]: \{ prefill, prefillShown, decode: null \}/);
+ assert.match(store, /move\(state\.liveGenerationByThreadId, "liveGenerationByThreadId"\);/);
+ const adapter = readSrc("features/chat/api/chat-adapter.ts");
+ assert.match(
+ adapter,
+ /clearActiveDiffusionCanvasForThread\(cleanupKey\);\s*runtime\.clearLiveGenerationForThread\(cleanupKey\);/,
+ );
+ // Handled before the first-token stamp, like the other control frames.
+ const progress = adapter.indexOf("_generationProgress");
+ const firstToken = adapter.indexOf("firstTokenTime = Date.now() - streamStartTime;");
+ assert.ok(progress !== -1 && progress < firstToken);
+});
+
+test("the placement readout names expert weights and gives both memory figures", () => {
+ // gemma-4-26B-A4B Q8_0 on a 3090: --fit's verdict "31 layers ( 8 overflowing)", the first of them GATE-only.
+ assert.deepEqual(
+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 8, gpuWeightsMib: 20709.15, hostWeightsMib: 5639.4, expertCacheMib: null }),
+ { layers: "All 31 layers on the GPU, expert weights of 8 layers in RAM", memory: "20 GiB in VRAM · 5.5 GiB in RAM" },
+ );
+ assert.equal(
+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 1, gpuWeightsMib: 20000, hostWeightsMib: 900, expertCacheMib: null })
+ .layers,
+ "All 31 layers on the GPU, expert weights of 1 layer in RAM",
+ );
+ assert.deepEqual(
+ describePlacement({ offloaded: 41, total: 41, overflowLayers: 0, gpuWeightsMib: 8161.75, hostWeightsMib: 417.3, expertCacheMib: null }),
+ { layers: "All 41 layers on the GPU", memory: "8.0 GiB in VRAM · 0.4 GiB in RAM" },
+ );
+ assert.deepEqual(
+ describePlacement({ offloaded: 20, total: 31, overflowLayers: null, gpuWeightsMib: 12288, hostWeightsMib: 5120, expertCacheMib: null }),
+ { layers: "20 of 31 layers on the GPU, 11 on the CPU", memory: "12 GiB in VRAM · 5.0 GiB in RAM" },
+ );
+ // A Manual load reports its expert offload as n_cpu_moe, not through --fit.
+ assert.equal(
+ loadedPlacementFrom({ is_gguf: true, offloaded_layers: 31, offload_total_layers: 31, gpu_memory_mode: "manual", n_cpu_moe: 15 })
+ ?.overflowLayers,
+ 15,
+ );
+ assert.equal(
+ loadedPlacementFrom({ is_gguf: true, offloaded_layers: 31, offload_total_layers: 31, gpu_memory_mode: "auto", n_cpu_moe: 15 })
+ ?.overflowLayers,
+ null,
+ );
+ assert.equal(loadedPlacementFrom({ is_gguf: false, offload_total_layers: 31 }), null);
+ assert.equal(loadedPlacementFrom({ is_gguf: true }), null);
+});
+
+test("a GPU build that fell back to the CPU is not reported as GPU layers", () => {
+ // CUDA build, no usable device: llama.cpp still logs "offloaded 29/29 layers to GPU", buffers AMX + CPU_Mapped.
+ assert.deepEqual(
+ describePlacement({ offloaded: 29, total: 29, overflowLayers: null, gpuWeightsMib: 0, hostWeightsMib: 499.8, expertCacheMib: null }),
+ { layers: "All 29 layers on the CPU", memory: "0.5 GiB in RAM" },
+ );
+});
+
+test("--moe-cache-mib auto reads as every expert in RAM with a GPU cache", () => {
+ // --fit names no layer on this path; the backend reports the model's MoE layer count instead.
+ const placement = loadedPlacementFrom({
+ is_gguf: true,
+ offloaded_layers: 31,
+ offload_total_layers: 31,
+ overflow_layers: 30,
+ gpu_weights_mib: 3900,
+ host_weights_mib: 22400,
+ expert_cache_mib: 6144,
+ });
+ assert.ok(placement);
+ assert.deepEqual(describePlacement(placement), {
+ layers: "All 31 layers on the GPU, all experts in RAM (6.0 GiB expert cache on the GPU)",
+ memory: "3.8 GiB in VRAM · 22 GiB in RAM",
+ });
+});
+
+test("one memory pool gets one figure, and amounts under 0.1 GiB read in MiB", () => {
+ // Qwen3.5-4B Q4_K_M under Metal on an M1 Mac mini: MTL0 2,603.5 MiB and CPU 497.3 MiB, one pool.
+ const metal = { offloaded: 33, total: 33, overflowLayers: null, gpuWeightsMib: 2603.5, hostWeightsMib: 497.31, expertCacheMib: null };
+ assert.deepEqual(describePlacement(metal, "unified memory"), {
+ layers: "All 33 layers on the GPU",
+ memory: "3.0 GiB in unified memory",
+ });
+ assert.equal(describePlacement(metal).memory, "2.5 GiB in VRAM · 0.5 GiB in RAM");
+ // A small model's token embeddings would otherwise print as "0.0 GiB in RAM".
+ assert.equal(
+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 0, gpuWeightsMib: 112.6, hostWeightsMib: 28.7, expertCacheMib: null }).memory,
+ "0.1 GiB in VRAM · 29 MiB in RAM",
+ );
+});
+
+test("the prefill label gives way once decoding started, not at 100%", () => {
+ const thread = readSrc("components/assistant-ui/thread.tsx");
+ const status = thread.slice(thread.indexOf("const GeneratingStatus"), thread.indexOf("// Placeholder when stop fires"));
+ assert.match(status, /!labelShown \|\| percent === null \|\| decoding\)/);
+});
+
+test("a followed run (reload, second tab) shows live figures and clears them on exit", () => {
+ const provider = readSrc("features/chat/runtime-provider.tsx");
+ assert.match(provider, /if \("_generationProgress" in chunk\) \{\s*if \(caughtUp\(update\.run\)\)/);
+ assert.match(provider, /\} finally \{\s*useChatRuntimeStore\.getState\(\)\.clearLiveGenerationForThread\(threadId\);/);
+});
+
+test("a background load cannot drop the visible model's loaded placement", () => {
+ // setCheckpoint clears loadedPlacement when the visible model is restored, so the snapshot must carry it back.
+ const adapter = readSrc("features/chat/api/chat-adapter.ts");
+ const keys = adapter.slice(
+ adapter.indexOf("const VISIBLE_MODEL_RUNTIME_KEYS = ["),
+ adapter.indexOf("] as const satisfies"),
+ );
+ assert.match(keys, /"loadedPlacement"/);
+});