Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: show prompt progress, live generation details and where a GGUF loaded (#14909)
oobaboogae139d53d2fb2
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 63c2ebd01ce..7f804da5f6b 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -1049,6 +1049,165 @@ _GPU_DEVICE_PREFIXES = ()+# Any backend's "model buffer size" line (CUDA0, CPU_Mapped, RPC0[host:port], ...).+_MODEL_BUFFER_RE = re.compile(+ r"\b([A-Za-z][^\s=]*)\s+model buffer size\s*=\s*([0-9]+(?:\.[0-9]+)?)\s+MiB\b"+)+# --fit's verdict per device, e.g. "- CUDA0 (NVIDIA GeForce RTX 3090): 31 layers ( 8 overflowing), 22815 MiB used".+# An overflowing layer keeps its MoE experts (or part of a layer) out of that device's memory.+_FIT_DEVICE_RE = re.compile(+ r"common_params_fit_impl:\s+-\s+(?P<device>[A-Za-z]+\d+)\s+\(.*?\):\s+(?P<layers>\d+)\s+layers?"+ r"\s*(?:\(\s*(?P<overflow>\d+)\s+overflowing\s*\))?\s*,\s*\d+\s+MiB used"+)+# The MoE-only fill: printed for a model with experts, whose GPU layers then hold only their dense weights.+_FIT_DENSE_ONLY_MARK = "filling dense-only layers back-to-front"+# From here the per-device lines carry an explicit "(N overflowing)" count.+_FIT_FULL_LAYERS_MARK = "converting dense-only layers to full layers"+# One extra partial layer squeezed onto device i; on any device but the last it overflows to the NEXT GPU, not RAM.+_FIT_EXTRA_LAYER_RE = re.compile(+ r"set ngl_per_device\[(?P<index>\d+)\]\.\(n_layer, n_part, overflow_type\)=\(\d+, \d+, (?:UP|GATE|ATTN)\)"+)+# --moe-cache-mib auto: every routed expert stays in host RAM and a cache of the busiest ones goes on the GPU. --fit+# returns before any per-device line on this path.+_FIT_MOE_CACHE_RE = re.compile(+ r"moe cache auto: all routed experts in host RAM, cache (?P<mib>\d+) MiB"+)+++def _host_buffer(name: str) -> bool:+ """CPU* buffers, AMX and a GPU backend's pinned host buffer (CUDA_Host) live in system memory."""+ upper = name.upper()+ return "CPU" in upper or upper.endswith("_HOST") or upper == "AMX"+++def parse_weight_placement(+ lines: "list[str]", model_bytes: Optional[int] = None+) -> "Optional[dict]":+ """Where llama.cpp put the main model's weights, from its startup log.++ ``gpu_mib`` / ``host_mib`` sum the main model's buffer block, the first allocated one (dry runs log 0.00 MiB).+ "offloaded 31/31" hides an expert spill, so ``overflow_layers`` comes from --fit's final per-device+ lines; the last device's count includes the output layer, which spills nothing (a 3090 gemma-4-26B+ load printing "8 overflowing" holds exactly seven layers' worth in CUDA_Host). None without --fit.+ A ``*_Mapped`` buffer logs its mmap span, first to last host tensor, which covers GPU tensors in+ between when experts spill; ``model_bytes`` (the main GGUF set) caps host at the file minus the GPU side."""+ blocks: list[list] = []+ current: Optional[list] = None+ # A build without GPU offload logs no "offloaded N/M layers" line, only the buffers: each run of them is a block.+ unmarked = False+ moe = False+ full_phase = False+ # device -> (layers, overflow or None for a dense-only line), in the order of the final verdict+ verdict: dict[str, tuple[int, Optional[int]]] = {}+ dense_only_order: list[str] = []+ extra_layer_on: set[int] = set()+ expert_cache_mib: Optional[float] = None+ for line in lines:+ if _FIT_DENSE_ONLY_MARK in line:+ moe = True+ continue+ if _FIT_FULL_LAYERS_MARK in line:+ full_phase = True+ verdict = {}+ continue+ cache = _FIT_MOE_CACHE_RE.search(line)+ if cache is not None:+ expert_cache_mib = float(cache.group("mib"))+ continue+ extra = _FIT_EXTRA_LAYER_RE.search(line)+ if extra is not None and full_phase:+ extra_layer_on.add(int(extra.group("index")))+ continue+ fit = _FIT_DEVICE_RE.search(line)+ if fit is not None:+ device = fit.group("device")+ overflow = fit.group("overflow")+ verdict[device] = (+ int(fit.group("layers")),+ int(overflow) if overflow is not None else None,+ )+ if not full_phase and device not in dense_only_order:+ dense_only_order.append(device)+ continue+ counts = _OFFLOADED_LAYERS_RE.search(line)+ if counts is not None:+ current, unmarked = [int(counts.group(2)), 0.0, 0.0, False], False+ blocks.append(current)+ continue+ buffer = _MODEL_BUFFER_RE.search(line)+ if buffer is None:+ if unmarked:+ current, unmarked = None, False+ continue+ if current is None:+ current, unmarked = [0, 0.0, 0.0, False], True+ blocks.append(current)+ mib = float(buffer.group(2))+ if _host_buffer(buffer.group(1)):+ current[2] += mib+ current[3] = current[3] or (mib > 0 and buffer.group(1).upper().endswith("_MAPPED"))+ else:+ current[1] += mib+ # A no-alloc dry run logs the same block with every buffer at 0.00 MiB.+ loaded = [block for block in blocks if block[1] + block[2] > 0]+ if not loaded:+ return None+ # llama-server loads the main model before any drafter, which can have MORE layers (Gemma 4 E2B: 35 vs 26B A4B: 30).+ main = loaded[0]+ host_mib = main[2]+ if main[3] and main[1] > 0 and model_bytes:+ host_mib = min(host_mib, max(0.0, model_bytes / (1024 * 1024) - main[1]))++ overflow: Optional[int] = None+ if verdict:+ devices = list(verdict)+ overflow = 0+ for index, device in enumerate(devices):+ layers, counted = verdict[device]+ if counted is None:+ # Dense-only fill (filled back to front, so the first device it printed is the last one). A dense+ # model's layers are full. The last device keeps one layer full and its output slot spills nothing.+ if not moe:+ continue+ last = bool(dense_only_order) and device == dense_only_order[0]+ counted = max(0, layers - 2) if last else layers+ elif index < len(devices) - 1:+ if index in extra_layer_on and counted > 0:+ counted -= 1+ elif counted > 0:+ counted -= 1+ overflow += counted+ return {+ "gpu_mib": round(main[1], 1),+ "host_mib": round(host_mib, 1),+ "overflow_layers": overflow,+ "all_experts_in_ram": expert_cache_mib is not None,+ "expert_cache_mib": expert_cache_mib,+ }+++def _extra_weight_file_bytes(argv, env: Optional[Mapping[str, str]] = None) -> int:+ """GGUF sets of the projector, local drafter and adapters a llama-server launch loads. arg.cpp reads+ the ``LLAMA_ARG_*`` env first and lets argv override it, so ``env`` is the one the child got."""+ tokens = [str(token) for token in argv] if isinstance(argv, (list, tuple)) else []+ env_map = env or {}+ projector: Optional[str] = None+ for index, token in enumerate(tokens):+ if _flag_name(token) not in ("--mmproj", "-mm"):+ continue+ _, eq, value = token.partition("=")+ if eq:+ projector = value+ elif index + 1 < len(tokens):+ projector = tokens[index + 1]+ if projector is None:+ projector = (env_map.get("LLAMA_ARG_MMPROJ") or "").strip() or None+ drafter, remote = _extra_args_mtp_draft_source(tokens, env_map)+ paths = [path for path in (projector, None if remote else drafter) if path]+ paths += _sidecar_adapter_paths(tokens)+ return sum(_gguf_set_bytes(Path(path)) for path in dict.fromkeys(paths))++def parse_gpu_offload_counts(lines: "list[str]") -> "Optional[tuple[int, int]]":"""``(offloaded, total)`` for the largest-M line, or None; ties keep the firstline since llama.cpp logs the main model before its drafter."""@@ -2758,6 +2917,39 @@ _SHARD_FULL_RE = re.compile(r"^(.*)-(\d{5})-of-(\d{5})\.gguf$", re.IGNORECASE)_SHARD_RE = re.compile(r"^(.*)-\d{5}-of-\d{5}\.gguf$", re.IGNORECASE)+def _gguf_set_bytes(path: Path) -> int:+ """Size of a GGUF plus, when it is a split one, its sibling shards: llama-server opens the whole set."""+ total = 0+ try:+ if path.is_file():+ total += path.stat().st_size+ m = _SHARD_FULL_RE.match(path.name)+ if m is None or not path.parent.is_dir():+ return total+ # llama.cpp names a shard "<prefix>-<i>-of-<count>.gguf" from the first one's, so both must match: a+ # drafter split as "<main prefix>-MTP-..." or a stale set with another count is not this set.+ prefix, count = m.group(1).lower(), int(m.group(3))+ for sibling in path.parent.iterdir():+ if sibling.name == path.name:+ continue+ sibling_match = _SHARD_FULL_RE.match(sibling.name)+ if (+ sibling_match is None+ or sibling_match.group(1).lower() != prefix+ or int(sibling_match.group(3)) != count+ or not 1 <= int(sibling_match.group(2)) <= count+ ):+ continue+ try:+ if sibling.is_file():+ total += sibling.stat().st_size+ except OSError:+ pass+ except OSError:+ pass+ return total++# ── Sliding-window-pattern resolver ───────────────────────────# Resolves the per-layer SWA mask when a GGUF reports a sliding window but# no `sliding_window_pattern` field. Tier order in `_resolve_swa_pattern`:@@ -8585,6 +8777,8 @@ class LlamaCppBackend:self._gpu_offload_active: Optional[bool] = None# (offloaded, total) layers llama.cpp reported for this load, when it said.self._gpu_offload_layers: Optional[tuple[int, int]] = None+ # parse_weight_placement's result for this load.+ self._weight_placement: Optional[dict] = None# Whether the user's own extras pinned the layer split for this load.self._offload_overridden: bool = False# Studio saw a GPU for this load but llama.cpp's device table did not.@@ -9112,6 +9306,29 @@ class LlamaCppBackend:def offload_total_layers(self) -> Optional[int]:return self._gpu_offload_layers[1] if self._gpu_offload_layers else None+ def _placement_value(self, key: str):+ placement = getattr(self, "_weight_placement", None)+ return placement.get(key) if placement else None++ @property+ def gpu_weights_mib(self) -> Optional[float]:+ return self._placement_value("gpu_mib")++ @property+ def host_weights_mib(self) -> Optional[float]:+ return self._placement_value("host_mib")++ @property+ def overflow_layers(self) -> Optional[int]:+ """Under --moe-cache-mib auto every routed expert is in RAM, so this is every MoE layer."""+ if self._placement_value("all_experts_in_ram"):+ return self.n_moe_layers or None+ return self._placement_value("overflow_layers")++ @property+ def expert_cache_mib(self) -> Optional[float]:+ return self._placement_value("expert_cache_mib")+@propertydef gpu_backend_unavailable(self) -> bool:"""Studio found a GPU for this load but llama.cpp reported none."""@@ -9228,38 +9445,104 @@ class LlamaCppBackend:@staticmethoddef _read_rss_bytes(pid: int) -> Optional[int]:- """Resident set size of ``pid`` in bytes, from /proc/<pid>/status (Linux).- 0 when the status has no VmRSS line (zombie / kernel thread); None where- /proc is unavailable (macOS/Windows) or the value is unreadable."""+ """``RssFile`` of ``pid`` (Linux), ``VmRSS`` before kernel 4.5; 0 when neither, None without /proc.+ Not ``RssAnon``: CUDA pins the host weight buffer (``CUDA_Host``) before a byte of it is read."""+ vm_rss = Nonetry:with open(f"/proc/{pid}/status", "r", encoding = "utf-8") as f:for line in f:- if line.startswith("VmRSS:"):- # IndexError guards a "VmRSS:" line with no value column.+ # IndexError guards a line with no value column.+ if line.startswith("RssFile:"):return int(line.split()[1]) * 1024 # kB -> bytes+ if line.startswith("VmRSS:"):+ vm_rss = int(line.split()[1]) * 1024except (FileNotFoundError, PermissionError, ValueError, IndexError, OSError):return None- return 0 # readable but no VmRSS line+ return 0 if vm_rss is None else vm_rss++ @staticmethod+ def _read_load_bytes(pid: int) -> Optional[int]:+ """Bytes ``pid`` has taken in, for a load that streams weights to VRAM and keeps little RSS.++ Linux: ``rchar`` (page cache hits included). Windows: psutil read_bytes. macOS: the larger of+ disk bytes read (a cold mmap Metal load) and physical footprint (a copy into Metal or CPU+ buffers)."""+ if os.path.isdir("/proc"):+ try:+ with open(f"/proc/{pid}/io", "r", encoding = "utf-8") as f:+ for line in f:+ if line.startswith("rchar:"):+ return int(line.split()[1])+ except (FileNotFoundError, PermissionError, ValueError, IndexError, OSError):+ return None+ return None+ if sys.platform == "darwin":+ usage = LlamaCppBackend._darwin_rusage(pid)+ return max(usage) if usage else None+ if sys.platform != "win32":+ return None+ try:+ import psutil+ return int(psutil.Process(pid).io_counters().read_bytes)+ except Exception:+ return None++ @staticmethod+ def _darwin_rusage(pid: int) -> Optional[tuple]:+ """``(ri_diskio_bytesread, ri_phys_footprint)`` of ``pid`` from proc_pid_rusage, or None."""+ import ctypes++ class _RusageInfoV2(ctypes.Structure):+ _fields_ = [("ri_uuid", ctypes.c_uint8 * 16)] + [+ (name, ctypes.c_uint64)+ for name in (+ "ri_user_time",+ "ri_system_time",+ "ri_pkg_idle_wkups",+ "ri_interrupt_wkups",+ "ri_pageins",+ "ri_wired_size",+ "ri_resident_size",+ "ri_phys_footprint",+ "ri_proc_start_abstime",+ "ri_proc_exit_abstime",+ "ri_child_user_time",+ "ri_child_system_time",+ "ri_child_pkg_idle_wkups",+ "ri_child_interrupt_wkups",+ "ri_child_pageins",+ "ri_child_elapsed_abstime",+ "ri_diskio_bytesread",+ "ri_diskio_byteswritten",+ )+ ]++ try:+ info = _RusageInfoV2()+ # RUSAGE_INFO_V2 = 2+ if (+ ctypes.CDLL("/usr/lib/libproc.dylib").proc_pid_rusage(pid, 2, ctypes.byref(info))+ != 0+ ):+ return None+ return int(info.ri_diskio_bytesread), int(info.ri_phys_footprint)+ except (OSError, AttributeError, ValueError):+ return Nonedef load_progress(self) -> Optional[dict]:"""Return live model-load progress, or None if not loading.- During warm-up llama-server mmaps weight shards into page cache before- pushing layers to VRAM, a window where status only reports ``loading``- and the UI spinner looks stuck for minutes on large MoEs. Samples- ``/proc/<pid>/status VmRSS`` against the sum of GGUF shard sizes for a- real progress bar. Returns ``None`` when no load is in flight.+ ``bytes_loaded`` is the larger of the bytes read and Linux file-backed RSS; never a working set,+ which counts buffers allocated before any read. The total includes projector, drafter and adapters.Shape::{"phase": "mmap" | "ready",- "bytes_loaded": int, # VmRSS of the llama-server- "bytes_total": int, # sum of shard file sizes+ "bytes_loaded": int, # as above, never decreasing+ "bytes_total": int, # weight files the server loads"fraction": float, # bytes_loaded / bytes_total, 0..1}-- Linux-only; returns ``None`` where ``/proc/<pid>/status`` is unavailable."""proc = self._processif proc is None:@@ -9268,42 +9551,19 @@ class LlamaCppBackend:if pid is None:return None- # Sum shard sizes (primary + any extras alongside).- bytes_total = 0gguf_path = self._gguf_path- if gguf_path:- primary = Path(gguf_path)- try:- if primary.is_file():- bytes_total += primary.stat().st_size- except OSError:- pass- # Extra shards share the primary's prefix before the shard index.- try:- parent = primary.parent- stem = primary.name- m = _SHARD_RE.match(stem)- prefix = m.group(1) if m else None- if prefix and parent.is_dir():- prefix_lower = prefix.lower()- for sibling in parent.iterdir():- if (- sibling.is_file()- and sibling.name.lower().startswith(prefix_lower)- and sibling.name != stem- and sibling.suffix.lower() == ".gguf"- ):- try:- bytes_total += sibling.stat().st_size- except OSError:- pass- except OSError:- pass+ bytes_total = _gguf_set_bytes(Path(gguf_path)) if gguf_path else 0+ launched = getattr(self, "_process_env", None)+ bytes_total += _extra_weight_file_bytes(+ getattr(proc, "args", None),+ launched[1] if launched is not None and launched[0] is proc else None,+ )- # VmRSS of the llama-server; None where /proc is unavailable.- bytes_loaded = LlamaCppBackend._read_rss_bytes(pid)- if bytes_loaded is None:+ rss = LlamaCppBackend._read_rss_bytes(pid)+ read = LlamaCppBackend._read_load_bytes(pid)+ if rss is None and read is None:return None+ bytes_loaded = max(rss or 0, read or 0)# RSS climbs as weights page in, then drops once -ngl offloads them to# VRAM and the mmap pages are freed. Hold a per-process high-water mark@@ -9316,6 +9576,8 @@ class LlamaCppBackend:phase = "ready" if self._healthy else "mmap"fraction = 0.0if bytes_total > 0:+ # Reads past the weights (headers, a second pass) must not push the figure past the total.+ bytes_loaded = min(bytes_loaded, bytes_total)fraction = min(1.0, bytes_loaded / bytes_total)# Once llama-server is healthy the load is complete by definition. With# layers offloaded to VRAM (-ngl) the process releases the mmap'd weight@@ -25791,6 +26053,8 @@ class LlamaCppBackend:))self._process = _spawned+ # The env picks companions too (LLAMA_ARG_MMPROJ, ...): load_progress sizes them from it.+ self._process_env = (_spawned, env)_invalidate_gpu_memory("llama-server spawned")# Cross-session backstop for when parent-death cleanup did not run.# Under the lock: see _spawn_and_wait.@@ -33548,6 +33812,7 @@ class LlamaCppBackend:))self._process = _spawned+ self._process_env = (_spawned, env)_invalidate_gpu_memory("llama-server spawned")# Inside the lock: written after a sweep reaped the child,# _pid_start_identity yields no start time, and the bare pid@@ -35560,6 +35825,10 @@ class LlamaCppBackend:):_offload_counts = Noneself._gpu_offload_layers = _offload_counts+ self._weight_placement = parse_weight_placement(+ self._stdout_lines,+ _gguf_set_bytes(Path(self._gguf_path)) if self._gguf_path else None,+ )self._gpu_backend_unavailable = bool(_detected_gpus or _metal_capable_host()) and (llama_saw_gpu_device(self._stdout_lines) is False)@@ -37141,6 +37410,7 @@ class LlamaCppBackend:# server's stale zero-offload flag until the health probe reclassifies.self._gpu_offload_active = Noneself._gpu_offload_layers = None+ self._weight_placement = Noneself._offload_overridden = Falseself._gpu_backend_unavailable = False# Drives _wait_for_vram_settle in the next load_model; set in finallydiff --git a/studio/backend/core/inference/sse_control_frames.py b/studio/backend/core/inference/sse_control_frames.pyindex 7777083896d..5cf9a6f7408 100644--- a/studio/backend/core/inference/sse_control_frames.py+++ b/studio/backend/core/inference/sse_control_frames.py@@ -44,6 +44,7 @@ _CONTROL_TYPES = frozenset("diffusion_frame","reasoning_summary","context_usage",+ "generation_progress",})@@ -54,6 +55,7 @@ _CONTROL_KEYS = ("_diffusionFrame","_reasoningDurationMs","_contextUsage",+ "_generationProgress","_mcp_provenance","quote_cut",)diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 7e725aef751..fdf7b443a81 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -1938,6 +1938,33 @@ class _InferenceRuntimeFields(BaseModel):None,description = "Total layers in the model, alongside offloaded_layers.",)+ overflow_layers: Optional[int] = Field(+ None,+ description = (+ "Layers with MoE expert weights that llama.cpp's --fit left in system memory: their "+ "routed experts, or part of the FFN for the first partial layer on a device. "+ "offloaded_layers still counts them as on the GPU, so a spill shows only here. Every "+ "MoE layer when expert_cache_mib is set. None when --fit did not run."+ ),+ )+ expert_cache_mib: Optional[float] = Field(+ None,+ description = (+ "--moe-cache-mib auto: every routed expert is in system memory and this many MiB of "+ "the most used ones are cached on the GPU. None when no MoE cache is in use."+ ),+ )+ gpu_weights_mib: Optional[float] = Field(+ None,+ description = "MiB of model weights llama.cpp placed in GPU buffers, from its load log.",+ )+ host_weights_mib: Optional[float] = Field(+ None,+ description = (+ "MiB of model weights in system memory (CPU, memory-mapped or pinned host buffers). "+ "Token embeddings stay here even when every layer is on the GPU."+ ),+ )gpu_backend_unavailable: bool = Field(False,description = (diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex aa38294dc07..013593afad3 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2004,6 +2004,10 @@ _TEARDOWN_TASK_RECANCEL_S = 0.01# (e.g. prompt prefill between tool iterations). A second layer atop the# tool_stream_exec heartbeats, keeping proxies (Cloudflare drops idle at ~100s)._LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S = 15.0+# How often a UI stream looks for new prefill progress while the generator is silent.+_LIVE_PROGRESS_POLL_S = 0.25+# Minimum gap between two live decode frames: the client only redraws one line of text.+_LIVE_DECODE_FRAME_INTERVAL_S = 1.0_APPROVAL_CACHE_RECLAIM_POLL_S = 0.25@@ -7697,22 +7701,62 @@ def _monitor_perf_callback(monitor_id: Optional[str], context_length):return _callback+def _live_count(value: Any) -> Optional[int]:+ if isinstance(value, bool):+ return None+ try:+ number = float(value)+ except (TypeError, ValueError):+ return None+ if not math.isfinite(number) or number < 0:+ return None+ return int(number)+++def _live_rate(value: Any) -> Optional[float]:+ if isinstance(value, bool):+ return None+ try:+ number = float(value)+ except (TypeError, ValueError):+ return None+ if not math.isfinite(number) or number <= 0:+ return None+ return round(number, 1)++class _PrefillProgressSignal:"""Counts advancing ``prompt_progress`` so the stall loop can send ``: prefill-progress``.- Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease."""+ Native GGUF generators yield nothing until the first token, and ``: keep-alive`` does not renew the lease.++ With ``live`` it also feeds ``live_frame``, the chat UI's ``generation_progress`` frames."""- def __init__(self, inner):+ def __init__(+ self,+ inner,+ live: bool = False,+ ):self._inner = innerself._last_processed = Noneself._advances = 0self._seen = 0- self.wants_timings = inner is not None+ self._live = live+ # Written from the generator thread, read on the event loop: each is replaced whole, never mutated.+ self._prefill: Optional[dict] = None+ self._prefill_sent: Optional[dict] = None+ self._decode: Optional[dict] = None+ self._decode_sent_at = 0.0+ self.wants_timings = inner is not None or live@propertydef needs_phase(self) -> bool:return self._inner is not None and getattr(self._inner, "needs_phase", True)+ @property+ def live(self) -> bool:+ return self._live+def __call__(self, sample: dict) -> None:progress = sample.get("prompt_progress")if isinstance(progress, dict):@@ -7721,9 +7765,66 @@ class _PrefillProgressSignal:if processed is not None and processed != self._last_processed:self._last_processed = processedself._advances += 1+ if self._live:+ self._record_prefill(progress, sample.get("prompt_per_second"))+ elif self._live:+ self._record_decode(sample)if self._inner is not None:self._inner(sample)+ def _record_prefill(self, progress: dict, rate: Any) -> None:+ total = _live_count(progress.get("total"))+ processed = _live_count(progress.get("processed"))+ if processed is None:+ return+ if total:+ processed = min(processed, total)+ cached = _live_count(progress.get("cache"))+ # A new prompt pass (the next round of a tool loop): the last round's decode figures must not follow it.+ self._decode = None+ self._prefill = {+ "type": "generation_progress",+ "phase": "prefill",+ "processed": processed,+ "total": total,+ "cached": min(cached, processed) if cached is not None else None,+ "tok_per_s": _live_rate(rate),+ }++ def _record_decode(self, sample: dict) -> None:+ tokens = _live_count(sample.get("predicted_n"))+ if not tokens:+ return+ frame = {+ "type": "generation_progress",+ "phase": "decode",+ "tokens": tokens,+ "tok_per_s": _live_rate(sample.get("predicted_per_second")),+ }+ draft_n = _live_count(sample.get("draft_n"))+ draft_accepted = _live_count(sample.get("draft_n_accepted"))+ if draft_n and draft_accepted is not None:+ frame["draft_n"] = draft_n+ frame["draft_accepted"] = min(draft_accepted, draft_n)+ self._decode = frame++ def live_frame(self) -> Optional[str]:+ """The next ``generation_progress`` line: the newest unsent prefill sample, else decode figures at most+ once per ``_LIVE_DECODE_FRAME_INTERVAL_S``. None when nothing is due."""+ if not self._live:+ return None+ prefill = self._prefill+ if prefill is not None and prefill is not self._prefill_sent:+ self._prefill_sent = prefill+ return f"data: {json.dumps(prefill, separators = (',', ':'))}\n\n"+ decode = self._decode+ now = time.monotonic()+ if decode is not None and now - self._decode_sent_at >= _LIVE_DECODE_FRAME_INTERVAL_S:+ self._decode = None+ self._decode_sent_at = now+ return f"data: {json.dumps(decode, separators = (',', ':'))}\n\n"+ return None+def advanced(self) -> bool:advances = self._advancesif advances == self._seen:@@ -7732,6 +7833,34 @@ class _PrefillProgressSignal:return True+async def _stall_wait_frames(next_task, signal: _PrefillProgressSignal, first_timeout: float):+ """Wait for a GGUF generator step, yielding keep-alives on stall and, when live, progress frames.+ Never cancels ``next_task``: the caller's finally drains it."""+ timeout = first_timeout+ waited = 0.0+ while True:+ poll = min(timeout - waited, _LIVE_PROGRESS_POLL_S) if signal.live else timeout+ done_tasks, _ = await asyncio.wait({next_task}, timeout = poll)+ if done_tasks:+ # Steps faster than the poll get their pending prefill sample, then decode figures, sent here.+ for _ in range(2):+ frame = signal.live_frame()+ if frame is not None:+ yield frame+ return+ frame = signal.live_frame()+ if frame is not None:+ yield frame+ waited += poll+ if waited < timeout:+ continue+ yield (+ _OPENAI_PREFILL_PROGRESS_SSE if signal.advanced() else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE+ )+ waited = 0.0+ timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S++def _monitor_call_text(name: Any, arguments: Any = None) -> str:call_name = str(name or "tool")if arguments is None or arguments == "":@@ -34175,7 +34304,10 @@ async def produce_openai_chat_completions(else None)# Only a stream has a stall loop to feed; elsewhere the monitor callback (or None) stands.- _gguf_prefill_signal = _PrefillProgressSignal(_gguf_monitor_callback)+ _gguf_prefill_signal = _PrefillProgressSignal(+ _gguf_monitor_callback,+ live = bool(payload.stream and _ui_events and not _wants_multiple_choices(payload)),+ )_gguf_perf_callback = _gguf_prefill_signal if payload.stream else _gguf_monitor_callbackdef _gguf_chat_delta_line(delta: ChoiceDelta, finish_reason = None) -> str:@@ -34704,20 +34836,10 @@ async def produce_openai_chat_completions(if approval_flush_pendingelse _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S)- while True:- done_tasks, _ = await asyncio.wait(- {next_task},- timeout = wait_timeout,- )- if done_tasks:- break- yield (- _OPENAI_PREFILL_PROGRESS_SSE- if _gguf_prefill_signal.advanced()- else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE- )- approval_flush_pending = False- wait_timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S+ async for stall_line in _stall_wait_frames(+ next_task, _gguf_prefill_signal, wait_timeout+ ):+ yield stall_lineevent = next_task.result()approval_flush_pending = Falsefinally:@@ -35429,18 +35551,12 @@ async def produce_openai_chat_completions(# Stall-timeout wait: keepalive while the generator stays# silent (e.g. no-tool prefill). asyncio.wait never cancels# next_task, matching the finally-drain shield (see GGUF stream).- while True:- done_tasks, _ = await asyncio.wait(- {next_task},- timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S,- )- if done_tasks:- break- yield (- _OPENAI_PREFILL_PROGRESS_SSE- if _gguf_prefill_signal.advanced()- else _OPENAI_PASSTHROUGH_SSE_KEEPALIVE- )+ async for stall_line in _stall_wait_frames(+ next_task,+ _gguf_prefill_signal,+ _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S,+ ):+ yield stall_linecumulative = next_task.result()finally:if next_task.done():diff --git a/studio/backend/tests/test_live_generation_progress.py b/studio/backend/tests/test_live_generation_progress.pynew file mode 100644index 00000000000..f1d31a8ac5b--- /dev/null+++ b/studio/backend/tests/test_live_generation_progress.py@@ -0,0 +1,207 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Live prefill / decode frames for the Unsloth chat UI (``generation_progress``)."""++from __future__ import annotations++import asyncio+import json+import sys+from pathlib import Path++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference.sse_control_frames import ( # noqa: E402+ is_ui_control_sse_line,+ sanitize_provider_sse_line,+)+from routes import inference as route # noqa: E402+++def _frame(line: str) -> dict:+ assert line.startswith("data: ") and line.endswith("\n\n")+ return json.loads(line[len("data: ") :])+++def test_prefill_samples_become_frames_once_each():+ signal = route._PrefillProgressSignal(None, live = True)+ assert signal.wants_timings, "a live signal must ask llama-server for per-token timings"+ assert signal.live_frame() is None++ signal(+ {+ "prompt_progress": {"total": 5316, "processed": 2048, "cache": 0, "time_ms": 1500},+ "prompt_per_second": 1365.33,+ }+ )+ frame = _frame(signal.live_frame())+ assert frame == {+ "type": "generation_progress",+ "phase": "prefill",+ "processed": 2048,+ "total": 5316,+ "cached": 0,+ "tok_per_s": 1365.3,+ }+ # Sent once; the stall loop polls far more often than llama-server reports a batch.+ assert signal.live_frame() is None+++def test_prefill_counts_are_clamped_and_bad_rates_dropped():+ signal = route._PrefillProgressSignal(None, live = True)+ signal(+ {+ "prompt_progress": {"total": 100, "processed": 250, "cache": 400},+ "prompt_per_second": float("inf"),+ }+ )+ frame = _frame(signal.live_frame())+ assert frame["processed"] == 100+ assert frame["cached"] == 100+ assert frame["tok_per_s"] is None+++def test_decode_frames_are_throttled(monkeypatch):+ clock = {"now": 1000.0}+ monkeypatch.setattr(route.time, "monotonic", lambda: clock["now"])+ signal = route._PrefillProgressSignal(None, live = True)++ signal({"predicted_n": 10, "predicted_per_second": 48.7})+ assert _frame(signal.live_frame()) == {+ "type": "generation_progress",+ "phase": "decode",+ "tokens": 10,+ "tok_per_s": 48.7,+ }+ signal({"predicted_n": 11, "predicted_per_second": 49.0})+ assert signal.live_frame() is None, "a second decode frame inside the interval"+ clock["now"] += route._LIVE_DECODE_FRAME_INTERVAL_S+ assert _frame(signal.live_frame())["tokens"] == 11+ assert signal.live_frame() is None, "the same sample twice"+++def test_a_new_prompt_pass_drops_the_last_rounds_decode_figures():+ signal = route._PrefillProgressSignal(None, live = True)+ signal({"predicted_n": 50, "predicted_per_second": 40.0})+ signal({"prompt_progress": {"total": 3000, "processed": 2048}})+ assert _frame(signal.live_frame())["phase"] == "prefill"+ assert signal.live_frame() is None, "round 1's decode frame must not follow round 2's prefill"+++def test_decode_frame_carries_draft_acceptance():+ signal = route._PrefillProgressSignal(None, live = True)+ signal({"predicted_n": 40, "predicted_per_second": 90.0, "draft_n": 23, "draft_n_accepted": 14})+ frame = _frame(signal.live_frame())+ assert (frame["draft_n"], frame["draft_accepted"]) == (23, 14)+++def test_not_live_keeps_the_old_behaviour():+ calls = []++ def inner(sample):+ calls.append(sample)++ signal = route._PrefillProgressSignal(inner)+ signal({"prompt_progress": {"total": 10, "processed": 5}})+ signal({"predicted_n": 3, "predicted_per_second": 20.0})+ assert signal.live_frame() is None+ assert signal.advanced() is True+ assert len(calls) == 2, "the monitor callback still sees every sample"+ assert route._PrefillProgressSignal(None).wants_timings is False+++def test_stall_wait_sends_progress_while_the_generator_is_silent(monkeypatch):+ monkeypatch.setattr(route, "_LIVE_PROGRESS_POLL_S", 0.01)+ signal = route._PrefillProgressSignal(None, live = True)++ async def scenario():+ release = asyncio.Event()++ async def step():+ await release.wait()+ return "token"++ task = asyncio.create_task(step())+ lines = []++ async def collect():+ async for line in route._stall_wait_frames(task, signal, first_timeout = 15.0):+ lines.append(line)++ collector = asyncio.create_task(collect())+ await asyncio.sleep(0.03)+ signal({"prompt_progress": {"total": 4000, "processed": 2048}})+ await asyncio.sleep(0.05)+ release.set()+ await collector+ return lines, task.result()++ lines, result = asyncio.run(scenario())+ assert result == "token"+ assert [_frame(line)["processed"] for line in lines] == [2048]+++def test_a_step_faster_than_the_poll_still_sends_what_it_recorded():+ # Fast tokens (or tool-call markup stripped to no text) never reach the poll.+ signal = route._PrefillProgressSignal(None, live = True)++ async def scenario():+ async def step():+ signal({"prompt_progress": {"total": 4000, "processed": 4000}})+ signal({"predicted_n": 3, "predicted_per_second": 20.0})+ return "token"++ task = asyncio.create_task(step())+ await task+ return [line async for line in route._stall_wait_frames(task, signal, first_timeout = 15.0)]++ lines = asyncio.run(scenario())+ assert [_frame(line)["phase"] for line in lines] == ["prefill", "decode"]+ assert _frame(lines[0])["processed"] == 4000 and _frame(lines[1])["tokens"] == 3+++def test_stall_wait_still_sends_the_comment_after_the_stall_interval(monkeypatch):+ monkeypatch.setattr(route, "_LIVE_PROGRESS_POLL_S", 0.01)++ async def scenario(live: bool):+ signal = route._PrefillProgressSignal(None, live = live)+ signal({"prompt_progress": {"total": 4000, "processed": 1}})+ signal.live_frame()+ release = asyncio.Event()++ async def step():+ await release.wait()++ task = asyncio.create_task(step())+ lines = []++ async def collect():+ async for line in route._stall_wait_frames(task, signal, first_timeout = 0.05):+ lines.append(line)++ collector = asyncio.create_task(collect())+ await asyncio.sleep(0.12)+ release.set()+ await collector+ return lines++ for live in (False, True):+ lines = asyncio.run(scenario(live))+ assert lines and lines[0] == route._OPENAI_PREFILL_PROGRESS_SSE, (live, lines)+ assert all(not line.startswith("data:") for line in lines)+++ "line",+ [+ 'data: {"type":"generation_progress","phase":"prefill","processed":1,"total":2}',+ ],+)+def test_providers_cannot_forge_the_frame(line):+ assert sanitize_provider_sse_line(line) is None+ assert is_ui_control_sse_line(line)diff --git a/studio/backend/tests/test_llama_cpp_load_progress_live.py b/studio/backend/tests/test_llama_cpp_load_progress_live.pyindex dff8334314c..21f4b4abfca 100644--- a/studio/backend/tests/test_llama_cpp_load_progress_live.py+++ b/studio/backend/tests/test_llama_cpp_load_progress_live.py@@ -71,14 +71,19 @@ def _make_backend(return inst-def test_live_rss_matches_kernel_vmrss(tmp_path):- """Spawn a real child, let it allocate real bytes, confirm ``bytes_loaded``- tracks the kernel's VmRSS within a sane tolerance."""+def test_live_mapped_pages_count_and_allocations_do_not(tmp_path):+ """Spawn a real child that pages in a mapped file and fills an allocation: ``bytes_loaded``+ follows the mapped pages, not the allocation (a pinned upload buffer is one)."""+ data = tmp_path / "weights.bin"+ data.write_bytes(b"\1" * (100 * 1024 * 1024))script = tmp_path / "burn.py"script.write_text(- "import time, sys\n"- "buf = bytearray(100 * 1024 * 1024)\n"- "# touch every page so RSS actually grows\n"+ "import mmap, sys, time\n"+ "f = open(sys.argv[1], 'rb')\n"+ "m = mmap.mmap(f.fileno(), 0, access = mmap.ACCESS_READ)\n"+ "# touch every page of the mapping and of the allocation so both are resident\n"+ "seen = sum(m[i] for i in range(0, len(m), 4096))\n"+ "buf = bytearray(200 * 1024 * 1024)\n""for i in range(0, len(buf), 4096):\n"" buf[i] = 1\n""sys.stdout.write('ready\\n')\n"@@ -86,7 +91,7 @@ def test_live_rss_matches_kernel_vmrss(tmp_path):"time.sleep(10)\n")proc = subprocess.Popen(- [sys.executable, str(script)],+ [sys.executable, str(script), str(data)],stdout = subprocess.PIPE,stderr = subprocess.PIPE,)@@ -96,20 +101,22 @@ def test_live_rss_matches_kernel_vmrss(tmp_path):gguf = tmp_path / "model.gguf"with open(gguf, "wb") as f:- f.truncate(200 * 1024 * 1024)+ f.truncate(400 * 1024 * 1024)inst = _make_backend(proc.pid, str(gguf), healthy = False)out = inst.load_progress()assert out is not None, "load_progress returned None for live pid"assert out["phase"] == "mmap"- assert out["bytes_total"] == 200 * 1024 * 1024- # VmRSS includes the interpreter plus the 100MB buffer.+ assert out["bytes_total"] == 400 * 1024 * 1024+ # The 100 MB mapping plus the interpreter's own files; the 200 MB allocation is left out.assert (- out["bytes_loaded"] >= 50 * 1024 * 1024+ out["bytes_loaded"] >= 90 * 1024 * 1024), f"bytes_loaded unexpectedly low: {out['bytes_loaded']}"- assert out["bytes_loaded"] <= 200 * 1024 * 1024- assert 0.0 < out["fraction"] <= 1.0+ assert (+ out["bytes_loaded"] < 200 * 1024 * 1024+ ), f"the allocation counted: {out['bytes_loaded']}"+ assert 0.0 < out["fraction"] < 0.5finally:proc.terminate()try:diff --git a/studio/backend/tests/test_llama_weight_placement.py b/studio/backend/tests/test_llama_weight_placement.pynew file mode 100644index 00000000000..a7439ad51a3--- /dev/null+++ b/studio/backend/tests/test_llama_weight_placement.py@@ -0,0 +1,406 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Where llama.cpp put a model's weights, and load progress that sees weights headed for VRAM."""++from __future__ import annotations++import io+import os+import subprocess+import sys+from pathlib import Path+from unittest.mock import patch++import pytest++_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)+if _BACKEND_DIR not in sys.path:+ sys.path.insert(0, _BACKEND_DIR)++from core.inference.llama_cpp import ( # noqa: E402+ LlamaCppBackend,+ _extra_weight_file_bytes,+ _gguf_set_bytes,+ parse_weight_placement,+)++# Gemma 4 26B A4B Q8_0 on a 24 GB RTX 3090 under --fit on (trimmed from a real llama-server log).+_MOE_SPILL_LOG = """\+0.00.997.437 I common_params_fit_impl: filling dense-only layers back-to-front:+0.01.282.617 I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 3090): 31 layers, 5550 MiB used, 18347 MiB free+0.02.623.039 I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 3090): 31 layers ( 8 overflowing), 22815 MiB used, 1082 MiB free+0.02.623.041 I common_fit_params: successfully fit params to free device memory+0.03.897.106 I load_tensors: offloading output layer to GPU+0.03.897.109 I load_tensors: offloading 29 repeating layers to GPU+0.03.897.109 I load_tensors: offloaded 31/31 layers to GPU+0.03.897.113 I load_tensors: CUDA0 model buffer size = 20709.15 MiB+0.03.897.113 I load_tensors: CUDA_Host model buffer size = 5639.40 MiB+0.23.263.171 I llama_kv_cache: CUDA0 KV buffer size = 160.00 MiB+""".splitlines()+++def test_moe_spill_is_read_from_fit_not_from_the_layer_count():+ assert parse_weight_placement(_MOE_SPILL_LOG) == {+ "gpu_mib": 20709.2,+ "host_mib": 5639.4,+ "overflow_layers": 7,+ "all_experts_in_ram": False,+ "expert_cache_mib": None,+ }+++def test_dense_model_fully_on_gpu():+ lines = [+ "0.00.365.476 I load_tensors: offloaded 41/41 layers to GPU",+ "0.00.365.484 I load_tensors: CUDA0 model buffer size = 8161.75 MiB",+ "0.00.365.484 I load_tensors: CUDA_Host model buffer size = 417.30 MiB",+ ]+ assert parse_weight_placement(lines) == {+ "gpu_mib": 8161.8,+ "host_mib": 417.3,+ "overflow_layers": None,+ "all_experts_in_ram": False,+ "expert_cache_mib": None,+ }+++def test_dry_run_and_drafter_blocks_do_not_count():+ lines = [+ # --fit's no-alloc pass logs the same block with nothing allocated.+ "I load_tensors: offloaded 31/31 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 0.00 MiB",+ "I load_tensors: CUDA_Host model buffer size = 0.00 MiB",+ "I load_tensors: offloaded 31/31 layers to GPU",+ "I load_tensors: CPU_Mapped model buffer size = 13986.81 MiB",+ "I load_tensors: CUDA0 model buffer size = 13258.54 MiB",+ # A drafter logs its own, smaller block after the main model.+ "I load_tensors: offloaded 3/3 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 600.00 MiB",+ ]+ # A drafter can also be DEEPER than its target (Gemma 4 E2B, 35 layers, for 26B A4B, 30): still the main model.+ deeper = lines + [+ "I load_tensors: offloaded 36/36 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 2000.00 MiB",+ ]+ assert parse_weight_placement(deeper)["gpu_mib"] == 13258.5+ assert parse_weight_placement(lines) == {+ "gpu_mib": 13258.5,+ "host_mib": 13986.8,+ "overflow_layers": None,+ "all_experts_in_ram": False,+ "expert_cache_mib": None,+ }+++def test_multi_gpu_extra_layer_overflows_to_the_next_gpu_not_ram():+ lines = [+ "I common_params_fit_impl: filling dense-only layers back-to-front:",+ "I common_params_fit_impl: converting dense-only layers to full layers and filling them front-to-back with overflow to next device/system memory:",+ # Device 0 took one extra partial layer; it overflows into CUDA1, not into RAM.+ "I common_params_fit_impl: set ngl_per_device[0].(n_layer, n_part, overflow_type)=(20, 1, GATE), id_dense_start=1",+ "I common_params_fit_impl: - CUDA0 (A): 20 layers ( 1 overflowing), 23000 MiB used, 900 MiB free",+ "I common_params_fit_impl: - CUDA1 (B): 11 layers ( 4 overflowing), 15000 MiB used, 800 MiB free",+ "I load_tensors: offloaded 31/31 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 20000.00 MiB",+ "I load_tensors: CUDA1 model buffer size = 12000.00 MiB",+ "I load_tensors: CPU model buffer size = 4000.00 MiB",+ ]+ out = parse_weight_placement(lines)+ # CUDA1's 4 partial slots end at the output layer, which spills nothing.+ assert (out["gpu_mib"], out["host_mib"], out["overflow_layers"]) == (32000.0, 4000.0, 3)+++def test_moe_fit_that_stops_after_the_dense_only_fill_has_every_gpu_layer_spilled():+ """The dense-only lines carry no count, but every GPU layer there holds dense weights only."""+ lines = [+ "I common_params_fit_impl: filling dense-only layers back-to-front:",+ "I common_params_fit_impl: - CUDA0 (NVIDIA GeForce RTX 5060 Ti): 60 layers, 15000 MiB used, 900 MiB free",+ "I load_tensors: offloaded 60/95 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 14000.00 MiB",+ "I load_tensors: CUDA_Host model buffer size = 120000.00 MiB",+ ]+ # One device, so it is also the last: its output layer carries no experts.+ # 60 slots, 59 partial; the last partial slot is the output layer and the first layer stays full.+ assert parse_weight_placement(lines)["overflow_layers"] == 58+ dense = [line.replace("dense-only layers", "dense layers") for line in lines]+ assert parse_weight_placement(dense)["overflow_layers"] == 0, "a dense model's layers are full"+++def test_moe_cache_auto_puts_every_expert_in_ram():+ lines = [+ "I common_params_fit_impl: moe cache auto: all routed experts in host RAM, cache 6144 MiB (margin 1024 MiB)",+ "I common_fit_params: successfully fit params to free device memory",+ "I load_tensors: offloaded 31/31 layers to GPU",+ "I load_tensors: CUDA0 model buffer size = 3900.00 MiB",+ "I load_tensors: CUDA_Host model buffer size = 22400.00 MiB",+ ]+ out = parse_weight_placement(lines)+ assert out["all_experts_in_ram"] is True+ assert out["expert_cache_mib"] == 6144.0+ assert (+ out["overflow_layers"] is None+ ), "no per-device line on this path; the backend supplies the MoE layer count"+++def test_cpu_repack_buffers_count_as_ram_and_remote_buffers_parse():+ lines = [+ "I load_tensors: offloaded 0/49 layers to GPU",+ "I load_tensors: AMX model buffer size = 9000.00 MiB",+ "I load_tensors: CPU_Mapped model buffer size = 500.00 MiB",+ "I load_tensors: RPC0[192.168.1.2:50052] model buffer size = 1000.00 MiB",+ ]+ out = parse_weight_placement(lines)+ assert (out["gpu_mib"], out["host_mib"]) == (1000.0, 9500.0)+++def test_no_buffer_lines_means_nothing_to_report():+ assert parse_weight_placement(["I srv load_model: loading model"]) is None+++def test_a_cpu_only_build_reports_its_ram_placement():+ # llama.cpp prints "offloaded N/M layers" only when the build supports GPU offload.+ lines = [+ "load_tensors: CPU_Mapped model buffer size = 21000.0 MiB",+ "load_tensors: CPU model buffer size = 0.6 MiB",+ "llama_context: n_ctx = 4096",+ # A drafter's buffers after other lines are its own block, not the main model's.+ "load_tensors: CPU_Mapped model buffer size = 500.0 MiB",+ ]+ placement = parse_weight_placement(lines)+ assert placement is not None+ assert (placement["gpu_mib"], placement["host_mib"]) == (0.0, 21000.6)+ assert placement["overflow_layers"] is None+++def test_projector_and_drafter_on_the_command_line_count_toward_the_load(tmp_path):+ projector = tmp_path / "mmproj-F16.gguf"+ projector.write_bytes(b"x" * 1000)+ drafter = tmp_path / "draft.gguf"+ drafter.write_bytes(b"x" * 300)+ argv = [+ "llama-server",+ "-m",+ "model.gguf",+ "--mmproj",+ str(projector),+ "-md",+ str(drafter),+ "--port",+ "1",+ ]+ assert _extra_weight_file_bytes(argv) == 1300+ assert _extra_weight_file_bytes(None) == 0+ assert (+ _extra_weight_file_bytes(["llama-server", "--mmproj", str(tmp_path / "missing.gguf")]) == 0+ )+++def test_a_split_drafter_counts_every_shard(tmp_path):+ # llama-server opens a split drafter's siblings from the named first shard.+ for index in (1, 2, 3):+ (tmp_path / f"Model-Q4_K_M-MTP-{index:05d}-of-00003.gguf").write_bytes(b"x" * 100)+ # The main model's set in the same folder, whose prefix the drafter's starts with, is not the drafter's.+ for index in (1, 2):+ (tmp_path / f"Model-Q4_K_M-{index:05d}-of-00002.gguf").write_bytes(b"x" * 7)+ # Nor is a stale set under the same prefix with another shard count.+ for index in (1, 2, 3, 4):+ (tmp_path / f"Model-Q4_K_M-MTP-{index:05d}-of-00004.gguf").write_bytes(b"x" * 1000)+ argv = ["llama-server", "-md", str(tmp_path / "Model-Q4_K_M-MTP-00001-of-00003.gguf")]+ assert _extra_weight_file_bytes(argv) == 300+ assert _gguf_set_bytes(tmp_path / "Model-Q4_K_M-00001-of-00002.gguf") == 14+++def test_companions_picked_by_the_launch_env_count_too(tmp_path):+ projector = tmp_path / "mmproj-F16.gguf"+ projector.write_bytes(b"x" * 1000)+ drafter = tmp_path / "draft.gguf"+ drafter.write_bytes(b"x" * 300)+ argv = ["llama-server", "-m", "model.gguf"]+ env = {"LLAMA_ARG_MMPROJ": str(projector), "LLAMA_ARG_SPEC_DRAFT_MODEL": str(drafter)}+ assert _extra_weight_file_bytes(argv, env) == 1300+ # Pre-b8955 drafter spelling.+ assert _extra_weight_file_bytes(argv, {"LLAMA_ARG_MODEL_DRAFT": str(drafter)}) == 300+ # argv overrides the env, as in arg.cpp; a Hugging Face repo is not a local file.+ other = tmp_path / "mmproj-BF16.gguf"+ other.write_bytes(b"x" * 50)+ assert _extra_weight_file_bytes(argv + ["--mmproj", str(other)], env) == 350+ assert _extra_weight_file_bytes(argv, {"LLAMA_ARG_SPEC_DRAFT_HF_REPO": "org/draft"}) == 0+++def test_adapters_on_the_command_line_count_toward_the_load(tmp_path, monkeypatch):+ lora = tmp_path / "lora.gguf"+ lora.write_bytes(b"x" * 400)+ (tmp_path / "vector.gguf").write_bytes(b"x" * 20)+ # Relative: upstream splits FNAME:SCALE on every ':', so a drive-lettered path is its own error.+ monkeypatch.chdir(tmp_path)+ argv = ["llama-server", "-m", "model.gguf", "--lora", str(lora)]+ assert _extra_weight_file_bytes(argv + ["--control-vector-scaled", "vector.gguf:0.5"]) == 420+++def test_load_progress_sizes_companions_from_the_env_its_process_got(tmp_path, monkeypatch):+ inst = _fake_load(tmp_path, 1000)+ projector = tmp_path / "mmproj-F16.gguf"+ projector.write_bytes(b"x" * 1000)+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: 500))+ inst._process_env = (inst._process, {"LLAMA_ARG_MMPROJ": str(projector)})+ assert inst.load_progress()["bytes_total"] == 2000+ # An env recorded for an earlier process does not describe this one.+ inst._process_env = (object(), {"LLAMA_ARG_MMPROJ": str(projector)})+ assert inst.load_progress()["bytes_total"] == 1000++[email protected](not Path("/proc").exists(), reason = "live /proc test is Linux-only")+def test_load_progress_follows_bytes_read_when_rss_stays_flat(tmp_path):+ """A server that streams its weights to VRAM reads every byte but keeps little resident:+ progress must follow the reads (it froze at the RSS share before)."""+ weights = tmp_path / "model.gguf"+ weights.write_bytes(os.urandom(64 * 1024 * 1024))+ script = tmp_path / "reader.py"+ script.write_text(+ "import sys, time\n"+ "with open(sys.argv[1], 'rb') as f:\n"+ " while f.read(1 << 20):\n"+ " pass\n"+ "print('read', flush=True)\n"+ "time.sleep(10)\n"+ )+ proc = subprocess.Popen([sys.executable, str(script), str(weights)], stdout = subprocess.PIPE)+ try:+ assert proc.stdout.readline().strip() == b"read"+ inst = LlamaCppBackend.__new__(LlamaCppBackend)+ inst._process = proc+ inst._gguf_path = str(weights)+ inst._healthy = False+ rss = LlamaCppBackend._read_rss_bytes(proc.pid)+ assert rss is not None and rss < 64 * 1024 * 1024, "the reader never holds the file"+ out = inst.load_progress()+ assert out is not None+ assert out["bytes_total"] == 64 * 1024 * 1024+ assert out["fraction"] == 1.0, out+ finally:+ proc.kill()+ proc.wait(timeout = 5)+++def _fake_load(tmp_path, size):+ weights = tmp_path / "model.gguf"+ weights.write_bytes(b"\0" * size)+ inst = LlamaCppBackend.__new__(LlamaCppBackend)+ inst._process = type("P", (), {"pid": 4242, "args": []})()+ inst._gguf_path = str(weights)+ inst._healthy = False+ return inst+++def test_windows_load_progress_follows_bytes_read_alone(tmp_path, monkeypatch):+ """A Windows working set takes in the buffers llama.cpp allocates before reading a byte (DevLab:+ 9.1 of 9.0 GB a second into the load, reads still at 0.01 GB), so only the read counter counts."""+ inst = _fake_load(tmp_path, 1000)+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: 250))+ assert inst.load_progress()["fraction"] == 0.25+++def test_linux_rss_term_leaves_out_the_pinned_weight_buffer():+ """With --load-mode none, CUDA pins the host buffer for the weights kept in RAM before a byte is+ read (3090: VmRSS 6.39 GB, 5.99 GB of it RssShmem, 0.16 GB read), which put the bar at 23% and held+ it there until the reads passed it. Only the file-backed share is what a mapped load pages in."""+ status = "VmRSS:\t6390000 kB\nRssAnon:\t250000 kB\nRssFile:\t150000 kB\nRssShmem:\t5990000 kB\n"+ with patch("builtins.open", side_effect = lambda *a, **kw: io.StringIO(status)):+ assert LlamaCppBackend._read_rss_bytes(4242) == 150000 * 1024+ # A kernel without the split (before 4.5) still gives VmRSS.+ with patch("builtins.open", side_effect = lambda *a, **kw: io.StringIO("VmRSS:\t2048 kB\n")):+ assert LlamaCppBackend._read_rss_bytes(4242) == 2048 * 1024+++def test_no_load_signal_reports_no_figure(tmp_path, monkeypatch):+ inst = _fake_load(tmp_path, 1000)+ monkeypatch.setattr(LlamaCppBackend, "_read_rss_bytes", staticmethod(lambda pid: None))+ monkeypatch.setattr(LlamaCppBackend, "_read_load_bytes", staticmethod(lambda pid: None))+ assert inst.load_progress() is None+++def test_each_os_reads_its_own_load_counter(monkeypatch):+ """Windows: psutil read bytes. macOS: the larger of disk bytes read (a cold memory-mapped Metal+ load) and physical footprint (a copy into Metal or CPU buffers); RSS stays at ~20% of a Metal load."""+ import core.inference.llama_cpp as llama_cpp++ calls = []++ class _Process:+ def __init__(self, pid):+ calls.append(pid)++ def io_counters(self):+ return type("C", (), {"read_bytes": 123})()++ monkeypatch.setitem(sys.modules, "psutil", type("M", (), {"Process": _Process}))+ monkeypatch.setattr(llama_cpp.os.path, "isdir", lambda path: False)+ monkeypatch.setattr(LlamaCppBackend, "_darwin_rusage", staticmethod(lambda pid: (700, 300)))+ monkeypatch.setattr(llama_cpp.sys, "platform", "win32")+ assert LlamaCppBackend._read_load_bytes(7) == 123+ monkeypatch.setattr(llama_cpp.sys, "platform", "darwin")+ assert LlamaCppBackend._read_load_bytes(7) == 700+ monkeypatch.setattr(LlamaCppBackend, "_darwin_rusage", staticmethod(lambda pid: None))+ assert LlamaCppBackend._read_load_bytes(7) is None+ monkeypatch.setattr(llama_cpp.sys, "platform", "freebsd14")+ assert LlamaCppBackend._read_load_bytes(7) is None+ assert calls == [7]+++def test_load_progress_follows_a_real_process_on_this_os(tmp_path):+ """No mocks: a child reads a weight file into memory, and load_progress must see all of it.+ Linux rchar and Windows read_bytes count the read; on macOS a page-cached file reads nothing from+ disk, so the physical footprint of the held bytes carries it. Before, macOS and Windows got None."""+ if not (os.path.isdir("/proc") or sys.platform in ("darwin", "win32")):+ pytest.skip("no load counter on this platform")+ size = 64 * 1024 * 1024+ weights = tmp_path / "model.gguf"+ weights.write_bytes(os.urandom(size))+ child = subprocess.Popen(+ [+ sys.executable,+ "-c",+ "import sys, time\n"+ "data = open(sys.argv[1], 'rb').read()\n"+ "print(len(data), flush = True)\n"+ "time.sleep(60)\n",+ str(weights),+ ],+ stdout = subprocess.PIPE,+ text = True,+ )+ try:+ assert int(child.stdout.readline()) == size+ backend = LlamaCppBackend.__new__(LlamaCppBackend)+ backend._process = type("P", (), {"pid": child.pid, "args": None})()+ backend._gguf_path = str(weights)+ backend._healthy = False+ progress = backend.load_progress()+ assert progress is not None+ assert progress["phase"] == "mmap"+ assert progress["bytes_total"] == size+ assert progress["fraction"] >= 0.99+ finally:+ child.kill()+ child.wait(10)+++def test_an_mmap_span_over_gpu_tensors_is_capped_at_the_file_minus_the_gpu_side():+ """With experts spilled under mmap, CPU_Mapped logs the span from the first to the last host tensor,+ which covers the GPU tensors in between: 25 GiB "in RAM" for a 25.3 GiB file with 21 GiB on the GPU."""+ lines = [+ "load_tensors: offloaded 49/49 layers to GPU",+ "load_tensors: CUDA0 model buffer size = 21504.00 MiB",+ "load_tensors: CPU_Mapped model buffer size = 25600.00 MiB",+ ]+ model_bytes = 26419 * 1024 * 1024+ out = parse_weight_placement(lines, model_bytes)+ assert out["gpu_mib"] == 21504.0+ assert out["host_mib"] == 26419 - 21504.0+ # Without the file size, and for a buffer that is not a mapping, the logged size stands.+ assert parse_weight_placement(lines)["host_mib"] == 25600.0+ host = [line.replace("CPU_Mapped", "CUDA_Host") for line in lines]+ assert parse_weight_placement(host, model_bytes)["host_mib"] == 25600.0diff --git a/studio/backend/tests/test_provider_control_frame_spoofing.py b/studio/backend/tests/test_provider_control_frame_spoofing.pyindex 235790b79ef..61e4c3d14e1 100644--- a/studio/backend/tests/test_provider_control_frame_spoofing.py+++ b/studio/backend/tests/test_provider_control_frame_spoofing.py@@ -29,6 +29,7 @@ from core.inference.sse_control_frames import sanitize_provider_sse_line"diffusion_frame","reasoning_summary","context_usage",+ "generation_progress",],)def test_every_control_type_is_dropped(frame_type):@@ -45,6 +46,7 @@ def test_every_control_type_is_dropped(frame_type):"_diffusionFrame","_reasoningDurationMs","_contextUsage",+ "_generationProgress","_mcp_provenance","quote_cut",],diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex dcff9a82f87..a310131adeb 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -416,8 +416,12 @@ export const BUDGET = {// in the entry and settings chunks naming the example box, now a lazy chunk shared by the API page and Settings// (6,932,829 -> 6,932,967 B raw, 2,088,616 -> 2,088,669 B transfer against cfc593f0bd, Node 26, 99 chunks).// Per-account chat preferences (#15025): +338 B raw / +93 B transfer over the line above (Node 22.23.2, 99 chunks).- transferBytes: 2_088_800,- rawBytes: 6_933_450,+ // Live prefill progress, the Generation details setting and the loaded placement readout (#14582): the+ // stream frame reader, live store slice, reply footer line and timing tooltip rows in the eager chat+ // chunk, +8,585 B raw / +2,268 B transfer over main (6,932,829 -> 6,941,414 B raw, 2,088,586 ->+ // 2,090,854 B transfer against 1492952a09, Node 22, 99 chunks). Raised by that on top of #15084 and #15025.+ transferBytes: 2_091_068,+ rawBytes: 6_942_035,};// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering thediff --git a/studio/frontend/src/components/assistant-ui/live-generation-line.tsx b/studio/frontend/src/components/assistant-ui/live-generation-line.tsxnew file mode 100644index 00000000000..094a3e24ecb--- /dev/null+++ b/studio/frontend/src/components/assistant-ui/live-generation-line.tsx@@ -0,0 +1,44 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import {+ formatLiveDecode,+ useChatPreferencesStore,+ useChatRuntimeStore,+} from "@/features/chat";+import { useLocale } from "@/i18n";+import { useAuiState } from "@assistant-ui/react";+import type { FC } from "react";++/** Gated on the preference first so a reply mounts no assistant-ui selector while it is off+ * (composer-keystroke-subscription-budget), like AssistantMessageTime. */+export const LiveGenerationLine: FC = () => {+ const enabled = useChatPreferencesStore((state) => state.showGenerationDetails);+ return enabled ? <RunningGate /> : null;+};++const RunningGate: FC = () => {+ const running = useAuiState(({ message }) => message.status?.type === "running");+ return running ? <LiveDecodeFigures /> : null;+};++const LiveDecodeFigures: FC = () => {+ const locale = useLocale();+ // Keyed like the prefill indicator: a first turn reads "__default" until the thread persists.+ const threadKey =+ useAuiState(({ threadListItem }) => threadListItem.remoteId) ?? "__default";+ const decode = useChatRuntimeStore(+ (s) => s.liveGenerationByThreadId[threadKey]?.decode ?? null,+ );+ if (!decode) {+ return null;+ }+ return (+ <span+ data-slot="live-generation-line"+ className="aui-live-generation-line ml-[var(--icon-btn-inset)] mr-2 min-w-0 select-none font-mono text-ui-11p5 text-muted-foreground tabular-nums"+ >+ {formatLiveDecode(decode, locale)}+ </span>+ );+};diff --git a/studio/frontend/src/components/assistant-ui/message-timing.tsx b/studio/frontend/src/components/assistant-ui/message-timing.tsxindex 307a4d93d81..caffeb1c1ee 100644--- a/studio/frontend/src/components/assistant-ui/message-timing.tsx+++ b/studio/frontend/src/components/assistant-ui/message-timing.tsx@@ -19,6 +19,27 @@ const formatNumber = (n: number): string => {return n.toLocaleString();};+/** A user stop wins over the stream's last finish_reason. */+function stopReasonLabel(+ finishReason: unknown,+ incompleteReason?: unknown,+): string | null {+ if (incompleteReason === "cancelled") return "Stopped by you";+ switch (finishReason) {+ case "stop":+ return "End of turn";+ case "length":+ return "Length limit";+ case "tool_calls":+ case "function_call":+ return "Tool call";+ case "content_filter":+ return "Content filter";+ default:+ return typeof finishReason === "string" && finishReason ? finishReason : null;+ }+}+const formatRate = (r: number | undefined): string => {if (r === undefined || !Number.isFinite(r)) return "—";return `${Math.round(r).toLocaleString()} tok/s`;@@ -48,9 +69,18 @@ export const MessageTiming: FC<{cachedTokens?: number;cacheWriteTokens?: number;};+ stopReason?: unknown;+ incomplete?: { reason?: unknown };}| undefined;const st = custom?.serverTimings;+ const stopReason = stopReasonLabel(custom?.stopReason, custom?.incomplete?.reason);+ const draftN = st?.draft_n ?? 0;+ const draftAccepted = st?.draft_n_accepted;+ const draftText =+ draftN > 0 && typeof draftAccepted === "number" && Number.isFinite(draftAccepted)+ ? `${formatNumber(draftAccepted)} / ${formatNumber(draftN)} (${Math.round((draftAccepted / draftN) * 100)}%)`+ : null;// `??` (not `||`) so an explicit cache_n=0 isn't replaced by a stale// contextUsage.cachedTokens from a prior turn.const cacheHits =@@ -239,6 +269,12 @@ export const MessageTiming: FC<{</span></div>)}+ {draftText !== null && (+ <div className="flex items-center justify-between gap-4">+ <span className="text-muted-foreground">Draft accepted</span>+ <span className="font-mono tabular-nums">{draftText}</span>+ </div>+ )}{timing.firstTokenTime !== undefined && (<div className="flex items-center justify-between gap-4"><span className="text-muted-foreground">First token</span>@@ -292,6 +328,12 @@ export const MessageTiming: FC<{{timing.totalChunks}</span></div>+ {stopReason !== null && (+ <div className="flex items-center justify-between gap-4">+ <span className="text-muted-foreground">Stop reason</span>+ <span>{stopReason}</span>+ </div>+ )}</>)) : (@@ -333,6 +375,12 @@ export const MessageTiming: FC<{{timing.totalChunks}</span></div>+ {stopReason !== null && (+ <div className="flex items-center justify-between gap-4">+ <span className="text-muted-foreground">Stop reason</span>+ <span>{stopReason}</span>+ </div>+ )}</>)}</div>diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsxindex 49956beb174..4bc91960c21 100644--- a/studio/frontend/src/components/assistant-ui/thread.tsx+++ b/studio/frontend/src/components/assistant-ui/thread.tsx@@ -53,6 +53,7 @@ import { QueueResumeIcon } from "@/components/assistant-ui/queue-resume-icon";import { ProgressiveMessages } from "@/components/assistant-ui/progressive-messages";import { MessageMenuTime } from "@/components/assistant-ui/message-menu-time";import { AssistantMessageTime } from "@/components/assistant-ui/assistant-message-time";+import { LiveGenerationLine } from "@/components/assistant-ui/live-generation-line";import { UserMessageActionBar, UserMessageFooter } from "@/components/assistant-ui/user-message-actions";import { useActionBarFocusReveal } from "@/components/assistant-ui/use-action-bar-focus-reveal";import {@@ -213,6 +214,7 @@ import {providerModelSupportsStudioTools,} from "@/features/chat/external-providers";import { toolStatusKind } from "@/features/chat/utils/tool-status";+import { formatPrefillDetails, formatPrefillTimeLeft, prefillPercent } from "@/features/chat";import { replySourceMarkdown } from "@/features/chat/utils/reply-source-markdown";import { toolResultModelText } from "@/features/chat/api/chat-adapter";import {@@ -260,7 +262,7 @@ import {resolveBindings,} from "@/features/settings";import { FIND_SKIP_ATTRIBUTE } from "@/features/find-in-page";-import { translate, useT } from "@/i18n";+import { translate, useLocale, useT } from "@/i18n";import type { TranslationKey } from "@/i18n";import {clampReasoningEffortToLevels,@@ -7888,9 +7890,44 @@ const GeneratingIndicator: FC = () => {if (!show) {return null;}+ return <GeneratingStatus />;+};++// Mounted only under the empty running reply, so the thread's other messages subscribe to nothing new.+const GeneratingStatus: FC = () => {+ // Keyed like DiffusionCanvas: a first turn reads "__default" until the thread persists.+ const threadKey =+ useAuiState(({ threadListItem }) => threadListItem.remoteId) ?? "__default";+ const prefill = useChatRuntimeStore(+ (s) => s.liveGenerationByThreadId[threadKey]?.prefill ?? null,+ );+ const labelShown = useChatRuntimeStore(+ (s) => s.liveGenerationByThreadId[threadKey]?.prefillShown === true,+ );+ // A reply can decode with nothing to show yet (a tool call being written): past the prompt, back to Working.+ const decoding = useChatRuntimeStore(+ (s) => s.liveGenerationByThreadId[threadKey]?.decode != null,+ );+ const showDetails = useChatPreferencesStore((s) => s.showGenerationDetails);+ const locale = useLocale();+ const percent = prefill ? prefillPercent(prefill) : null;+ // Held at 100% until the first token: going back to Working... for that moment would flash too.+ if (prefill === null || !labelShown || percent === null || decoding) {+ return (+ <span className="generating-glint text-sm text-muted-foreground">+ Working...+ </span>+ );+ }+ const timeLeft = formatPrefillTimeLeft(prefill);return (- <span className="generating-glint text-sm text-muted-foreground">- Working...+ <span+ data-slot="prefill-progress"+ className="generating-glint text-sm text-muted-foreground tabular-nums whitespace-normal!"+ >+ Processing prompt {percent}%+ {timeLeft ? ` · ${timeLeft}` : null}+ {showDetails ? ` · ${formatPrefillDetails(prefill, locale)}` : null}</span>);};@@ -8550,6 +8587,8 @@ const AssistantMessage: FC = () => {<div className="aui-assistant-message-footer mt-1.5 -ml-[var(--icon-btn-inset)] flex min-h-8 min-w-0 flex-wrap items-center gap-y-1">{/* First, so the action bar mounting on hover never moves it. */}<AssistantMessageTime />+ {/* Only while streaming, where the action bar is hidden; the timing badge takes over after. */}+ <LiveGenerationLine /><BranchPicker className="mr-0.5" /><AssistantActionBar /></div>diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex d889771d1f1..1807e3ab74c 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -2,6 +2,7 @@// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0import { attachedMediaUnavailableReason } from "../lib/attached-media-gate";+import { readGenerationProgress } from "../lib/generation-progress";import { externalModelLabel } from "../lib/external-model-label";import { mlxRuntimeStateFrom } from "../lib/mlx-runtime-state";import { minPSamplingPayload } from "../lib/min-p-policy";@@ -2792,6 +2793,7 @@ const VISIBLE_MODEL_RUNTIME_KEYS = ["speculativeType","loadedSpeculativeType","specFallbackReason",+ "loadedPlacement","specDraftNMax","loadedSpecDraftNMax","specDraftModel",@@ -5884,6 +5886,7 @@ export function createOpenAIStreamAdapter(let responseModelId = externalSelection?.modelId ?? params.checkpoint;let firstTokenTime: number | undefined;let totalChunks = 0;+ let lastFinishReason: string | null = null;let resolveFirstToken: (() => void) | null = null;let rejectFirstToken: ((err: unknown) => void) | null = null;const firstTokenPromise = new Promise<void>((resolve, reject) => {@@ -7502,6 +7505,20 @@ export function createOpenAIStreamAdapter(continue;}+ // Transient like the diffusion canvas: never transcript text, never a first token.+ const generationProgress = (+ chunk as unknown as { _generationProgress?: unknown }+ )._generationProgress;+ if (generationProgress !== undefined) {+ const update = readGenerationProgress(generationProgress);+ if (update?.phase === "prefill") {+ runtime.setLivePrefill(liveThreadKey(serverCancel), update.prefill);+ } else if (update?.phase === "decode") {+ runtime.setLiveDecode(liveThreadKey(serverCancel), update.decode);+ }+ continue;+ }+// Diffusion frame: a transient canvas snapshot routed to the transient store and skipped; it// has no assistant text, so it never enters the transcript.const diffusionFrame = (@@ -8092,6 +8109,10 @@ export function createOpenAIStreamAdapter(} else if (chunk.choices?.[0]?.finish_reason) {incompleteReason = null;}+ const chunkFinishReason = chunk.choices?.[0]?.finish_reason;+ if (typeof chunkFinishReason === "string" && chunkFinishReason) {+ lastFinishReason = chunkFinishReason;+ }// Latch the chunk's `model` field so the openrouter/free chip shows the underlying model.if (isExternalRequest &&@@ -9029,7 +9050,8 @@ export function createOpenAIStreamAdapter(const finalTiming = buildTiming(streamStartTime,totalChunks,- serverPromptEvalTime ?? firstTokenTime,+ // What the user waited, as a resumed run records it. Server prompt eval has its own row.+ firstTokenTime ?? serverPromptEvalTime,finishedAt - streamStartTime,finalTokenCount,toolCallParts.length,@@ -9078,6 +9100,7 @@ export function createOpenAIStreamAdapter(// Persisted refusal flag driving the two-pass prune.anthropicRefusal: anthropicRefusalSeen || undefined,serverTimings: meta?.timings ?? undefined,+ stopReason: lastFinishReason ?? undefined,contextUsage: meta?.usage? {promptTokens: meta.usage.prompt_tokens,@@ -9326,6 +9349,7 @@ export function createOpenAIStreamAdapter(// Drop the transient denoising canvas so the finished bubble shows only the committed answer;// scoped, since a global clear wiped another chat's frame.runtime.clearActiveDiffusionCanvasForThread(cleanupKey);+ runtime.clearLiveGenerationForThread(cleanupKey);clearTimeout(warmupTimer);if (waitingFirstChunk) {if (firstTokenSettled) {diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.tsindex 37adc248c81..b1e9a5d47a5 100644--- a/studio/frontend/src/features/chat/api/chat-api.ts+++ b/studio/frontend/src/features/chat/api/chat-api.ts@@ -2064,6 +2064,11 @@ export async function* streamChatCompletions(separatorIndex = buffer.search(/\r?\n\r?\n/);continue;}+ if ("type" in parsed && parsed.type === "generation_progress") {+ yield { _generationProgress: parsed } as unknown as OpenAIChatChunk;+ separatorIndex = buffer.search(/\r?\n\r?\n/);+ continue;+ }const {usage: parsedUsage,timings: parsedTimings,diff --git a/studio/frontend/src/features/chat/api/chat-generation-api.ts b/studio/frontend/src/features/chat/api/chat-generation-api.tsindex 79bad7f040d..476b81a7fc8 100644--- a/studio/frontend/src/features/chat/api/chat-generation-api.ts+++ b/studio/frontend/src/features/chat/api/chat-generation-api.ts@@ -102,6 +102,9 @@ export function normalizeChatGenerationChunkPayload(if (frameType === "context_usage") {return { _contextUsage: payload } as unknown as OpenAIChatChunk;}+ if (frameType === "generation_progress") {+ return { _generationProgress: payload } as unknown as OpenAIChatChunk;+ }}return payload;}diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.tsindex dab2197b65d..4432dcc89fd 100644--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts@@ -3707,7 +3707,8 @@ export function useChatModelRuntime() {const totalGb = prog.bytes_total / 1e9;const pct = Math.min(99, Math.round(prog.fraction * 100));const est = estimate(mmapSamples, prog.bytes_loaded, prog.bytes_total);- const base = `${loadedGb.toFixed(1)} of ${totalGb.toFixed(1)} GB in memory`;+ // Bytes the server has read or paged in, not bytes resident in RAM: weights headed for VRAM pass through.+ const base = `${loadedGb.toFixed(1)} of ${totalGb.toFixed(1)} GB loaded`;const label = est.stable? `${base} • ${formatRate(est.rate)}${formatEta(est.eta) !== "--" ? ` • ${formatEta(est.eta)} left` : ""@@ -3728,7 +3729,7 @@ export function useChatModelRuntime() {...modelLoadToastOptions(renderLoadDescription("Starting model…",- "Paging weights into memory.",+ "Loading model weights.",pct,label,),diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.tsindex 797a8b79a94..8b4c68b0d85 100644--- a/studio/frontend/src/features/chat/index.ts+++ b/studio/frontend/src/features/chat/index.ts@@ -90,7 +90,14 @@ export {GPU_LAYERS_AUTO,} from "./stores/chat-runtime-store";export { resolveStagedDiffusionClassification } from "./lib/gpu-placement";-export { offloadCountsFrom, offloadWarning } from "./lib/partial-offload";+export { describePlacement, offloadCountsFrom, offloadWarning } from "./lib/partial-offload";+export {+ formatLiveDecode,+ formatPrefillDetails,+ formatPrefillTimeLeft,+ prefillPercent,+} from "./lib/generation-progress";+export type { LiveDecodeProgress } from "./stores/chat-runtime-store";export {preferFullToolOutput,preferSanitizedFullToolOutput,diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.tsindex 02e38f13655..0c188a65952 100644--- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts+++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts@@ -9,6 +9,7 @@ import {resolveResidentInitialConfig,savedContextPin,} from "@/features/model-picker";+import { loadedPlacementFrom } from "./partial-offload";// eslint-disable-next-line no-restricted-imports -- Avoid the hub barrel's React and download-manager exports.import { normalizeMlxKvQuant } from "@/features/model-picker/model-config/per-model-config";import { modelDisplayName } from "@/features/hub/lib/model-identity";@@ -502,6 +503,7 @@ export function applyActiveModelStatusToStore(activeModelIsLocal: status.is_local_model ?? false,specFallbackReason: status.spec_fallback_reason ?? null,mmprojFallbackReason: status.mmproj_fallback_reason ?? null,+ loadedPlacement: loadedPlacementFrom(status),specDrafterKind: status.spec_drafter_kind ?? null,// Controls follow the server while clean; loaded baselines always describe// the settled resident, including same-model reloads from another client.diff --git a/studio/frontend/src/features/chat/lib/generation-progress.ts b/studio/frontend/src/features/chat/lib/generation-progress.tsnew file mode 100644index 00000000000..8ef980b2ddf--- /dev/null+++ b/studio/frontend/src/features/chat/lib/generation-progress.ts@@ -0,0 +1,149 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import type {+ LiveDecodeProgress,+ LiveGenerationProgress,+ LivePrefillProgress,+} from "../stores/chat-runtime-store";+import { formatEta } from "../utils/format-transfer";++export type GenerationProgressUpdate =+ | { phase: "prefill"; prefill: LivePrefillProgress }+ | { phase: "decode"; decode: LiveDecodeProgress };++function count(value: unknown): number | null {+ return typeof value === "number" && Number.isFinite(value) && value >= 0+ ? Math.floor(value)+ : null;+}++function rate(value: unknown): number | null {+ return typeof value === "number" && Number.isFinite(value) && value > 0+ ? value+ : null;+}++/** Null for anything malformed, so a bad frame is dropped instead of drawing NaN. */+export function readGenerationProgress(+ frame: unknown,+): GenerationProgressUpdate | null {+ if (frame === null || typeof frame !== "object") return null;+ const f = frame as Record<string, unknown>;+ if (f.phase === "prefill") {+ const processed = count(f.processed);+ if (processed === null) return null;+ const total = count(f.total);+ return {+ phase: "prefill",+ prefill: {+ processed: total ? Math.min(processed, total) : processed,+ total: total || null,+ cached: count(f.cached),+ tokPerS: rate(f.tok_per_s),+ },+ };+ }+ if (f.phase === "decode") {+ const tokens = count(f.tokens);+ if (!tokens) return null;+ const draftN = count(f.draft_n);+ const draftAccepted = count(f.draft_accepted);+ return {+ phase: "decode",+ decode: {+ tokens,+ tokPerS: rate(f.tok_per_s),+ draftN: draftN || null,+ draftAccepted:+ draftN && draftAccepted !== null ? Math.min(draftAccepted, draftN) : null,+ },+ };+ }+ return null;+}++/** "9,216 / 23,700 tokens · 1,326 tok/s" for the Generation details setting, counted past the+ * cached prefix like the percent. */+export function formatPrefillDetails(+ prefill: LivePrefillProgress,+ locale?: string,+): string {+ const cached = prefill.cached ?? 0;+ const done = Math.max(0, prefill.processed - cached).toLocaleString(locale);+ const parts = [+ prefill.total+ ? `${done} / ${Math.max(0, prefill.total - cached).toLocaleString(locale)} tokens`+ : `${done} tokens`,+ ];+ // The label shows only once prefillLabelShown has a rate it trusts, so the rate is past the first sample.+ if (prefill.tokPerS !== null) {+ parts.push(`${Math.round(prefill.tokPerS).toLocaleString(locale)} tok/s`);+ }+ return parts.join(" · ");+}++const MIN_TOKENS_FOR_PREFILL_RATE = 512;+const PREFILL_LABEL_MIN_SECONDS = 2;++/** Seconds left at the rate so far, or null while llama.cpp's first sample (a few tokens in, before+ * the first batch) is all there is, since its rate is noise. */+function prefillSecondsLeft(prefill: LivePrefillProgress): number | null {+ if (!prefill.total || prefill.tokPerS === null) return null;+ if (prefill.processed - (prefill.cached ?? 0) < MIN_TOKENS_FOR_PREFILL_RATE) return null;+ return (prefill.total - prefill.processed) / prefill.tokPerS;+}++/** Whether "Processing prompt" shows after this sample. Only a pass expected to keep it up for+ * PREFILL_LABEL_MIN_SECONDS turns it on, so a fast prompt stays on Working... instead of flashing it,+ * and it then stays for the rest of that pass instead of dropping as the estimate shrinks. */+export function prefillLabelShown(+ previous: LiveGenerationProgress | undefined,+ prefill: LivePrefillProgress,+): boolean {+ const samePass =+ previous?.prefill != null &&+ previous.decode === null &&+ previous.prefill.total === prefill.total &&+ previous.prefill.processed <= prefill.processed;+ if (samePass && previous.prefillShown) return true;+ const left = prefillSecondsLeft(prefill);+ return left !== null && left >= PREFILL_LABEL_MIN_SECONDS;+}++/** "12s left" at the rate so far, like the load toast; null without a trusted rate or under a second. */+export function formatPrefillTimeLeft(prefill: LivePrefillProgress): string | null {+ const left = prefillSecondsLeft(prefill);+ return left !== null && left >= 1 ? `${formatEta(left)} left` : null;+}++/** "4,306 tokens · 58.3 tok/s · draft 1,791/5,030 accepted (36%)", every number in the reader's locale. */+export function formatLiveDecode(decode: LiveDecodeProgress, locale?: string): string {+ const parts = [+ `${decode.tokens.toLocaleString(locale)} ${decode.tokens === 1 ? "token" : "tokens"}`,+ ];+ if (decode.tokPerS !== null) {+ const rate = decode.tokPerS.toLocaleString(locale, {+ minimumFractionDigits: 1,+ maximumFractionDigits: 1,+ });+ parts.push(`${rate} tok/s`);+ }+ if (decode.draftN && decode.draftAccepted !== null) {+ const share = Math.round((decode.draftAccepted / decode.draftN) * 100);+ parts.push(+ `draft ${decode.draftAccepted.toLocaleString(locale)}/${decode.draftN.toLocaleString(locale)} accepted (${share}%)`,+ );+ }+ return parts.join(" · ");+}++/** Percent of the part of the prompt the cache did not cover, llama.cpp's "timed progress": a+ * follow-up in a long chat would otherwise start near 100% and crawl through the last few. */+export function prefillPercent(prefill: LivePrefillProgress): number | null {+ if (!prefill.total) return null;+ const cached = prefill.cached ?? 0;+ const todo = prefill.total - cached;+ if (todo <= 0) return 100;+ return Math.min(100, Math.max(0, Math.floor(((prefill.processed - cached) / todo) * 100)));+}diff --git a/studio/frontend/src/features/chat/lib/partial-offload.ts b/studio/frontend/src/features/chat/lib/partial-offload.tsindex ce5401809c1..7f91ab545d5 100644--- a/studio/frontend/src/features/chat/lib/partial-offload.ts+++ b/studio/frontend/src/features/chat/lib/partial-offload.ts@@ -1,6 +1,8 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import { formatGiB } from "../../../lib/memory/format";+import type { LoadedPlacement } from "../stores/chat-runtime-store";import { CPU_FALLBACK_MESSAGE } from "../utils/mmproj-fallback";// React-free so node:test can import it; shared by every model load path.@@ -97,3 +99,98 @@ export function offloadWarning(counts: OffloadCounts): OffloadWarning | null {description: `${offloaded} of ${total} layers are on the GPU. The rest run on CPU, so generation will be slower. A smaller quantization or a shorter context may let more of it fit on the GPU.`,};}++function count(value: unknown): number | null {+ return typeof value === "number" && Number.isFinite(value) && value >= 0+ ? value+ : null;+}++/** GiB like the rest of the panel, or MiB under 0.1 GiB, which would otherwise print as "0.0 GiB". */+function weightAmount(mib: number): string {+ return mib < 102.4+ ? `${Math.max(1, Math.round(mib))} MiB`+ : formatGiB(mib / 1024);+}++function layerCount(n: number): string {+ return `${n} ${n === 1 ? "layer" : "layers"}`;+}++/** --fit reports its own expert spill; a Manual load says it through n_cpu_moe instead. */+export function loadedPlacementFrom(response: {+ is_gguf?: boolean | null;+ offloaded_layers?: number | null;+ offload_total_layers?: number | null;+ overflow_layers?: number | null;+ gpu_weights_mib?: number | null;+ host_weights_mib?: number | null;+ expert_cache_mib?: number | null;+ gpu_memory_mode?: string | null;+ n_cpu_moe?: number | null;+}): LoadedPlacement | null {+ if (response.is_gguf === false) return null;+ const manualExperts =+ response.gpu_memory_mode === "manual" ? count(response.n_cpu_moe) : null;+ const placement: LoadedPlacement = {+ offloaded: count(response.offloaded_layers),+ total: count(response.offload_total_layers),+ overflowLayers:+ count(response.overflow_layers) ?? (manualExperts ? manualExperts : null),+ gpuWeightsMib: count(response.gpu_weights_mib),+ hostWeightsMib: count(response.host_weights_mib),+ expertCacheMib: count(response.expert_cache_mib) || null,+ };+ return placement.total === null &&+ placement.gpuWeightsMib === null &&+ placement.hostWeightsMib === null+ ? null+ : placement;+}++/** An overflowing layer has expert weights in RAM: all its routed experts, except --fit's first partial layer+ * per device, which keeps only part of its FFN there, so the text says expert weights, not experts. */+function describeLayers(p: LoadedPlacement): string | null {+ const { offloaded, total, overflowLayers, expertCacheMib } = p;+ if (offloaded === null || total === null || total <= 0) return null;+ // A GPU build whose backend failed to start still logs "offloaded N/N"; its buffers are all host ones.+ const noGpuBuffers = p.gpuWeightsMib === 0 && (p.hostWeightsMib ?? 0) > 0;+ if (offloaded <= 0 || noGpuBuffers) return `All ${total} layers on the CPU`;+ if (offloaded < total)+ return `${offloaded} of ${total} layers on the GPU, ${total - offloaded} on the CPU`;+ if (expertCacheMib !== null && expertCacheMib > 0) {+ return `All ${total} layers on the GPU, all experts in RAM (${weightAmount(expertCacheMib)} expert cache on the GPU)`;+ }+ if (overflowLayers !== null && overflowLayers > 0) {+ return `All ${total} layers on the GPU, expert weights of ${layerCount(overflowLayers)} in RAM`;+ }+ return `All ${total} layers on the GPU`;+}++/** Both figures are shown: token embeddings stay in RAM even when every layer is on the GPU. On a machine whose+ * GPU and CPU share one pool, `pool` names it ("unified memory") and the weights are one figure. */+function describeWeights(+ gpuWeightsMib: number | null,+ hostWeightsMib: number | null,+ pool: string | null,+): string | null {+ const gpu = gpuWeightsMib !== null && gpuWeightsMib > 0 ? gpuWeightsMib : 0;+ const host =+ hostWeightsMib !== null && hostWeightsMib > 0 ? hostWeightsMib : 0;+ if (pool !== null && gpu + host > 0)+ return `${weightAmount(gpu + host)} in ${pool}`;+ const parts: string[] = [];+ if (gpu > 0) parts.push(`${weightAmount(gpu)} in VRAM`);+ if (host > 0) parts.push(`${weightAmount(host)} in RAM`);+ return parts.length > 0 ? parts.join(" · ") : null;+}++export function describePlacement(+ p: LoadedPlacement,+ pool: string | null = null,+): { layers: string | null; memory: string | null } {+ return {+ layers: describeLayers(p),+ memory: describeWeights(p.gpuWeightsMib, p.hostWeightsMib, pool),+ };+}diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsxindex 055a8862899..81d4a0a6307 100644--- a/studio/frontend/src/features/chat/runtime-provider.tsx+++ b/studio/frontend/src/features/chat/runtime-provider.tsx@@ -1,6 +1,7 @@// SPDX-License-Identifier: AGPL-3.0-only// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+import { readGenerationProgress } from "./lib/generation-progress";import { useAppShellReadySignal } from "@/components/app-readiness";import { authFetch, getAuthSessionEpoch } from "@/features/auth";import { isMcpToolOnly, mcpImageMappingsEnabled } from "./api/mcp-image";@@ -1782,6 +1783,7 @@ function scheduleGenerationRecovery(cursor = update.event.seq;if (update.event.type === "chunk") {const chunk = update.event.payload as {+ _generationProgress?: unknown;_reasoningDurationMs?: unknown;usage?: {prompt_tokens?: unknown;@@ -1801,6 +1803,19 @@ function scheduleGenerationRecovery(context_truncated?: OpenAIChatChunk["context_truncated"];quote_cut?: boolean;};+ // Live figures only once caught up: a replayed backlog would flash every old sample.+ if ("_generationProgress" in chunk) {+ if (caughtUp(update.run)) {+ const progress = readGenerationProgress(chunk._generationProgress);+ const store = useChatRuntimeStore.getState();+ if (progress?.phase === "prefill") {+ store.setLivePrefill(threadId, progress.prefill);+ } else if (progress?.phase === "decode") {+ store.setLiveDecode(threadId, progress.decode);+ }+ }+ continue;+ }if ("_reasoningDurationMs" in chunk) {currentMetadata = recoveredReasoningSummaryMetadata(currentMetadata,@@ -1948,6 +1963,7 @@ function scheduleGenerationRecovery();}} finally {+ useChatRuntimeStore.getState().clearLiveGenerationForThread(threadId);// EVERY exit, not just the terminal one: a permanent follower error (another tab deletes the// thread, the run row cascades, the request 404s) throws past the terminal branch into the// outer catch, leaving a seeded card in the global store for the session. `soleRequest` countsdiff --git a/studio/frontend/src/features/chat/stores/chat-preferences-store.ts b/studio/frontend/src/features/chat/stores/chat-preferences-store.tsindex b5963ed9364..b59bff80b78 100644--- a/studio/frontend/src/features/chat/stores/chat-preferences-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-preferences-store.ts@@ -29,6 +29,8 @@ import {// showInlineEditResponse: on keeps Edit response in each response's action bar instead of its// More menu. notifyWhenReplyFinishes: on sends a system notification when a reply ends while the// window is hidden or unfocused. showReplyTimes: on shows when each reply finished under it.+// showGenerationDetails: on shows live prefill / decode figures while a reply streams and where+// a local model's layers ended up after it loads.// filesPanelCollapsed: on shrinks Chat with files to a header and// a row of pills.export interface ChatPreferencesState {@@ -50,6 +52,8 @@ export interface ChatPreferencesState {setShowResponseModel: (value: boolean) => void;showReplyTimes: boolean;setShowReplyTimes: (value: boolean) => void;+ showGenerationDetails: boolean;+ setShowGenerationDetails: (value: boolean) => void;showInlineEditResponse: boolean;setShowInlineEditResponse: (value: boolean) => void;thinkingVisibility: DisplayVisibility;@@ -103,6 +107,10 @@ export const useChatPreferencesStore = create<ChatPreferencesState>()(// Off by default: it adds text under every reply, so it stays opt in.showReplyTimes: false,setShowReplyTimes: (showReplyTimes) => set({ showReplyTimes }),+ // Off by default: it adds a line of numbers under streaming replies, so it stays opt in.+ showGenerationDetails: false,+ setShowGenerationDetails: (showGenerationDetails) =>+ set({ showGenerationDetails }),showInlineEditResponse: true,setShowInlineEditResponse: (showInlineEditResponse) =>set({ showInlineEditResponse }),@@ -157,6 +165,7 @@ export const useChatPreferencesStore = create<ChatPreferencesState>()(showModelDisclaimer: saved?.showModelDisclaimer ?? false,showResponseModel: saved?.showResponseModel ?? false,showReplyTimes: saved?.showReplyTimes === true,+ showGenerationDetails: saved?.showGenerationDetails === true,showInlineEditResponse: saved?.showInlineEditResponse ?? true,thinkingVisibility: migrateVisibility(saved?.thinkingVisibility,diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex 73679877ad0..3b84af1d85e 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -120,6 +120,7 @@ import {type ModelLifecyclePhase,} from "../utils/model-lifecycle-gate";import { shouldAdvanceQueuedSettingsEpoch } from "../utils/queued-settings-epoch";+import { prefillLabelShown } from "../lib/generation-progress";import { adoptFirstTurnConfirmations } from "../utils/adopt-first-turn-confirmations";import type { MmprojFallbackReason } from "../types/api";import type {@@ -427,6 +428,36 @@ export type DiffusionCanvasFrame = {total: number;text: string;};+/** Where llama.cpp placed a loaded GGUF's weights (counts from its log; MiB from its buffers). */+export type LoadedPlacement = {+ offloaded: number | null;+ total: number | null;+ /** Layers whose MoE experts --fit kept in system memory; null when --fit did not run. */+ overflowLayers: number | null;+ gpuWeightsMib: number | null;+ hostWeightsMib: number | null;+ /** Set when every routed expert is in RAM with a GPU cache of the busiest (--moe-cache-mib auto). */+ expertCacheMib: number | null;+};+export type LivePrefillProgress = {+ processed: number;+ total: number | null;+ cached: number | null;+ tokPerS: number | null;+};+export type LiveDecodeProgress = {+ tokens: number;+ tokPerS: number | null;+ draftN: number | null;+ draftAccepted: number | null;+};+/** Transient, like the diffusion canvas: set from generation_progress frames, cleared when the run ends. */+export type LiveGenerationProgress = {+ prefill: LivePrefillProgress | null;+ /** Whether "Processing prompt" is up for this pass; see prefillLabelShown. */+ prefillShown: boolean;+ decode: LiveDecodeProgress | null;+};export type PendingImageEditReference = {threadId: string | null;openaiImageGenerationCallId: string;@@ -2683,6 +2714,8 @@ type ChatRuntimeStore = {specFallbackReason: string | null;/** Projector recovery outcome for the active GGUF, or null. */mmprojFallbackReason: MmprojFallbackReason | null;+ /** Where the loaded GGUF's weights went, from llama.cpp's load log; null for other backends. */+ loadedPlacement: LoadedPlacement | null;/** Which drafter the speculative resolution was about ("mtp", "dspark", "dflash").* specFallbackReason alone cannot name the file to fix, since Auto resolves it server-side. */specDrafterKind: string | null;@@ -2777,6 +2810,8 @@ type ChatRuntimeStore = {/** Live denoising frame per conversation ("__default" until the id exists). Transient and* keyed, since two denoising chats overwrote each other's frame. */activeDiffusionCanvasByThreadId: Record<string, DiffusionCanvasFrame>;+ /** Live prefill / decode progress per conversation, keyed like the diffusion canvas. */+ liveGenerationByThreadId: Record<string, LiveGenerationProgress>;customContextLength: number | null;/** The pinned context the loaded model used (null = Auto), so dirty-tracking and a later fit* Apply can tell an explicit pin from Auto. */@@ -2966,6 +3001,9 @@ type ChatRuntimeStore = {) => void;/** Drop only `threadId`'s canvas: a run ending in a background chat must not wipe another's. */clearActiveDiffusionCanvasForThread: (threadId: string | null) => void;+ setLivePrefill: (threadId: string | null, prefill: LivePrefillProgress) => void;+ setLiveDecode: (threadId: string | null, decode: LiveDecodeProgress) => void;+ clearLiveGenerationForThread: (threadId: string | null) => void;setAutoHealToolCalls: (enabled: boolean) => void;setNudgeToolCalls: (enabled: boolean) => void;setDeduplicateToolCalls: (enabled: boolean) => void;@@ -4622,6 +4660,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({toolFullOutput: {},generatingStatus: null,activeDiffusionCanvasByThreadId: {},+ liveGenerationByThreadId: {},autoHealToolCalls: true,nudgeToolCalls: true,deduplicateToolCalls: true,@@ -4651,6 +4690,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({loadedSpeculativeType: null,specFallbackReason: null,mmprojFallbackReason: null,+ loadedPlacement: null,specDrafterKind: null,specDraftNMax: null,loadedSpecDraftNMax: null,@@ -5162,6 +5202,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({state.activeDiffusionCanvasByThreadId,"activeDiffusionCanvasByThreadId",);+ move(state.liveGenerationByThreadId, "liveGenerationByThreadId");// Its parked calls too, or the row never trades its spinner for the approval mark.if (moved.runningByThreadId) {const confirmations = adoptFirstTurnConfirmations(@@ -5345,6 +5386,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({activeModelIsLocal: false,specFallbackReason: null,mmprojFallbackReason: null,+ loadedPlacement: null,specDrafterKind: null,}: {}),@@ -5639,6 +5681,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({toolLiveOutput: {},toolFullOutput: {},activeDiffusionCanvasByThreadId: {},+ liveGenerationByThreadId: {},kvCacheDtype: null,mlxKvQuant: null,loadedMlxKvQuantRequested: null,@@ -5661,6 +5704,7 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({loadedSpeculativeType: null,specFallbackReason: null,mmprojFallbackReason: null,+ loadedPlacement: null,specDrafterKind: null,specDraftNMax: null,loadedSpecDraftNMax: null,@@ -6339,6 +6383,41 @@ export const useChatRuntimeStore = create<ChatRuntimeStore>((set, get) => ({delete next[key];return { activeDiffusionCanvasByThreadId: next };}),+ setLivePrefill: (threadId, prefill) =>+ set((state) => {+ const key = threadId || "__default";+ const prefillShown = prefillLabelShown(state.liveGenerationByThreadId[key], prefill);+ // A new prompt pass (the next round of a tool loop) starts over, so its decode figures go too.+ return {+ liveGenerationByThreadId: {+ ...state.liveGenerationByThreadId,+ [key]: { prefill, prefillShown, decode: null },+ },+ };+ }),+ setLiveDecode: (threadId, decode) =>+ set((state) => {+ const key = threadId || "__default";+ const current = state.liveGenerationByThreadId[key];+ return {+ liveGenerationByThreadId: {+ ...state.liveGenerationByThreadId,+ [key]: {+ prefill: current?.prefill ?? null,+ prefillShown: current?.prefillShown ?? false,+ decode,+ },+ },+ };+ }),+ clearLiveGenerationForThread: (threadId) =>+ set((state) => {+ const key = threadId || "__default";+ if (state.liveGenerationByThreadId[key] === undefined) return state;+ const next = { ...state.liveGenerationByThreadId };+ delete next[key];+ return { liveGenerationByThreadId: next };+ }),setGeneratingStatus: (generatingStatus) => set({ generatingStatus }),setAutoHealToolCalls: (autoHealToolCalls) =>set((state) => {diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex aa9dad0285f..ecbe754700c 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -493,6 +493,12 @@ export interface InferenceStatusResponse {/** Set while the active model is a recovered CPU-only Vulkan load. */offloaded_layers?: number | null;offload_total_layers?: number | null;+ /** Layers --fit left partly in system memory (MoE experts first); null without --fit. */+ overflow_layers?: number | null;+ gpu_weights_mib?: number | null;+ host_weights_mib?: number | null;+ /** --moe-cache-mib auto: every routed expert in RAM, this many MiB cached on the GPU. */+ expert_cache_mib?: number | null;offload_overridden?: boolean | null;gpu_backend_unavailable?: boolean | null;cpu_fallback_reason?: CpuFallbackReason | null;diff --git a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts b/studio/frontend/src/features/chat/utils/chat-generation-recovery.tsindex 3503317f0ff..58878121faa 100644--- a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts+++ b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts@@ -52,6 +52,7 @@ export function generationChunkCountsTowardTiming(payload: unknown): boolean {| {_reasoningDurationMs?: unknown;_contextUsage?: unknown;+ _generationProgress?: unknown;context_truncated?: unknown;quote_cut?: unknown;usage?: unknown;@@ -63,6 +64,7 @@ export function generationChunkCountsTowardTiming(payload: unknown): boolean {if ("_reasoningDurationMs" in chunk ||"_contextUsage" in chunk ||+ "_generationProgress" in chunk ||chunk.context_truncated ||chunk.quote_cut) {@@ -181,6 +183,8 @@ type RecoveryTimings = {[key: string]: unknown;};+const RECOVERED_STOP_REASONS = new Set(["stop", "length", "tool_calls", "content_filter"]);+export function recoveredGenerationFinalMetadata(options: {current: Record<string, unknown>;run: {@@ -189,6 +193,7 @@ export function recoveredGenerationFinalMetadata(options: {createdAt: number;startedAt: number | null;completedAt: number | null;+ finishReason?: string | null;};usage?: RecoveryUsage;timings?: RecoveryTimings;@@ -219,6 +224,10 @@ export function recoveredGenerationFinalMetadata(options: {if (next.serverTimings === undefined && timings !== undefined) {next.serverTimings = timings;}+ // Only the stream's own vocabulary: "cancelled" / "interrupted" are run states the reply already shows.+ if (next.stopReason === undefined && RECOVERED_STOP_REASONS.has(run.finishReason ?? "")) {+ next.stopReason = run.finishReason;+ }if (next.contextUsage === undefined &&typeof usage?.prompt_tokens === "number" &&diff --git a/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsx b/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsxnew file mode 100644index 00000000000..150970cc84f--- /dev/null+++ b/studio/frontend/src/features/model-picker/components/loaded-placement-row.tsx@@ -0,0 +1,38 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import {+ describePlacement,+ useChatPreferencesStore,+ useChatRuntimeStore,+} from "@/features/chat";++/** Where the resident GGUF's weights landed. Under Auto, --fit decides, and the layer count alone hides a+ * MoE spill, so the experts and the RAM side are shown too. `pool` is the panel's single memory pool, if any. */+export function LoadedPlacementRow({ pool }: { pool: string | null }) {+ const enabled = useChatPreferencesStore((s) => s.showGenerationDetails);+ const placement = useChatRuntimeStore((s) => s.loadedPlacement);+ if (!enabled || placement === null) return null;+ const { layers, memory } = describePlacement(placement, pool);+ if (layers === null && memory === null) return null;+ return (+ <div+ data-slot="loaded-placement"+ className="flex flex-col gap-1 border-b border-border pb-5"+ >+ <span className="flex min-h-8 items-center text-ui-13 font-medium leading-[1.25] tracking-nav text-foreground">+ Weight placement+ </span>+ {layers !== null && (+ <span className="text-ui-12 leading-snug text-muted-foreground">+ {layers}+ </span>+ )}+ {memory !== null && (+ <span className="text-ui-12 leading-snug text-muted-foreground tabular-nums">+ {memory}+ </span>+ )}+ </div>+ );+}diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsxindex 0d4d154ad81..6e73837fe93 100644--- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx+++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx@@ -229,6 +229,7 @@ import {} from "../sharing";import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog";import { MemoryEstimateRow } from "./memory-estimate-row";+import { LoadedPlacementRow } from "./loaded-placement-row";import type { ModelPickTarget } from "./model-selector/types";import type { ModelConfigFocus } from "../model-config/model-config-handoff";import { reconcileTensorSplit } from "@/hooks/gpu-tensor-split";@@ -4018,6 +4019,12 @@ export function ModelConfigPage({onExpandedChange={setMemoryBreakdownOpen}/>)}+ {isActiveModel && target.isGguf && !audioRuntimeGguf ? (+ // The same pool the estimate above names: "Unified" on Apple Silicon, "Shared" on other one-pool machines.+ <LoadedPlacementRow+ pool={singleMemoryPool ? (isAppleUnifiedMemory ? "unified memory" : "shared memory") : null}+ />+ ) : null}{target.isGguf && !audioRuntimeGguf && (<><GgufContextLengthSettingdiff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.tsindex 7826c1c03c9..fa11332e89b 100644--- a/studio/frontend/src/features/settings/settings-search.ts+++ b/studio/frontend/src/features/settings/settings-search.ts@@ -147,6 +147,7 @@ export const SETTINGS_SEARCH_INDEX: Record<SettingsTab, TranslationKey[]> = {"settings.chat.modelDisclaimer","settings.chat.inlineEditResponse","settings.chat.showReplyTimes",+ "settings.chat.showGenerationDetails","settings.chat.groups.menu.title",],browser: [@@ -478,6 +479,7 @@ export const SETTINGS_SEARCH_KEYWORDS: Partial<"settings.chat.scrollToBottomButton": "settings.chat.scrollToBottomButtonKeywords","settings.chat.replyNotification": "settings.chat.replyNotificationKeywords","settings.chat.showReplyTimes": "settings.chat.showReplyTimesKeywords",+ "settings.chat.showGenerationDetails": "settings.chat.showGenerationDetailsKeywords",// match protocol and agent names omitted from the title."settings.apiKeys.mcp.title": "settings.apiKeys.mcp.keywords",// the intro names only some agents; every listed one has to find this page.diff --git a/studio/frontend/src/features/settings/tabs/chat-tab.tsx b/studio/frontend/src/features/settings/tabs/chat-tab.tsxindex 1153c16f0c0..2bece356672 100644--- a/studio/frontend/src/features/settings/tabs/chat-tab.tsx+++ b/studio/frontend/src/features/settings/tabs/chat-tab.tsx@@ -369,6 +369,12 @@ export function ChatTab() {const setShowReplyTimes = useChatPreferencesStore((state) => state.setShowReplyTimes,);+ const showGenerationDetails = useChatPreferencesStore(+ (state) => state.showGenerationDetails,+ );+ const setShowGenerationDetails = useChatPreferencesStore(+ (state) => state.setShowGenerationDetails,+ );const autoScrollWhileGenerating = useChatPreferencesStore((state) => state.autoScrollWhileGenerating,);@@ -796,6 +802,16 @@ export function ChatTab() {onCheckedChange={setShowReplyTimes}/></SettingsRow>+ <SettingsRow+ label={t("settings.chat.showGenerationDetails")}+ description={t("settings.chat.showGenerationDetailsDescription")}+ >+ <Switch+ aria-label={t("settings.chat.showGenerationDetails")}+ checked={showGenerationDetails}+ onCheckedChange={setShowGenerationDetails}+ />+ </SettingsRow><SettingsRowlabel={t("settings.chat.inlineEditResponse")}description={t("settings.chat.inlineEditResponseDescription")}diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.tsindex e269265cb47..1798315f4fe 100644--- a/studio/frontend/src/i18n/locales/ar.ts+++ b/studio/frontend/src/i18n/locales/ar.ts@@ -2755,6 +2755,11 @@ export const ar = {showReplyTimesDescription: "إظهار وقت انتهاء كل رد أسفله.",showReplyTimesKeywords:"طابع زمني وقت ساعة تاريخ متى انتهى رد رسالة timestamp time clock date reply",+ showGenerationDetails: "تفاصيل التوليد",+ showGenerationDetailsDescription:+ "عرض السرعة وعدد الرموز مباشرةً أثناء بث الرد، ومكان وضع طبقات النموذج المحلي.",+ showGenerationDetailsKeywords:+ "تفاصيل سرعة رموز إحصاءات طبقات ذاكرة verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "تعديل الرد في الردود",inlineEditResponseDescription: "إبقاء تعديل الرد على كل رد بدلًا من قائمة المزيد.",modelDisclaimer: "إظهار إخلاء مسؤولية النموذج",diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.tsindex 41ce024aea1..4737d6a6a73 100644--- a/studio/frontend/src/i18n/locales/de.ts+++ b/studio/frontend/src/i18n/locales/de.ts@@ -2796,6 +2796,11 @@ export const de = {showReplyTimesDescription: "Unter jeder Antwort anzeigen, wann sie fertig war.",showReplyTimesKeywords:"Zeitstempel Uhrzeit Datum wann fertig Antwort Nachricht timestamp time clock date reply",+ showGenerationDetails: "Generierungsdetails",+ showGenerationDetailsDescription:+ "Tempo und Tokenzahl live anzeigen, während eine Antwort gestreamt wird, und wo die Schichten eines lokalen Modells liegen.",+ showGenerationDetailsKeywords:+ "Details Geschwindigkeit Token Statistik Schichten Auslagerung verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Antwort bearbeiten bei Antworten",inlineEditResponseDescription: "Antwort bearbeiten bei jeder Antwort anzeigen statt im Menü „Mehr“.",modelDisclaimer: "Modell-Hinweis anzeigen",diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.tsindex f1a61a8e6e1..80930b9fcec 100644--- a/studio/frontend/src/i18n/locales/en.ts+++ b/studio/frontend/src/i18n/locales/en.ts@@ -2813,6 +2813,11 @@ export const en = {showReplyTimesDescription: "Show when each reply finished, below the reply.",showReplyTimesKeywords:"timestamp time clock date hour when finished arrived reply response message",+ showGenerationDetails: "Generation details",+ showGenerationDetailsDescription:+ "Show live prompt and reply speeds with token counts, and where a model's layers loaded.",+ showGenerationDetailsKeywords:+ "verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Edit response on responses",inlineEditResponseDescription:"Keep Edit response on every response, instead of in the More menu.",diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.tsindex ef5b0f5bb28..1c6322487ec 100644--- a/studio/frontend/src/i18n/locales/es.ts+++ b/studio/frontend/src/i18n/locales/es.ts@@ -2787,6 +2787,11 @@ export const es = {showReplyTimesDescription: "Mostrar debajo de cada respuesta cuándo terminó.",showReplyTimesKeywords:"marca de tiempo hora fecha cuándo terminó respuesta mensaje timestamp time clock date reply",+ showGenerationDetails: "Detalles de generación",+ showGenerationDetailsDescription:+ "Muestra la velocidad y los tokens en vivo mientras se genera una respuesta, y dónde quedaron las capas de un modelo local.",+ showGenerationDetailsKeywords:+ "detalles velocidad tokens estadísticas capas descarga verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Editar respuesta en las respuestas",inlineEditResponseDescription: "Mostrar Editar respuesta en cada respuesta, en lugar de en el menú Más.",modelDisclaimer: "Mostrar aviso del modelo",diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.tsindex ba38c337482..9c6d3b16217 100644--- a/studio/frontend/src/i18n/locales/fr.ts+++ b/studio/frontend/src/i18n/locales/fr.ts@@ -2793,6 +2793,11 @@ export const fr = {showReplyTimesDescription: "Afficher sous chaque réponse l'heure à laquelle elle s'est terminée.",showReplyTimesKeywords:"horodatage heure date quand terminé réponse message timestamp time clock date reply",+ showGenerationDetails: "Détails de génération",+ showGenerationDetailsDescription:+ "Affiche la vitesse et le nombre de jetons en direct pendant une réponse, et où se trouvent les couches d'un modèle local.",+ showGenerationDetailsKeywords:+ "détails vitesse jetons statistiques couches déchargement verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Modifier la réponse sur les réponses",inlineEditResponseDescription: "Afficher Modifier la réponse sur chaque réponse, plutôt que dans le menu Plus.",modelDisclaimer: "Afficher l'avertissement du modèle",diff --git a/studio/frontend/src/i18n/locales/he.ts b/studio/frontend/src/i18n/locales/he.tsindex 647fea5fe42..9fe7695fab9 100644--- a/studio/frontend/src/i18n/locales/he.ts+++ b/studio/frontend/src/i18n/locales/he.ts@@ -2871,6 +2871,11 @@ export const he = {showReplyTimesDescription: "הצג מתחת לכל תשובה מתי היא הסתיימה.",showReplyTimesKeywords:"חותמת זמן שעה תאריך מתי הסתיים תשובה הודעה timestamp time clock date reply",+ showGenerationDetails: "פרטי יצירה",+ showGenerationDetailsDescription:+ "הצגת מהירות ומספר אסימונים בזמן אמת בזמן שתשובה נוצרת, והיכן הוצבו השכבות של מודל מקומי.",+ showGenerationDetailsKeywords:+ "פרטים מהירות אסימונים סטטיסטיקה שכבות verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "עריכת תגובה על גבי התגובות",inlineEditResponseDescription:"הצג את 'עריכת תגובה' בכל תגובה, במקום בתפריט 'עוד'.",diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.tsindex 9e8aeae3c58..b21def95af8 100644--- a/studio/frontend/src/i18n/locales/hi.ts+++ b/studio/frontend/src/i18n/locales/hi.ts@@ -2764,6 +2764,11 @@ export const hi = {showReplyTimesDescription: "हर जवाब के नीचे दिखाएँ कि वह कब पूरा हुआ।",showReplyTimesKeywords:"टाइमस्टैम्प समय घड़ी तारीख कब पूरा जवाब संदेश timestamp time clock date reply",+ showGenerationDetails: "जनरेशन विवरण",+ showGenerationDetailsDescription:+ "जवाब बनते समय लाइव गति और टोकन गिनती दिखाएँ, और लोकल मॉडल की परतें कहाँ रखी गईं।",+ showGenerationDetailsKeywords:+ "विवरण गति टोकन आँकड़े परतें verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "जवाबों पर जवाब संपादित करें",inlineEditResponseDescription: "जवाब संपादित करें को More मेन्यू के बजाय हर जवाब पर दिखाएँ।",modelDisclaimer: "मॉडल अस्वीकरण दिखाएं",diff --git a/studio/frontend/src/i18n/locales/it.ts b/studio/frontend/src/i18n/locales/it.tsindex 7ffdea8431e..c696c90930e 100644--- a/studio/frontend/src/i18n/locales/it.ts+++ b/studio/frontend/src/i18n/locales/it.ts@@ -2699,6 +2699,11 @@ export const it = {showReplyTimesDescription: "Mostra sotto ogni risposta quando è terminata.",showReplyTimesKeywords:"orario ora data quando terminata risposta messaggio timestamp time clock date reply",+ showGenerationDetails: "Dettagli generazione",+ showGenerationDetailsDescription:+ "Mostra velocità e token in tempo reale mentre una risposta viene generata, e dove sono stati collocati i livelli di un modello locale.",+ showGenerationDetailsKeywords:+ "dettagli velocità token statistiche livelli scaricamento verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Modifica risposta nelle risposte",inlineEditResponseDescription: "Mostra Modifica risposta su ogni risposta, invece che nel menu Altro.",modelDisclaimer: "Mostra l'avviso sul modello",diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.tsindex f3ba6571bcb..e3e0d357d66 100644--- a/studio/frontend/src/i18n/locales/ja.ts+++ b/studio/frontend/src/i18n/locales/ja.ts@@ -2741,6 +2741,11 @@ export const ja = {showReplyTimesDescription: "各応答の下に、完了した時刻を表示します。",showReplyTimesKeywords:"タイムスタンプ 時刻 時間 日付 完了 応答 メッセージ timestamp time clock date reply",+ showGenerationDetails: "生成の詳細",+ showGenerationDetailsDescription:+ "応答の生成中に速度とトークン数をリアルタイムで表示し、ローカルモデルのレイヤーの配置も表示します。",+ showGenerationDetailsKeywords:+ "詳細 速度 トークン 統計 レイヤー オフロード verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "応答に応答の編集を表示",inlineEditResponseDescription: "応答の編集を「その他」メニューではなく、各応答に表示します。",modelDisclaimer: "モデルの免責事項を表示",diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.tsindex a4aeda644fa..48a8497073d 100644--- a/studio/frontend/src/i18n/locales/ko.ts+++ b/studio/frontend/src/i18n/locales/ko.ts@@ -2746,6 +2746,11 @@ export const ko = {showReplyTimesDescription: "각 응답 아래에 완료된 시간을 표시합니다.",showReplyTimesKeywords:"타임스탬프 시간 시각 날짜 완료 응답 메시지 timestamp time clock date reply",+ showGenerationDetails: "생성 세부 정보",+ showGenerationDetailsDescription:+ "응답이 생성되는 동안 속도와 토큰 수를 실시간으로 표시하고, 로컬 모델의 레이어가 어디에 배치되었는지 보여줍니다.",+ showGenerationDetailsKeywords:+ "세부 속도 토큰 통계 레이어 오프로드 verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "응답에 응답 편집 표시",inlineEditResponseDescription: "응답 편집을 더보기 메뉴 대신 모든 응답에 표시합니다.",modelDisclaimer: "모델 고지 표시",diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.tsindex 79c20db9b71..13f10b0881b 100644--- a/studio/frontend/src/i18n/locales/pt-br.ts+++ b/studio/frontend/src/i18n/locales/pt-br.ts@@ -2774,6 +2774,11 @@ export const ptBR = {showReplyTimesDescription: "Mostrar abaixo de cada resposta quando ela terminou.",showReplyTimesKeywords:"carimbo de data hora data quando terminou resposta mensagem timestamp time clock date reply",+ showGenerationDetails: "Detalhes da geração",+ showGenerationDetailsDescription:+ "Mostrar ao vivo a velocidade do prompt e da resposta com a contagem de tokens, e onde as camadas de um modelo foram carregadas.",+ showGenerationDetailsKeywords:+ "detalhes geração velocidade tokens estatísticas camadas especialistas desenvolvedor verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Editar resposta nas respostas",inlineEditResponseDescription: "Mostrar Editar resposta em cada resposta, em vez de no menu Mais.",modelDisclaimer: "Mostrar aviso do modelo",diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.tsindex 51261942772..71741fd1208 100644--- a/studio/frontend/src/i18n/locales/ru.ts+++ b/studio/frontend/src/i18n/locales/ru.ts@@ -2772,6 +2772,11 @@ export const ru = {showReplyTimesDescription: "Показывать под каждым ответом, когда он завершился.",showReplyTimesKeywords:"время метка дата когда завершено ответ сообщение timestamp time clock date reply",+ showGenerationDetails: "Подробности генерации",+ showGenerationDetailsDescription:+ "Показывать скорость и число токенов в реальном времени во время ответа и где размещены слои локальной модели.",+ showGenerationDetailsKeywords:+ "подробности скорость токены статистика слои выгрузка verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Изменение ответа в ответах",inlineEditResponseDescription: "Показывать «Изменить ответ» у каждого ответа, а не в меню «Ещё».",modelDisclaimer: "Показывать предупреждение о модели",diff --git a/studio/frontend/src/i18n/locales/sv.ts b/studio/frontend/src/i18n/locales/sv.tsindex 1d89e28cac1..95a4ea693c6 100644--- a/studio/frontend/src/i18n/locales/sv.ts+++ b/studio/frontend/src/i18n/locales/sv.ts@@ -2954,6 +2954,11 @@ export const sv = {showReplyTimesDescription: "Visa under varje svar när det blev klart.",showReplyTimesKeywords:"tidsstämpel tid klocka datum när klar svar meddelande timestamp time clock date reply",+ showGenerationDetails: "Genereringsdetaljer",+ showGenerationDetailsDescription:+ "Visa hastighet och antal token live medan ett svar strömmas, och var en lokal modells lager placerades.",+ showGenerationDetailsKeywords:+ "detaljer hastighet token statistik lager avlastning verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "Redigera svar på svaren",inlineEditResponseDescription:"Visa Redigera svar på varje svar i stället för i menyn Mer.",diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.tsindex ad00ea959a9..a9db0f9d320 100644--- a/studio/frontend/src/i18n/locales/zh-CN.ts+++ b/studio/frontend/src/i18n/locales/zh-CN.ts@@ -2712,6 +2712,11 @@ export const zhCN = {showReplyTimesDescription: "在每条回复下方显示其完成的时间。",showReplyTimesKeywords:"时间戳 时间 时刻 日期 完成 回复 消息 timestamp time clock date reply",+ showGenerationDetails: "生成详情",+ showGenerationDetailsDescription:+ "在回复生成时实时显示速度和 token 数,并显示本地模型各层的放置位置。",+ showGenerationDetailsKeywords:+ "详情 速度 统计 层 卸载 verbose developer stats speed tok/s prefill decode layers offload vram experts",inlineEditResponse: "在回复上显示编辑回复",inlineEditResponseDescription: "在每条回复上显示编辑回复,而不是放在“更多”菜单中。",modelDisclaimer: "显示模型免责声明",diff --git a/studio/frontend/tests/generation-details.test.ts b/studio/frontend/tests/generation-details.test.tsnew file mode 100644index 00000000000..870d1ed25e9--- /dev/null+++ b/studio/frontend/tests/generation-details.test.ts@@ -0,0 +1,279 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// Live prefill / decode figures, the Generation details setting, and the loaded placement readout.++import assert from "node:assert/strict";+import test from "node:test";++import type { LiveDecodeProgress, LivePrefillProgress } from "../src/features/chat/stores/chat-runtime-store.ts";+import { readSrc, registerBundlerResolver } from "./helpers/kit.ts";++registerBundlerResolver();+const {+ formatLiveDecode,+ formatPrefillDetails,+ formatPrefillTimeLeft,+ prefillLabelShown,+ prefillPercent,+ readGenerationProgress,+} = await import("../src/features/chat/lib/generation-progress.ts");+const { describePlacement, loadedPlacementFrom } = await import(+ "../src/features/chat/lib/partial-offload.ts"+);+const { useChatPreferencesStore } = await import(+ "../src/features/chat/stores/chat-preferences-store.ts"+);+const { normalizeChatGenerationChunkPayload } = await import(+ "../src/features/chat/api/chat-generation-api.ts"+);+const { generationChunkCountsTowardTiming } = await import(+ "../src/features/chat/utils/chat-generation-recovery.ts"+);++test("prefill frames read into store shape, clamped, and give a whole percent", () => {+ const update = readGenerationProgress({+ type: "generation_progress",+ phase: "prefill",+ processed: 6000,+ total: 5316,+ cached: 7000,+ tok_per_s: 1326.4,+ });+ assert.equal(update?.phase, "prefill");+ if (update?.phase !== "prefill") return;+ assert.deepEqual(update.prefill, { processed: 5316, total: 5316, cached: 7000, tokPerS: 1326.4 });+ assert.equal(prefillPercent({ processed: 2048, total: 5316, cached: 0, tokPerS: null }), 38);+ assert.equal(prefillPercent({ processed: 10, total: null, cached: null, tokPerS: null }), null);+});++test("a follow-up in a long chat counts only the part the cache did not cover", () => {+ // 29,000 of 30,000 tokens reused: halfway through the new 1,000, not 98% of the whole prompt.+ const followUp = { processed: 29500, total: 30000, cached: 29000, tokPerS: 1000 };+ assert.equal(prefillPercent(followUp), 50);+ assert.equal(formatPrefillDetails(followUp, "en-US"), "500 / 1,000 tokens · 1,000 tok/s");+ // llama.cpp evaluates at least one token, but a cache that covers it all is done, not NaN.+ assert.equal(prefillPercent({ processed: 30000, total: 30000, cached: 30000, tokPerS: null }), 100);+});++test("the time left follows the rate so far and drops out under a second", () => {+ assert.equal(formatPrefillTimeLeft({ processed: 4118, total: 18880, cached: 0, tokPerS: 1401.2 }), "11s left");+ assert.equal(formatPrefillTimeLeft({ processed: 5000, total: 200000, cached: 0, tokPerS: 1500 }), "2m 10s left");+ // The first sample's rate is noise, and at the end there is nothing worth counting down.+ assert.equal(formatPrefillTimeLeft({ processed: 22, total: 18844, cached: 0, tokPerS: 242 }), null);+ assert.equal(formatPrefillTimeLeft({ processed: 18500, total: 18880, cached: 0, tokPerS: 1400 }), null);+});++test("malformed frames are dropped rather than drawn as NaN", () => {+ assert.equal(readGenerationProgress(null), null);+ assert.equal(readGenerationProgress({ phase: "prefill", processed: "12" }), null);+ assert.equal(readGenerationProgress({ phase: "decode", tokens: 0 }), null);+ assert.equal(readGenerationProgress({ phase: "warmup" }), null);+});++test("decode frames keep draft acceptance only when both counts are there", () => {+ const update = readGenerationProgress({ phase: "decode", tokens: 40, tok_per_s: 90, draft_n: 23, draft_accepted: 30 });+ assert.equal(update?.phase, "decode");+ if (update?.phase !== "decode") return;+ assert.deepEqual(update.decode, { tokens: 40, tokPerS: 90, draftN: 23, draftAccepted: 23 });+});++test("the prefill details give the rate when there is one", () => {+ assert.equal(+ formatPrefillDetails({ processed: 4118, total: 18880, cached: 0, tokPerS: 1401.2 }, "en-US"),+ "4,118 / 18,880 tokens · 1,401 tok/s",+ );+ assert.equal(formatPrefillDetails({ processed: 900, total: null, cached: null, tokPerS: null }, "en-US"), "900 tokens");+});++test("the prefill label shows only for a pass expected to keep it up, then stays for that pass", () => {+ const sample = (processed: number, total: number, tokPerS: number | null, cached = 0) => ({+ processed,+ total,+ cached,+ tokPerS,+ });+ const after = (prefill: LivePrefillProgress, prefillShown: boolean, decode: LiveDecodeProgress | null = null) => ({+ prefill,+ prefillShown,+ decode,+ });+ // llama.cpp's first sample, a few tokens in: its rate is noise, so no estimate yet.+ assert.equal(prefillLabelShown(undefined, sample(22, 18844, 242)), false);+ // One batch into a long prompt: about 10 s left.+ const long = sample(4118, 18880, 1401.2);+ assert.equal(prefillLabelShown(after(sample(22, 18844, 242), false), long), true);+ // Stays up near the end, where the estimate alone would drop it.+ assert.equal(prefillLabelShown(after(long, true), sample(18000, 18880, 1390)), true);+ // A 5,000-token prompt at 2,600 tok/s has about 1 s left after its first batch, so it never shows.+ const fast = sample(2048, 5000, 2600);+ assert.equal(prefillLabelShown(after(sample(0, 5000, null), false), fast), false);+ assert.equal(prefillLabelShown(after(fast, false), sample(4096, 5000, 2600)), false);+ // A cache hit leaves too few fresh tokens for a rate.+ assert.equal(prefillLabelShown(undefined, sample(29000, 30000, 0.5, 29000)), false);+ // The next pass of a tool loop starts over: decode figures came in between.+ const decode = { tokens: 40, tokPerS: 50, draftN: null, draftAccepted: null };+ assert.equal(prefillLabelShown(after(long, true, decode), sample(2048, 5000, 2600)), false);+});++test("the live line writes every number in the reader's locale", () => {+ const decode = { tokens: 3060, tokPerS: 58.25, draftN: 5030, draftAccepted: 1791 };+ assert.equal(+ formatLiveDecode(decode, "en-US"),+ "3,060 tokens · 58.3 tok/s · draft 1,791/5,030 accepted (36%)",+ );+ // A locale that groups with "." must not get "." as the rate's decimal point too.+ assert.equal(+ formatLiveDecode(decode, "de-DE"),+ "3.060 tokens · 58,3 tok/s · draft 1.791/5.030 accepted (36%)",+ );+});++test("both live lines wrap instead of clipping on a narrow chat column", () => {+ // .generating-glint is nowrap and overflow: hidden; at 390 px the prefill details and the decode line ran past the edge.+ const thread = readSrc("components/assistant-ui/thread.tsx");+ const prefill = thread.slice(thread.indexOf('data-slot="prefill-progress"'), thread.indexOf("Processing prompt"));+ assert.match(prefill, /whitespace-normal!/);+ const line = readSrc("components/assistant-ui/live-generation-line.tsx");+ assert.doesNotMatch(line, /\bshrink-0\b/);+ assert.match(line, /\bmin-w-0\b/);+});++test("Generation details is off by default and wired into Settings > Chat > Display", () => {+ assert.equal(useChatPreferencesStore.getInitialState().showGenerationDetails, false);+ const store = readSrc("features/chat/stores/chat-preferences-store.ts");+ assert.match(store, /showGenerationDetails: saved\?\.showGenerationDetails === true/);+ const tab = readSrc("features/settings/tabs/chat-tab.tsx");+ const times = tab.indexOf('t("settings.chat.showReplyTimes")');+ const details = tab.indexOf('t("settings.chat.showGenerationDetails")');+ assert.ok(times !== -1 && details > times, "the switch sits after Reply times");+ assert.match(readSrc("features/settings/settings-search.ts"), /"settings\.chat\.showGenerationDetailsKeywords"/);+});++test("the live line gates on the preference before any assistant-ui selector", () => {+ const line = readSrc("components/assistant-ui/live-generation-line.tsx");+ const gate = line.slice(line.indexOf("export const LiveGenerationLine"), line.indexOf("const RunningGate"));+ assert.match(gate, /return enabled \? <RunningGate \/> : null;/);+ assert.doesNotMatch(gate, /useAuiState/);+});++test("persisted runs re-tag generation_progress and keep it out of timing", () => {+ const frame = { type: "generation_progress", phase: "prefill", processed: 1, total: 2 };+ const normalized = normalizeChatGenerationChunkPayload(frame) as { _generationProgress?: unknown };+ assert.deepEqual(normalized._generationProgress, frame);+ assert.equal(generationChunkCountsTowardTiming(normalized), false);+});++// The runtime store pulls in the app shell, so like chat-model-residency.test.ts this reads its source.+test("live progress is per thread, transient, and re-keyed with the thread's other run handles", () => {+ const store = readSrc("features/chat/stores/chat-runtime-store.ts");+ // A new prompt pass (the next round of a tool loop) drops the previous round's decode figures.+ assert.match(store, /\[key\]: \{ prefill, prefillShown, decode: null \}/);+ assert.match(store, /move\(state\.liveGenerationByThreadId, "liveGenerationByThreadId"\);/);+ const adapter = readSrc("features/chat/api/chat-adapter.ts");+ assert.match(+ adapter,+ /clearActiveDiffusionCanvasForThread\(cleanupKey\);\s*runtime\.clearLiveGenerationForThread\(cleanupKey\);/,+ );+ // Handled before the first-token stamp, like the other control frames.+ const progress = adapter.indexOf("_generationProgress");+ const firstToken = adapter.indexOf("firstTokenTime = Date.now() - streamStartTime;");+ assert.ok(progress !== -1 && progress < firstToken);+});++test("the placement readout names expert weights and gives both memory figures", () => {+ // gemma-4-26B-A4B Q8_0 on a 3090: --fit's verdict "31 layers ( 8 overflowing)", the first of them GATE-only.+ assert.deepEqual(+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 8, gpuWeightsMib: 20709.15, hostWeightsMib: 5639.4, expertCacheMib: null }),+ { layers: "All 31 layers on the GPU, expert weights of 8 layers in RAM", memory: "20 GiB in VRAM · 5.5 GiB in RAM" },+ );+ assert.equal(+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 1, gpuWeightsMib: 20000, hostWeightsMib: 900, expertCacheMib: null })+ .layers,+ "All 31 layers on the GPU, expert weights of 1 layer in RAM",+ );+ assert.deepEqual(+ describePlacement({ offloaded: 41, total: 41, overflowLayers: 0, gpuWeightsMib: 8161.75, hostWeightsMib: 417.3, expertCacheMib: null }),+ { layers: "All 41 layers on the GPU", memory: "8.0 GiB in VRAM · 0.4 GiB in RAM" },+ );+ assert.deepEqual(+ describePlacement({ offloaded: 20, total: 31, overflowLayers: null, gpuWeightsMib: 12288, hostWeightsMib: 5120, expertCacheMib: null }),+ { layers: "20 of 31 layers on the GPU, 11 on the CPU", memory: "12 GiB in VRAM · 5.0 GiB in RAM" },+ );+ // A Manual load reports its expert offload as n_cpu_moe, not through --fit.+ assert.equal(+ loadedPlacementFrom({ is_gguf: true, offloaded_layers: 31, offload_total_layers: 31, gpu_memory_mode: "manual", n_cpu_moe: 15 })+ ?.overflowLayers,+ 15,+ );+ assert.equal(+ loadedPlacementFrom({ is_gguf: true, offloaded_layers: 31, offload_total_layers: 31, gpu_memory_mode: "auto", n_cpu_moe: 15 })+ ?.overflowLayers,+ null,+ );+ assert.equal(loadedPlacementFrom({ is_gguf: false, offload_total_layers: 31 }), null);+ assert.equal(loadedPlacementFrom({ is_gguf: true }), null);+});++test("a GPU build that fell back to the CPU is not reported as GPU layers", () => {+ // CUDA build, no usable device: llama.cpp still logs "offloaded 29/29 layers to GPU", buffers AMX + CPU_Mapped.+ assert.deepEqual(+ describePlacement({ offloaded: 29, total: 29, overflowLayers: null, gpuWeightsMib: 0, hostWeightsMib: 499.8, expertCacheMib: null }),+ { layers: "All 29 layers on the CPU", memory: "0.5 GiB in RAM" },+ );+});++test("--moe-cache-mib auto reads as every expert in RAM with a GPU cache", () => {+ // --fit names no layer on this path; the backend reports the model's MoE layer count instead.+ const placement = loadedPlacementFrom({+ is_gguf: true,+ offloaded_layers: 31,+ offload_total_layers: 31,+ overflow_layers: 30,+ gpu_weights_mib: 3900,+ host_weights_mib: 22400,+ expert_cache_mib: 6144,+ });+ assert.ok(placement);+ assert.deepEqual(describePlacement(placement), {+ layers: "All 31 layers on the GPU, all experts in RAM (6.0 GiB expert cache on the GPU)",+ memory: "3.8 GiB in VRAM · 22 GiB in RAM",+ });+});++test("one memory pool gets one figure, and amounts under 0.1 GiB read in MiB", () => {+ // Qwen3.5-4B Q4_K_M under Metal on an M1 Mac mini: MTL0 2,603.5 MiB and CPU 497.3 MiB, one pool.+ const metal = { offloaded: 33, total: 33, overflowLayers: null, gpuWeightsMib: 2603.5, hostWeightsMib: 497.31, expertCacheMib: null };+ assert.deepEqual(describePlacement(metal, "unified memory"), {+ layers: "All 33 layers on the GPU",+ memory: "3.0 GiB in unified memory",+ });+ assert.equal(describePlacement(metal).memory, "2.5 GiB in VRAM · 0.5 GiB in RAM");+ // A small model's token embeddings would otherwise print as "0.0 GiB in RAM".+ assert.equal(+ describePlacement({ offloaded: 31, total: 31, overflowLayers: 0, gpuWeightsMib: 112.6, hostWeightsMib: 28.7, expertCacheMib: null }).memory,+ "0.1 GiB in VRAM · 29 MiB in RAM",+ );+});++test("the prefill label gives way once decoding started, not at 100%", () => {+ const thread = readSrc("components/assistant-ui/thread.tsx");+ const status = thread.slice(thread.indexOf("const GeneratingStatus"), thread.indexOf("// Placeholder when stop fires"));+ assert.match(status, /!labelShown \|\| percent === null \|\| decoding\)/);+});++test("a followed run (reload, second tab) shows live figures and clears them on exit", () => {+ const provider = readSrc("features/chat/runtime-provider.tsx");+ assert.match(provider, /if \("_generationProgress" in chunk\) \{\s*if \(caughtUp\(update\.run\)\)/);+ assert.match(provider, /\} finally \{\s*useChatRuntimeStore\.getState\(\)\.clearLiveGenerationForThread\(threadId\);/);+});++test("a background load cannot drop the visible model's loaded placement", () => {+ // setCheckpoint clears loadedPlacement when the visible model is restored, so the snapshot must carry it back.+ const adapter = readSrc("features/chat/api/chat-adapter.ts");+ const keys = adapter.slice(+ adapter.indexOf("const VISIBLE_MODEL_RUNTIME_KEYS = ["),+ adapter.indexOf("] as const satisfies"),+ );+ assert.match(keys, /"loadedPlacement"/);+});