Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Long-chat gate: measure streaming a reply into the chat (#17269)
Daniel Han0739e88a8eca
diff --git a/tests/studio/_area_matrix.py b/tests/studio/_area_matrix.pyindex eab067e2b42..4802ccc8e46 100644--- a/tests/studio/_area_matrix.py+++ b/tests/studio/_area_matrix.py@@ -58,6 +58,9 @@ LENGTH_RATIO = float(os.environ.get("LONG_THREAD_AREAS_LENGTH_RATIO", "2.0"))# Counts below this many elements / layout objects at LONG turns never fail the length check.COUNT_FLOOR = int(os.environ.get("LONG_THREAD_AREAS_COUNT_FLOOR", "3000"))MAX_RATIO = float(os.environ.get("LONG_THREAD_AREAS_MAX_RATIO", "4.0"))+# Also judged on CPU growth with chat length: a streamed reply restyles only its own message, so its+# element / layout counts stay flat while per-token work over the thread grows.+CPU_LENGTH = {"stream": float(os.environ.get("LONG_THREAD_AREAS_STREAM_LENGTH_RATIO", "3.0"))}# CPU ms a mode must add on top of the ratio before it fails.CPU_FLOOR_MS = float(os.environ.get("LONG_THREAD_AREAS_CPU_FLOOR_MS", "150"))# Ratios are taken over at least this many ms of the base's CPU: a 20 ms click is not a fair yardstick.@@ -141,11 +144,100 @@ def known_limits(text: str) -> dict[tuple[str, str], float]:# --- areas ---------------------------------------------------------------------------------------# name: {open, ready, close, gone, setup, teardown, covers}# actions: ("key", chord) | ("click", css) | ("rclick", css) | ("hover", css) | ("js", expr) | ("type", text)+# | ("compose", text) the composer holds text (inserted at once)# | ("scroll", where) | ("tabs", None) | None ; chords use Mod for Meta on macOS, Control elsewhere# ready / gone: ("overlay",) a new visible dialog/menu/listbox | ("overlay_gone",) | ("sel", css) | ("gone", css)-# | ("settle",) wait only+# | ("settle",) wait only | ("streamed", text) a streamed reply ended with text# setup / teardown run outside the measured window; covers: trigger keys (see trigger_key) it measures.# "last|css": inside the last assistant message; "js|expr": an expression returning the element.+# Fixed frames at a fixed cadence, answered in-page, so every run renders the same tokens on the same clock.+STREAM_PROMPT = "Explain the scheduler buffer in detail."+# The model the stub reports resident (/api/inference/status); its chip is the model picker trigger.+STUB_MODEL = "unsloth/Qwen3-4B"+STREAM_END = "GATE-STREAM-END"+STREAM_MS = 25+++def stream_frames() -> list[dict]:+ """The reply as SSE payloads: reasoning, a web search tool call, prose, a code fence, a list."""++ def delta(d: dict) -> dict:+ return {+ "id": "gate",+ "object": "chat.completion.chunk",+ "created": 0,+ "model": STUB_MODEL,+ "choices": [{"index": 0, "delta": d, "finish_reason": None}],+ }++ words = PARA.split(" ")+ frames = [delta({"role": "assistant"})]+ frames += [+ delta({"reasoning_content": f"Step {k}: " + " ".join(words[:8]) + ". "}) for k in range(20)+ ]+ q = {"query": "scheduler buffer stream"}+ frames.append(+ {"type": "tool_start", "tool_name": "web_search", "tool_call_id": "gate_ws", "arguments": q}+ )+ frames.append(+ {"type": "tool_end", "tool_name": "web_search", "tool_call_id": "gate_ws", "result": PARA}+ )+ for para in range(3):+ frames += [+ delta({"content": " ".join(words[i : i + 4]) + " "}) for i in range(0, len(words), 4)+ ]+ frames.append(delta({"content": "\n\n"}))+ frames.append(delta({"content": "```python\n"}))+ frames += [delta({"content": line + "\n"}) for line in CODE.splitlines()]+ frames.append(delta({"content": "```\n\n"}))+ frames += [delta({"content": f"- item {k}: {PARA[:60]}\n"}) for k in range(6)]+ frames.append(delta({"content": f"\nDone. {STREAM_END}"}))+ frames.append(+ {+ "id": "gate",+ "object": "chat.completion.chunk",+ "created": 0,+ "model": STUB_MODEL,+ "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],+ }+ )+ return frames+++STREAM_TIMEOUT_MS = 30_000+# Polled inside the measured window: no DOM walk until the shim has sent its last frame.+STREAMED = """(end) => {+ if (!window.__gateStreamDone) return false;+ const all = document.querySelectorAll('[data-role="assistant"]');+ const a = all[all.length - 1];+ return !!a && (a.textContent || '').includes(end) && !document.querySelector('.aui-composer-cancel');+}"""++# Answers /v1/chat/completions from the page with STREAM's frames, one per STREAM_MS.+STREAM_SHIM = """(([frames, ms]) => {+ const realFetch = window.fetch.bind(window);+ window.__gateStreams = 0;+ window.fetch = (input, init) => {+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;+ if (!/\\/v1\\/chat\\/completions(\\?|$)/.test(url)) return realFetch(input, init);+ window.__gateStreams++;+ window.__gateStreamDone = false;+ const enc = new TextEncoder();+ const out = frames.map((f) => 'data: ' + JSON.stringify(f) + '\\n\\n').concat(['data: [DONE]\\n\\n']);+ let i = 0;+ const body = new ReadableStream({+ pull(ctrl) {+ return new Promise((done) => setTimeout(() => {+ if (i < out.length) ctrl.enqueue(enc.encode(out[i++]));+ else { ctrl.close(); window.__gateStreamDone = true; }+ done();+ }, ms));+ },+ });+ return Promise.resolve(new Response(body, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }));+ };+})"""+LAST = "last|"CODE_COPY = "js|(() => { const a = [...document.querySelectorAll('[data-role=\"assistant\"]')].pop(); const pre = a && a.querySelector('pre'); let n = pre; while (n && n !== a && !n.querySelector('button')) n = n.parentElement; return n && n !== a ? n.querySelector('button') : null; })()"AREAS: dict[str, dict] = {@@ -217,7 +309,7 @@ AREAS: dict[str, dict] = {"ready": ("overlay",),"close": ("key", "Escape"),"gone": ("overlay_gone",),- "covers": ["popover-trigger:Select model"],+ "covers": ["popover-trigger:Select model", f"popover-trigger:{STUB_MODEL.split('/')[-1]}"],},"composer_menu": {"open": ("key", "Mod+Shift+Period"),@@ -372,6 +464,15 @@ AREAS: dict[str, dict] = {"gone": ("gone", '[data-slot="hover-card-content"]'),"covers": ["hover-card-trigger:*"],},+ # Last on purpose: it adds a turn to the thread.+ "stream": {+ "setup": ("compose", STREAM_PROMPT),+ "open": ("key", "Enter"),+ "ready": ("streamed", STREAM_END),+ "close": None,+ "gone": ("settle",),+ "covers": [],+ },}# Popup triggers on the chat page that are not measured, with why. A trigger that is neither here@@ -379,6 +480,9 @@ AREAS: dict[str, dict] = {NOT_AREAS = {"popover-trigger:Request failed": "the gate's API stub has no model runtime; this chip only shows a failed status call","dropdown-menu-trigger:Open preset list": "inside the run settings panel, measured as run_settings",+ "dropdown-menu-trigger:Run settings for this model": "in the run settings sheet (mounted off screen once a model is resident), measured as run_settings",+ "select-trigger:Inference engine": "in the run settings sheet (mounted off screen once a model is resident), measured as run_settings",+ "message-timing-trigger:Message timing": "a hover readout on the reply the stream area streamed; the seeded turns carry no timing",}# Keyboard shortcuts (features/settings/lib/keyboard-shortcuts.ts): the area that measures what it@@ -569,6 +673,13 @@ class Api:"/api/hub/hidden-models": {"hidden": []},"/api/settings/pinned-models": {"pinned": []},"/api/health": {"status": "ok"},+ "/api/inference/status": {+ "active_model": STUB_MODEL,+ "model_identifier": STUB_MODEL,+ "is_vision": False,+ "is_gguf": False,+ },+ "/api/chat/settings": {"settings": {}},}def handle(self, route) -> None:@@ -578,6 +689,10 @@ class Api:body = {"messagesByThreadId": {self.rec["id"]: self.msgs}}elif req.method == "GET" and path in self.bodies:body = self.bodies[path]+ elif req.method == "PUT" and path.startswith(+ f"/api/chat/threads/{self.rec['id']}/messages"+ ):+ body = {} # the streamed turn is saved; nothing reads it backelse:return route.fulfill(status = 404, content_type = "application/json", body = "{}")route.fulfill(status = 200, content_type = "application/json", body = json.dumps(body))@@ -794,6 +909,7 @@ class Page:"(() => { const v = %s; for (const k in v) localStorage.setItem(k, v[k]); })()"% json.dumps(init))+ self.ctx.add_init_script(f"{STREAM_SHIM}({json.dumps([stream_frames(), STREAM_MS])})")self.ctx.route(lambda url: urlparse(url).path.startswith("/api/"), self.api.handle)self.ctx.route(lambda url: not url.startswith(base), lambda route: route.fulfill(status = 204, body = "")@@ -841,6 +957,10 @@ class Page:page.keyboard.press(ch)page.wait_for_timeout(40)return True+ if kind == "compose":+ page.click(".aui-composer-input", timeout = 3_000)+ page.keyboard.insert_text(arg)+ return Trueif kind == "scroll":return bool(page.evaluate(SCROLL, [arg, 20, 60 if arg == "thread" else 40]))if kind == "tabs":@@ -910,6 +1030,11 @@ class Page:arg = [pred[1], pred[2], self.attr_before],timeout = 3_000,)+ elif kind == "streamed":+ # A reply that never draws is NOT_RUN, which fails on the default page.+ self.page.wait_for_function(+ STREAMED, arg = pred[1], timeout = STREAM_TIMEOUT_MS, polling = 50+ )elif kind == "focusin":self.page.wait_for_function("([s, n]) => window.__gateFocusins.slice(n).some((t) => t instanceof Element && t.closest(s))",@@ -1511,6 +1636,24 @@ def judge(result: dict, known: str | None = None) -> list[dict]:continuelimit = limits.get(("length", area), LENGTH_RATIO)cap = limits.get(("count", area))+ if area in CPU_LENGTH:+ lv, sv = cheapest(lc, "cpu_ms"), cheapest(sc, "cpu_ms")+ ratio = lv / max(sv, CPU_BASE_FLOOR_MS)+ cpu_limit = limits.get(("length", area), CPU_LENGTH[area])+ out.append(+ {+ "kind": "length",+ "metric": "cpu_ms",+ "turns": turns,+ "mode": mode,+ "area": area,+ "value": round(lv),+ "base_value": round(sv),+ "ratio": ratio,+ "limit": cpu_limit,+ "fail": ratio > cpu_limit and lv - sv >= CPU_FLOOR_MS,+ }+ )for metric in ("elements", "layout_objects"):lv, sv = lc.get(metric, 0), sc.get(metric, 0)ratio = lv / max(sv, 1)@@ -1627,7 +1770,7 @@ def run(for v in verdictsif v["fail"]and (- v["kind"] == "appearance"+ v["kind"] in ("appearance", "length")and v["metric"] == "cpu_ms"or v["kind"] == "broken"and v["area"] != "*"@@ -1667,6 +1810,15 @@ def run(if "cap" in velse f"limit {v['limit']:.1f}x; LONG_THREAD_AREAS_LENGTH_RATIO / _KNOWN length:{v['area']}=...")+ if v["metric"] == "cpu_ms":+ failures.append(+ f"area {v['area']} ({v['mode']}) costs more as the chat grows: {v['base_value']:,} ms CPU at "+ f"{SHORT} turns -> {v['value']:,} at {LONG} turns = {v['ratio']:.1f}x (limit {v['limit']:.1f}x; "+ f"LONG_THREAD_AREAS_STREAM_LENGTH_RATIO / _KNOWN length:{v['area']}=...). Per-token work walks the "+ "whole thread: a selector or effect over every message, a whole-thread layout read, or more "+ "publishes than frames (src/features/chat/utils/stream-pacing.ts)"+ )+ continuefailures.append(f"area {v['area']} ({v['mode']}) costs more as the chat grows: {v['metric'].replace('_', ' ')} "f"{v['base_value']:,} at {SHORT} turns -> {v['value']:,} at {LONG} turns = {v['ratio']:.1f}x ({bound}). "diff --git a/tests/studio/test_area_matrix.py b/tests/studio/test_area_matrix.pyindex d1bad6742fd..2f79f9e081c 100644--- a/tests/studio/test_area_matrix.py+++ b/tests/studio/test_area_matrix.py@@ -465,3 +465,76 @@ def test_settings_ready_marker_is_set_from_inside_the_panel_suspense():"data-settings-panel=" not in src), "a pane-level marker would read ready before the panel resolves"assert "data-settings-panel-ready=" in am.TABS and "data-settings-panel-failed" in am.TABS+++def _stream_result(pages):+ return {"pages": pages, "modes": {m["name"]: m for m in am.app_modes()}, "areas": ["stream"]}+++ "short, long_, fails",+ # main reads 1.4-2.1x for a 2 s stream; 3x is the limit; a small absolute rise never fails+ [(2500, 5200, False), (2500, 9000, True), (40, 180, False), (2500, 7400, False)],+)+def test_stream_is_judged_on_cpu_growth_with_the_chat(short, long_, fails):+ cell = lambda cpu: {"elements": 1200, "layout_objects": 3000, "cpu_ms": cpu}+ pages = {+ (am.SHORT, "default"): _page({"stream": cell(short)}),+ (am.LONG, "default"): _page({"stream": cell(long_)}),+ }+ rows = [+ v+ for v in am.judge(_stream_result(pages), known = "")+ if v["kind"] == "length" and v["metric"] == "cpu_ms"+ ]+ assert [v["fail"] for v in rows] == [fails]+ counts = [v for v in am.judge(_stream_result(pages), known = "") if v["metric"] == "elements"]+ assert counts and not any(v["fail"] for v in counts), "flat counts never fail"+++def test_stream_area_is_last_sends_a_fixed_reply_and_waits_for_its_end():+ assert list(am.AREAS)[-1] == "stream", "it adds a turn, so it runs after every other area"+ spec = am.AREAS["stream"]+ assert spec["setup"] == ("compose", am.STREAM_PROMPT)+ assert spec["open"] == ("key", "Enter") and spec["ready"] == ("streamed", am.STREAM_END)+ frames = am.stream_frames()+ text = "".join(f["choices"][0]["delta"].get("content", "") for f in frames if "choices" in f)+ assert text.rstrip().endswith(am.STREAM_END), "the end text is the last thing drawn"+ assert "```python" in text and "- item 5" in text, "prose, a code fence and a list"+ assert any(f["choices"][0]["delta"].get("reasoning_content") for f in frames if "choices" in f)+ kinds = [f.get("type") for f in frames]+ assert kinds.count("tool_start") == 1 and kinds.count("tool_end") == 1+ assert frames[-1]["choices"][0]["finish_reason"] == "stop"+ assert am.stream_frames() == frames, "deterministic"+++def test_the_stub_has_a_resident_model_and_saves_the_streamed_turn():+ api = am.Api(2)+ assert api.bodies["/api/inference/status"]["active_model"]++ class Route:+ def __init__(self, method, path):+ self.request = type("R", (), {"method": method, "url": "http://x" + path})()+ self.got = None++ def fulfill(+ self,+ status,+ content_type = None,+ body = "",+ ):+ self.got = status++ put = Route("PUT", f"/api/chat/threads/{api.rec['id']}/messages/abc")+ api.handle(put)+ assert put.got == 200+ other = Route("PUT", "/api/chat/threads/other/messages")+ api.handle(other)+ assert other.got == 404, "only the seeded thread's saves are answered"+++def test_stream_wait_is_cheap_while_the_reply_streams():+ # polled inside the measured window: no DOM walk until the shim has sent its last frame+ body = am.STREAMED.split("if (!window.__gateStreamDone) return false;")+ assert len(body) == 2 and "querySelector" not in body[0]+ assert "__gateStreamDone = true" in am.STREAM_SHIM