Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stream durable chat run snapshots without re-parsing the stored request (#16302)
Daniel Han2c2870ff5265
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.pyindex 643e0acb3b6..f84c968f1cc 100644--- a/studio/backend/core/inference/chat_generation_runs.py+++ b/studio/backend/core/inference/chat_generation_runs.py@@ -657,10 +657,9 @@ class ChatGenerationSupervisor:return failuretry:- worker_run = await asyncio.to_thread(db.get_worker_run, run_id)- if worker_run is None:+ worker_token = await asyncio.to_thread(db.get_worker_token, run_id)+ if worker_token is None:return- run, owner, worker_token = worker_runif cancel_event.is_set():shutting_down = run_id in self._shutdown_runsawait asyncio.to_thread(diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.pyindex 2f6ae35c3d4..958d800bf45 100644--- a/studio/backend/routes/chat_generation_runs.py+++ b/studio/backend/routes/chat_generation_runs.py@@ -269,6 +269,12 @@ def _require_run(run_id: str) -> dict[str, Any]:return run+def _require_run_exists(run_id: str) -> None:+ """_require_run's 404 without reading or parsing the stored request."""+ if db.get_run_state(run_id) is None:+ raise HTTPException(status_code = 404, detail = "Chat generation run not found")++def cancel_account_run(request: Request, run_id: str, *, supervisor_name: str) -> None:"""Signal only the caller's registration: another account may legitimately reuse a barecancel ID, so it is never stashed."""@@ -405,7 +411,7 @@ def cancel_chat_generation_run(request: Request,current_subject: str = Depends(get_current_subject),):- _require_run(run_id)+ _require_run_exists(run_id)run = db.request_cancel(run_id)if run is None:raise HTTPException(status_code = 404, detail = "Chat generation run not found")@@ -425,7 +431,7 @@ async def chat_generation_events(last_event_id: str | None = Header(None, alias = "Last-Event-ID"),current_subject: str = Depends(get_current_subject),):- _require_run(run_id)+ _require_run_exists(run_id)cursor = _event_cursor(after, last_event_id)wait_for_events = db.wait_for_eventsif policy.installation_has_managed_accounts():@@ -440,7 +446,7 @@ async def chat_generation_events(nonlocal cursorloop = asyncio.get_running_loop()# A settled run has nothing to replay; don't hold an event-wait worker for it.- opening = await asyncio.to_thread(db.get_run, run_id)+ opening = await asyncio.to_thread(db.get_run_state, run_id)if opening is None:returnif opening["status"] in db.TERMINAL_STATUSES and cursor >= int(opening["lastEventSeq"]):@@ -456,16 +462,15 @@ async def chat_generation_events(15,True,)- # Only non-chunk events carry the run, so a chunk-only batch skips parsing request_json.- snapshot = await asyncio.to_thread(- db.get_run- if any(event["type"] != "chunk" for event in events)- else db.get_run_state,- run_id,- )+ # Only non-chunk events carry the run, spliced in as already encoded text.+ run_json = None+ if any(event["type"] != "chunk" for event in events):+ frame = await asyncio.to_thread(db.get_run_frame, run_id)+ snapshot, run_json = frame if frame is not None else (None, None)+ else:+ snapshot = await asyncio.to_thread(db.get_run_state, run_id)if snapshot is None:return- run_json = Nonefor event in events:cursor = int(event["seq"])event_type = event["type"]@@ -476,8 +481,6 @@ async def chat_generation_events(f'"payload":{event["payloadJson"]},"createdAt":{int(event["createdAt"])}')if event_type != "chunk":- if run_json is None:- run_json = _encode_json(snapshot)encoded += f',"run":{run_json}'yield f"id: {cursor}\nevent: {event_type}\ndata: {encoded}}}\n\n"if snapshot["status"] in db.TERMINAL_STATUSES and cursor >= int(diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.pyindex 8b002944876..f1db5f0ae72 100644--- a/studio/backend/storage/chat_generation_runs_db.py+++ b/studio/backend/storage/chat_generation_runs_db.py@@ -277,10 +277,22 @@ def _prepare_connection() -> tuple[sqlite3.Connection, bool]:return conn, db_path.resolve() in _schema_ready-_SPLICE_OK_SQL = (- "json_valid(payload_json) AND instr(payload_json, char(10))=0"- " AND instr(payload_json, char(13))=0"-)+_SPLICE_OK_SQL = "json_valid({0}) AND instr({0}, char(10))=0 AND instr({0}, char(13))=0"+++def _execute_splice(+ conn: sqlite3.Connection, query: str, column: str, params: tuple+) -> sqlite3.Cursor:+ """Run query with {} as whether column is one line of valid JSON, safe to splice into a frame."""+ global _SPLICE_OK_SQL+ try:+ return conn.execute(query.format(_SPLICE_OK_SQL.format(column)), params)+ except sqlite3.OperationalError as exc:+ # SQLite built without JSON1: re-encode every row, as the parsed read does.+ if "json_valid" not in str(exc):+ raise+ _SPLICE_OK_SQL = "0"+ return conn.execute(query.format("0"), params)def _dumps_payload(payload: Any) -> str:@@ -333,14 +345,14 @@ def canonical_request(return request_json, hashlib.sha256(identity.encode("utf-8")).hexdigest()-def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:+def _run_fields(row: sqlite3.Row, request_payload: Any) -> dict[str, Any]:return {"id": row["id"],"threadId": row["thread_id"],"userMessageId": row["user_message_id"],"assistantMessageId": row["assistant_message_id"],"requestHash": row["request_hash"],- "requestPayload": _loads(row["request_json"], {}),+ "requestPayload": request_payload,"status": row["status"],"cancelRequested": bool(row["cancel_requested"]),"lastEventSeq": int(row["last_event_seq"]),@@ -353,6 +365,27 @@ def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:}+def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:+ return _run_fields(row, _loads(row["request_json"], {}))+++def _run_json_from_row(row: sqlite3.Row) -> str:+ """json.dumps(_run_from_row(row), ensure_ascii=False, separators=(",", ":")) without parsing+ request_json: canonical_request's output is a fixed point of loads then dumps with these options."""+ fields = _run_fields(row, None)+ keys = list(fields)+ split = keys.index("requestPayload")+ head, tail = (+ json.dumps(+ {key: fields[key] for key in part},+ ensure_ascii = False,+ separators = (",", ":"),+ )+ for part in (keys[:split], keys[split + 1 :])+ )+ return f'{head[:-1]},"requestPayload":{row["request_json"]},{tail[1:]}'++def _append_events_locked(conn: sqlite3.Connection, run_id: str, events: Iterable[ChatGenerationEventInput]) -> list[int]:@@ -676,6 +709,33 @@ def get_run_state(run_id: str) -> dict[str, Any] | None:conn.close()+def get_run_frame(run_id: str) -> tuple[dict[str, Any], str] | None:+ """get_run_state's fields plus get_run's row as its compact JSON text, without parsing request_json.++ A request_json SQLite does not see as one line of valid JSON (corrupt, hand edited) takes the old+ parse and re-encode, so the route never splices text the client cannot parse or that splits the frame."""+ conn = _connect()+ try:+ row = _execute_splice(+ conn,+ "SELECT *, {} AS splice_ok FROM chat_generation_runs WHERE id=?",+ "request_json",+ (run_id,),+ ).fetchone()+ if row is None:+ return None+ state = {+ "status": row["status"],+ "lastEventSeq": int(row["last_event_seq"]),+ "updatedAt": int(row["updated_at"]),+ }+ if row["splice_ok"]:+ return state, _run_json_from_row(row)+ return state, json.dumps(_run_from_row(row), ensure_ascii = False, separators = (",", ":"))+ finally:+ conn.close()++def get_worker_token(run_id: str) -> str | None:conn = _connect()try:@@ -950,21 +1010,13 @@ def list_events("""raw=True returns the stored text as "payloadJson": it is already _dumps_payload output, the SSEroute's encoding. Rows that are not one line of valid JSON (corrupt, hand edited, NaN) arere-encoded so the route never splices text the client cannot parse or that splits the frame."""- global _SPLICE_OK_SQLconn = _connect()try:if raw:query = """SELECT seq, event_type, payload_json, created_at, {} AS splice_okFROM chat_generation_eventsWHERE run_id=? AND seq>? ORDER BY seq LIMIT ?"""- try:- rows = conn.execute(query.format(_SPLICE_OK_SQL), (run_id, after, limit)).fetchall()- except sqlite3.OperationalError as exc:- # SQLite built without JSON1: re-encode every row, as the parsed read does.- if "json_valid" not in str(exc):- raise- _SPLICE_OK_SQL = "0"- rows = conn.execute(query.format("0"), (run_id, after, limit)).fetchall()+ rows = _execute_splice(conn, query, "payload_json", (run_id, after, limit)).fetchall()return [{"seq": int(row["seq"]),diff --git a/studio/backend/tests/test_chat_generation_events_reads.py b/studio/backend/tests/test_chat_generation_events_reads.pyindex e7c38e81542..8a2962a56ff 100644--- a/studio/backend/tests/test_chat_generation_events_reads.py+++ b/studio/backend/tests/test_chat_generation_events_reads.py@@ -73,6 +73,13 @@ def test_chunk_only_batches_do_not_parse_the_request(running_run, monkeypatch):monkeypatch.setattr(runs_db, "get_run", lambda *a, **k: full_reads.append(a) or original_get(*a, **k))+ frame_reads = []+ original_frame = runs_db.get_run_frame+ monkeypatch.setattr(+ runs_db,+ "get_run_frame",+ lambda *a, **k: frame_reads.append(a) or original_frame(*a, **k),+ )raw = asyncio.run(_stream())@@ -85,8 +92,76 @@ def test_chunk_only_batches_do_not_parse_the_request(running_run, monkeypatch):assert "run" not in frameelse:assert len(frame["run"]["requestPayload"]["messages"]) == 20- # Ownership + opening checks, plus one per run-event batch (3).- assert len(full_reads) == 5+ # Ownership and opening checks read only the state; each run-event batch (3) splices the stored text.+ assert full_reads == []+ assert len(frame_reads) == 3+++def test_run_frames_match_encoding_the_parsed_run(running_run):+ runs_db.append_events("run-1", running_run, [("tool.status", {"name": "web"})])+ runs_db.finish_run("run-1", worker_token = running_run, status = "completed", finish_reason = "stop")+ run = runs_db.get_run("run-1")+ state, run_json = runs_db.get_run_frame("run-1")+ assert run_json == json.dumps(run, ensure_ascii = False, separators = (",", ":"))+ assert state == runs_db.get_run_state("run-1")+ raw = asyncio.run(_stream())+ frames = [line[6:] for line in raw.splitlines() if line.startswith("data: ")]+ for event, frame in zip(runs_db.list_events("run-1"), frames, strict = True):+ data = {key: event[key] for key in ("seq", "type", "payload", "createdAt")}+ data["run"] = run+ assert frame == json.dumps(data, ensure_ascii = False, separators = (",", ":"))+++ "extra",+ [+ {"temperature": float("nan"), "top_p": float("inf"), "x": -0.0},+ {"big": 2**70, "tiny": 5e-324, "e": 1e16, "neg": -(10**30)},+ {"text": 'é你好😀 "q" \\ \n\t\u0000\x1f\u2028', "nested": [{"a": [None, True, {}]}]},+ ],+)+def test_run_frame_matches_get_run_for_awkward_values(extra):+ studio_db.upsert_chat_thread(+ {"id": "thread-2", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}+ )+ studio_db.upsert_chat_message(+ {"id": "user-2", "threadId": "thread-2", "role": "user", "content": "hi", "createdAt": 2}+ )+ runs_db.create_run(+ run_id = "run-2",+ owner_subject = "alice",+ thread_id = "thread-2",+ user_message_id = "user-2",+ assistant_message_id = "assistant-2",+ request_payload = {"model": "local", "messages": [], **extra},+ )+ _, run_json = runs_db.get_run_frame("run-2")+ assert run_json == json.dumps(+ runs_db.get_run("run-2"), ensure_ascii = False, separators = (",", ":")+ )+++ "stored",+ ["not json", '{"model":"local"', '{"model":\n"local"}', '{"model":"local"}\r'],+)+def test_run_frame_falls_back_for_a_stored_request_that_is_not_one_line_of_json(+ running_run, stored+):+ conn = runs_db._connect()+ try:+ conn.execute("UPDATE chat_generation_runs SET request_json=? WHERE id='run-1'", (stored,))+ conn.commit()+ finally:+ conn.close()+ runs_db.append_events("run-1", running_run, [("tool.status", {"name": "web"})])+ runs_db.finish_run("run-1", worker_token = running_run, status = "completed", finish_reason = "stop")+ run = runs_db.get_run("run-1")+ _, run_json = runs_db.get_run_frame("run-1")+ assert run_json == json.dumps(run, ensure_ascii = False, separators = (",", ":"))+ raw = asyncio.run(_stream())+ frame = next(line[6:] for line in raw.splitlines() if line.startswith("data: "))+ assert json.loads(frame)["run"] == rundef test_caught_up_wait_reads_events_twice(running_run, monkeypatch):@@ -172,3 +247,13 @@ def test_concurrent_followers_never_miss_a_commit(running_run, monkeypatch):assert not stallsassert all(not thread.is_alive() for thread in threads)assert seen == {i: [str(n) for n in range(total)] for i in range(6)}+++def test_run_frame_without_json1_encodes_the_parsed_run(running_run, monkeypatch):+ monkeypatch.setattr(runs_db, "_SPLICE_OK_SQL", "json_valid_missing({0})")+ state, run_json = runs_db.get_run_frame("run-1")+ assert run_json == json.dumps(+ runs_db.get_run("run-1"), ensure_ascii = False, separators = (",", ":")+ )+ assert state == runs_db.get_run_state("run-1")+ assert runs_db._SPLICE_OK_SQL == "0"