Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stream durable chat run snapshots without re-parsing the stored request (#16302)

Daniel Han2c2870ff5265
diff --git a/studio/backend/core/inference/chat_generation_runs.py b/studio/backend/core/inference/chat_generation_runs.py
index 643e0acb3b6..f84c968f1cc 100644
--- a/studio/backend/core/inference/chat_generation_runs.py
+++ b/studio/backend/core/inference/chat_generation_runs.py
@@ -657,10 +657,9 @@ class ChatGenerationSupervisor:
return failure
try:
- worker_run = await asyncio.to_thread(db.get_worker_run, run_id)
- if worker_run is None:
+ worker_token = await asyncio.to_thread(db.get_worker_token, run_id)
+ if worker_token is None:
return
- run, owner, worker_token = worker_run
if cancel_event.is_set():
shutting_down = run_id in self._shutdown_runs
await asyncio.to_thread(
diff --git a/studio/backend/routes/chat_generation_runs.py b/studio/backend/routes/chat_generation_runs.py
index 2f6ae35c3d4..958d800bf45 100644
--- a/studio/backend/routes/chat_generation_runs.py
+++ b/studio/backend/routes/chat_generation_runs.py
@@ -269,6 +269,12 @@ def _require_run(run_id: str) -> dict[str, Any]:
return run
+def _require_run_exists(run_id: str) -> None:
+ """_require_run's 404 without reading or parsing the stored request."""
+ if db.get_run_state(run_id) is None:
+ raise HTTPException(status_code = 404, detail = "Chat generation run not found")
+
+
def cancel_account_run(request: Request, run_id: str, *, supervisor_name: str) -> None:
"""Signal only the caller's registration: another account may legitimately reuse a bare
cancel ID, so it is never stashed."""
@@ -405,7 +411,7 @@ def cancel_chat_generation_run(
request: Request,
current_subject: str = Depends(get_current_subject),
):
- _require_run(run_id)
+ _require_run_exists(run_id)
run = db.request_cancel(run_id)
if run is None:
raise HTTPException(status_code = 404, detail = "Chat generation run not found")
@@ -425,7 +431,7 @@ async def chat_generation_events(
last_event_id: str | None = Header(None, alias = "Last-Event-ID"),
current_subject: str = Depends(get_current_subject),
):
- _require_run(run_id)
+ _require_run_exists(run_id)
cursor = _event_cursor(after, last_event_id)
wait_for_events = db.wait_for_events
if policy.installation_has_managed_accounts():
@@ -440,7 +446,7 @@ async def chat_generation_events(
nonlocal cursor
loop = asyncio.get_running_loop()
# A settled run has nothing to replay; don't hold an event-wait worker for it.
- opening = await asyncio.to_thread(db.get_run, run_id)
+ opening = await asyncio.to_thread(db.get_run_state, run_id)
if opening is None:
return
if opening["status"] in db.TERMINAL_STATUSES and cursor >= int(opening["lastEventSeq"]):
@@ -456,16 +462,15 @@ async def chat_generation_events(
15,
True,
)
- # Only non-chunk events carry the run, so a chunk-only batch skips parsing request_json.
- snapshot = await asyncio.to_thread(
- db.get_run
- if any(event["type"] != "chunk" for event in events)
- else db.get_run_state,
- run_id,
- )
+ # Only non-chunk events carry the run, spliced in as already encoded text.
+ run_json = None
+ if any(event["type"] != "chunk" for event in events):
+ frame = await asyncio.to_thread(db.get_run_frame, run_id)
+ snapshot, run_json = frame if frame is not None else (None, None)
+ else:
+ snapshot = await asyncio.to_thread(db.get_run_state, run_id)
if snapshot is None:
return
- run_json = None
for event in events:
cursor = int(event["seq"])
event_type = event["type"]
@@ -476,8 +481,6 @@ async def chat_generation_events(
f'"payload":{event["payloadJson"]},"createdAt":{int(event["createdAt"])}'
)
if event_type != "chunk":
- if run_json is None:
- run_json = _encode_json(snapshot)
encoded += f',"run":{run_json}'
yield f"id: {cursor}\nevent: {event_type}\ndata: {encoded}}}\n\n"
if snapshot["status"] in db.TERMINAL_STATUSES and cursor >= int(
diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.py
index 8b002944876..f1db5f0ae72 100644
--- a/studio/backend/storage/chat_generation_runs_db.py
+++ b/studio/backend/storage/chat_generation_runs_db.py
@@ -277,10 +277,22 @@ def _prepare_connection() -> tuple[sqlite3.Connection, bool]:
return conn, db_path.resolve() in _schema_ready
-_SPLICE_OK_SQL = (
- "json_valid(payload_json) AND instr(payload_json, char(10))=0"
- " AND instr(payload_json, char(13))=0"
-)
+_SPLICE_OK_SQL = "json_valid({0}) AND instr({0}, char(10))=0 AND instr({0}, char(13))=0"
+
+
+def _execute_splice(
+ conn: sqlite3.Connection, query: str, column: str, params: tuple
+) -> sqlite3.Cursor:
+ """Run query with {} as whether column is one line of valid JSON, safe to splice into a frame."""
+ global _SPLICE_OK_SQL
+ try:
+ return conn.execute(query.format(_SPLICE_OK_SQL.format(column)), params)
+ except sqlite3.OperationalError as exc:
+ # SQLite built without JSON1: re-encode every row, as the parsed read does.
+ if "json_valid" not in str(exc):
+ raise
+ _SPLICE_OK_SQL = "0"
+ return conn.execute(query.format("0"), params)
def _dumps_payload(payload: Any) -> str:
@@ -333,14 +345,14 @@ def canonical_request(
return request_json, hashlib.sha256(identity.encode("utf-8")).hexdigest()
-def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:
+def _run_fields(row: sqlite3.Row, request_payload: Any) -> dict[str, Any]:
return {
"id": row["id"],
"threadId": row["thread_id"],
"userMessageId": row["user_message_id"],
"assistantMessageId": row["assistant_message_id"],
"requestHash": row["request_hash"],
- "requestPayload": _loads(row["request_json"], {}),
+ "requestPayload": request_payload,
"status": row["status"],
"cancelRequested": bool(row["cancel_requested"]),
"lastEventSeq": int(row["last_event_seq"]),
@@ -353,6 +365,27 @@ def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:
}
+def _run_from_row(row: sqlite3.Row) -> dict[str, Any]:
+ return _run_fields(row, _loads(row["request_json"], {}))
+
+
+def _run_json_from_row(row: sqlite3.Row) -> str:
+ """json.dumps(_run_from_row(row), ensure_ascii=False, separators=(",", ":")) without parsing
+ request_json: canonical_request's output is a fixed point of loads then dumps with these options."""
+ fields = _run_fields(row, None)
+ keys = list(fields)
+ split = keys.index("requestPayload")
+ head, tail = (
+ json.dumps(
+ {key: fields[key] for key in part},
+ ensure_ascii = False,
+ separators = (",", ":"),
+ )
+ for part in (keys[:split], keys[split + 1 :])
+ )
+ return f'{head[:-1]},"requestPayload":{row["request_json"]},{tail[1:]}'
+
+
def _append_events_locked(
conn: sqlite3.Connection, run_id: str, events: Iterable[ChatGenerationEventInput]
) -> list[int]:
@@ -676,6 +709,33 @@ def get_run_state(run_id: str) -> dict[str, Any] | None:
conn.close()
+def get_run_frame(run_id: str) -> tuple[dict[str, Any], str] | None:
+ """get_run_state's fields plus get_run's row as its compact JSON text, without parsing request_json.
+
+ A request_json SQLite does not see as one line of valid JSON (corrupt, hand edited) takes the old
+ parse and re-encode, so the route never splices text the client cannot parse or that splits the frame."""
+ conn = _connect()
+ try:
+ row = _execute_splice(
+ conn,
+ "SELECT *, {} AS splice_ok FROM chat_generation_runs WHERE id=?",
+ "request_json",
+ (run_id,),
+ ).fetchone()
+ if row is None:
+ return None
+ state = {
+ "status": row["status"],
+ "lastEventSeq": int(row["last_event_seq"]),
+ "updatedAt": int(row["updated_at"]),
+ }
+ if row["splice_ok"]:
+ return state, _run_json_from_row(row)
+ return state, json.dumps(_run_from_row(row), ensure_ascii = False, separators = (",", ":"))
+ finally:
+ conn.close()
+
+
def get_worker_token(run_id: str) -> str | None:
conn = _connect()
try:
@@ -950,21 +1010,13 @@ def list_events(
"""raw=True returns the stored text as "payloadJson": it is already _dumps_payload output, the SSE
route's encoding. Rows that are not one line of valid JSON (corrupt, hand edited, NaN) are
re-encoded so the route never splices text the client cannot parse or that splits the frame."""
- global _SPLICE_OK_SQL
conn = _connect()
try:
if raw:
query = """SELECT seq, event_type, payload_json, created_at, {} AS splice_ok
FROM chat_generation_events
WHERE run_id=? AND seq>? ORDER BY seq LIMIT ?"""
- try:
- rows = conn.execute(query.format(_SPLICE_OK_SQL), (run_id, after, limit)).fetchall()
- except sqlite3.OperationalError as exc:
- # SQLite built without JSON1: re-encode every row, as the parsed read does.
- if "json_valid" not in str(exc):
- raise
- _SPLICE_OK_SQL = "0"
- rows = conn.execute(query.format("0"), (run_id, after, limit)).fetchall()
+ rows = _execute_splice(conn, query, "payload_json", (run_id, after, limit)).fetchall()
return [
{
"seq": int(row["seq"]),
diff --git a/studio/backend/tests/test_chat_generation_events_reads.py b/studio/backend/tests/test_chat_generation_events_reads.py
index e7c38e81542..8a2962a56ff 100644
--- a/studio/backend/tests/test_chat_generation_events_reads.py
+++ b/studio/backend/tests/test_chat_generation_events_reads.py
@@ -73,6 +73,13 @@ def test_chunk_only_batches_do_not_parse_the_request(running_run, monkeypatch):
monkeypatch.setattr(
runs_db, "get_run", lambda *a, **k: full_reads.append(a) or original_get(*a, **k)
)
+ frame_reads = []
+ original_frame = runs_db.get_run_frame
+ monkeypatch.setattr(
+ runs_db,
+ "get_run_frame",
+ lambda *a, **k: frame_reads.append(a) or original_frame(*a, **k),
+ )
raw = asyncio.run(_stream())
@@ -85,8 +92,76 @@ def test_chunk_only_batches_do_not_parse_the_request(running_run, monkeypatch):
assert "run" not in frame
else:
assert len(frame["run"]["requestPayload"]["messages"]) == 20
- # Ownership + opening checks, plus one per run-event batch (3).
- assert len(full_reads) == 5
+ # Ownership and opening checks read only the state; each run-event batch (3) splices the stored text.
+ assert full_reads == []
+ assert len(frame_reads) == 3
+
+
+def test_run_frames_match_encoding_the_parsed_run(running_run):
+ runs_db.append_events("run-1", running_run, [("tool.status", {"name": "web"})])
+ runs_db.finish_run("run-1", worker_token = running_run, status = "completed", finish_reason = "stop")
+ run = runs_db.get_run("run-1")
+ state, run_json = runs_db.get_run_frame("run-1")
+ assert run_json == json.dumps(run, ensure_ascii = False, separators = (",", ":"))
+ assert state == runs_db.get_run_state("run-1")
+ raw = asyncio.run(_stream())
+ frames = [line[6:] for line in raw.splitlines() if line.startswith("data: ")]
+ for event, frame in zip(runs_db.list_events("run-1"), frames, strict = True):
+ data = {key: event[key] for key in ("seq", "type", "payload", "createdAt")}
+ data["run"] = run
+ assert frame == json.dumps(data, ensure_ascii = False, separators = (",", ":"))
+
+
+ "extra",
+ [
+ {"temperature": float("nan"), "top_p": float("inf"), "x": -0.0},
+ {"big": 2**70, "tiny": 5e-324, "e": 1e16, "neg": -(10**30)},
+ {"text": 'é你好😀 "q" \\ \n\t\u0000\x1f\u2028', "nested": [{"a": [None, True, {}]}]},
+ ],
+)
+def test_run_frame_matches_get_run_for_awkward_values(extra):
+ studio_db.upsert_chat_thread(
+ {"id": "thread-2", "title": "Chat", "modelType": "base", "modelId": "local", "createdAt": 1}
+ )
+ studio_db.upsert_chat_message(
+ {"id": "user-2", "threadId": "thread-2", "role": "user", "content": "hi", "createdAt": 2}
+ )
+ runs_db.create_run(
+ run_id = "run-2",
+ owner_subject = "alice",
+ thread_id = "thread-2",
+ user_message_id = "user-2",
+ assistant_message_id = "assistant-2",
+ request_payload = {"model": "local", "messages": [], **extra},
+ )
+ _, run_json = runs_db.get_run_frame("run-2")
+ assert run_json == json.dumps(
+ runs_db.get_run("run-2"), ensure_ascii = False, separators = (",", ":")
+ )
+
+
+ "stored",
+ ["not json", '{"model":"local"', '{"model":\n"local"}', '{"model":"local"}\r'],
+)
+def test_run_frame_falls_back_for_a_stored_request_that_is_not_one_line_of_json(
+ running_run, stored
+):
+ conn = runs_db._connect()
+ try:
+ conn.execute("UPDATE chat_generation_runs SET request_json=? WHERE id='run-1'", (stored,))
+ conn.commit()
+ finally:
+ conn.close()
+ runs_db.append_events("run-1", running_run, [("tool.status", {"name": "web"})])
+ runs_db.finish_run("run-1", worker_token = running_run, status = "completed", finish_reason = "stop")
+ run = runs_db.get_run("run-1")
+ _, run_json = runs_db.get_run_frame("run-1")
+ assert run_json == json.dumps(run, ensure_ascii = False, separators = (",", ":"))
+ raw = asyncio.run(_stream())
+ frame = next(line[6:] for line in raw.splitlines() if line.startswith("data: "))
+ assert json.loads(frame)["run"] == run
def test_caught_up_wait_reads_events_twice(running_run, monkeypatch):
@@ -172,3 +247,13 @@ def test_concurrent_followers_never_miss_a_commit(running_run, monkeypatch):
assert not stalls
assert all(not thread.is_alive() for thread in threads)
assert seen == {i: [str(n) for n in range(total)] for i in range(6)}
+
+
+def test_run_frame_without_json1_encodes_the_parsed_run(running_run, monkeypatch):
+ monkeypatch.setattr(runs_db, "_SPLICE_OK_SQL", "json_valid_missing({0})")
+ state, run_json = runs_db.get_run_frame("run-1")
+ assert run_json == json.dumps(
+ runs_db.get_run("run-1"), ensure_ascii = False, separators = (",", ":")
+ )
+ assert state == runs_db.get_run_state("run-1")
+ assert runs_db._SPLICE_OK_SQL == "0"