Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(portal): a turn cut off by a restart settles by itself, with one Continue when it cannot be proven (#8206)

Mike Bradley09cd7f899d7d
diff --git a/ods/extensions/services/dashboard-api/pixel_chat_results.py b/ods/extensions/services/dashboard-api/pixel_chat_results.py
index 220d3f172..ebdee0b64 100644
--- a/ods/extensions/services/dashboard-api/pixel_chat_results.py
+++ b/ods/extensions/services/dashboard-api/pixel_chat_results.py
@@ -173,5 +173,12 @@ class ChatResultStore:
def has_pending(self, conversation):
return self.db.execute("SELECT 1 FROM attempts WHERE owner=? AND chat=? AND state IN ('active','unresolved')", conversation).fetchone() is not None
+ def pending_attempts(self, conversation):
+ """Keys of this conversation's unfinished attempts, newest first."""
+ rows = self.db.execute(
+ "SELECT attempt FROM attempts WHERE owner=? AND chat=? AND state IN ('active','unresolved') ORDER BY rowid DESC",
+ conversation).fetchall()
+ return [(*conversation, row["attempt"]) for row in rows]
+
def chunks(self, key, after=-1):
return self.db.execute("SELECT sequence,data FROM chunks WHERE owner=? AND chat=? AND attempt=? AND sequence>? ORDER BY sequence", (*key, after)).fetchall()
diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.py
index a4fe37aa1..8b05e86a5 100644
--- a/ods/extensions/services/dashboard-api/routers/pixel.py
+++ b/ods/extensions/services/dashboard-api/routers/pixel.py
@@ -248,6 +248,9 @@ _result_store: ChatResultStore | None = None
_result_tasks: dict[tuple[str, str, str], asyncio.Task] = {}
_result_preflights: set[tuple[str, str, str]] = set()
_result_stops: set[tuple[str, str]] = set()
+# Chats whose last turn an activity check is settling right now. Kept apart
+# from Stop's fence, so neither can remove the other's.
+_result_settles: set[tuple[str, str]] = set()
_result_abort_ack: set[tuple[str, str, str]] = set()
@@ -366,7 +369,7 @@ async def pixel_chat_result(body: ChatResultRequest, owner: str = Depends(verify
# the request identity while its exact cancellation is still pending.
return {"state": "active", "events": ""}
if row["state"] == "unresolved":
- activity = await pixel_chat_activity(ChatCancelRequest(chat_id=body.chat_id))
+ activity = await pixel_chat_activity(ChatCancelRequest(chat_id=body.chat_id), owner)
if activity["state"] == "terminal":
store.finish(key, "interrupted")
row = store.get(key)
@@ -1001,34 +1004,129 @@ async def pixel_chat_cancel(body: ChatCancelRequest, owner: str = Depends(verify
finally:
_result_abort_ack.discard(identity)
_result_stops.discard(identity[:2])
- if isinstance(owner, str) and _result_store is not None and _result_store.has_pending((owner_namespace(owner), body.chat_id)):
- return {"aborted": False}
+ if isinstance(owner, str):
+ conversation = (owner_namespace(owner), body.chat_id)
+ # A restarted API process has not opened the store yet, but receipts
+ # its predecessor left unfinished still hold this chat.
+ pending = _chat_results().pending_attempts(conversation)
+ if pending:
+ # A chat-level request ("Continue in this chat") never stops a turn
+ # this process is still streaming; that turn's own Stop names it.
+ # An attempt no producer owns any more is resolved by its exact ID.
+ if _owns_live_work(conversation):
+ return {"aborted": False}
+ return await pixel_chat_cancel(ChatCancelRequest(chat_id=body.chat_id, request_id=pending[0][2]), owner)
return {"aborted": await _cancel_edge_run(edge_url, key, body.chat_id)}
[email protected]("/chat/activity", dependencies=[Depends(verify_api_key)])
-async def pixel_chat_activity(body: ChatCancelRequest) -> dict[str, str]:
- """One authenticated chat lookup; global activity never supplies its state."""
- config = _pixel_config()
- if config is None:
- return {"state": "unknown"}
- edge_url, key = config
+def _owns_live_work(conversation) -> bool:
+ """True while this process streams, prepares or stops a turn of this chat."""
+ return (conversation in _result_stops
+ or any(identity[:2] == conversation for identity in _result_preflights)
+ or any(identity[:2] == conversation and not task.done() for identity, task in _result_tasks.items()))
+
+
+async def _edge_chat_activity(edge_url: str, key: str, chat_id: str) -> str | None:
+ """The edge's exact view of one chat, or None when it could not be read."""
try:
timeout = httpx.Timeout(connect=2.0, read=5.0, write=2.0, pool=2.0)
async with httpx.AsyncClient(timeout=timeout, trust_env=False, follow_redirects=False) as client:
async with client.stream("POST", f"{edge_url}/v1/chat/activity",
- json={"user": body.chat_id},
+ json={"user": chat_id},
headers=_edge_headers(key, accept="application/json")) as response:
if response.status_code != 200 or not response.headers.get("content-type", "").lower().startswith("application/json"):
- return {"state": "unknown"}
+ return None
raw = await _bounded_response_bytes(response, 1024)
parsed = json.loads(raw)
if (isinstance(parsed, dict) and set(parsed) == {"state"}
and isinstance(parsed["state"], str) and parsed["state"] in {"active", "terminal", "unknown"}):
- return {"state": parsed["state"]}
+ return parsed["state"]
except (httpx.HTTPError, asyncio.TimeoutError, ValueError, TypeError):
pass
- return {"state": "unknown"}
+ return None
+
+
+async def _settle_edge_chat(edge_url: str, key: str, chat_id: str) -> str | None:
+ """Ask the ingress, through the edge, to settle a chat it proves idle.
+
+ "settled" means no run of this chat can still be alive and any turn a
+ restart cut off is recorded as interrupted; "active" means work may still
+ run. Nothing is ever stopped. None means it could not be confirmed, for
+ example while Portal is still restarting or on an older edge or ingress.
+ """
+ try:
+ timeout = httpx.Timeout(connect=2.0, read=10.0, write=2.0, pool=2.0)
+ async with httpx.AsyncClient(timeout=timeout, trust_env=False, follow_redirects=False) as client:
+ async with client.stream("POST", f"{edge_url}/v1/chat/settle",
+ json={"user": chat_id},
+ headers=_edge_headers(key, accept="application/json")) as response:
+ if response.status_code != 200 or not response.headers.get("content-type", "").lower().startswith("application/json"):
+ return None
+ raw = await _bounded_response_bytes(response, 1024)
+ parsed = json.loads(raw)
+ if (isinstance(parsed, dict) and set(parsed) == {"state"}
+ and isinstance(parsed["state"], str) and parsed["state"] in {"settled", "active"}):
+ return parsed["state"]
+ except (httpx.HTTPError, asyncio.TimeoutError, ValueError, TypeError):
+ pass
+ return None
+
+
[email protected]("/chat/activity")
+async def pixel_chat_activity(body: ChatCancelRequest, owner: str = Depends(verify_api_key)) -> dict[str, str]:
+ """One authenticated chat lookup; global activity never supplies its state.
+
+ A turn this process still streams is active. Otherwise, when no stream of
+ this chat passes through the edge, the host ingress is asked to settle the
+ chat: under its history lock it proves the native session idle before it
+ records a turn that a restart cut off as interrupted, and it never stops a
+ run. Only that proof turns the chat terminal here, and it also releases
+ receipts no producer owns any more, so the chat can take its next message.
+ """
+ config = _pixel_config()
+ if config is None:
+ return {"state": "unknown"}
+ edge_url, key = config
+ conversation = (owner_namespace(owner), body.chat_id) if isinstance(owner, str) else None
+ if conversation is not None and _owns_live_work(conversation):
+ return {"state": "active"}
+ state = await _edge_chat_activity(edge_url, key, body.chat_id)
+ if state is None:
+ return {"state": "unknown"}
+ if state == "active" or conversation is None:
+ return {"state": state}
+ # A Stop, a turn or another check may have started here during that read.
+ # It owns the chat now; settling beside it could race its outcome.
+ if _owns_live_work(conversation) or conversation in _result_settles:
+ return {"state": "active"}
+ # Fence this process's own next attempt so it cannot race the history
+ # lock. A Stop keeps its own fence; only this check removes this one.
+ _result_settles.add(conversation)
+ try:
+ settled = await _settle_edge_chat(edge_url, key, body.chat_id)
+ if settled == "settled" and _owns_live_work(conversation):
+ # A Stop that started meanwhile settles its own receipt.
+ settled = "active"
+ if settled == "settled":
+ store = _chat_results()
+ for identity in store.pending_attempts(conversation):
+ row = _result_state(store, identity)
+ if row is not None and row["state"] == "unresolved":
+ # No producer owns this receipt and the ingress proved the
+ # chat idle. Keep every saved chunk; never call it complete.
+ store.finish(identity, "interrupted")
+ finally:
+ _result_settles.discard(conversation)
+ if settled == "settled":
+ return {"state": "terminal"}
+ if settled == "active":
+ return {"state": "active"}
+ # An older edge or ingress cannot settle. Its own memory that the chat
+ # ended does not free a chat that a receipt still holds: the next message
+ # would only be refused.
+ if state == "terminal" and _chat_results().pending_attempts(conversation):
+ return {"state": "unknown"}
+ return {"state": state}
_EXAMPLE_PROJECT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")
@@ -1157,6 +1255,8 @@ async def _retained_chat_stream(request, body, owner):
try:
if identity[:2] in _result_stops:
raise ResultConflict("Stop is still being confirmed")
+ if identity[:2] in _result_settles:
+ raise ResultConflict("The last turn of this chat is still being checked")
created = store.reserve(identity, fingerprint)
except ResultConflict as exc:
raise HTTPException(status_code=423, detail=str(exc)) from None
diff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.py b/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.py
new file mode 100644
index 000000000..b6128a40b
--- /dev/null
+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.py
@@ -0,0 +1,338 @@
+"""A turn cut off by a restart heals by itself once nothing can still run.
+
+Seen on the RTX Spark: an update recreated dashboard-api and pixel-edge and
+restarted the gateway while "Can you do things on cron timers?" streamed. The
+retained receipt stayed unresolved, the restarted edge had no record of the
+chat, and every later message was refused while the page polled forever.
+"""
+import asyncio
+import json
+import os
+from pathlib import Path
+import sys
+from typing import Any, cast
+from unittest.mock import patch
+
+import pytest
+from fastapi import HTTPException, Request
+
+os.environ.setdefault("DASHBOARD_API_KEY", "dashboard-test-key")
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+import pixel_chat_results as receipts
+from routers import pixel
+import pixel_chat_identity
+from test_pixel import EDGE_KEY, ConnectedRequest, FakeClient, FakeResponse
+
+OWNER = "dashboard-test-key"
+CHAT = "spark-chat"
+CUT_OFF = (receipts.owner_namespace(OWNER), CHAT, "cron-question")
+PARTIAL = b'data: {"choices":[{"delta":{"content":"Yes. Portal can run"}}]}\n\n'
+
+
+def stores(tmp_path, monkeypatch):
+ async def saved_identity(*_args, **_kwargs):
+ return {"schemaVersion": 1, "revision": 1, "displayName": "Portal"}
+ monkeypatch.setattr(pixel_chat_identity, "async_request_json", saved_identity)
+ monkeypatch.setattr(pixel, "_result_tasks", {})
+ monkeypatch.setattr(pixel, "_result_preflights", set())
+ monkeypatch.setattr(pixel, "_result_stops", set())
+ monkeypatch.setattr(pixel, "_result_settles", set())
+ monkeypatch.setattr(pixel, "_result_abort_ack", set())
+ monkeypatch.setenv("PIXEL_OPENWEBUI_KEY", EDGE_KEY)
+ async def ready():
+ return None
+ monkeypatch.setattr(pixel, "_model_readiness_issue", ready)
+ # The API process that was streaming the turn when the update recreated it.
+ before = receipts.ChatResultStore(tmp_path / "receipts")
+ before.reserve(CUT_OFF, "cron-question-input")
+ before.append(CUT_OFF, PARTIAL)
+ # The recreated API process: the receipt is unresolved, nobody owns it.
+ after = receipts.ChatResultStore(tmp_path / "receipts")
+ monkeypatch.setattr(pixel, "_result_store", after)
+ yield after
+ before.close()
+ after.close()
+
+
+def edge(monkeypatch):
+ """The restarted edge has no record of the chat; the ingress proves it idle."""
+ state: dict[str, Any] = {"activity": "unknown", "settle": "settled", "lookups": [], "settles": []}
+
+ async def activity(edge_url, key, chat_id):
+ state["lookups"].append(chat_id)
+ return state["activity"]
+
+ async def settle(edge_url, key, chat_id):
+ state["settles"].append(chat_id)
+ # The settle fence is its own; a Stop's fence is never borrowed for it.
+ assert (receipts.owner_namespace(OWNER), chat_id) in pixel._result_settles
+ assert (receipts.owner_namespace(OWNER), chat_id) not in pixel._result_stops
+ return state["settle"]
+
+ async def never_cancel(*_args):
+ raise AssertionError("Healing a restart-cut turn must never stop a run")
+
+ monkeypatch.setattr(pixel, "_edge_chat_activity", activity)
+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)
+ monkeypatch.setattr(pixel, "_cancel_edge_run", never_cancel)
+ return state
+
+
+def connected() -> Request:
+ # The handlers only ask whether the client is still connected.
+ return cast(Request, ConnectedRequest())
+
+
+def stream(request_id, text="Checking in"):
+ return pixel.ChatStreamRequest.model_validate(
+ {"chat_id": CHAT, "request_id": request_id, "messages": [{"role": "user", "content": text}]})
+
+
+def test_restart_cut_turn_settles_by_itself_and_the_chat_takes_its_next_message(stores, edge):
+ assert stores.get(CUT_OFF)["state"] == "unresolved"
+ lookup = pixel.ChatResultRequest(chat_id=CHAT, request_id=CUT_OFF[2])
+ result = asyncio.run(pixel.pixel_chat_result(lookup, OWNER))
+ # No terminal frame: the producer died with the old process. The saved
+ # partial answer stays exactly as it was received.
+ assert result == {"state": "interrupted", "events": PARTIAL.decode()}
+ assert edge["settles"] == [CHAT]
+ assert not stores.has_pending(CUT_OFF[:2])
+ assert not pixel._result_stops
+ assert stores.reserve((*CUT_OFF[:2], "checking-in"), "checking-in-input") is True
+
+
+def test_a_refused_follow_up_cannot_strand_the_chat_behind_the_cut_off_turn(stores, edge):
+ # "Checking in" was refused while the cut-off turn held the chat, so its
+ # request ID has no receipt. Its activity lookup still heals the chat.
+ with pytest.raises(HTTPException) as refused:
+ asyncio.run(pixel.pixel_chat_stream(connected(), stream("checking-in"), OWNER))
+ assert refused.value.status_code == 423
+ missing = pixel.ChatResultRequest(chat_id=CHAT, request_id="checking-in")
+ assert asyncio.run(pixel.pixel_chat_result(missing, OWNER)) == {"state": "unknown", "events": ""}
+ assert asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)) == {"state": "terminal"}
+ assert stores.get(CUT_OFF)["state"] == "interrupted"
+ assert b"".join(row["data"] for row in stores.chunks(CUT_OFF)) == PARTIAL
+ assert not stores.has_pending(CUT_OFF[:2])
+
+
[email protected]("activity,settle,expected", [
+ ("active", "settled", "active"), # a stream of this chat still passes through the edge
+ ("unknown", "active", "active"), # the native session is busy: a run may still be alive
+ ("unknown", None, "unknown"), # Portal still restarting, or an older edge or ingress
+ ("terminal", None, "unknown"), # an older edge remembers the end, but the receipt holds the chat
+ (None, "settled", "unknown"), # the edge could not be read at all
+])
+def test_work_that_may_still_run_keeps_the_chat_held(stores, edge, activity, settle, expected):
+ edge.update(activity=activity, settle=settle)
+ result = asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ assert result == {"state": expected}
+ assert edge["settles"] == ([CHAT] if activity in {"unknown", "terminal"} else [])
+ assert stores.get(CUT_OFF)["state"] == "unresolved"
+ assert stores.has_pending(CUT_OFF[:2])
+ with pytest.raises(HTTPException) as refused:
+ asyncio.run(pixel.pixel_chat_stream(connected(), stream("second-turn"), OWNER))
+ assert refused.value.status_code == 423
+
+
+def test_an_older_edge_that_remembers_the_end_frees_a_chat_no_receipt_holds(stores, edge):
+ stores.finish(CUT_OFF, "interrupted")
+ edge.update(activity="terminal", settle=None)
+ result = asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ assert result == {"state": "terminal"}
+ assert stores.reserve((*CUT_OFF[:2], "next-turn"), "next-input") is True
+
+
+def test_a_turn_this_process_still_streams_is_active_without_asking_anyone(stores, edge):
+ async def run():
+ live = asyncio.create_task(asyncio.sleep(3600))
+ pixel._result_tasks[(*CUT_OFF[:2], "live-turn")] = live
+ try:
+ return await pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)
+ finally:
+ live.cancel()
+ await asyncio.gather(live, return_exceptions=True)
+ assert asyncio.run(run()) == {"state": "active"}
+ assert edge["lookups"] == [] and edge["settles"] == []
+ assert stores.get(CUT_OFF)["state"] == "unresolved"
+
+
+def test_settle_fences_a_new_attempt_until_it_answers(stores, edge, monkeypatch):
+ async def run():
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def slow_settle(edge_url, key, chat_id):
+ entered.set()
+ await release.wait()
+ return "settled"
+
+ monkeypatch.setattr(pixel, "_settle_edge_chat", slow_settle)
+ healing = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ await entered.wait()
+ stores.finish(CUT_OFF, "interrupted") # even with nothing pending, the fence holds
+ with pytest.raises(HTTPException) as fenced:
+ await pixel.pixel_chat_stream(connected(), stream("during-settle"), OWNER)
+ assert fenced.value.status_code == 423
+ release.set()
+ assert await healing == {"state": "terminal"}
+ assert not pixel._result_settles and not pixel._result_stops
+ asyncio.run(run())
+
+
+def _soon(awaitable):
+ """A race that went the wrong way fails here instead of hanging the run."""
+ return asyncio.wait_for(awaitable, 5)
+
+
+def _blocked(result):
+ """A stand-in edge call that waits until the test releases it."""
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def call(*_args):
+ entered.set()
+ await release.wait()
+ return result
+ return call, entered, release
+
+
+async def _stop_holds_the_chat(stores):
+ """While a Stop's cancel runs, its fence holds: no new turn, result active."""
+ assert CUT_OFF[:2] in pixel._result_stops
+ lookup = pixel.ChatResultRequest(chat_id=CHAT, request_id=CUT_OFF[2])
+ assert await pixel.pixel_chat_result(lookup, OWNER) == {"state": "active", "events": ""}
+ with pytest.raises(HTTPException) as refused:
+ await pixel.pixel_chat_stream(connected(), stream("during-stop"), OWNER)
+ assert refused.value.status_code == 423
+ assert stores.get(CUT_OFF)["state"] == "unresolved"
+
+
+def test_a_stop_that_starts_during_the_edge_read_is_left_alone(stores, edge, monkeypatch):
+ async def run():
+ read, reading, answer = _blocked("unknown")
+ cancel, cancelling, acknowledge = _blocked(True)
+ monkeypatch.setattr(pixel, "_edge_chat_activity", read)
+ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)
+ check = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ await _soon(reading.wait())
+ stop = asyncio.create_task(pixel.pixel_chat_cancel(
+ pixel.ChatCancelRequest(chat_id=CHAT, request_id=CUT_OFF[2]), OWNER))
+ await _soon(cancelling.wait())
+ answer.set()
+ # The check finds the Stop and leaves the chat to it: no settle.
+ assert await _soon(check) == {"state": "active"}
+ assert edge["settles"] == []
+ await _stop_holds_the_chat(stores)
+ acknowledge.set()
+ assert await _soon(stop) == {"aborted": True}
+ assert stores.get(CUT_OFF)["state"] == "cancelled"
+ assert not pixel._result_stops and not pixel._result_settles
+ asyncio.run(run())
+
+
[email protected]("ingress", ["settled", "active"])
+def test_settle_never_removes_the_fence_of_a_stop_that_started_during_it(stores, edge, monkeypatch, ingress):
+ async def run():
+ settle, settling, answer = _blocked(ingress)
+ cancel, cancelling, acknowledge = _blocked(True)
+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)
+ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)
+ check = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ await _soon(settling.wait())
+ # The owner's Stop is not refused because a check is settling.
+ stop = asyncio.create_task(pixel.pixel_chat_cancel(
+ pixel.ChatCancelRequest(chat_id=CHAT, request_id=CUT_OFF[2]), OWNER))
+ await _soon(cancelling.wait())
+ answer.set()
+ # Whatever the ingress answers, the Stop now owns this receipt.
+ assert await _soon(check) == {"state": "active"}
+ assert not pixel._result_settles
+ await _stop_holds_the_chat(stores)
+ acknowledge.set()
+ assert await _soon(stop) == {"aborted": True}
+ assert stores.get(CUT_OFF)["state"] == "cancelled"
+ assert not pixel._result_stops
+ asyncio.run(run())
+
+
+def test_a_second_check_waits_for_the_settle_already_running(stores, edge, monkeypatch):
+ async def run():
+ settle, settling, answer = _blocked("settled")
+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)
+ first = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ await _soon(settling.wait())
+ assert await pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER) == {"state": "active"}
+ assert CUT_OFF[:2] in pixel._result_settles
+ answer.set()
+ assert await _soon(first) == {"state": "terminal"}
+ assert stores.get(CUT_OFF)["state"] == "interrupted"
+ assert not pixel._result_settles
+ asyncio.run(run())
+
+
+def test_continue_in_this_chat_resolves_the_cut_off_turn_by_its_exact_id(stores, monkeypatch):
+ calls = []
+
+ async def cancel(edge_url, key, chat_id):
+ calls.append(chat_id)
+ assert CUT_OFF[:2] in pixel._result_stops
+ return True
+
+ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)
+ # The page only knows the refused follow-up's ID, so it asks for the chat.
+ result = asyncio.run(pixel.pixel_chat_cancel(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))
+ assert result == {"aborted": True}
+ assert calls == [CHAT]
+ assert stores.get(CUT_OFF)["state"] == "cancelled"
+ assert b"".join(row["data"] for row in stores.chunks(CUT_OFF)) == PARTIAL
+ assert not stores.has_pending(CUT_OFF[:2])
+
+
+def test_continue_never_stops_a_turn_this_process_is_streaming(stores, monkeypatch):
+ async def forbidden(*_args):
+ raise AssertionError("A chat-level request must not stop a live turn")
+
+ monkeypatch.setattr(pixel, "_cancel_edge_run", forbidden)
+
+ async def run():
+ stores.finish(CUT_OFF, "interrupted")
+ live = (*CUT_OFF[:2], "live-turn")
+ stores.reserve(live, "live-input")
+ task = asyncio.create_task(asyncio.sleep(3600))
+ pixel._result_tasks[live] = task
+ try:
+ return await pixel.pixel_chat_cancel(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ assert asyncio.run(run()) == {"aborted": False}
+
+
[email protected]("chunks,expected", [
+ ([b'{"state":"settled"}'], "settled"),
+ ([b'{"state":"active"}'], "active"),
+ ([b'{"state":"terminal"}'], None),
+ ([b'{"state":"settled","secret":"hidden"}'], None),
+ ([b'{"aborted":true}'], None),
+ ([b"x" * 1025], None),
+])
+def test_settle_request_is_exact_and_only_trusts_a_closed_receipt(chunks, expected, monkeypatch):
+ monkeypatch.setenv("PIXEL_OPENWEBUI_KEY", EDGE_KEY)
+ capture: dict[str, Any] = {}
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(chunks=chunks), capture)) as client:
+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) == expected
+ assert capture["method"] == "POST"
+ assert capture["url"] == "http://pixel-edge:9595/v1/chat/settle"
+ assert capture["json"] == {"user": CHAT}
+ assert capture["headers"]["Authorization"] == f"Bearer {EDGE_KEY}"
+ assert client.call_args.kwargs["trust_env"] is False
+ assert client.call_args.kwargs["follow_redirects"] is False
+
+
[email protected]("response", [FakeResponse(status=404), FakeResponse(status=502), FakeResponse(content_type="text/plain")])
+def test_an_older_or_failing_edge_never_settles(response, monkeypatch):
+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(response)):
+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) is None
+ with patch.object(pixel.httpx, "AsyncClient", side_effect=pixel.httpx.ConnectError("private credential")):
+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) is None
+ assert EDGE_KEY not in json.dumps(response.headers)
diff --git a/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx b/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx
index 0e2f5ca9d..7347aac95 100644
--- a/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx
+++ b/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx
@@ -24,7 +24,11 @@ function useModelFailureCause(content) {
return cause
}
+// Replies saved by older versions carry only this marker. Its cause was never
+// known, so it is not blamed on the connection.
+export const UNDELIVERED_REPLY='This reply did not arrive. Your conversation is preserved; send your message again if you still need it.'
+
export default function PortalResponseError({content}) {
const cause=useModelFailureCause(content)
- return <div className="portal-response-error" role="status"><CircleAlert size={13}/><PortalStreamingText instant>{content==='Request failed'?'The response could not be received. Your conversation is preserved; check the connection before continuing.':content}</PortalStreamingText>{cause && <p className="portal-response-cause">{cause}</p>} <HelpLink label="Still stuck? Get help on Discord"/></div>
+ return <div className="portal-response-error" role="status"><CircleAlert size={13}/><PortalStreamingText instant>{content==='Request failed'?UNDELIVERED_REPLY:content}</PortalStreamingText>{cause && <p className="portal-response-cause">{cause}</p>} <HelpLink label="Still stuck? Get help on Discord"/></div>
}
diff --git a/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx b/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx
index 8a58f67ac..ef5bb3ee8 100644
--- a/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx
+++ b/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx
@@ -1,10 +1,11 @@
import {render,screen} from '@testing-library/react'
-import PortalResponseError from './PortalResponseError'
+import PortalResponseError, {UNDELIVERED_REPLY} from './PortalResponseError'
-it('explains a generic transport failure without claiming that the work succeeded',()=>{
+it('explains an older undelivered reply without claiming success or blaming the connection',()=>{
render(<PortalResponseError content="Request failed"/>);
- expect(screen.getByRole('status')).toHaveTextContent('The response could not be received.')
- expect(screen.getByRole('status')).toHaveTextContent('check the connection before continuing')
+ expect(screen.getByRole('status')).toHaveTextContent(UNDELIVERED_REPLY)
+ expect(screen.getByRole('status')).not.toHaveTextContent(/connection|could not be received/)
+ expect(screen.getByRole('link',{name:/Get help on Discord/})).toBeVisible()
expect(screen.queryByText('Request failed')).toBeNull()
})
it('names the cause of a generic failure when the latest model call failed',async()=>{
diff --git a/ods/extensions/services/dashboard/src/components/portal-agent-experience.css b/ods/extensions/services/dashboard/src/components/portal-agent-experience.css
index caffa7610..48ba41cdf 100644
--- a/ods/extensions/services/dashboard/src/components/portal-agent-experience.css
+++ b/ods/extensions/services/dashboard/src/components/portal-agent-experience.css
@@ -113,3 +113,7 @@
.portal-context-status > span { flex:1; min-width:180px; }
.portal-context-status > button { padding:3px 5px; background:transparent; color:rgb(var(--theme-text-secondary)); text-decoration:underline; text-underline-offset:3px; }
.portal-context-status > button:focus-visible { outline:1px solid currentColor; outline-offset:2px; }
+/* The one action that settles a held chat reads as a button, not a link. */
+.portal-context-status > button.portal-context-action { display:inline-flex; align-items:center; min-height:30px; padding:5px 11px; border:1px solid rgb(var(--theme-text) / .16); border-radius:8px; color:rgb(var(--theme-text)); background:rgb(var(--theme-text) / .06); font-weight:500; text-decoration:none; }
+.portal-context-status > button.portal-context-action:hover:not(:disabled) { background:rgb(var(--theme-text) / .11); }
+.portal-context-status > button.portal-context-action:disabled { opacity:.5; cursor:not-allowed; }
diff --git a/ods/extensions/services/dashboard/src/lib/pixelMascotState.js b/ods/extensions/services/dashboard/src/lib/pixelMascotState.js
index b9bcffc48..b47ccd878 100644
--- a/ods/extensions/services/dashboard/src/lib/pixelMascotState.js
+++ b/ods/extensions/services/dashboard/src/lib/pixelMascotState.js
@@ -15,5 +15,7 @@ export function pixelReplyPose(message, active) {
if (message.status === 'stopped' || message.goalState === 'cancelled') return 'idle'
// A restored/abandoned streaming placeholder is not proof of ongoing work.
if (message.status === 'streaming') return active ? (message.task?.state === 'running' && message.task?.calls > 0 ? 'working' : 'thinking') : 'waiting'
+ // A reply whose answer stopped arriving waits until Portal settles it.
+ if (message.status === 'unconfirmed') return 'waiting'
return message.content ? 'done' : 'idle'
}
diff --git a/ods/extensions/services/dashboard/src/lib/portalContext.js b/ods/extensions/services/dashboard/src/lib/portalContext.js
index a6fcef18d..42c7dd56d 100644
--- a/ods/extensions/services/dashboard/src/lib/portalContext.js
+++ b/ods/extensions/services/dashboard/src/lib/portalContext.js
@@ -25,6 +25,11 @@ const token=(value,maximum=100_000_000)=>Number.isSafeInteger(value) && value>=0
const text=(value,max)=>typeof value==='string' && value.length>0 && value.length<=max && !/[\u0000-\u001f\u007f]/.test(value)
const routeFingerprint=value=>typeof value==='string' && value.length===64 && /^[a-f0-9]{64}$/.test(value)
const contextRetryDelays=[2000,5000,10000]
+// An unconfirmed last turn is checked again on a growing interval, never every
+// second. Each check asks the server to settle the chat if it proves idle.
+const historyRecheckDelays=[1000,2000,4000,8000,15000,30000,60000]
+export const CONTINUE_FAILED='Portal could not confirm this chat is free yet. Try again in a moment.'
+export const STOP_UNCONFIRMED='Portal could not confirm the stop yet. Try Stop again in a moment.'
export function parseConversationContext(value) {
if(!value || value.schemaVersion!==1 || !['ready','missing','busy','unavailable'].includes(value.status))return null
const {context,compaction,history,model}=value
@@ -54,8 +59,12 @@ function requestId() {
}
/** Compaction belongs to the server session. The browser retains only its request
- * identity; neither the transcript nor an inferred model summary is replaced. */
-export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdentity,capacity,initialRequestId,blocked,onPendingChange}) {
+ * identity; neither the transcript nor an inferred model summary is replaced.
+ * `blocked` refuses every mutation. `held` means the page is still settling its
+ * own interrupted turn: compaction and automatic checks wait for it, but the
+ * owner's explicit Continue stays usable, because that is when it is needed. */
+export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdentity,capacity,initialRequestId,blocked:blockedMutation,held=false,onPendingChange}) {
+ const blocked=Boolean(blockedMutation || held)
const acceptedModel=useRef(null)
const runtime=useRef({chatId,identity:{model:'',source:'',provider:'',routeFingerprint:''},epoch:0})
const typed=runtimeIdentity!==undefined
@@ -79,14 +88,15 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
if(!sameChat)acceptedModel.current=null
}
const runtimeKey=typed?`identity-${runtime.current.epoch}`:legacyRuntimeKey
- const [view,setView]=useState({chatId,runtimeKey,context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,resolving:false,recoveryNotice:'',readUnavailable:false})
+ const [view,setView]=useState({chatId,runtimeKey,context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false,readUnavailable:false})
const current=useRef({}),pending=useRef(initialRequestId || null),requests=useRef(new Set()),query=useRef(null),mutation=useRef(false)
- const refreshQueued=useRef(false)
+ const refreshQueued=useRef(false),historyAttempts=useRef(0),historyPaused=useRef(false)
const generation=useRef(0),lastQuery=useRef(0),priorRuntime=useRef(runtimeKey)
const priorChat=useRef(chatId),lastMeasurement=useRef(null),invalidatedMeasurement=useRef(null)
const lastManual=useRef(initialRequestId || null),observedCompaction=useRef(null)
const contextRetries=useRef(0),priorAvailability=useRef({chatId,runtimeKey,blocked})
- current.current={chatId,runtimeKey,runtimeIdentity:typed?runtime.current.identity:null,capacity,blocked,onPendingChange,historyUnknown:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyUnknown,
+ current.current={chatId,runtimeKey,runtimeIdentity:typed?runtime.current.identity:null,capacity,blocked,explicitBlocked:Boolean(blockedMutation),onPendingChange,historyUnknown:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyUnknown,
+ historyActive:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyActive,
readUnavailable:view.chatId===chatId && view.runtimeKey===runtimeKey && view.readUnavailable}
const publish=useCallback(update=>setView(previous=>({...previous,...update,chatId:current.current.chatId,runtimeKey:current.current.runtimeKey})),[])
const persist=useCallback(id=>{current.current.onPendingChange(id,current.current.chatId);pending.current=id},[])
@@ -132,7 +142,8 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
const historyUnknown=snapshot.history.status==='unknown'
const readUnavailable=snapshot.status==='unavailable'
if(!readUnavailable)contextRetries.current=0
- publish({context,phase,notice,historyUnknown,readUnavailable,observedModel:modelMatches && ['ready','missing','busy'].includes(snapshot.status)?snapshot.model || null:null,...(snapshot.model?{observedCapacity:modelMatches?snapshot.model.contextWindow:null}:{}),...(!historyUnknown?{recoveryNotice:''}:{})})
+ if(!historyUnknown)historyAttempts.current=0
+ publish({context,phase,notice,historyUnknown,readUnavailable,observedModel:modelMatches && ['ready','missing','busy'].includes(snapshot.status)?snapshot.model || null:null,...(snapshot.model?{observedCapacity:modelMatches?snapshot.model.contextWindow:null}:{}),...(!historyUnknown?{recoveryNotice:'',recoveryFailed:false,historyChecking:true,historyActive:false}:{})})
},[persist,publish])
const refresh=useCallback(async(force=false)=>{
@@ -168,6 +179,36 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
return run
},[accept,publish])
+ // An unconfirmed last turn usually means a restart cut it off. The server
+ // settles the chat only when it proves nothing can still run there, and it
+ // never stops work, so this check may run without the owner. Work it finds
+ // still running is shown as such and waited for. Only a check that can tell
+ // neither asks the owner to act.
+ const settleHistory=useCallback(async()=>{
+ if(current.current.blocked || mutation.current || pending.current || !current.current.historyUnknown)return refresh(true)
+ const at=generation.current,identity=current.current.chatId,controller=new AbortController()
+ requests.current.add(controller)
+ const timer=setTimeout(()=>controller.abort(),15000)
+ let state='unknown'
+ try {
+ const response=await fetch('/api/pixel/chat/activity',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({chat_id:identity}),signal:controller.signal})
+ const value=await response.json()
+ if(response.ok && value && Object.keys(value).length===1 && ['active','terminal','unknown'].includes(value.state))state=value.state
+ }catch {
+ // An unanswered check proves nothing; the owner can still continue.
+ }finally {clearTimeout(timer);requests.current.delete(controller)}
+ if(at!==generation.current)return
+ // A failed Stop or Continue is shown until the chat's state changes kind.
+ const changed=(state==='active')!==Boolean(current.current.historyActive)?{recoveryFailed:false,recoveryNotice:''}:{}
+ if(state==='active')publish({historyChecking:false,historyActive:true,...changed})
+ else if(state==='unknown')publish({historyChecking:false,historyActive:false,...changed})
+ await refresh(true)
+ // "Nothing runs" settles the history too. If an older Portal could not
+ // record that, its history stays unconfirmed, and then the owner can
+ // continue: nothing is running.
+ if(state==='terminal' && at===generation.current)setView(previous=>previous.historyUnknown?{...previous,historyChecking:false,historyActive:false}:previous)
+ },[publish,refresh])
+
useEffect(()=>{
generation.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=false;mutation.current=false;lastQuery.current=0
pending.current=initialRequestId || null
@@ -176,7 +217,8 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
else if(priorRuntime.current && priorRuntime.current!==runtimeKey)invalidatedMeasurement.current=lastMeasurement.current
priorChat.current=chatId
contextRetries.current=0
- publish({context:null,observedCapacity:null,observedModel:null,phase:pending.current?'checking':'idle',notice:pending.current?'Checking compaction status…':'',historyUnknown:false,resolving:false,recoveryNotice:'',readUnavailable:false})
+ historyAttempts.current=0
+ publish({context:null,observedCapacity:null,observedModel:null,phase:pending.current?'checking':'idle',notice:pending.current?'Checking compaction status…':'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false,readUnavailable:false})
priorRuntime.current=runtimeKey
void refresh(true)
return ()=>{generation.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=false}
@@ -185,13 +227,25 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
useEffect(()=>{
if(view.chatId!==chatId || view.runtimeKey!==runtimeKey)return
- const observingOperation=pending.current || observedCompaction.current && view.phase==='unknown' || ['running','busy'].includes(view.phase) || view.historyUnknown
- const retryRead=!observingOperation && view.readUnavailable && contextRetries.current<contextRetryDelays.length
- if(!observingOperation && !retryRead)return
- const delay=retryRead?contextRetryDelays[contextRetries.current]:['unknown','unavailable'].includes(view.phase)?5000:2000
- const timer=setTimeout(()=>{if(retryRead)contextRetries.current+=1;void refresh(true)},delay)
+ const operation=pending.current || observedCompaction.current && view.phase==='unknown' || ['running','busy'].includes(view.phase)
+ const historyOnly=!operation && view.historyUnknown
+ // While the page works or checks its own interrupted turn, it refreshes
+ // this context when it is done; resume checking promptly after that.
+ if(historyOnly && blocked){historyPaused.current=true;return}
+ if(historyPaused.current){historyPaused.current=false;historyAttempts.current=0}
+ const retryRead=!operation && !historyOnly && view.readUnavailable && contextRetries.current<contextRetryDelays.length
+ if(!operation && !historyOnly && !retryRead)return
+ // Work still running is checked at least every 15 s, so the chat frees
+ // itself soon after that work ends.
+ const delay=historyOnly?Math.min(historyRecheckDelays[Math.min(historyAttempts.current,historyRecheckDelays.length-1)],view.historyActive?15000:Infinity)
+ :retryRead?contextRetryDelays[contextRetries.current]:['unknown','unavailable'].includes(view.phase)?5000:2000
+ const timer=setTimeout(()=>{
+ if(retryRead)contextRetries.current+=1
+ if(historyOnly){historyAttempts.current+=1;void settleHistory()}
+ else void refresh(true)
+ },delay)
return ()=>clearTimeout(timer)
- },[view,chatId,runtimeKey,refresh])
+ },[view,chatId,runtimeKey,blocked,refresh,settleHistory])
useEffect(()=>{
const previous=priorAvailability.current
@@ -237,31 +291,40 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden
}finally {clearTimeout(timer);requests.current.delete(controller);if(at===generation.current){mutation.current=false;if(requery)void refresh(true)}}
},[accept,persist,publish,refresh])
- const resolveInterrupted=useCallback(async()=>{
- if(current.current.blocked || !current.current.historyUnknown || mutation.current || pending.current)return false
+ // Both owner actions on an unconfirmed history stop whatever runs for this
+ // chat. Continue is for a history Portal cannot tell about; Stop is for
+ // work the server reports still running. Merely loading a conversation or
+ // reading its context never cancels work, and the hold is released only
+ // after the server confirms nothing is running for this chat.
+ const releaseChat=useCallback(async stop=>{
+ if(current.current.explicitBlocked || !current.current.historyUnknown || Boolean(current.current.historyActive)!==stop || mutation.current || pending.current)return false
generation.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=false
const at=generation.current,identity=current.current.chatId,controller=new AbortController()
mutation.current=true;requests.current.add(controller)
- publish({resolving:true,recoveryNotice:'Confirming the previous turn has stopped…'})
+ publish({resolving:true,recoveryFailed:false,recoveryNotice:stop?'Stopping…':'Making sure nothing is still running in this chat…'})
const timer=setTimeout(()=>controller.abort(),30000)
try {
- // This explicit owner action resolves uncertain history. Merely loading
- // a conversation or reading its context must never cancel work.
const response=await fetch('/api/pixel/chat/cancel',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({chat_id:identity}),signal:controller.signal})
const result=await response.json()
if(at!==generation.current)return false
if(!response.ok || result?.aborted!==true)throw new Error('stop unconfirmed')
- publish({recoveryNotice:'Stop confirmed. Checking conversation context…'})
+ publish({recoveryNotice:stop?'Stopping…':'Continuing in this chat…'})
return true
}catch {
- if(at===generation.current)publish({recoveryNotice:'The stop could not be confirmed. Your conversation is preserved; check status or try again.'})
+ if(at===generation.current)publish({recoveryFailed:true,recoveryNotice:stop?STOP_UNCONFIRMED:CONTINUE_FAILED})
return false
}finally {
clearTimeout(timer);requests.current.delete(controller)
if(at===generation.current){mutation.current=false;publish({resolving:false});void refresh(true)}
}
},[publish,refresh])
+ // Continue stops whatever is left over, so it is never offered, nor sent,
+ // while the server reports work still running in this chat.
+ const resolveInterrupted=useCallback(()=>releaseChat(false),[releaseChat])
+ const stopHistory=useCallback(()=>releaseChat(true),[releaseChat])
- const displayed=view.chatId===chatId && view.runtimeKey===runtimeKey?view:{context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,resolving:false,recoveryNotice:''}
- return {...displayed,busy:displayed.resolving || ['requesting','checking','running','unknown'].includes(displayed.phase),canResolve:displayed.historyUnknown && !blocked && !pending.current && !displayed.resolving,compact,refresh,resolveInterrupted}
+ const displayed=view.chatId===chatId && view.runtimeKey===runtimeKey?view:{context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false}
+ const releasable=displayed.historyUnknown && !blockedMutation && !pending.current && !displayed.resolving
+ return {...displayed,busy:displayed.resolving || ['requesting','checking','running','unknown'].includes(displayed.phase),
+ canResolve:releasable && !displayed.historyActive,canStop:releasable && displayed.historyActive,compact,refresh,resolveInterrupted,stopHistory}
}
diff --git a/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx b/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx
index e51e11c9b..32d4a897f 100644
--- a/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx
+++ b/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx
@@ -1,5 +1,5 @@
import {act,renderHook,waitFor} from '@testing-library/react'
-import {compactCommand,historySnapshot,parseConversationContext,usePortalContext} from './portalContext'
+import {compactCommand,historySnapshot,parseConversationContext,STOP_UNCONFIRMED,usePortalContext} from './portalContext'
const context=(used=1200)=>({schemaVersion:1,status:'ready',sessionRevision:null,model:{id:'small',provider:'local',contextWindow:4096},
context:{used,window:4096,measuredAt:'2026-09-16T12:00:00.000Z'},compaction:{status:'idle',count:0},history:{revision:null,acknowledgedMessages:2}})
@@ -131,6 +131,152 @@ it('never cancels from inspection and requires unknown history plus an explicit
expect(result.current.historyUnknown).toBe(false)
})
+it('settles an unconfirmed last turn through the server by itself, on a growing interval and without a stop',async()=>{
+ vi.useFakeTimers()
+ try {
+ let history='unknown',checks=0
+ globalThis.fetch=vi.fn().mockImplementation(async url=>{
+ if(url.endsWith('/activity')){checks+=1;if(checks===2)history='ready';return response({state:checks===2?'terminal':'unknown'})}
+ return response({...context(),status:history==='unknown'?'unavailable':'ready',history:{...context().history,status:history}})
+ })
+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))
+ await act(async()=>{})
+ expect(result.current.historyUnknown).toBe(true)
+ // Until a check fails to prove the turn is over, the owner is not asked.
+ expect(result.current.historyChecking).toBe(true)
+ await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(checks).toBe(1)
+ expect(result.current.historyChecking).toBe(false)
+ expect(result.current.canResolve).toBe(true)
+ await act(async()=>vi.advanceTimersByTimeAsync(1999))
+ expect(checks).toBe(1)
+ await act(async()=>vi.advanceTimersByTimeAsync(1))
+ expect(checks).toBe(2)
+ expect(result.current.historyUnknown).toBe(false)
+ const calls=fetch.mock.calls.length
+ await act(async()=>vi.advanceTimersByTimeAsync(10*60_000))
+ expect(fetch).toHaveBeenCalledTimes(calls)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'},{chat_id:'a'}])
+ }finally {vi.useRealTimers()}
+})
+
+it('checks an unprovable last turn at most every minute instead of every two seconds',async()=>{
+ vi.useFakeTimers()
+ try {
+ globalThis.fetch=vi.fn().mockImplementation(async url=>response(url.endsWith('/activity')?{state:'unknown'}:{...context(),status:'unavailable',history:{...context().history,status:'unknown'}}))
+ renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))
+ // Step the clock so each check's re-render schedules the next one.
+ for(let second=0;second<10*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ // 1, 2, 4, 8, 15, 30 s, then once a minute: 15 checks in ten minutes, not 300.
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).length).toBeLessThanOrEqual(16)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/context')).length).toBeLessThanOrEqual(17)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)
+ }finally {vi.useRealTimers()}
+})
+
+it('waits for work the server still runs, never offers Continue for it, and frees the chat when it ends',async()=>{
+ vi.useFakeTimers()
+ try {
+ let history='unknown',state='active'
+ globalThis.fetch=vi.fn().mockImplementation(async url=>{
+ if(url.endsWith('/activity')){if(state==='terminal')history='ready';return response({state})}
+ return response({...context(),status:history==='unknown'?'busy':'ready',history:{...context().history,status:history}})
+ })
+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))
+ await act(async()=>{})
+ expect(result.current.historyActive).toBe(false)
+ await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(result.current.historyUnknown).toBe(true)
+ expect(result.current.historyActive).toBe(true)
+ // Continue stops whatever is left over, so it is neither offered nor sent.
+ expect(result.current.canResolve).toBe(false)
+ await act(async()=>expect(await result.current.resolveInterrupted()).toBe(false))
+ // Running work is checked at least every 15 s, not once a minute.
+ for(let second=0;second<5*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ const checks=fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).length
+ expect(checks).toBeGreaterThanOrEqual(20)
+ expect(checks).toBeLessThanOrEqual(26)
+ expect(result.current.historyActive).toBe(true)
+ state='terminal'
+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(result.current.historyUnknown).toBe(false)
+ expect(result.current.historyActive).toBe(false)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)
+ }finally {vi.useRealTimers()}
+})
+
+it('stops work the server still runs only on the owner’s Stop, and frees the chat once that is confirmed',async()=>{
+ vi.useFakeTimers()
+ try {
+ let history='unknown',aborted=false
+ globalThis.fetch=vi.fn().mockImplementation(async url=>{
+ if(url.endsWith('/activity'))return response({state:'active'})
+ if(url.endsWith('/cancel')){if(aborted)history='ready';return response({aborted})}
+ return response({...context(),status:history==='unknown'?'busy':'ready',history:{...context().history,status:history}})
+ })
+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))
+ await act(async()=>{})
+ await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(result.current.historyActive).toBe(true)
+ expect(result.current.canStop).toBe(true)
+ expect(result.current.canResolve).toBe(false)
+ // Nothing is stopped without the owner.
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)
+ await act(async()=>expect(await result.current.stopHistory()).toBe(false))
+ expect(result.current.recoveryFailed).toBe(true)
+ expect(result.current.recoveryNotice).toBe(STOP_UNCONFIRMED)
+ expect(STOP_UNCONFIRMED).toBe('Portal could not confirm the stop yet. Try Stop again in a moment.')
+ aborted=true
+ await act(async()=>expect(await result.current.stopHistory()).toBe(true))
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'},{chat_id:'a'}])
+ expect(result.current.historyUnknown).toBe(false)
+ expect(result.current.historyActive).toBe(false)
+ }finally {vi.useRealTimers()}
+})
+
+it('offers Continue when nothing runs but an older Portal could not record that',async()=>{
+ vi.useFakeTimers()
+ try {
+ // An older pixel edge remembers the chat ended but cannot settle its history.
+ globalThis.fetch=vi.fn().mockImplementation(async url=>response(url.endsWith('/activity')?{state:'terminal'}:{...context(),status:'unavailable',history:{...context().history,status:'unknown'}}))
+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))
+ await act(async()=>{})
+ expect(result.current.historyChecking).toBe(true)
+ await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(result.current.historyUnknown).toBe(true)
+ expect(result.current.historyChecking).toBe(false)
+ expect(result.current.historyActive).toBe(false)
+ expect(result.current.canResolve).toBe(true)
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)
+ }finally {vi.useRealTimers()}
+})
+
+it('waits for the page to settle its own turn, but never blocks the owner’s Continue',async()=>{
+ vi.useFakeTimers()
+ try {
+ let history='unknown'
+ globalThis.fetch=vi.fn().mockImplementation(async url=>{
+ if(url.endsWith('/cancel')){history='ready';return response({aborted:true})}
+ return response(url.endsWith('/activity')?{state:'unknown'}:{...context(),status:'unavailable',history:{...context().history,status:history}})
+ })
+ const props={chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn(),held:true}
+ const {result,rerender}=renderHook(value=>usePortalContext(value),{initialProps:props})
+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))
+ expect(result.current.historyUnknown).toBe(true)
+ // The page checks its interrupted turn itself; nothing here doubles it.
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity'))).toHaveLength(0)
+ expect(await result.current.compact()).toBe(false)
+ // The explicit action is exactly what an unconfirmed turn needs.
+ expect(result.current.canResolve).toBe(true)
+ await act(async()=>expect(await result.current.resolveInterrupted()).toBe(true))
+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'}])
+ expect(result.current.historyUnknown).toBe(false)
+ rerender({...props,held:false,blocked:true})
+ expect(result.current.canResolve).toBe(false)
+ }finally {vi.useRealTimers()}
+})
+
it.each(['runtime-restarted','result-unconfirmed'])('distinguishes a stopped runtime from an unconfirmed running compaction (%s)',async reason=>{
const id='11111111-2222-4333-8444-555555555555',persist=vi.fn()
globalThis.fetch=vi.fn().mockResolvedValue(response({...context(),
diff --git a/ods/extensions/services/dashboard/src/lib/portalTaskState.js b/ods/extensions/services/dashboard/src/lib/portalTaskState.js
index 2f3ba372a..81725d714 100644
--- a/ods/extensions/services/dashboard/src/lib/portalTaskState.js
+++ b/ods/extensions/services/dashboard/src/lib/portalTaskState.js
@@ -17,10 +17,11 @@ export function newTaskBlockedReason({running, stopping, checking, launching, co
// The composer stays editable while Portal works. Sending waits, and says why
// instead of doing nothing.
-export function sendBlockedReason({deleted, running, stopping, waiting, ready}) {
+export function sendBlockedReason({deleted, running, stopping, held, waiting, ready}) {
if (deleted) return ''
if (stopping) return 'Portal is stopping the current task. Your message is kept here; send it when Portal has stopped.'
if (running) return 'Portal is finishing the current task. Your message is kept here; send it when Portal is done, or stop the task.'
+ if (held) return 'Choose Continue in this chat first. Your message is kept here.'
if (waiting || !ready) return 'Portal is not ready for a new message yet. Your message is kept here; send it when Portal is ready.'
return ''
}
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.jsx
index 6737eede2..bfb1b03ff 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.jsx
@@ -32,7 +32,7 @@ import PixelQuestions from '../components/PixelQuestions'
import PortalGoalPlan from '../components/PortalGoalPlan'
import {goalCommand,continueGoal} from '../lib/portalGoal'
import PortalContextRing from '../components/PortalContextRing'
-import {compactCommand,CONTEXT_REQUEST_ID,historySnapshot,usePortalContext} from '../lib/portalContext'
+import {compactCommand,CONTEXT_REQUEST_ID,CONTINUE_FAILED,historySnapshot,STOP_UNCONFIRMED,usePortalContext} from '../lib/portalContext'
import PortalModelSelector from '../components/PortalModelSelector'
import PortalRuntimeIdentity from '../components/PortalRuntimeIdentity'
import PortalReadiness from '../components/PortalReadiness'
@@ -143,6 +143,36 @@ const MODEL_SWITCH_DETAIL = 'Model switch in progress; Portal will be ready when
const CLEAN_CONTEXT_RECOVERY_REASON = 'operations-unavailable-zero-submissions'
const CLEAN_CONTEXT_RECOVERY_NOTICE = 'The first attempt did not reach the Operations Broker, and the host verified that no work was submitted. Retrying once with a clean context…'
const CLEAN_CONTEXT_RECOVERY_FAILED = 'Automatic recovery was attempted once, but Portal again did not reach the Operations Broker. The host verified that no Operations work was submitted. Check any other work before continuing; this does not confirm that other tools had no effects.'
+// The last turn's answer did not arrive. Calm wording: nothing failed that the
+// owner must repair, and the composer is free again. Its work may have run on
+// and finished after the cut-off (the gateway can outlive the dashboard API,
+// and delegated work outlives both), so nothing offers to send the message
+// again in one click: that could repeat every tool action and file write.
+export const RESTART_NOTICE = 'Portal restarted before its reply arrived. Your files are saved. Ask Portal what it did before you send your message again.'
+export const NO_REPLY_NOTICE = 'Portal finished working on your last request, but its reply couldn’t be shown here. Your files are saved. Ask Portal what it did, or send your next message.'
+export const NOT_RECEIVED_NOTICE = 'Portal didn’t receive your last message. It’s back in the message box, ready to send again.'
+const CHECKING_TURN = 'Checking on your last request…'
+const TURN_STILL_RUNNING = 'Portal is still working on your last request. Its answer will appear here when it’s done. You can stop it.'
+// Work still running in this chat whose answer the page cannot receive, for
+// example after a restart cut the reply off while delegated work went on.
+const TURN_RUNNING_UNSEEN = 'Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.'
+// Work the server found running for this chat when the page has no turn of
+// its own to follow there.
+const CHAT_STILL_RUNNING = 'Portal is still working on your last request. You can stop it, or send your next message when it’s done.'
+const TURN_UNCONFIRMED = 'Portal can’t confirm your last request has finished.'
+const CONTINUING_TURN = 'Making sure nothing is still running in this chat…'
+// A Stop of earlier work that was not acknowledged. The alert goes once that
+// turn settles: there is nothing left to stop.
+const RESTORED_STOP_UNCONFIRMED = STOP_UNCONFIRMED
+const clearRestoredStopError = current => current === RESTORED_STOP_UNCONFIRMED ? '' : current
+// A cut-off turn is checked again on a growing interval and the checks stop
+// once its state is proven. Work that is still running is checked at most
+// every 15 s so the chat frees up soon after it finishes.
+const TURN_RECHECK_MS = [1000, 2000, 4000, 8000, 15000, 30000, 60000]
+const ACTIVE_RECHECK_MAX_MS = 15000
+// A refused message is sent on its own only soon after the owner sent it.
+const HELD_SEND_MAX_MS = 2 * 60 * 1000
+const HELD_SEND_NOTE = ' Your new message will go out right after.'
const STATUS_POLL_MS = 3000
const OPS_STATUS_POLL_MS = 3000
const OPS_TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'cancelled', 'rejected'])
@@ -441,6 +471,25 @@ function stoppedContent(content) {
return `${partial}\n\n---\n\n_${STOPPED_NOTICE}_`
}
+// A reply cut off before its answer arrived keeps the text that did arrive,
+// followed by one calm line.
+function cutOffContent(content, notice) {
+ const partial = typeof content === 'string' ? content.trimEnd() : ''
+ if (!partial) return notice
+ if (partial.includes(notice)) return partial
+ return `${partial}\n\n_${notice}_`
+}
+
+// An activity receipt that recorded nothing would only add an empty log to a
+// cut-off reply.
+function recordedTask(task) {
+ return task && (task.events?.length || task.activities?.length || task.goal) ? task : undefined
+}
+
+// A reply whose outcome was never confirmed: the answer stopped arriving and
+// no receipt has settled it yet. It is not saved as a final outcome.
+const unsettledReply = message => message?.role === 'assistant' && !['done', 'error', 'stopped'].includes(message.status)
+
function messagePublication(message) {
const validate = preview => parseVerifiedPreviewFrame({choices:[{finish_reason:'stop'}],pixel:{schemaVersion:1,preview}})
if (message?.role !== 'assistant') return {}
@@ -633,6 +682,9 @@ export default function Pixel({ systemStatus = null }) {
if (status === 'recovery') window.dispatchEvent(new Event(PIXEL_RECOVERY_REQUIRED))
}, [status])
const [messages, setMessages] = useState(() => initialChat?.messages || [])
+ // Checks that settle a cut-off turn later read the conversation as rendered.
+ const messagesRef = useRef(messages)
+ messagesRef.current = messages
const [input, setInput] = useState(() => initialChat?.draft || '')
const [persistenceError, setPersistenceError] = useState('')
const [sending, setSending] = useState(false)
@@ -642,9 +694,26 @@ export default function Pixel({ systemStatus = null }) {
const [stopError, setStopError] = useState('')
// A turn Portal refused because it was busy; its draft is back in the composer.
const [refusal, setRefusal] = useState('')
+ // "Continue in this chat": '' | 'pending' | 'failed'.
+ const [continuing, setContinuing] = useState('')
+ // The chat whose owner chose Continue and whose server confirmed that
+ // nothing runs there: that confirmation alone releases its hold.
+ const continueRequested = useRef(null)
+ // A message refused because an earlier turn still held this chat. It is sent
+ // as soon as that turn is cleared up, unless the owner changes it first. The
+ // ref makes that send happen once, even if a render still shows it pending.
+ const [heldSend, setHeldSend] = useState(null)
+ const heldSendRef = useRef(null)
+ const holdSend = useCallback(value => { heldSendRef.current = value; setHeldSend(value) }, [])
+ // Set only while that automatic send starts: it is made once per owner
+ // action, never again for its own refusal.
+ const automaticSendRef = useRef(false)
const [sendHint, setSendHint] = useState(false)
const [exampleError, setExampleError] = useState(null)
const [restoredActivity, setRestoredActivity] = useState(() => initialChat?.interrupted ? 'checking' : 'idle')
+ // Whether the work still running there is this turn's own reply, which the
+ // server keeps receiving and the page then shows, or only work in the chat.
+ const [restoredReplyLive, setRestoredReplyLive] = useState(false)
const [activityRefresh, setActivityRefresh] = useState(0)
const [workingElapsedSeconds, setWorkingElapsedSeconds] = useState(0)
const [agentRuntime, setAgentRuntime] = useState(null)
@@ -753,13 +822,19 @@ export default function Pixel({ systemStatus = null }) {
const previewAccess = resolvePreviewAccess(currentPreview)
const restoredActive = interrupted && !sending && restoredActivity === 'active'
const restoredChecking = interrupted && !sending && restoredActivity === 'checking'
+ // Portal could not prove what happened to the last turn. Sending waits for
+ // the owner's Continue, which is offered right where they look.
+ const restoredUnconfirmed = interrupted && !sending && restoredActivity === 'unknown'
const contextCapacity=contextRuntime?.source==='external-host'
? Number(contextRuntime.contextLength) || null
: Number(contextRuntime?.contextLength || systemStatus?.inference?.contextSize || systemStatus?.model?.contextLength) || null
const contextControl=usePortalContext({chatId:chatIdRef.current,
runtimeIdentity:{model:contextRuntime?.model || activeModel,source:contextRuntime?.source || '',routeFingerprint:contextRuntime?.routeFingerprint},capacity:contextCapacity,
initialRequestId:compactionRequestRef.current,
- blocked:conversationDeleted || sending || modelSwitching || stopping || teams.busy || (interrupted && restoredActivity!=='terminal') || status!=='available',
+ blocked:conversationDeleted || sending || modelSwitching || stopping || teams.busy || status!=='available',
+ // The page settles its own interrupted turn first. That holds compaction
+ // and automatic checks, never the owner's explicit Continue.
+ held:interrupted && restoredActivity!=='terminal',
onPendingChange:(id,chatId)=>{
if(chatId!==chatIdRef.current)throw new Error('The conversation changed before compaction could be saved.')
historySnapshot(messages)
@@ -785,10 +860,60 @@ export default function Pixel({ systemStatus = null }) {
let controller = null
let disposed = false
let timer = null
+ let attempt = 0
+ let rereadReceipt = Boolean(requestId)
+ // Each check waits longer than the last, and a proven state ends them.
+ // Work still running is checked at least every 15 s.
+ const later = state => {
+ const delay = TURN_RECHECK_MS[Math.min(attempt, TURN_RECHECK_MS.length - 1)]
+ attempt += 1
+ timer = globalThis.setTimeout(checkActivity, state === 'active' ? Math.min(delay, ACTIVE_RECHECK_MAX_MS) : delay)
+ }
+ // No receipt holds this turn and the server proved that nothing still
+ // runs for this chat, so no answer will arrive. The work may well be done,
+ // so the reply keeps the text that did arrive and says so plainly. After
+ // the owner's Continue, which stops anything left over, it says stopped.
+ // A message that has no receipt and got nothing back never reached
+ // Portal: it goes back to the composer. The composer is free, with
+ // whatever the owner had typed.
+ function settleWithoutReceipt({stopped = false, received = true} = {}) {
+ continueRequested.current = null
+ requestIdRef.current = null
+ setStopError(clearRestoredStopError)
+ const latest = messagesRef.current
+ const last = latest.at(-1)
+ const user = latest.at(-2)
+ // Its pictures are still attached in the composer, so it goes back as
+ // a whole and is not left in the conversation to be sent twice.
+ const unsent = !stopped && !received && unsettledReply(last) && !last.content?.trim() && !recordedTask(last.task)
+ && user?.role === 'user' && typeof user.content === 'string' && Boolean(user.content.trim() || user.images?.length)
+ if (unsent) {
+ setMessages(latest.slice(0, -2))
+ setInput(current => restoreUnsentDraft(current, user.content))
+ setRefusal(NOT_RECEIVED_NOTICE)
+ } else if (unsettledReply(last)) {
+ setMessages(replaceLastAssistant(latest, {content:stopped ? stoppedContent(last.content) : cutOffContent(last.content, NO_REPLY_NOTICE),
+ status:'stopped', task:recordedTask(last.task)}))
+ }
+ setInterrupted(false)
+ updateRestoredActivity('idle')
+ void contextControl.refresh(true)
+ }
async function checkActivity() {
+ if (continueRequested.current === chatId) {
+ // The owner chose Continue and the server confirmed that nothing runs
+ // for this chat. That confirmation releases the hold by itself, even
+ // while the receipt or the chat lookup cannot be read.
+ settleWithoutReceipt({stopped:true})
+ return
+ }
controller = new AbortController()
const deadline = globalThis.setTimeout(() => controller.abort(), 15000)
let state = 'unknown'
+ let receiptPending = false
+ // Only a receipt lookup that answered can say this turn has none; one
+ // that failed proves nothing, and then the turn is not settled here.
+ let noReceipt = false
try {
if (requestId) {
const resultResponse = await fetch('/api/pixel/chat/result', {
@@ -800,8 +925,9 @@ export default function Pixel({ systemStatus = null }) {
if (resultResponse.ok && result && Object.keys(result).sort().join(',') === 'events,state'
&& typeof result.events === 'string' && result.events.length <= 8 * 1024 * 1024) {
if (result.state === 'active') {
+ setRestoredReplyLive(true)
updateRestoredActivity('active')
- timer = globalThis.setTimeout(checkActivity, 2000)
+ later('active')
return
}
if (['complete', 'interrupted', 'cancelled', 'rejected'].includes(result.state)) {
@@ -816,7 +942,21 @@ export default function Pixel({ systemStatus = null }) {
let chat = loadStoredChat(readConversations().find(item => item.chatId === chatId))
if (!chat || chat.chatId !== chatId) throw new Error('Saved conversation is unavailable')
const recovered = retainedResult(result.events)
- const successful = result.state === 'complete' && recovered.done && !recovered.failed
+ // The ingress sends a turn's whole answer in one frame, after it
+ // recorded the turn complete. A receipt that lost its producer
+ // after saving that text lost only its closing frames: the turn
+ // finished, and its answer is shown as such.
+ const answered = result.state === 'interrupted' && !recovered.failed && Boolean(recovered.content.trim())
+ const successful = (result.state === 'complete' && recovered.done && !recovered.failed) || answered
+ // Its closing frame carried the final activity log; the running
+ // snapshot saved before it is kept only if it recorded something,
+ // and it replaces the page's own snapshot, never adds to it.
+ const recoveredTask = answered ? {task:recordedTask(recovered.task)} : recovered.task ? {task:recovered.task} : {}
+ // A receipt without its answer lost its producer with the
+ // dashboard API process, and the server then proved nothing
+ // still runs: Portal restarted while it was answering. Whether
+ // its work ran on cannot be told, so it is never re-sent here.
+ const cutOff = result.state === 'interrupted' && !recovered.done && !recovered.failed && !answered
if (chat.requestId === requestId && chat.interrupted) {
const source = chat.persistenceSnapshot
const publication = successful ? recovered.preview : null
@@ -824,11 +964,15 @@ export default function Pixel({ systemStatus = null }) {
const writer = createConversationWriter(source)
try {
let saved = writer.recover({...source, requestId:null, inFlight:false, interrupted:false,
- messages:replaceLastAssistant(source.messages, {
+ messages:replaceLastAssistant(source.messages, cutOff ? {
+ content: cutOffContent(recovered.content, RESTART_NOTICE),
+ status: 'stopped',
+ task: recordedTask(recovered.task || source.messages.at(-1)?.task),
+ } : {
content: result.state === 'cancelled' ? stoppedContent(recovered.content)
: recovered.content || (successful ? 'Completed without a text response.' : 'Portal could not complete the response. Check saved work before continuing.'),
status: result.state === 'cancelled' ? 'stopped' : successful ? 'done' : 'error',
- ...(recovered.task ? {task:recovered.task} : {}),
+ ...recoveredTask,
...(successful && recovered.questions ? {questions:recovered.questions} : {}),
...(successful && recovered.artifacts ? {artifacts:recovered.artifacts} : {}),
...(publication ? {publication,beforePublication:before} : {}),
@@ -850,6 +994,10 @@ export default function Pixel({ systemStatus = null }) {
}
// The sender may already have saved, or another request may now
// own this chat. Adopt its authoritative record without writing.
+ // The receipt settled this turn, so a Continue still answering
+ // for it has nothing left to release, nor a Stop to retry.
+ continueRequested.current = null
+ setStopError(clearRestoredStopError)
conversationWriter.current = createConversationWriter(chat.persistenceSnapshot)
hydrating.current = true; hydrationAction.current = null; savedView.current = null
requestIdRef.current = chat.requestId
@@ -866,6 +1014,10 @@ export default function Pixel({ systemStatus = null }) {
void contextControl.refresh(true)
return
}
+ // An unresolved receipt: the chat lookup below may prove the turn
+ // is over, and that also settles this receipt on the server.
+ receiptPending = result.state === 'unresolved'
+ noReceipt = result.state === 'unknown'
}
}
const response = await fetch('/api/pixel/chat/activity', {
@@ -879,8 +1031,23 @@ export default function Pixel({ systemStatus = null }) {
// A failed lookup or edge restart is not evidence that work finished.
} finally { globalThis.clearTimeout(deadline) }
if (disposed || chatIdRef.current !== chatId) return
+ if (state === 'terminal' && receiptPending && rereadReceipt) {
+ // The server settled the receipt as it proved the chat idle; read it
+ // once more so whatever was saved of the answer is kept.
+ rereadReceipt = false
+ timer = globalThis.setTimeout(checkActivity, 0)
+ return
+ }
+ if (state === 'terminal' && (!requestId || noReceipt || receiptPending)) {
+ settleWithoutReceipt({received:!noReceipt})
+ return
+ }
+ // Nothing runs, but this turn's receipt could not be read: it may still
+ // hold the answer. Check again rather than settle without it.
+ if (state === 'terminal') state = 'unknown'
+ setRestoredReplyLive(false)
updateRestoredActivity(state)
- if (state !== 'terminal') timer = globalThis.setTimeout(checkActivity, 2000)
+ later(state)
}
checkActivity()
return () => {
@@ -890,6 +1057,15 @@ export default function Pixel({ systemStatus = null }) {
}
}, [interrupted, sending, activityRefresh, updateRestoredActivity])
+ // Portal coming back after an update or restart is when a cut-off turn can
+ // settle: check it then, not at the end of a long interval.
+ const priorStatus = useRef(status)
+ useEffect(() => {
+ const previous = priorStatus.current
+ priorStatus.current = status
+ if (status === 'available' && !['available', 'loading'].includes(previous) && interrupted && !sending) setActivityRefresh(value => value + 1)
+ }, [status, interrupted, sending])
+
useEffect(() => {
let controller = null
let stopped = false
@@ -1097,12 +1273,14 @@ export default function Pixel({ systemStatus = null }) {
}, [messages, preview, workspaceOpen, sending, interrupted, input, imageDraftKey, conversationDeleted])
const sendMessage = useCallback(async (answerOverride, continuationId = null) => {
+ const automatic = automaticSendRef.current
+ automaticSendRef.current = false
if (conversationDeleted) return
const exactDraft = typeof answerOverride === 'string' ? answerOverride : input
const trimmed = (typeof answerOverride === 'string' ? answerOverride : input).trim()
if(compactCommand(trimmed)){await compactConversation();return}
if(contextControl.busy || contextControl.historyUnknown)return
- if ((!trimmed && !images.items.length) || sending || modelSwitching || abortRef.current || restoredActive || restoredChecking || status !== 'available' || trimmed.length > MAX_INPUT_LEN) return
+ if ((!trimmed && !images.items.length) || sending || modelSwitching || abortRef.current || restoredActive || restoredChecking || restoredUnconfirmed || status !== 'available' || trimmed.length > MAX_INPUT_LEN) return
if(images.busy){setStopError('Finish uploading or remove the failed image before sending. Your draft is preserved.');return}
const turnImages=typeof answerOverride==='string'?[]:images.receipts.map(({id,sha256})=>({id,sha256}))
const needsImageRoute=turnImages.length>0 || hasImageHistory
@@ -1151,6 +1329,9 @@ export default function Pixel({ systemStatus = null }) {
userMessage,
]
const visibleConversation = [...messages, userMessage]
+ // A refused send must not replace the identity of a turn that still holds
+ // this chat; it is the one that can still be recovered.
+ const heldRequestId = requestIdRef.current
setMessages([...visibleConversation, { role: 'assistant', content: '', status: 'streaming', revealResponse:makeChatId() }])
if (typeof answerOverride !== 'string') setInput('')
setSending(true)
@@ -1159,6 +1340,10 @@ export default function Pixel({ systemStatus = null }) {
setStopping(false)
setStopError('')
setRefusal('')
+ holdSend(null)
+ setContinuing('')
+ // A Continue for an earlier turn never releases this one unchecked.
+ continueRequested.current = null
setExampleError(null)
// Text typed while this turn runs is a new draft. A turn that does not
// start returns its own text to the composer without replacing that draft.
@@ -1255,13 +1440,22 @@ export default function Pixel({ systemStatus = null }) {
requestIdRef.current = null
throw new Error('chat-request-rejected')
}
+ if (response.status === 423) {
+ // An earlier turn still holds this chat, most often one a restart
+ // cut off. Nothing started. Keep following that turn, never this
+ // refused attempt, which has no receipt to recover.
+ requestIdRef.current = heldRequestId
+ return { kind: 'held' }
+ }
if (response.status === 429) {
// Portal was busy (for example preparing another picture) and
// refused this turn before starting it. Nothing ran.
requestIdRef.current = null
return { kind: 'busy' }
}
- if (!response.ok) throw new Error('chat unavailable')
+ // A gateway error here usually means the dashboard API is restarting;
+ // any other refusal is a real failure the owner may need help with.
+ if (!response.ok) throw new Error(response.status >= 502 && response.status <= 504 ? 'chat-connection-lost' : 'chat unavailable')
reader = response.body?.getReader()
if (!reader) throw new Error('stream unavailable')
@@ -1369,7 +1563,7 @@ export default function Pixel({ systemStatus = null }) {
// An acknowledged Stop or page disposal can close a reader normally.
// Its late close must not overwrite the explicit cancellation outcome.
if (!isCurrentTurn()) return
- if (attempt.receivedError) { setInterrupted(true); return }
+ if (attempt.receivedError) { setInterrupted(true); updateRestoredActivity('checking'); return }
if (attempt.receivedDone) requestIdRef.current = null
if (attempt.receivedDone) {
const previousPublication = [...messages].reverse().find(message => message.publication?.relativeDirectory === attempt.verifiedPreview?.relativeDirectory)?.publication || preview
@@ -1389,11 +1583,13 @@ export default function Pixel({ systemStatus = null }) {
}))
return
}
- const content = attempt.assistantText
- ? `${attempt.assistantText}\n\n_Response interrupted._`
- : 'Connection interrupted'
+ // The answer stopped arriving before Portal said it was done, for
+ // example because Portal restarted. Keep what arrived; the check that
+ // follows settles it. The server may hold the whole answer, may still
+ // be working, or may prove the turn was cut off.
setInterrupted(true)
- setMessages(previous => replaceLastAssistant(previous, { content, status: 'error' }))
+ updateRestoredActivity('checking')
+ setMessages(previous => replaceLastAssistant(previous, { content: attempt.assistantText, status: 'unconfirmed' }))
}
try {
@@ -1417,6 +1613,25 @@ export default function Pixel({ systemStatus = null }) {
setRefusal(busyRefusalText(turnImages.length > 0))
return
}
+ if (attempt.kind === 'held') {
+ // Not a failure either, and never the connection: this message did
+ // not start because an earlier turn still holds the chat. It waits in
+ // the composer while that turn is cleared up, then goes out.
+ restoreDraft()
+ contextStartRef.current = originalContextStart
+ setMessages(messages)
+ if (automatic) {
+ // That automatic send was refused too. It is not repeated: the
+ // message waits in the composer, one click from going out.
+ setRefusal(busyRefusalText(false))
+ return
+ }
+ if (typeof answerOverride !== 'string' && !turnImages.length) holdSend({chatId:chatIdRef.current, text:trimmed, at:Date.now()})
+ setInterrupted(true)
+ updateRestoredActivity('checking')
+ setActivityRefresh(value => value + 1)
+ return
+ }
if (attempt.kind === 'rejected') {
restoreDraft()
contextStartRef.current = originalContextStart
@@ -1472,7 +1687,8 @@ export default function Pixel({ systemStatus = null }) {
updateRestoredActivity('idle')
return
}
- if (attempt.kind === 'busy') {
+ if (attempt.kind === 'busy' || attempt.kind === 'held') {
+ if (attempt.kind === 'held') requestIdRef.current = null
contextStartRef.current = originalContextStart
setMessages(messages)
restoreDraft()
@@ -1522,10 +1738,18 @@ export default function Pixel({ systemStatus = null }) {
const storageFailed = conversationChanged || error?.message === 'chat-recovery-storage-unavailable'
const historyTooLarge=error?.message==='history-request-too-large'
const requestRejected=error?.message==='chat-request-rejected'
- setInterrupted(!storageFailed && !historyTooLarge && !requestRejected)
- if (storageFailed || historyTooLarge || requestRejected || turnImages.length) restoreDraft()
- setMessages(previous => replaceLastAssistant(previous, {
- content: requestRejected?'Portal rejected the request format (HTTP 422). No task was started. Your draft is preserved. Refresh Portal before retrying.':historyTooLarge?'The encoded conversation exceeds the 8 MB request limit. No task was started. Export this conversation before starting a new chat.':conversationChanged ? 'This conversation changed in another tab. No task was started. Download a recovery copy, then reload to read the saved version.' : storageFailed ? 'Could not save the request for recovery. No task was started. Check browser storage and try again.' : latestAssistantText || 'Request failed',
+ const refused=error?.message==='chat unavailable'
+ // A lost connection is not yet a failure: the turn may have started,
+ // finished or been cut off by a restart. Keep what arrived and let
+ // the check that follows settle it.
+ const uncertain=!storageFailed && !historyTooLarge && !requestRejected && !refused
+ // A refusal may still have saved a receipt that explains it; check.
+ const recover=!storageFailed && !historyTooLarge && !requestRejected
+ setInterrupted(recover)
+ if (recover) updateRestoredActivity('checking')
+ if (!uncertain || turnImages.length) restoreDraft()
+ setMessages(previous => replaceLastAssistant(previous, uncertain ? {content:latestAssistantText, status:'unconfirmed'} : {
+ content: requestRejected?'Portal rejected the request format (HTTP 422). No task was started. Your draft is preserved. Refresh Portal before retrying.':historyTooLarge?'The encoded conversation exceeds the 8 MB request limit. No task was started. Export this conversation before starting a new chat.':conversationChanged ? 'This conversation changed in another tab. No task was started. Download a recovery copy, then reload to read the saved version.' : storageFailed ? 'Could not save the request for recovery. No task was started. Check browser storage and try again.' : 'Portal could not start this message. Your draft is kept; try sending it again.',
status: 'error',
}))
}
@@ -1543,7 +1767,7 @@ export default function Pixel({ systemStatus = null }) {
void contextControl.refresh(true)
}
}
- }, [input, messages, preview, workspaceOpen, sending, modelSwitching, status, restoredActive, restoredChecking, updateRestoredActivity, teams.busy, teams.start,compactConversation,contextControl.busy,contextControl.historyUnknown,contextControl.refresh,startExtensionInstallation,startGithubExtensionRequest,images,imageModel,hasImageHistory,conversationDeleted])
+ }, [input, messages, preview, workspaceOpen, sending, modelSwitching, status, restoredActive, restoredChecking, restoredUnconfirmed, updateRestoredActivity, teams.busy, teams.start,compactConversation,contextControl.busy,contextControl.historyUnknown,contextControl.refresh,startExtensionInstallation,startGithubExtensionRequest,images,imageModel,hasImageHistory,conversationDeleted])
const stopStreaming = useCallback(async () => {
const controller = abortRef.current
@@ -1596,10 +1820,13 @@ export default function Pixel({ systemStatus = null }) {
controller?.abort()
abortRef.current = null
requestIdRef.current = null
- setMessages(previous => replaceLastAssistant(previous, {
+ // The live stream's own reply is the one stopped, whatever its last
+ // frame said. Without one, only a reply still unresolved is: a held
+ // chat's Stop ends older work, and its last reply may be finished.
+ setMessages(previous => controller || unsettledReply(previous.at(-1)) ? replaceLastAssistant(previous, {
content: stoppedContent(controller?.responseText?.() ?? previous.at(-1)?.content),
status: 'stopped',
- }))
+ }) : previous)
setSending(false)
setInterrupted(false)
updateRestoredActivity('terminal')
@@ -1608,7 +1835,7 @@ export default function Pixel({ systemStatus = null }) {
// without an exact acknowledgement could leave tools or inference active.
if (stopRequestRef.current === stopRequest && chatIdRef.current === chatId && abortRef.current === controller) {
setStopError(restored
- ? 'Stop was not confirmed. This chat may still have work in progress; check its activity or retry the stop request.'
+ ? RESTORED_STOP_UNCONFIRMED
: 'Stop was not confirmed. Portal is still connected; retry Stop.')
if (restored) setActivityRefresh(value => value + 1)
}
@@ -1622,6 +1849,47 @@ export default function Pixel({ systemStatus = null }) {
}
}, [stopping, interrupted, updateRestoredActivity])
+ // "Continue in this chat" is the one action offered when Portal cannot prove
+ // what happened to the last turn. It asks the server to stop anything left
+ // over in this chat and releases the hold only after the server confirms
+ // that nothing is running there.
+ const continueChat = useCallback(async () => {
+ if (!interrupted) { void contextControl.resolveInterrupted(); return }
+ if (continuing === 'pending' || sending || stopping) return
+ const chatId = chatIdRef.current
+ const controller = new AbortController()
+ const timeout = setTimeout(() => controller.abort(), 30000)
+ setContinuing('pending')
+ try {
+ const response = await fetch('/api/pixel/chat/cancel', {
+ method: 'POST', signal: controller.signal, headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ chat_id: chatId }),
+ })
+ const payload = await response.json()
+ if (chatIdRef.current !== chatId) return
+ if (!response.ok || !payload || Object.keys(payload).length !== 1 || payload.aborted !== true) throw new Error('continue unconfirmed')
+ continueRequested.current = chatId
+ setContinuing('')
+ // The next check settles the turn at once and frees the composer.
+ setActivityRefresh(value => value + 1)
+ } catch {
+ if (chatIdRef.current === chatId) setContinuing('failed')
+ } finally {
+ clearTimeout(timeout)
+ }
+ }, [interrupted, continuing, sending, stopping, contextControl.resolveInterrupted, updateRestoredActivity])
+
+ // Settle, then send: the message a held chat refused goes out once the
+ // earlier turn is cleared up, while the composer still holds it unchanged.
+ useEffect(() => {
+ if (!heldSend || heldSendRef.current !== heldSend || interrupted || sending || stopping || status !== 'available'
+ || contextControl.historyUnknown || contextControl.busy) return
+ holdSend(null)
+ if (heldSend.chatId !== chatIdRef.current || Date.now() - heldSend.at > HELD_SEND_MAX_MS || input.trim() !== heldSend.text) return
+ automaticSendRef.current = true
+ void sendMessage()
+ }, [heldSend, interrupted, sending, stopping, status, contextControl.historyUnknown, contextControl.busy, input, sendMessage, holdSend])
+
// One live task streams into this page; see portalTaskState for why a new
// task waits for it. Every New task control shows this reason, never nothing.
const newTaskBlocked = newTaskBlockedReason({running:sending || restoredActive, stopping, checking:restoredChecking,
@@ -1632,6 +1900,9 @@ export default function Pixel({ systemStatus = null }) {
const startNewChat = useCallback((preserveDraft = false) => {
if (newTaskBlocked) { setStopError(newTaskBlocked); return }
setRefusal('')
+ holdSend(null)
+ setContinuing('')
+ continueRequested.current = null
setSendHint(false)
chatIdRef.current = makeChatId()
images.replace(chatIdRef.current)
@@ -1649,7 +1920,7 @@ export default function Pixel({ systemStatus = null }) {
setExampleError(null)
updateRestoredActivity('idle')
inputRef.current?.focus?.()
- }, [newTaskBlocked, updateRestoredActivity, images.replace, input])
+ }, [newTaskBlocked, updateRestoredActivity, images.replace, input, holdSend])
useEffect(() => {
const remove = async event => {
@@ -1771,6 +2042,9 @@ export default function Pixel({ systemStatus = null }) {
setInput(chat.draft)
setStopError('')
setRefusal('')
+ holdSend(null)
+ setContinuing('')
+ continueRequested.current = null
setSendHint(false)
setExampleError(null)
setInterrupted(chat.interrupted)
@@ -1782,7 +2056,7 @@ export default function Pixel({ systemStatus = null }) {
}
window.addEventListener(SELECT_EVENT, select)
return () => window.removeEventListener(SELECT_EVENT, select)
- }, [sending, restoredActive, restoredChecking, stopping, updateRestoredActivity, teams.launching,contextControl.busy,images.replace])
+ }, [sending, restoredActive, restoredChecking, stopping, updateRestoredActivity, teams.launching,contextControl.busy,images.replace,holdSend])
const inputOver = input.length > MAX_INPUT_LEN
const inputEmpty = !(command?.task ?? goalDraft?.task ?? input).trim() && !images.items.length
@@ -1791,6 +2065,7 @@ export default function Pixel({ systemStatus = null }) {
// editable and is saved with the chat. Only sending waits, and says why.
const composerLocked = conversationDeleted || modelSwitching || status !== 'available'
const sendBlocked = sendBlockedReason({deleted:conversationDeleted, stopping, running:sending || restoredActive || teams.busy,
+ held:restoredUnconfirmed || (contextControl.historyUnknown && !contextControl.historyChecking && !contextControl.historyActive),
waiting:restoredChecking || contextControl.busy || contextControl.historyUnknown || modelSwitching, ready:status === 'available'})
useEffect(() => { if (!sendBlocked) setSendHint(false) }, [sendBlocked])
const composerVisible = !(workspaceExpanded && workspaceOpen && !previewCollapsed)
@@ -1844,9 +2119,7 @@ export default function Pixel({ systemStatus = null }) {
: restoredActive
? 'Working in this chat'
: restoredChecking
- ? 'Checking previous work'
- : interrupted && restoredActivity === 'unknown'
- ? 'Activity unknown'
+ ? 'Checking last request'
: status === 'available'
? (readinessView.attention ? 'Needs attention' : 'Available')
: status === 'switching'
@@ -1862,6 +2135,27 @@ export default function Pixel({ systemStatus = null }) {
: status === 'loading'
? 'Connecting...'
: 'Degraded'
+ // One calm line beside the composer while the last turn is settled. Its one
+ // action, Continue in this chat, stops whatever is left over, so it appears
+ // only after a check found that Portal cannot tell: never while a check runs
+ // or while work is still running.
+ const turnHold = sending ? null : interrupted ? (
+ continuing === 'pending' ? {text:CONTINUING_TURN, busy:true}
+ : restoredActive ? {text:restoredReplyLive ? TURN_STILL_RUNNING : TURN_RUNNING_UNSEEN}
+ : restoredUnconfirmed ? {text:continuing === 'failed' ? CONTINUE_FAILED : TURN_UNCONFIRMED,
+ failed:continuing === 'failed', action:continueChat, disabled:stopping}
+ : {text:CHECKING_TURN, busy:true}
+ ) : contextControl.historyUnknown ? (
+ contextControl.resolving ? {text:contextControl.recoveryNotice || CONTINUING_TURN, busy:true}
+ // Here the page follows no turn of its own, so Stop sits in this line.
+ : contextControl.historyActive ? {text:contextControl.recoveryFailed ? STOP_UNCONFIRMED : CHAT_STILL_RUNNING,
+ busy:!contextControl.recoveryFailed, failed:contextControl.recoveryFailed,
+ action:contextControl.stopHistory, label:'Stop', disabled:!contextControl.canStop}
+ : contextControl.recoveryFailed ? {text:CONTINUE_FAILED, failed:true, action:continueChat, disabled:!contextControl.canResolve}
+ : contextControl.historyChecking ? {text:CHECKING_TURN, busy:true}
+ : {text:TURN_UNCONFIRMED, action:continueChat, disabled:!contextControl.canResolve}
+ ) : null
+ const heldNote = heldSend?.chatId === chatIdRef.current && turnHold ? HELD_SEND_NOTE : ''
// Native Windows Core has local model chat but no Pixel host runtime. The
// status API distinguishes that deliberate selection from a temporary
@@ -1981,27 +2275,6 @@ export default function Pixel({ systemStatus = null }) {
onClick={() => startNewChat(true)}>New chat with draft</button>
{(images.items.length > 0 || hasImageHistory) && <p>Attachments will not be copied. Reattach the original files in the new chat.</p>}
</div>}
- {interrupted && !sending && (
- <div role="status" className="mx-auto w-full max-w-5xl rounded-xl border border-theme-border bg-theme-text-secondary/10 px-4 py-3 text-sm text-theme-text-secondary">
- {restoredActivity === 'active'
- ? 'The previous request is still active in this chat. Your saved conversation and preview are preserved. You can stop that work below; its live response cannot be reattached.'
- : restoredActivity === 'terminal'
- ? 'The previous request is no longer active. Its final response was not recovered; check the saved files and results before continuing.'
- : restoredActivity === 'checking'
- ? 'Checking whether this chat’s previous request is still active. Your saved conversation and preview are preserved.'
- : 'Completion was not confirmed. This chat’s activity is unknown. Your request and partial response are saved; check its results before continuing. You can attempt to stop previous work in this chat without resending it.'}
- {restoredActivity === 'unknown' && (
- <>
- <button type="button" className="ml-2 underline" disabled={stopping} onClick={() => {
- updateRestoredActivity('checking')
- setActivityRefresh(value => value + 1)
- }}>Check activity again</button>
- <button type="button" className="ml-2 underline" disabled={stopping}
- onClick={stopStreaming}>Try Stop previous work</button>
- </>
- )}
- </div>
- )}
{status === 'loading' && messages.length === 0 && (
<div className="flex h-full flex-col items-center justify-center text-theme-text-muted">
<Loader2 className="mb-3 h-8 w-8 animate-spin" />
@@ -2094,7 +2367,7 @@ export default function Pixel({ systemStatus = null }) {
{message.role === 'assistant' && message.content ? (
<>
{message.publication && <PixelSnapshotChanges preview={message.publication} before={message.beforePublication} variant="summary" onPreview={()=>openPublication(message.publication,'preview')} onReview={path=>openPublication(message.publication,'review',path)}/>}
- {!message.questions && (shownContent || message.status==='streaming') && (message.status==='error' ? <PortalResponseError content={shownContent}/> : <PortalStreamingText key={message.revealResponse || chatIdRef.current} active={message.status==='streaming'} animate={Boolean(message.revealResponse) || message.status==='streaming'} instant={['stopped','recovering'].includes(message.status)} onReveal={chatScroll.onContentResize} components={MARKDOWN_COMPONENTS} artifacts={message.role==='assistant' && message.status==='done' ? message.artifacts : null}>{shownContent}</PortalStreamingText>)}
+ {!message.questions && (shownContent || message.status==='streaming') && (message.status==='error' ? <PortalResponseError content={shownContent}/> : <PortalStreamingText key={message.revealResponse || chatIdRef.current} active={message.status==='streaming'} animate={Boolean(message.revealResponse) || message.status==='streaming'} instant={['stopped','recovering','unconfirmed'].includes(message.status)} onReveal={chatScroll.onContentResize} components={MARKDOWN_COMPONENTS} artifacts={message.role==='assistant' && message.status==='done' ? message.artifacts : null}>{shownContent}</PortalStreamingText>)}
<OperationsApprovalCard content={message.content} />
{!message.questions && message.status!=='streaming' && <PortalResponseActions content={displayedContent}/>}
</>
@@ -2168,7 +2441,7 @@ export default function Pixel({ systemStatus = null }) {
) : (
<button
onClick={sendFromComposer}
- disabled={isDisabled || inputOver || inputEmpty || images.busy}
+ disabled={isDisabled || restoredUnconfirmed || inputOver || inputEmpty || images.busy}
className="inline-flex h-11 w-11 items-center justify-center rounded-xl bg-theme-accent text-white transition hover:bg-theme-accent-hover disabled:cursor-not-allowed disabled:opacity-40"
title="Send"
>
@@ -2184,10 +2457,11 @@ export default function Pixel({ systemStatus = null }) {
<button type="button" onClick={sendFromComposer}>Send again</button>
</div>}
{exampleError && <p role="alert" className="mt-1.5 px-1 text-xs text-theme-text-secondary">{exampleError.text}{exampleError.help && <> <HelpLink /></>}</p>}
- {contextControl.historyUnknown && <div className="portal-context-status" role="status">
- {contextControl.resolving && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}
- <span>{contextControl.recoveryNotice || 'The previous turn could not be confirmed. Resolve it before continuing; your conversation is preserved.'}</span>
- <button type="button" disabled={!contextControl.canResolve} onClick={()=>void contextControl.resolveInterrupted()}>Resolve interrupted turn</button>
+ {turnHold && <div className="portal-context-status" role="status" aria-label="Last message">
+ {turnHold.busy && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}
+ <span>{turnHold.text}{heldNote}</span>
+ {turnHold.action && <button type="button" className="portal-context-action" disabled={turnHold.disabled} onClick={()=>void turnHold.action()}>{turnHold.label || 'Continue in this chat'}</button>}
+ {turnHold.failed && <HelpLink />}
</div>}
{contextControl.notice && <div className="portal-context-status" role={contextControl.phase==='failed'?'alert':'status'}>
{contextControl.busy && contextControl.phase!=='unknown' && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}
@@ -2206,7 +2480,7 @@ export default function Pixel({ systemStatus = null }) {
</div>
</div>
<div className="mt-1.5 flex items-center justify-between gap-3 px-1 text-[10px] text-theme-text-muted/70">
- <span>{stopping ? 'Waiting for exact cancellation acknowledgement' : restoredActive ? 'Earlier work is active in this chat; Stop targets only this chat.' : sending ? `${displayName} is using the active ODS model and tools · ${workingElapsed} elapsed` : sendKey.mode === 'mod-enter' ? 'Ctrl/⌘+Enter to send • Enter for a new line' : 'Enter to send • Shift+Enter for a new line'}</span>
+ <span>{stopping ? 'Waiting for exact cancellation acknowledgement' : restoredActive ? 'Stop ends only the work in this chat.' : sending ? `${displayName} is using the active ODS model and tools · ${workingElapsed} elapsed` : sendKey.mode === 'mod-enter' ? 'Ctrl/⌘+Enter to send • Enter for a new line' : 'Enter to send • Shift+Enter for a new line'}</span>
</div>
</div>
{inputOver && (
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx
index 4228a61b5..9bd5f89ab 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx
@@ -61,7 +61,7 @@ it('opening a second tab on an active request does not overwrite the streaming s
const before = localStorage.getItem(CHAT_KEY)
const observer = render(<Pixel/>)
try {
- await within(observer.container).findByText(/The previous request is still active in this chat/)
+ await within(observer.container).findByText(/Portal is still working on your last request/)
expect(localStorage.getItem(CHAT_KEY)).toBe(before)
} finally {
observer.unmount()
@@ -157,7 +157,7 @@ it.each(['ack-first','stream-first'])('recovers observer Stop without stale conf
})
await deliver(first)
const observer = render(<Pixel/>)
- await within(observer.container).findByText(/previous request is still active/)
+ await within(observer.container).findByText(/Portal is still working on your last request/)
// The source advances after observer hydration; Stop must retain that work.
await deliver(second)
fireEvent.click(within(observer.container).getByTitle('Stop'))
diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx
index ad092a212..f8c01629e 100644
--- a/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx
@@ -14,6 +14,7 @@ const previewManifests=new Map()
// The repository's base ESLint profile does not mark JSX identifiers as uses.
// eslint-disable-next-line no-unused-vars
import Pixel, {
+ NO_REPLY_NOTICE,
OperationsApprovalCard,
formatElapsed,
isCleanContextRecoveryFrame,
@@ -28,6 +29,8 @@ const response = (body, status = 200) => ({
status,
json: async () => body,
})
+// The calm line a reply ends with once the server proved its turn is over.
+const CUT_OFF = (_content, node) => node?.tagName === 'EM' && node.textContent === NO_REPLY_NOTICE
it('resolves old cards to the latest verified version of their own project',()=>{
const publication=(letter,directory)=>{
@@ -2044,10 +2047,11 @@ describe('Pixel', () => {
await act(async () => {})
for (let reload = 0; reload < 2; reload += 1) {
const restored = render(<Pixel />)
- await screen.findByText('Activity unknown')
+ await screen.findByText('Portal can’t confirm your last request has finished.')
+ expect(screen.getByRole('button', { name: 'Continue in this chat' })).toBeEnabled()
expect(screen.getByText('Create a report, then verify it.')).toBeInTheDocument()
expect(screen.getByText('Created report.md; checking its contents.')).toBeInTheDocument()
- expect(screen.getByText(/Completion was not confirmed/)).toBeInTheDocument()
+ expect(screen.queryByText(/Activity unknown|Completion was not confirmed/)).not.toBeInTheDocument()
expect(screen.queryByText(/Stopped by you/)).not.toBeInTheDocument()
expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()
expect(globalThis.fetch.mock.calls.filter(([url]) => url === '/api/pixel/chat/stream')).toHaveLength(1)
@@ -2202,9 +2206,10 @@ describe('Pixel', () => {
expect(signals).toHaveLength(1)
await act(async () => { await vi.advanceTimersByTimeAsync(15000) })
expect(signals[0].aborted).toBe(true)
- expect(screen.getByRole('button',{name:'Check activity again'})).toBeEnabled()
+ expect(screen.getByRole('button',{name:'Continue in this chat'})).toBeEnabled()
expect(screen.getByText('Saved partial result')).toBeVisible()
- await act(async () => { fireEvent.click(screen.getByRole('button',{name:'Check activity again'})) })
+ // The next check starts by itself, with a fresh deadline.
+ await act(async () => { await vi.advanceTimersByTimeAsync(1000) })
expect(signals).toHaveLength(2)
expect(signals[1]).not.toBe(signals[0])
expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()
@@ -2230,12 +2235,19 @@ describe('Pixel', () => {
expect(screen.getByRole('button', {name:'New chat'})).toHaveAttribute('aria-disabled', 'true')
expect(screen.getByRole('button', {name:'New chat'})).toHaveAttribute('title', expect.stringContaining('Portal is finishing the current task'))
expect(screen.getByText('Saved partial result')).toBeInTheDocument()
+ // No receipt brings this turn's answer back here, so the line never promises it.
+ expect(screen.getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()
+ expect(screen.queryByRole('button', {name:'Continue in this chat'})).not.toBeInTheDocument()
state = 'terminal'
- await screen.findByText(/previous request is no longer active/, {}, { timeout: 3000 })
+ // The server proved the turn is over: the reply settles calmly, it is
+ // never claimed as the owner's Stop, and the composer is free again.
+ await screen.findByText(CUT_OFF, {}, { timeout: 3000 })
expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()
- expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()
- expect(screen.getByText(/final response was not recovered/)).toBeInTheDocument()
+ expect(screen.queryByText(/Stopped by you|no longer active|was not recovered/)).not.toBeInTheDocument()
expect(screen.getByText('Saved partial result')).toBeInTheDocument()
+ expect(screen.queryByRole('status', { name: 'Last message' })).not.toBeInTheDocument()
+ fireEvent.change(screen.getByPlaceholderText('Message Portal...'), { target: { value: 'What did you change?' } })
+ expect(screen.getByTitle('Send')).toBeEnabled()
expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/stream')).toBe(false)
})
@@ -2249,7 +2261,7 @@ describe('Pixel', () => {
return response({state:chat_id === 'second_interrupted' ? 'active' : 'terminal'})
})
render(<Pixel />)
- await screen.findByText(/previous request is no longer active/)
+ await screen.findByText(CUT_OFF)
act(()=>window.dispatchEvent(new CustomEvent('ods:pixel-select-conversation',{detail:'second_interrupted'})))
await screen.findByText('Working in this chat')
expect(fetch).toHaveBeenCalledWith('/api/pixel/chat/activity',expect.objectContaining({body:JSON.stringify({chat_id:'second_interrupted'})}))
@@ -2270,11 +2282,11 @@ describe('Pixel', () => {
return response({ aborted })
})
render(<Pixel />)
- await screen.findByText('Activity unknown')
+ await screen.findByText('Portal can’t confirm your last request has finished.')
expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()
state = 'active'
- fireEvent.click(screen.getByText('Check activity again'))
- await screen.findByText('Working in this chat')
+ // The next check, on a growing interval, finds the work still running.
+ await screen.findByText('Working in this chat', {}, { timeout: 3000 })
expect(screen.getByRole('textbox')).toBeDisabled()
fireEvent.click(screen.getByTitle('Stop'))
await screen.findByRole('alert')
@@ -2301,9 +2313,11 @@ describe('Pixel', () => {
await screen.findByTitle('Stop')
fireEvent.click(screen.getByTitle('Stop'))
state = 'terminal'
- await screen.findByText(/previous request is no longer active/, {}, { timeout: 3000 })
+ await screen.findByText(CUT_OFF, {}, { timeout: 3000 })
await act(async () => { finishCancel(response({ aborted: true })) })
- expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()
+ // The late acknowledgement never rewrites the settled reply as a Stop.
+ expect(screen.queryByText(/Stopped by you/)).not.toBeInTheDocument()
+ expect(screen.getByText(CUT_OFF)).toBeInTheDocument()
expect(screen.getByText('Saved partial result')).toBeInTheDocument()
expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()
})
@@ -2320,15 +2334,15 @@ describe('Pixel', () => {
return response({ state: 'active' })
})
render(<Pixel />)
- await screen.findByText('Activity unknown')
- fireEvent.click(screen.getByText('Check activity again'))
- await screen.findByText('Activity unknown')
+ await screen.findByText('Portal can’t confirm your last request has finished.')
+ // Neither a failed lookup nor global activity counts as this chat's work.
+ await waitFor(() => expect(calls).toBe(2), { timeout: 3000 })
+ expect(screen.getByText('Portal can’t confirm your last request has finished.')).toBeVisible()
expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()
- fireEvent.click(screen.getByText('Check activity again'))
- await screen.findByText('Working in this chat')
+ await screen.findByText('Working in this chat', {}, { timeout: 4000 })
})
- it('restored unknown activity permits an explicit exact-chat stop attempt without replay or false stopped claims', async () => {
+ it('restored unknown activity permits an explicit exact-chat Continue without replay, and says stopped only once confirmed', async () => {
saveInterruptedChat()
let aborted = false
globalThis.fetch.mockImplementation(async (url, options) => {
@@ -2339,15 +2353,19 @@ describe('Pixel', () => {
return response({ aborted })
})
render(<Pixel />)
- await screen.findByText('Activity unknown')
+ await screen.findByText('Portal can’t confirm your last request has finished.')
expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/cancel')).toBe(false)
- fireEvent.click(screen.getByText('Try Stop previous work'))
- await screen.findByRole('alert')
+ fireEvent.click(screen.getByRole('button', { name: 'Continue in this chat' }))
+ await screen.findByText('Portal could not confirm this chat is free yet. Try again in a moment.')
expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()
expect(screen.getByText('Saved partial result')).toBeInTheDocument()
aborted = true
- fireEvent.click(screen.getByText('Try Stop previous work'))
+ fireEvent.click(screen.getByRole('button', { name: 'Continue in this chat' }))
+ // Continue stops anything left over: only its confirmation says stopped,
+ // and the reply never claims Portal finished.
await screen.findByText('Response stopped')
+ expect(screen.getByText(/Stopped by you/)).toBeInTheDocument()
+ expect(screen.queryByText(CUT_OFF)).not.toBeInTheDocument()
expect(screen.getByText('Saved partial result')).toBeInTheDocument()
expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/stream')).toBe(false)
})
@@ -2688,7 +2706,7 @@ describe('Pixel', () => {
expect(screen.queryByText(/private-upstream-secret/)).toBeNull()
})
- it('marks a stream that closes without DONE as interrupted', async () => {
+ it('keeps a stream that closes without DONE calm and checks it instead of calling it failed', async () => {
globalThis.fetch.mockResolvedValueOnce(
response({ available: true, model: 'pixel/default', detail: 'local' })
)
@@ -2701,8 +2719,12 @@ describe('Pixel', () => {
fireEvent.change(screen.getByPlaceholderText('Message Portal...'), { target: { value: 'test' } })
fireEvent.click(screen.getByTitle('Send'))
- await waitFor(() => expect(screen.getByText('Response interrupted.')).toBeInTheDocument())
+ expect(await screen.findByRole('status', { name: 'Last message' })).toBeVisible()
expect(screen.getByText('Partial answer')).toBeInTheDocument()
+ expect(screen.getByRole('button', { name: 'Continue in this chat' })).toBeEnabled()
+ expect(screen.queryByText(/Response interrupted|Connection interrupted|could not be received|Ended early/)).not.toBeInTheDocument()
+ await waitFor(() => expect(JSON.parse(localStorage.getItem('ods.pixel.chat.v1'))).toMatchObject({ interrupted: true }))
+ expect(JSON.parse(localStorage.getItem('ods.pixel.chat.v1')).messages.at(-1).status).toBeUndefined()
})
it('preserves partial answers and ignores late content after an upstream error', async () => {
diff --git a/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx
index bd242152e..de96af374 100644
--- a/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx
@@ -262,23 +262,25 @@ it.each([true,false])('resolves uncertain history only after an explicit stop an
return Promise.resolve(response(url==='/api/pixel/chat/context'?context:{available:true,runtime}))
})
render(<Pixel/>);await screen.findByText('Available')
- const resolveButton=await screen.findByRole('button',{name:'Resolve interrupted turn'})
+ const resolveButton=await screen.findByRole('button',{name:'Continue in this chat'})
expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
// The draft stays editable; sending waits until the turn is resolved.
expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()
expect(screen.getByTitle('Send')).toBeDisabled()
expect(screen.queryByText(/Context compaction failed/)).toBeNull()
fireEvent.click(resolveButton)
- expect(resolveButton).toBeDisabled()
+ expect(await screen.findByText('Making sure nothing is still running in this chat…')).toBeVisible()
+ expect(screen.queryByRole('button',{name:'Continue in this chat'})).toBeNull()
expect(screen.getByRole('button',{name:'New chat'})).toHaveAttribute('aria-disabled','true')
await act(async()=>finish())
if(acknowledged) {
- await waitFor(()=>expect(screen.queryByRole('button',{name:'Resolve interrupted turn'})).toBeNull())
+ await waitFor(()=>expect(screen.queryByRole('status',{name:'Last message'})).toBeNull())
expect(screen.getByTitle('Send')).toBeEnabled()
}else {
- await screen.findByText(/The stop could not be confirmed/)
+ await screen.findByText('Portal could not confirm this chat is free yet. Try again in a moment.')
+ expect(screen.getByRole('link',{name:/Get help on Discord/})).toBeVisible()
expect(screen.getByTitle('Send')).toBeDisabled()
- expect(screen.getByRole('button',{name:'Resolve interrupted turn'})).toBeEnabled()
+ expect(screen.getByRole('button',{name:'Continue in this chat'})).toBeEnabled()
}
expect(stored()).toMatchObject({chatId:'context-chat',draft:'Keep the next request',messages:initial})
expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)
diff --git a/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx
index 3e07ce2e0..ee82160ef 100644
--- a/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx
+++ b/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx
@@ -1,5 +1,5 @@
/* global atob */
-import {act,fireEvent,screen,waitFor} from '@testing-library/react'
+import {act,fireEvent,screen,waitFor,within} from '@testing-library/react'
import {render} from '../test/test-utils'
import Pixel from './Pixel'
import {sha256} from '../lib/pixelArtifacts'
@@ -218,11 +218,12 @@ it('waits for an observer image upload before adopting terminal request recovery
return originalFetch(url,options)
})
render(<Pixel/>)
- await screen.findByRole('button',{name:'Check activity again'})
+ await screen.findByRole('button',{name:'Continue in this chat'})
fireEvent.change(screen.getByLabelText('Choose images'),{target:{files:[fixtureFile()]}})
await waitFor(()=>expect(finishUpload).toBeTypeOf('function'))
+ // The next check finds the completed receipt by itself.
terminal=true
- fireEvent.click(screen.getByRole('button',{name:'Check activity again'}))
+ await waitFor(()=>expect(calls('/api/pixel/chat/result').length).toBeGreaterThan(1),{timeout:3000})
await act(async()=>{})
expect(signal.aborted).toBe(false)
expect(screen.getByRole('list',{name:'Attached images'})).toBeVisible()
@@ -382,3 +383,27 @@ it('preserves the draft after preflight validation rejection without claiming a
expect(screen.queryByRole('button',{name:'Resolve interrupted turn'})).toBeNull()
expect(stored().requestId).toBeNull()
})
+
+it('puts an image message that never reached Portal back once, pictures and all, and says so',async()=>{
+ // nginx answered 502 while the dashboard API was recreated: no receipt.
+ stream=async()=>response({detail:'Bad Gateway'},502)
+ const prior=fetchMock.getMockImplementation()
+ fetchMock.mockImplementation(async(url,options)=>url==='/api/pixel/chat/result'?response({state:'unknown',events:''})
+ :url==='/api/pixel/chat/activity'?response({state:'terminal'}):prior(url,options))
+ await ready();await choose()
+ fireEvent.change(screen.getByPlaceholderText('Message Portal...'),{target:{value:'What is in this picture?'}})
+ fireEvent.click(screen.getByTitle('Send'))
+ const note=await screen.findByRole('status',{name:'Message not started'})
+ expect(note).toHaveTextContent('Portal didn’t receive your last message. It’s back in the message box, ready to send again.')
+ expect(screen.queryByText(/Portal finished working/)).toBeNull()
+ expect(screen.getByPlaceholderText('Message Portal...')).toHaveValue('What is in this picture?')
+ expect(screen.getByText(/1 × 1 · Ready/)).toBeVisible()
+ await waitFor(()=>expect(stored().messages.some(message=>message.content==='What is in this picture?')).toBe(false))
+ // Sending it again adds the turn, with its picture, to the history once.
+ stream=async()=>completed('A single pixel.')
+ fireEvent.click(within(note).getByRole('button',{name:'Send again'}))
+ expect(await screen.findByText('A single pixel.')).toBeVisible()
+ const sent=JSON.parse(calls('/api/pixel/chat/stream').at(-1)[1].body)
+ expect(sent.history_snapshot.messages.filter(message=>message.role==='user')).toEqual([
+ {role:'user',content:'What is in this picture?',images:[{id:imageId,sha256:receipt.sha256}]}])
+})
diff --git a/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsx
new file mode 100644
index 000000000..2a70eb079
--- /dev/null
+++ b/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsx
@@ -0,0 +1,534 @@
+// Seen on the RTX Spark: an update recreated the dashboard and its API and
+// restarted Portal's gateway while "Can you do things on cron timers?"
+// streamed. The chat showed "Ended early", "No tool activity was recorded"
+// and "The response could not be received", the next message failed the same
+// way, and the page then sat behind "Activity unknown" with a disabled
+// "Resolve interrupted turn" while it polled three endpoints every second.
+import {act,fireEvent,screen,waitFor,within} from '@testing-library/react'
+import {render} from '../test/test-utils'
+// The repository's base ESLint profile does not mark JSX identifiers as uses.
+// eslint-disable-next-line no-unused-vars
+import Pixel, {NO_REPLY_NOTICE, NOT_RECEIVED_NOTICE, RESTART_NOTICE} from './Pixel'
+import {CONTINUE_FAILED} from '../lib/portalContext'
+
+const QUESTION='Can you do things on cron timers or scheduled work tasks?'
+const PARTIAL='Yes. Portal can run'
+const STALE=/Ended early|No tool activity was recorded|could not be received|check the connection|Activity unknown|could not be confirmed|Resolve interrupted turn|Check activity again|Try Stop previous work|Completion was not confirmed/
+const STOPPED='Stopped by you. Workspace changes completed before cancellation were preserved.'
+const STILL_WORKING='Portal is still working on your last request. You can stop it, or send your next message when it’s done.'
+const CONTINUE={name:'Continue in this chat'}
+const response=(body,status=200)=>({ok:status>=200 && status<300,status,json:async()=>body})
+const encode=value=>new TextEncoder().encode(`data: ${typeof value==='string'?value:JSON.stringify(value)}\n\n`)
+const task={schemaVersion:1,runId:'chatcmpl_11111111-2222-4333-8444-555555555555',startedAt:'2026-10-26T01:34:20.000Z',finishedAt:null,state:'running',calls:0,failures:0,blocked:0,truncated:false,activities:[]}
+const context=(history='ready')=>({schemaVersion:1,status:history==='unknown'?'unavailable':'ready',sessionRevision:null,context:null,model:null,
+ compaction:{status:'idle',count:0},history:{status:history,revision:null,acknowledgedMessages:0}})
+// The ingress sends a turn's whole answer in one frame, after it recorded
+// the turn complete. Before that, only task activity arrives.
+const TASK_ONLY=`data: ${JSON.stringify({object:'ods.task.activity',id:task.runId,pixel_task:task})}\n\n`
+// A turn whose connection drops when the dashboard API is recreated, before
+// its answer arrived or, with text, right after it.
+function cutOffStream(text) {
+ const chunks=[encode({object:'ods.task.activity',id:task.runId,pixel_task:task}),...(text?[encode({choices:[{delta:{content:text}}]})]:[])]
+ return {ok:true,status:200,headers:new Map([['content-type','text/event-stream']]),body:{getReader:()=>({
+ read:async()=>{
+ if(chunks.length)return {done:false,value:chunks.shift()}
+ throw new TypeError('network error')
+ },releaseLock(){},
+ })}}
+}
+const answer=text=>{
+ const chunks=[encode({choices:[{delta:{content:text}}]}),encode('[DONE]')]
+ return {ok:true,status:200,headers:new Map([['content-type','text/event-stream']]),body:{getReader:()=>({
+ read:async()=>chunks.length?{done:false,value:chunks.shift()}:{done:true},releaseLock(){},
+ })}}
+}
+const stored=()=>JSON.parse(localStorage.getItem('ods.pixel.chat.v1'))
+const field=()=>screen.getByPlaceholderText('Message Portal...')
+const row=()=>screen.queryByRole('status',{name:'Last message'})
+
+let server,fetchMock
+const calls=url=>fetchMock.mock.calls.filter(([called])=>called===url)
+beforeEach(()=>{
+ localStorage.clear()
+ server={result:()=>response({state:'unknown',events:''}),activity:()=>response({state:'unknown'}),cancel:()=>response({aborted:true}),
+ context:()=>response(context()),streams:[]}
+ fetchMock=vi.fn(async(url,options)=>{
+ if(url==='/api/pixel/status')return response({available:true})
+ if(url==='/api/pixel/chat/context')return server.context(options)
+ if(url==='/api/pixel/chat/result')return server.result(JSON.parse(options.body),options)
+ if(url==='/api/pixel/chat/activity')return server.activity(JSON.parse(options.body))
+ if(url==='/api/pixel/chat/cancel')return server.cancel(JSON.parse(options.body))
+ if(url==='/api/pixel/chat/stream')return (server.streams.shift() || (()=>answer('Done')))(JSON.parse(options.body))
+ return response({})
+ })
+ vi.stubGlobal('fetch',fetchMock)
+})
+afterEach(()=>{vi.useRealTimers();vi.unstubAllGlobals();vi.restoreAllMocks()})
+
+function saveCutOffChat(extra={}) {
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:'checking-in',inFlight:false,interrupted:true,
+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:PARTIAL}],...extra}))
+}
+
+it('settles a turn a restart cut off by itself, and never offers to repeat it in one click',async()=>{
+ let attempt
+ server.streams.push(body=>{attempt=body;return cutOffStream()})
+ // The restarted server proved nothing still runs and settled the receipt:
+ // the producer died with the old process before the answer was sent. The
+ // gateway may well have finished the turn meanwhile, files and all.
+ server.result=body=>{
+ expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id})
+ return response({state:'interrupted',events:TASK_ONLY})
+ }
+ render(<Pixel/>);await screen.findByText('Available')
+ fireEvent.change(field(),{target:{value:QUESTION}})
+ fireEvent.click(screen.getByTitle('Send'))
+ const reply=await screen.findByText(RESTART_NOTICE)
+ expect(RESTART_NOTICE).toBe('Portal restarted before its reply arrived. Your files are saved. Ask Portal what it did before you send your message again.')
+ const bubble=reply.closest('[data-pixel-response]')
+ expect(within(bubble).getByText('Response stopped')).toBeVisible()
+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()
+ expect(screen.queryByText(STALE)).toBeNull()
+ expect(row()).toBeNull()
+ // The composer is free, and nothing is put back to be sent again.
+ expect(field()).toBeEnabled()
+ expect(field()).toHaveValue('')
+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,inFlight:false,requestId:null}))
+ expect(stored().messages.at(-1)).toMatchObject({status:'stopped',content:RESTART_NOTICE})
+ expect(stored().messages.at(-1).task).toBeUndefined()
+ expect(stored().draft || '').toBe('')
+ // The next message is the owner's own.
+ server.streams.push(()=>answer('I created the scheduled task "cron-check".'))
+ fireEvent.change(field(),{target:{value:'What did you do?'}})
+ fireEvent.click(screen.getByTitle('Send'))
+ expect(await screen.findByText('I created the scheduled task "cron-check".')).toBeVisible()
+ expect(calls('/api/pixel/chat/stream')).toHaveLength(2)
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('settles a receipt that kept the answer as finished, with nothing to send again',async()=>{
+ // The dashboard API was recreated after it saved the answer, before the
+ // closing frames. The ingress had recorded the turn complete before sending.
+ const ANSWER='Yes. Portal can run scheduled tasks: open Automations and add one.'
+ let attempt
+ server.streams.push(body=>{attempt=body;return cutOffStream(ANSWER)})
+ server.result=body=>{
+ expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id})
+ return response({state:'interrupted',events:`${TASK_ONLY}data: ${JSON.stringify({choices:[{delta:{content:ANSWER}}]})}\n\n`})
+ }
+ render(<Pixel/>);await screen.findByText('Available')
+ fireEvent.change(field(),{target:{value:QUESTION}})
+ fireEvent.click(screen.getByTitle('Send'))
+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,inFlight:false,requestId:null}))
+ expect(stored().messages.at(-1)).toMatchObject({role:'assistant',status:'done',content:ANSWER})
+ const bubble=screen.getByText(ANSWER).closest('[data-pixel-response]')
+ expect(within(bubble).queryByText('Response stopped')).toBeNull()
+ expect(within(bubble).queryByRole('button',{name:'Send again'})).toBeNull()
+ expect(screen.queryByText(RESTART_NOTICE,{exact:false})).toBeNull()
+ expect(screen.queryByText(STALE)).toBeNull()
+ expect(row()).toBeNull()
+ // Nothing is put back to be sent again.
+ expect(field()).toHaveValue('')
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('frees the Spark’s window whose own request has no receipt once the server settles the chat',async()=>{
+ // The window kept the refused "Checking in" request, which never got a
+ // receipt, while an older dead attempt held the chat on the server.
+ saveCutOffChat({draft:'checking in'})
+ server.result=body=>{expect(body).toEqual({chat_id:'spark_chat',request_id:'checking-in'});return response({state:'unknown',events:''})}
+ server.activity=()=>response({state:'terminal'})
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ expect(screen.getByText('Available')).toBeVisible()
+ // The work may well be done, so the reply says so plainly and asks for
+ // nothing to be repeated; the owner's own draft stays as it was.
+ expect(screen.getByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE)).toBeVisible()
+ expect(row()).toBeNull()
+ expect(field()).toBeEnabled()
+ expect(field()).toHaveValue('checking in')
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()
+ expect(screen.queryByText(STALE)).toBeNull()
+ // A proven state ends the checks: nothing is polled any more.
+ const checks=calls('/api/pixel/chat/result').length+calls('/api/pixel/chat/activity').length
+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))
+ expect(calls('/api/pixel/chat/result').length+calls('/api/pixel/chat/activity').length).toBe(checks)
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+ expect(stored()).toMatchObject({interrupted:false,requestId:null})
+})
+
+it('puts a message that never reached Portal back in the composer, and says so',async()=>{
+ // nginx answered 502 while the dashboard API was recreated: the request
+ // never arrived, so no receipt exists and nothing came back.
+ let attempt
+ server.streams.push(body=>{attempt=body;return response({detail:'Bad Gateway'},502)})
+ server.result=body=>{expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id});return response({state:'unknown',events:''})}
+ server.activity=()=>response({state:'terminal'})
+ render(<Pixel/>);await screen.findByText('Available')
+ fireEvent.change(field(),{target:{value:QUESTION}})
+ fireEvent.click(screen.getByTitle('Send'))
+ const note=await screen.findByRole('status',{name:'Message not started'})
+ expect(note).toHaveTextContent(NOT_RECEIVED_NOTICE)
+ expect(screen.queryByText(NO_REPLY_NOTICE,{exact:false})).toBeNull()
+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText(QUESTION)).toBeNull()
+ expect(row()).toBeNull()
+ expect(field()).toHaveValue(QUESTION)
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,requestId:null,draft:QUESTION}))
+ expect(stored().messages.some(message=>message.content===QUESTION)).toBe(false)
+ // One click sends it as a new turn.
+ server.streams.push(()=>answer('Yes: use a scheduled task.'))
+ fireEvent.click(within(note).getByRole('button',{name:'Send again'}))
+ expect(await screen.findByText('Yes: use a scheduled task.')).toBeVisible()
+ const resent=JSON.parse(calls('/api/pixel/chat/stream')[1][1].body)
+ expect(resent.messages.filter(message=>message.role==='user')).toEqual([{role:'user',content:QUESTION}])
+ expect(resent.request_id).not.toBe(attempt.request_id)
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('never settles a turn whose receipt could not be read, even when nothing runs',async()=>{
+ saveCutOffChat()
+ server.result=()=>response({detail:'Portal results are unavailable'},503)
+ server.activity=()=>response({state:'terminal'})
+ render(<Pixel/>);await screen.findByText('Available')
+ const hold=await screen.findByRole('status',{name:'Last message'})
+ await within(hold).findByText('Portal can’t confirm your last request has finished.')
+ expect(screen.queryByText(NO_REPLY_NOTICE,{exact:false})).toBeNull()
+ expect(screen.queryByRole('status',{name:'Message not started'})).toBeNull()
+ // Once the receipt can be read, it decides what the reply says. Every
+ // check before that said "unknown", which proves nothing about whether
+ // the turn finished meanwhile, so nothing is put back to send again.
+ server.result=()=>response({state:'interrupted',events:TASK_ONLY})
+ await screen.findByText(RESTART_NOTICE,{},{timeout:4000})
+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()
+ expect(field()).toHaveValue('')
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('backs off instead of polling every second while a turn cannot be proven',async()=>{
+ saveCutOffChat()
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(3*60_000))
+ // Main checked /result and /activity every 2 s: about 90 times each here.
+ expect(calls('/api/pixel/chat/activity').length).toBeLessThanOrEqual(9)
+ expect(calls('/api/pixel/chat/result').length).toBeLessThanOrEqual(9)
+ expect(calls('/api/pixel/chat/context').length).toBeLessThanOrEqual(3)
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('keeps one working Continue in this chat while activity stays unknown, and frees the composer when the server confirms',async()=>{
+ saveCutOffChat({draft:'checking in'})
+ render(<Pixel/>);await screen.findByText('Available')
+ const hold=await screen.findByRole('status',{name:'Last message'})
+ await within(hold).findByText('Portal can’t confirm your last request has finished.')
+ const proceed=within(hold).getByRole('button',{name:'Continue in this chat'})
+ expect(proceed).toBeEnabled()
+ expect(screen.queryByText(STALE)).toBeNull()
+ // The draft stays editable, and sending waits for the owner's choice.
+ expect(field()).toHaveValue('checking in')
+ expect(screen.getByTitle('Send')).toBeDisabled()
+ fireEvent.keyDown(field(),{key:'Enter'})
+ expect(screen.getByText('Choose Continue in this chat first. Your message is kept here.')).toBeVisible()
+ // The server's history lock is unusable at first (an empty lock file left
+ // by a crash), so it cannot confirm: the control stays and links help.
+ server.cancel=()=>response({detail:'Portal cancellation is unavailable'},503)
+ fireEvent.click(proceed)
+ await within(hold).findByText(CONTINUE_FAILED)
+ expect(within(hold).getByRole('link',{name:/Get help on Discord/})).toBeVisible()
+ expect(within(hold).getByRole('button',{name:'Continue in this chat'})).toBeEnabled()
+ expect(screen.getByRole('button',{name:'New chat'})).not.toHaveAttribute('aria-disabled')
+ // Once it can, its confirmation alone releases the hold, even though
+ // the activity lookup still answers unknown. Continue stops anything left
+ // over, so the reply says it was stopped, never that Portal finished.
+ server.cancel=body=>{expect(body).toEqual({chat_id:'spark_chat'});return response({aborted:true})}
+ fireEvent.click(within(hold).getByRole('button',{name:'Continue in this chat'}))
+ await waitFor(()=>expect(row()).toBeNull())
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(2)
+ expect(screen.getByText(STOPPED)).toBeVisible()
+ expect(screen.queryByText(NO_REPLY_NOTICE)).toBeNull()
+ expect(field()).toBeEnabled()
+ expect(field()).toHaveValue('checking in')
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ // The conversation is saved after it renders; wait for that save.
+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,requestId:null}))
+ await waitFor(()=>expect(stored().messages.at(-1)).toMatchObject({status:'stopped',content:`${PARTIAL}\n\n---\n\n_${STOPPED}_`}))
+ server.streams.push(()=>answer('I am here.'))
+ fireEvent.click(screen.getByTitle('Send'))
+ expect(await screen.findByText('I am here.')).toBeVisible()
+ expect(calls('/api/pixel/chat/stream')).toHaveLength(1)
+})
+
+it('a Continue that the receipt answered first never settles a later turn unchecked',async()=>{
+ saveCutOffChat()
+ let finishCancel,cancelled=false
+ server.cancel=()=>new Promise(resolve=>{finishCancel=resolve})
+ server.result=body=>response(body.request_id==='checking-in'
+ ?(cancelled?{state:'cancelled',events:TASK_ONLY}:{state:'unknown',events:''})
+ :{state:'active',events:''})
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ await act(async()=>{fireEvent.click(within(row()).getByRole('button',CONTINUE))})
+ // The server stops the old turn, and the next scheduled check reads that
+ // receipt before the owner's Continue hears back.
+ cancelled=true
+ await act(async()=>vi.advanceTimersByTimeAsync(5000))
+ expect(row()).toBeNull()
+ await act(async()=>{finishCancel(response({aborted:true}))})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ // A later turn is cut off. It is checked against its own receipt, which
+ // is still being written, and its answer can still arrive.
+ server.streams.push(()=>cutOffStream())
+ fireEvent.change(field(),{target:{value:'Next question'}})
+ await act(async()=>{fireEvent.click(screen.getByTitle('Send'))})
+ await act(async()=>vi.advanceTimersByTimeAsync(100))
+ const next=JSON.parse(calls('/api/pixel/chat/stream')[0][1].body).request_id
+ expect(calls('/api/pixel/chat/result').some(([,options])=>JSON.parse(options.body).request_id===next)).toBe(true)
+ expect(within(row()).getByText('Portal is still working on your last request. Its answer will appear here when it’s done. You can stop it.')).toBeVisible()
+ expect(stored().messages.at(-1).status).not.toBe('stopped')
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)
+})
+
+it('offers Continue in this chat only once a check cannot tell, never while that check runs',async()=>{
+ saveCutOffChat()
+ // The restarting API does not answer; each check gives up after 15 s.
+ server.result=(_body,options)=>new Promise((_resolve,reject)=>options.signal.addEventListener('abort',()=>reject(new DOMException('aborted','AbortError'))))
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ // Continue stops anything left over, and this check may still find work running.
+ expect(within(row()).getByText('Checking on your last request…')).toBeVisible()
+ expect(within(row()).queryByRole('button',CONTINUE)).toBeNull()
+ await act(async()=>vi.advanceTimersByTimeAsync(14_000))
+ expect(within(row()).queryByRole('button',CONTINUE)).toBeNull()
+ await act(async()=>vi.advanceTimersByTimeAsync(1_000))
+ expect(within(row()).getByText('Portal can’t confirm your last request has finished.')).toBeVisible()
+ expect(within(row()).getByRole('button',CONTINUE)).toBeEnabled()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('never offers Continue while the server finds this turn still running',async()=>{
+ saveCutOffChat()
+ let state='active'
+ server.activity=()=>response({state})
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ // This turn's reply was cut off, so its answer cannot appear here; the line
+ // does not promise that it will.
+ expect(within(row()).getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()
+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()
+ expect(screen.getByTitle('Stop')).toBeEnabled()
+ for(let second=0;second<2*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()
+ // It was checked at least every 15 s, and the end of that work frees the chat.
+ expect(calls('/api/pixel/chat/activity').length).toBeGreaterThanOrEqual(8)
+ state='terminal'
+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(row()).toBeNull()
+ expect(screen.getByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE)).toBeVisible()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('never asks to send again when the cut-off turn’s work went on, so nothing is done twice',async()=>{
+ // The turn delegated work. A restart cut its reply off while that work
+ // went on, and later ended. The settled reply says what it says for any
+ // cut-off turn, claiming nothing about what ran.
+ saveCutOffChat()
+ let running=true
+ server.result=()=>response(running?{state:'unresolved',events:''}:{state:'interrupted',events:TASK_ONLY})
+ server.activity=()=>response({state:running?'active':'terminal'})
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ expect(within(row()).getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()
+ running=false
+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(row()).toBeNull()
+ const reply=screen.getByText(RESTART_NOTICE)
+ const bubble=reply.closest('[data-pixel-response]')
+ expect(within(bubble).queryByRole('button',{name:'Send again'})).toBeNull()
+ expect(screen.queryByText(/kept working on this/)).toBeNull()
+ // The question is not put back to be sent again; the composer is free.
+ expect(field()).toHaveValue('')
+ expect(field()).toBeEnabled()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+})
+
+it('offers Stop in place for work found running in the chat, and frees the chat once it is stopped',async()=>{
+ let history='unknown'
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))
+ server.context=()=>response({...context(history),status:history==='unknown'?'busy':'ready'})
+ server.activity=()=>response({state:'active'})
+ let aborted=false
+ server.cancel=body=>{expect(body).toEqual({chat_id:'history_chat'});if(aborted)history='ready';return response({aborted})}
+ render(<Pixel/>);await screen.findByText('Available')
+ const hold=await screen.findByRole('status',{name:'Last message'})
+ await within(hold).findByText(STILL_WORKING,{},{timeout:4000})
+ expect(within(hold).queryByRole('button',CONTINUE)).toBeNull()
+ // A Stop that is not confirmed says so, links help, and can be tried again.
+ fireEvent.click(within(hold).getByRole('button',{name:'Stop'}))
+ await within(hold).findByText('Portal could not confirm the stop yet. Try Stop again in a moment.')
+ expect(within(hold).getByRole('link',{name:/Get help on Discord/})).toBeVisible()
+ aborted=true
+ fireEvent.click(within(hold).getByRole('button',{name:'Stop'}))
+ await waitFor(()=>expect(row()).toBeNull())
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ expect(screen.getByText('Earlier answer')).toBeVisible()
+ expect(screen.queryByText(/Stopped by you/)).toBeNull()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(2)
+})
+
+it('says Portal is still working, with no Continue, when it finds the chat busy, and frees the chat when that work ends',async()=>{
+ let history='unknown',state='active'
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))
+ server.context=()=>response({...context(history),status:history==='unknown'?'busy':'ready'})
+ server.activity=body=>{expect(body).toEqual({chat_id:'history_chat'});if(state==='terminal')history='ready';return response({state})}
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(1_100))
+ expect(within(row()).getByText(STILL_WORKING)).toBeVisible()
+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()
+ expect(screen.getByTitle('Send')).toBeDisabled()
+ fireEvent.keyDown(field(),{key:'Enter'})
+ expect(screen.getByText('Portal is not ready for a new message yet. Your message is kept here; send it when Portal is ready.')).toBeVisible()
+ // It stays that way while the work runs, checked at least every 15 s.
+ for(let second=0;second<2*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(within(row()).getByText(STILL_WORKING)).toBeVisible()
+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()
+ expect(calls('/api/pixel/chat/activity').length).toBeGreaterThanOrEqual(8)
+ state='terminal'
+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))
+ expect(row()).toBeNull()
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ expect(field()).toHaveValue('checking in')
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+ expect(screen.queryByText(STALE)).toBeNull()
+})
+
+it('does not blame the connection when an older turn holds the chat, and sends the new message once it is cleared up',async()=>{
+ // The window believes the chat is free, but the server still holds an
+ // older dead attempt and refuses the new one before starting it.
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,
+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))
+ let settled=false
+ server.streams.push(()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423))
+ server.activity=()=>response({state:settled?'terminal':'unknown'})
+ render(<Pixel/>);await screen.findByText('Available')
+ fireEvent.change(field(),{target:{value:'Checking in'}})
+ fireEvent.click(screen.getByTitle('Send'))
+ const hold=await screen.findByRole('status',{name:'Last message'})
+ await waitFor(()=>expect(hold).toHaveTextContent('Your new message will go out right after.'))
+ expect(await within(hold).findByRole('button',{name:'Continue in this chat'})).toBeEnabled()
+ expect(screen.queryByText(STALE)).toBeNull()
+ // The refused message is not left in the conversation; it waits in the composer.
+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText('Checking in')).toBeNull()
+ expect(field()).toHaveValue('Checking in')
+ // The server then proves the older attempt is over: the held message goes out.
+ settled=true
+ server.streams.push(()=>answer('Hi, I am here.'))
+ expect(await screen.findByText('Hi, I am here.',{},{timeout:4000})).toBeVisible()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+ const sent=calls('/api/pixel/chat/stream').map(([,options])=>JSON.parse(options.body).messages.at(-1))
+ expect(sent).toEqual([{role:'user',content:'Checking in'},{role:'user',content:'Checking in'}])
+ expect(screen.getByText('Earlier answer')).toBeVisible()
+ expect(screen.queryByText(STALE)).toBeNull()
+})
+
+it.each(['without a receipt','from its receipt'])('clears a Stop that could not be confirmed once the turn settles %s',async source=>{
+ saveCutOffChat()
+ let state='active'
+ server.activity=()=>response({state})
+ if(source==='from its receipt')server.result=()=>response(state==='active'?{state:'unresolved',events:''}:{state:'interrupted',events:TASK_ONLY})
+ server.cancel=()=>response({aborted:false})
+ render(<Pixel/>)
+ fireEvent.click(await screen.findByTitle('Stop'))
+ const failed='Portal could not confirm the stop yet. Try Stop again in a moment.'
+ expect(await screen.findByText(failed)).toBeVisible()
+ // The work then ends by itself, and the chat frees itself.
+ state='terminal'
+ if(source==='without a receipt')await screen.findByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE,{},{timeout:4000})
+ else await screen.findByText(RESTART_NOTICE,{},{timeout:4000})
+ expect(row()).toBeNull()
+ expect(screen.queryByText(failed)).toBeNull()
+})
+
+it('a Stop in a held chat never stamps the earlier, finished reply as stopped',async()=>{
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,
+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))
+ // The send is refused because older work still holds the chat, and the
+ // check finds that work running, so Stop is offered.
+ server.streams.push(()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423))
+ let running=true
+ server.activity=()=>response({state:running?'active':'terminal'})
+ server.cancel=body=>{expect(body).toEqual({chat_id:'spark_chat'});running=false;return response({aborted:true})}
+ render(<Pixel/>);await screen.findByText('Available')
+ fireEvent.change(field(),{target:{value:'Checking in'}})
+ fireEvent.click(screen.getByTitle('Send'))
+ fireEvent.click(await screen.findByTitle('Stop'))
+ await waitFor(()=>expect(screen.queryByTitle('Stop')).toBeNull())
+ expect(screen.queryByText(/Stopped by you/)).toBeNull()
+ expect(screen.getByText('Earlier answer')).toBeVisible()
+ await waitFor(()=>expect(stored().messages.find(message=>message.content.startsWith('Earlier answer'))).toMatchObject({content:'Earlier answer',status:'done'}))
+})
+
+it('sends a held message by itself once per owner action, never in a loop',async()=>{
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,
+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))
+ const held=()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423)
+ // The chat looks free each time it is checked, yet every send is refused.
+ server.activity=()=>response({state:'terminal'})
+ server.streams.push(held,held,held,held)
+ vi.useFakeTimers()
+ await act(async()=>{render(<Pixel/>)})
+ await act(async()=>vi.advanceTimersByTimeAsync(10))
+ fireEvent.change(field(),{target:{value:'Checking in'}})
+ await act(async()=>{fireEvent.click(screen.getByTitle('Send'))})
+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))
+ // It went out by itself once more, was refused again, and now waits.
+ expect(calls('/api/pixel/chat/stream')).toHaveLength(2)
+ const note=screen.getByRole('status',{name:'Message not started'})
+ expect(note).toHaveTextContent('Portal is busy with another request, so it did not start this message. Your message is kept; send it again in a moment.')
+ expect(field()).toHaveValue('Checking in')
+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText('Checking in')).toBeNull()
+ // Send again is the owner's next action, and it too is retried once at most.
+ await act(async()=>{fireEvent.click(within(note).getByRole('button',{name:'Send again'}))})
+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))
+ expect(calls('/api/pixel/chat/stream')).toHaveLength(4)
+ expect(screen.getByRole('status',{name:'Message not started'})).toBeVisible()
+ expect(field()).toHaveValue('Checking in')
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+ expect(screen.queryByText(STALE)).toBeNull()
+})
+
+it('heals an unconfirmed history by itself when the server proves the chat idle',async()=>{
+ let unknown=true
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))
+ server.context=()=>response(context(unknown?'unknown':'ready'))
+ server.activity=body=>{expect(body).toEqual({chat_id:'history_chat'});unknown=false;return response({state:'terminal'})}
+ render(<Pixel/>);await screen.findByText('Available')
+ await screen.findByRole('status',{name:'Last message'})
+ await waitFor(()=>expect(row()).toBeNull(),{timeout:4000})
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)
+ expect(screen.queryByText(STALE)).toBeNull()
+})
+
+it('asks once, in place, when an unconfirmed history cannot be proven idle',async()=>{
+ let unknown=true
+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))
+ server.context=()=>response(context(unknown?'unknown':'ready'))
+ server.cancel=body=>{expect(body).toEqual({chat_id:'history_chat'});unknown=false;return response({aborted:true})}
+ render(<Pixel/>);await screen.findByText('Available')
+ const hold=await screen.findByRole('status',{name:'Last message'})
+ const proceed=await within(hold).findByRole('button',{name:'Continue in this chat'})
+ await within(hold).findByText('Portal can’t confirm your last request has finished.',{},{timeout:4000})
+ expect(screen.getByTitle('Send')).toBeDisabled()
+ fireEvent.click(proceed)
+ await waitFor(()=>expect(row()).toBeNull())
+ expect(screen.getByTitle('Send')).toBeEnabled()
+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)
+})
diff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
index d15989879..62eec58db 100644
--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs
@@ -60,6 +60,9 @@ const GATEWAY_ABORT_MAX_ATTEMPTS = 30;
const DOCKER_TIMEOUT_MS = 10000;
const MAX_TOKEN_LEN = 4096;
const MAX_CANCEL_BODY = 256;
+// Settle reads one native context under the history lock; keep it well inside
+// the edge's relay deadline so a slow gateway answers "unconfirmed", not a hang.
+const SETTLE_CONTEXT_TIMEOUT_MS = 5000;
const MAX_STATUS_INTERVAL_MS = 86400000; // 1 day
const STATUS_MODE = 0o640; // group-readable, service-owner-writable projection
const ODS_VERSION_RE = /^(?:unknown|[0-9]+(?:\.[0-9]+){1,3}(?:[-+][A-Za-z0-9.-]+)?)$/;
@@ -1984,6 +1987,24 @@ export function createIngressServer({ token, gatewayPort, deps = defaultDeps, hi
return;
}
+ if (pathname === "/v1/chat/settle") {
+ if (req.method !== "POST") {
+ sendError(res, 405, "method not allowed");
+ return;
+ }
+ if (req.url !== pathname) {
+ sendError(res, 400, "invalid settle request");
+ return;
+ }
+ const contentType = String(req.headers["content-type"] || "").toLowerCase();
+ if (contentType.split(";", 1)[0].trim() !== "application/json") {
+ sendError(res, 415, "content type must be application/json");
+ return;
+ }
+ void handleSettle(req, res, token, gatewayPort, deps, historyLedger);
+ return;
+ }
+
sendError(res, 404, "not found");
});
}
@@ -2046,6 +2067,60 @@ async function handleCancel(req, res, token, gatewayPort, deps, ledger, historyA
} catch {sendError(res,503,'cancellation-unconfirmed');}
}
+// A restart can cut a turn off and leave its history outcome unknown while
+// nothing runs any more. Settle proves that without stopping anything: under
+// the history lock no turn of this ingress is in flight, and an idle native
+// session has no run of this chat. Only then is the unresolved turn recorded
+// as interrupted. A held lock or a busy session answers "active" and changes
+// nothing; Stop stays the only way to end work that may still be running.
+async function handleSettle(req, res, token, gatewayPort, deps, ledger) {
+ let parsed;
+ try {
+ parsed = JSON.parse((await readBody(req, MAX_CANCEL_BODY)).toString("utf8"));
+ } catch (error) {
+ sendError(res, error instanceof HttpError ? error.status : 400,
+ error instanceof HttpError ? error.message : "invalid settle request");
+ return;
+ }
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || Object.keys(parsed).length !== 1
+ || typeof parsed.user !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(parsed.user)) {
+ sendError(res, 400, "invalid settle request");
+ return;
+ }
+ if (!ledger) {
+ sendError(res, 503, "history-storage-unavailable");
+ return;
+ }
+ const user = computeSessionUser({ user: parsed.user });
+ let release;
+ try {
+ try {
+ release = ledger.lock(user);
+ } catch (error) {
+ if (error instanceof HistoryError && error.status === 409) {
+ sendJson(res, 200, { state: "active" });
+ return;
+ }
+ throw error;
+ }
+ const native = await nativeContextRequest("context", { user }, token, gatewayPort, deps, undefined, SETTLE_CONTEXT_TIMEOUT_MS);
+ if (native.status === "busy") {
+ sendJson(res, 200, { state: "active" });
+ return;
+ }
+ if (!["ready", "missing"].includes(native.status)) throw new HistoryError("context-unavailable", 503);
+ const current = ledger.read(user);
+ if (current && ["pending", "unknown"].includes(current.status) && !ledger.interrupt(user, current.requestId)) {
+ throw new HistoryError("history-settle-unconfirmed", 503);
+ }
+ sendJson(res, 200, { state: "settled" });
+ } catch {
+ sendError(res, 503, "settle-unconfirmed");
+ } finally {
+ try { release?.(); } catch { /* The next lock() reclaims a lock it could not remove, once removal works. */ }
+ }
+}
+
async function handleImageDelete(req,res,{imageStore,historyLedger,imageRoutes,token,gatewayPort,deps}) {
let release;
try {
diff --git a/ods/extensions/services/pixel-agent/plugin/index.js b/ods/extensions/services/pixel-agent/plugin/index.js
index b74c44ec6..81e46eafe 100644
--- a/ods/extensions/services/pixel-agent/plugin/index.js
+++ b/ods/extensions/services/pixel-agent/plugin/index.js
@@ -694,8 +694,14 @@ export default definePluginEntry({
handler:async (req,res) => {
const parsed = await readContextRequest(req, operation === 'compact');
if (parsed.status !== 200) {sendJson(res,parsed.status,{error:'invalid context request'});return true;}
- const result = operation === 'compact' ? await contextCompaction.compact(parsed.user,parsed.requestId)
+ let result = operation === 'compact' ? await contextCompaction.compact(parsed.user,parsed.requestId)
: contextCompaction.context(parsed.user);
+ // The owner session can be idle while children it delegated to still
+ // run or will report back into this chat. Then the chat is busy, so
+ // restart recovery and a new turn wait for that work instead of
+ // treating the turn as over.
+ if (operation === 'context' && ['ready','missing'].includes(result?.status)
+ && delegationDelivery.pending(parsed.user,key => Boolean(resolveActiveEmbeddedRunSessionId(key)))) result = {...result,status:'busy'};
sendJson(res,200,result);return true;
}});
}
diff --git a/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs b/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs
index 297a0d91d..68e3f6df9 100644
--- a/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs
+++ b/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs
@@ -266,6 +266,24 @@ export function createSubagentDelivery({agentId = 'pixel', now = Date.now,
const chain=roots.get(runId);
return chain && ownerMatches(chain,user) && valid(chain) ? chain.ready?.runId : undefined;
}
+ // Whether this user's chat may still have delegated work while its owner
+ // session is idle: a child that one of its turns started still runs, or a
+ // registered child will still report back into the chat. Only an explicit
+ // false from isActive proves a child idle. A delegation that can no longer
+ // report back (Stop, access change, expiry) is over once its children have
+ // ended; valid() records that, so a later announcement is refused as well.
+ function pending(user, isActive) {
+ if (!USER.test(user ?? '')) return false;
+ const running = key => { try { return isActive(key) !== false; } catch { return true; } };
+ for (const chain of roots.values()) {
+ if (chain.sessionKey !== prefix + user) continue;
+ const started = [...spawned.entries()].filter(([,record]) => record.chainId === chain.id || record.cancelledChainIds?.includes(chain.id));
+ if ([...chain.children.keys(), ...started.map(([key]) => key)].some(running)) return true;
+ if (!chain.delivered && !chain.ready && valid(chain)
+ && (pendingSpawn(chain) || [...chain.children.values()].some(child => !child.announced))) return true;
+ }
+ return false;
+ }
function hasPendingChildren(context) {
const run = owned(context);
// Advice must have the same live owner custody as delivery. A prompt,
@@ -369,6 +387,6 @@ export function createSubagentDelivery({agentId = 'pixel', now = Date.now,
return {block:true,blockReason:'No registered child completion event is pending. Review the current event and the earlier child results supplied in this run, then consolidate the owner response. This yield was not executed.'};
}
}
- return {observe,before,after,nativeSpawn,finalize,end,read,finalRun,cancel,blocked,promptContext,ownerRunFor,ownerRunOf,admission,hasPendingChildren,
+ return {observe,before,after,nativeSpawn,finalize,end,read,finalRun,cancel,pending,blocked,promptContext,ownerRunFor,ownerRunOf,admission,hasPendingChildren,
invalidate:() => {for (const chain of roots.values()) fail(chain);}};
}
diff --git a/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjs b/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjs
new file mode 100644
index 000000000..1c22d9f12
--- /dev/null
+++ b/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjs
@@ -0,0 +1,112 @@
+// A restart that cuts a turn off leaves its history outcome unknown. Settle
+// must prove nothing still runs before recording that turn as interrupted, and
+// must never stop work itself: Stop stays the only way to end running work.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import {createIngressServer,computeSessionUser} from '../host/pixel_ingress.mjs';
+import {createChatHistoryLedger} from '../host/chat_history_ledger.mjs';
+
+const u=content=>({role:'user',content});
+const rawUser='settle-canary', user=computeSessionUser({user:rawUser});
+const runId='chatcmpl_11111111-2222-4333-8444-555555555555';
+const nativeState=()=>({schemaVersion:1,status:'ready',sessionExists:true,sessionRevision:'a'.repeat(64),context:{used:100,window:32000,measuredAt:1},model:{id:'test',provider:'local',contextWindow:32000},compaction:{status:'idle',requestId:null,tokensBefore:null,tokensAfter:null,reason:null,count:0}});
+const listen=server=>new Promise(resolve=>server.listen(0,'127.0.0.1',()=>resolve(server.address().port)));
+
+async function fixture(t) {
+ const dir=fs.mkdtempSync(path.join(os.tmpdir(),'ods-ingress-settle-'));fs.chmodSync(dir,0o700);
+ // The turn that a restart cut off: prepared (pending) by the previous ingress process.
+ const before=createChatHistoryLedger(dir);
+ before.prepare(user,'cut-off',{schemaVersion:1,messages:[u('Can you do things on cron timers?')]},nativeState());
+ // The restarted ingress has no record of that turn in flight.
+ const ledger=createChatHistoryLedger(dir),calls=[],native=nativeState();
+ const gateway=http.createServer(async(req,res)=>{
+ if(req.url==='/health') {res.setHeader('content-type','application/json');return res.end('{"ok":true}');}
+ let raw='';for await(const part of req) raw+=part;
+ const body=JSON.parse(raw||'{}');calls.push({path:req.url,body});res.setHeader('content-type','application/json');
+ if(req.url==='/pixel-ods/context') return res.end(JSON.stringify(native));
+ if(req.url==='/pixel-ods/abort') return res.end(JSON.stringify({aborted:true}));
+ if(req.url==='/pixel-ods/verification') return res.end(JSON.stringify({status:'none'}));
+ if(req.url==='/pixel-ods/subagent-delivery') return res.end(JSON.stringify({schemaVersion:1,kind:'ods-subagent-delivery',runId:body.runId,status:'not-delegated'}));
+ if(req.url==='/v1/chat/completions') return res.end(JSON.stringify({id:runId,choices:[{finish_reason:'stop',message:{role:'assistant',content:'Yes, I can schedule work.'}}]}));
+ res.statusCode=404;res.end('{}');
+ });
+ const port=await listen(gateway),ingress=createIngressServer({token:'test-token',gatewayPort:port,historyLedger:ledger});
+ const ingressPort=await listen(ingress);
+ t.after(async()=>{await Promise.all([new Promise(r=>ingress.close(r)),new Promise(r=>gateway.close(r))]);fs.rmSync(dir,{recursive:true,force:true})});
+ async function request(route,body,{method='POST',contentType='application/json'}={}) {
+ const response=await fetch(`http://127.0.0.1:${ingressPort}${route}`,{method,headers:{'content-type':contentType},...(method==='POST'?{body:typeof body==='string'?body:JSON.stringify(body)}:{})});
+ return {status:response.status,value:await response.json()};
+ }
+ const settle=(body={user:rawUser})=>request('/v1/chat/settle',body);
+ const chat=(request_id,messages)=>request('/v1/chat/completions',{user:rawUser,request_id,history_snapshot:{schemaVersion:1,messages},messages:[{role:'system',content:'Trusted identity'},u(messages.at(-1).content)],stream:false});
+ const stops=()=>calls.filter(call=>call.path==='/pixel-ods/abort' || call.path==='/v1/chat/completions');
+ return {dir,ledger,calls,native,request,settle,chat,stops};
+}
+
+test('a turn cut off by a restart settles without stopping anything, and the next turn runs',async t=>{
+ const f=await fixture(t);
+ assert.equal(f.ledger.projection(user).status,'unknown');
+ const refused=await f.chat('checking-in',[u('Can you do things on cron timers?'),u('Checking in')]);
+ assert.equal(refused.status,409,'an unresolved turn still refuses a silent resubmission');
+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});
+ assert.deepEqual(f.stops(),[],'settle never aborts a run or starts one');
+ const settled=f.ledger.read(user);
+ assert.equal(settled.status,'ready');
+ assert.equal(settled.reason,'history-interrupted');
+ assert.equal(settled.requests.at(-1).status,'interrupted');
+ // Settling twice is harmless: there is nothing left to record.
+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});
+ const next=await f.chat('checking-in-again',[u('Can you do things on cron timers?'),u('Checking in')]);
+ assert.equal(next.status,200);
+ assert.deepEqual(f.calls.filter(call=>call.path==='/v1/chat/completions').at(-1).body.messages,[{role:'system',content:'Trusted identity'},u('Checking in')]);
+});
+
+test('a busy native session answers active and leaves the unresolved turn alone',async t=>{
+ const f=await fixture(t);
+ f.native.status='busy';
+ assert.deepEqual(await f.settle(),{status:200,value:{state:'active'}});
+ assert.equal(f.ledger.projection(user).status,'unknown');
+ assert.deepEqual(f.stops(),[]);
+});
+
+test('a turn holding the history lock answers active without reading or changing history',async t=>{
+ const f=await fixture(t);
+ const release=f.ledger.lock(user);
+ try {assert.deepEqual(await f.settle(),{status:200,value:{state:'active'}});}
+ finally {release();}
+ assert.equal(f.calls.filter(call=>call.path==='/pixel-ods/context').length,0);
+ assert.equal(f.ledger.projection(user).status,'unknown');
+});
+
+test('an unusable history lock answers unconfirmed, never settled and never active forever',async t=>{
+ const f=await fixture(t);
+ // Something that is not a lock file sits where the lock belongs, as after a
+ // crash. Settle cannot take the lock, so it must not claim anything.
+ const lockPath=path.join(f.dir,`${user}.lock`);
+ fs.mkdirSync(lockPath);
+ const answer=await f.settle();
+ assert.equal(answer.status,503);
+ assert.equal(answer.value.error.message,'settle-unconfirmed');
+ assert.equal(f.ledger.projection(user).status,'unknown');
+ assert.deepEqual(f.stops(),[]);
+ fs.rmdirSync(lockPath);
+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});
+});
+
+test('settle fails closed when idle cannot be proven or the request is not exact',async t=>{
+ const f=await fixture(t);
+ f.native.status='unavailable';
+ assert.equal((await f.settle()).status,503);
+ assert.equal(f.ledger.projection(user).status,'unknown');
+ for(const body of [{},{user:rawUser,request_id:'cut-off'},{user:'../escape'},{user:7},'not json'])
+ assert.equal((await f.settle(body)).status,400,JSON.stringify(body));
+ assert.equal((await f.request('/v1/chat/settle?user=other',{user:rawUser})).status,400);
+ assert.equal((await f.request('/v1/chat/settle',{user:rawUser},{contentType:'text/plain'})).status,415);
+ assert.equal((await f.request('/v1/chat/settle',undefined,{method:'GET'})).status,405);
+ assert.equal(f.ledger.projection(user).status,'unknown');
+ assert.deepEqual(f.stops(),[]);
+});
diff --git a/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjs b/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjs
new file mode 100644
index 000000000..756fdd4fe
--- /dev/null
+++ b/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjs
@@ -0,0 +1,156 @@
+// Restart recovery (the ingress's /v1/chat/settle) and a new owner turn treat
+// an idle owner session as proof that no run of the chat is left. A turn that
+// delegated can leave the owner session idle while its children still work or
+// will still report back into the chat. These tests pin when the delegation
+// registry keeps such a chat busy, and that the plugin's context report says so.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import vm from 'node:vm';
+import {Readable} from 'node:stream';
+import {createSubagentDelivery} from '../plugin/subagent-delivery.mjs';
+import {readContextRequest} from '../plugin/context-compaction.mjs';
+
+const user='ods-'+'a'.repeat(64), other='ods-'+'b'.repeat(64);
+const id='chatcmpl_11111111-2222-4333-8444-555555555555';
+const child='agent:pixel:subagent:22222222-2222-4333-8444-555555555555';
+const childRun='33333333-2222-4333-8444-555555555555';
+const owner={agentId:'pixel',runId:id,sessionId:'owner-session',sessionKey:'agent:pixel:openai-user:'+user,trigger:'user'};
+const continuation={...owner,runId:`announce:v1:${child}:${childRun}`,
+ inputProvenance:{kind:'inter_session',sourceTool:'subagent_announce',sourceSessionKey:child}};
+
+function fixture() {
+ let clock=1, access='current';
+ // Native sessions with an active embedded run, as resolveActiveEmbeddedRunSessionId reports them.
+ const running=new Set();
+ const registry=createSubagentDelivery({now:()=>clock,accessIdentity:()=>access,
+ finalText:message=>message.content.filter(block=>block.type==='text').map(block=>block.text).join('\n'),
+ resolveOwnerSession:key=>key===owner.sessionKey?{sessionId:owner.sessionId}:null,
+ abortSession:async key=>!running.has(key)});
+ registry.observe({},owner);
+ function spawn({key=child,runId=childRun,accepted=true}={}) {
+ const context={...owner,toolName:'sessions_spawn',toolCallId:'spawn-'+key};
+ registry.before({params:{runtime:'subagent',mode:'run'}},context);
+ registry.nativeSpawn({runId,childSessionKey:key},{runId,childSessionKey:key,requesterSessionKey:owner.sessionKey});
+ running.add(key);
+ if (accepted) registry.after({result:{details:{status:'accepted',runId,childSessionKey:key}}},context);
+ }
+ function yieldTurn() {
+ const context={...owner,toolName:'sessions_yield',toolCallId:'yield'};
+ registry.before({params:{}},context);
+ registry.after({result:{details:{status:'yielded'}}},context);
+ registry.end({success:true},owner);
+ }
+ function final(text='Both reviews are done.') {
+ registry.finalize({lastAssistantMessage:text},continuation);
+ registry.end({success:true,messages:[{role:'assistant',stopReason:'stop',content:[{type:'text',text}]}]},continuation);
+ }
+ return {registry,running,spawn,yieldTurn,final,
+ pending:(who=user,isActive=key=>running.has(key))=>registry.pending(who,isActive),
+ tick:()=>{clock+=33*60*1000;},revoke:()=>{access='downgraded';}};
+}
+
+test('a delegated turn stays pending while its child runs and until the child reports back', () => {
+ const f=fixture();
+ f.spawn();f.yieldTurn();
+ assert.equal(f.pending(),true,'the child still runs while the owner session waits');
+ f.running.delete(child);
+ assert.equal(f.pending(),true,'the child ended but its result has not reached the chat yet');
+ f.registry.observe({prompt:'Review result: nothing to fix.'},continuation);
+ f.final();
+ assert.equal(f.pending(),false,'the consolidated answer is ready and nothing runs');
+ assert.equal(f.registry.read(user,id).status,'ready');
+ assert.equal(f.pending(),false,'a delivered answer is settled');
+});
+
+test('a native child that started before its spawn reply arrived keeps the turn pending', () => {
+ const f=fixture();
+ f.spawn({accepted:false});
+ assert.equal(f.pending(),true,'the spawn call is still open and its child already runs');
+ f.running.delete(child);
+ assert.equal(f.pending(),true,'the spawn call has not answered yet');
+});
+
+test('Stop ends the wait; only a child that still runs keeps the chat busy', async () => {
+ const f=fixture();
+ f.spawn();f.yieldTurn();
+ assert.deepEqual(await f.registry.cancel(user),{tracked:true,aborted:false},'the child could not be stopped yet');
+ assert.equal(f.pending(),true,'never treat a child that still runs as finished');
+ f.running.delete(child);
+ assert.equal(f.pending(),false,'a stopped delegation can no longer report back');
+});
+
+test('a delegation that can no longer report back is over once its child ends', () => {
+ for (const end of ['access change','expiry']) {
+ const f=fixture();
+ f.spawn();f.yieldTurn();
+ if (end==='access change') f.revoke(); else f.tick();
+ assert.equal(f.pending(),true,`${end}: the child still runs`);
+ f.running.delete(child);
+ assert.equal(f.pending(),false,end);
+ assert.equal(f.registry.admission(continuation)?.outcome,'block',`${end}: its late result cannot start a turn either`);
+ }
+});
+
+test('ordinary turns, other chats and malformed users are never pending', () => {
+ const f=fixture();
+ assert.equal(f.pending(),false,'an owner turn that never delegated');
+ f.spawn();f.yieldTurn();
+ assert.equal(f.pending(other),false,'another chat');
+ for (const malformed of [undefined,'',owner.sessionKey,'ods-'+'A'.repeat(64)]) assert.equal(f.registry.pending(malformed,()=>true),false,String(malformed));
+});
+
+test('only an explicit idle answer proves a child has ended', () => {
+ const f=fixture();
+ f.spawn();f.yieldTurn();f.revoke();
+ assert.equal(f.pending(user,()=>{throw new Error('registry unavailable');}),true);
+ assert.equal(f.pending(user,()=>undefined),true);
+ assert.equal(f.pending(user,()=>false),false);
+});
+
+// Values built inside the vm context have that context's prototypes.
+const plain=value=>JSON.parse(JSON.stringify(value));
+
+function contextRoute(registry,{active=new Set(),report}) {
+ const source=fs.readFileSync(new URL('../plugin/index.js',import.meta.url),'utf8');
+ const start=source.indexOf(" for (const operation of ['context','compact']) {");
+ const end=source.indexOf(' // The OpenAI-compatible gateway route does not dispatch',start);
+ assert.ok(start>=0 && end>start);
+ const routes={}, compacted=[];
+ vm.runInNewContext(source.slice(start,end),{api:{registerHttpRoute:value=>{routes[value.path]=value;}},
+ readContextRequest,delegationDelivery:registry,resolveActiveEmbeddedRunSessionId:key=>active.has(key)?'native-session':undefined,
+ contextCompaction:{context:()=>({...report()}),compact:async (who,requestId)=>{compacted.push([who,requestId]);return {...report()};}},
+ sendJson:(res,status,value)=>{res.status=status;res.value=value;}});
+ const call=async (path,body)=>{
+ const req=Readable.from([Buffer.from(JSON.stringify(body))]);
+ req.method='POST';req.headers={'content-type':'application/json'};const res={};
+ await routes[path].handler(req,res);return res;
+ };
+ return {context:who=>call('/pixel-ods/context',{user:who}),compact:(who,requestId)=>call('/pixel-ods/compact',{user:who,request_id:requestId}),compacted};
+}
+
+test('the plugin reports an idle owner session busy while its delegation is pending', async () => {
+ const f=fixture();
+ let status='ready';
+ const report=()=>({schemaVersion:1,status,sessionExists:status!=='missing',sessionRevision:status==='missing'?null:'r1',context:null,model:null,
+ compaction:{status:'idle',count:0}});
+ const route=contextRoute(f.registry,{active:f.running,report});
+ assert.deepEqual(plain((await route.context(user)).value),report(),'nothing delegated: the report is unchanged');
+ f.spawn();f.yieldTurn();
+ const busy=await route.context(user);
+ assert.equal(busy.status,200);
+ assert.deepEqual(plain(busy.value),{...report(),status:'busy'},'only the status changes');
+ status='missing';
+ assert.equal((await route.context(user)).value.status,'busy');
+ status='unavailable';
+ assert.equal((await route.context(user)).value.status,'unavailable','an unavailable report stays unavailable');
+ status='ready';
+ assert.equal((await route.context(other)).value.status,'ready','another chat is not affected');
+ assert.equal((await route.compact(user,'manual-1')).value.status,'ready','a compaction receipt is passed through as it is');
+ assert.deepEqual(route.compacted,[[user,'manual-1']]);
+ f.running.delete(child);
+ assert.equal((await route.context(user)).value.status,'busy','the child result has not reached the chat yet');
+ f.registry.observe({prompt:'Review result: nothing to fix.'},continuation);
+ f.final();
+ assert.equal((await route.context(user)).value.status,'ready');
+});
diff --git a/ods/extensions/services/pixel-edge/pixel_edge.py b/ods/extensions/services/pixel-edge/pixel_edge.py
index 713150de5..4fc237a37 100644
--- a/ods/extensions/services/pixel-edge/pixel_edge.py
+++ b/ods/extensions/services/pixel-edge/pixel_edge.py
@@ -997,6 +997,58 @@ async def handle_chat_activity(request: web.Request):
return web.json_response({"state": state}, headers={"Cache-Control": "no-store"})
+async def handle_chat_settle(request: web.Request):
+ """Ask the ingress to settle a chat it proves idle; never stop a run.
+
+ A restart can leave a turn's outcome unknown while nothing runs any more.
+ The ingress proves that under its history lock and records the turn as
+ interrupted. A stream still passing through this edge, a held lock or a
+ busy native session answers "active" and nothing changes.
+ """
+ fail = _check_auth(request)
+ if fail is not None:
+ return fail
+ if request.content_type != "application/json":
+ return web.json_response({"error": "Content-Type must be application/json"}, status=415)
+ if request.query_string or (request.content_length and request.content_length > _MAX_CANCEL_BODY):
+ return web.json_response({"error": "invalid settle request"}, status=400)
+ try:
+ raw = await request.read()
+ if len(raw) > _MAX_CANCEL_BODY:
+ raise ValueError("request too large")
+ data = strict_json(raw)
+ if (not isinstance(data, dict) or set(data) != {"user"}
+ or not isinstance(data["user"], str) or not _SAFE_CHAT_ID.fullmatch(data["user"])):
+ raise ValueError("invalid settle request")
+ except (ValueError, UnicodeDecodeError, TypeError):
+ return web.json_response({"error": "invalid settle request"}, status=400)
+ chat_id = data["user"]
+ if request.app[_CANCEL_EVENTS_KEY].get(chat_id):
+ return web.json_response({"state": "active"}, headers={"Cache-Control": "no-store"})
+ try:
+ connector = UnixConnector(path=_SOCKET_PATH)
+ # The ingress bounds its own native read at 5 s under the history lock.
+ timeout = ClientTimeout(total=8, sock_connect=2, sock_read=7)
+ async with ClientSession(connector=connector, timeout=timeout) as session:
+ async with session.post(
+ "http://pixel-upstream/v1/chat/settle",
+ json={"user": chat_id},
+ headers={"Content-Type": "application/json", "Accept": "application/json"},
+ ) as resp:
+ if resp.status != 200 or "application/json" not in resp.headers.get("Content-Type", "").lower():
+ raise ValueError("settle unconfirmed")
+ result = strict_json(await _read_bounded(resp.content, _MAX_CANCEL_RESPONSE_BYTES))
+ if (not isinstance(result, dict) or set(result) != {"state"}
+ or result["state"] not in ("settled", "active")):
+ raise ValueError("invalid settle receipt")
+ except (ConnectionError, OSError, asyncio.TimeoutError, ClientError, ValueError, TypeError,
+ UnicodeError, RecursionError):
+ return web.json_response({"error": "Portal settle unconfirmed"}, status=502)
+ if result["state"] == "settled" and not request.app[_CANCEL_EVENTS_KEY].get(chat_id):
+ _remember_chat_activity(request.app, chat_id, "terminal")
+ return web.json_response({"state": result["state"]}, headers={"Cache-Control": "no-store"})
+
+
async def _context_upstream(data, *, compact=False):
connector = UnixConnector(path=_SOCKET_PATH)
timeout = ClientTimeout(total=18, sock_connect=2, sock_read=16)
@@ -1970,6 +2022,7 @@ def create_app() -> web.Application:
app.router.add_post("/v1/chat/completions", handle_chat_completions)
app.router.add_post("/v1/chat/cancel", handle_chat_cancel)
app.router.add_post("/v1/chat/activity", handle_chat_activity)
+ app.router.add_post("/v1/chat/settle", handle_chat_settle)
app.router.add_post("/v1/chat/context", handle_chat_context)
app.router.add_post("/v1/chat/images-delete", handle_chat_images_delete)
app.router.add_post("/v1/chat/compact", handle_chat_context)
diff --git a/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py b/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py
index e9c4bd466..4a06f83af 100644
--- a/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py
+++ b/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py
@@ -183,6 +183,19 @@ async def _upstream_health(_request):
return web.json_response({"status": "ok"})
+async def _upstream_settle(request):
+ data = await request.json()
+ request.app["settle_users"].append(data.get("user"))
+ native = request.app["native_runs"].get(data.get("user"))
+ if native is not None and not native[1].done():
+ # The ingress reads its native session: a detached run is still busy.
+ return web.json_response({"state": "active"})
+ reply = request.app["settle_reply"]
+ if reply["status"] != 200:
+ return web.json_response({"error": "not found"}, status=reply["status"])
+ return web.json_response(reply["receipt"])
+
+
async def _upstream_cancel(request):
data = await request.json()
request.app["cancel_users"].append(data.get("user"))
@@ -244,6 +257,8 @@ async def _start_upstream():
app = web.Application(client_max_size=8 * 1024 * 1024 + 1)
app["chat_requests"] = []
app["cancel_users"] = []
+ app["settle_users"] = []
+ app["settle_reply"] = {"status": 200, "receipt": {"state": "settled"}}
app["native_runs"] = {}
app["stream_started"] = asyncio.Event()
app["release_stream"] = asyncio.Event()
@@ -252,6 +267,7 @@ async def _start_upstream():
app["preview_paths"] = []
app.router.add_post("/v1/chat/completions", _upstream_chat)
app.router.add_post("/v1/chat/cancel", _upstream_cancel)
+ app.router.add_post("/v1/chat/settle", _upstream_settle)
app.router.add_get("/v1/models", _upstream_models)
app.router.add_get("/health", _upstream_health)
app.router.add_get("/{site_id}/{tail:.*}", _upstream_preview)
@@ -2188,6 +2204,78 @@ class TestChatActivity(BaseEdgeTest):
self.assertEqual(await self.activity(user), {"state": "terminal"})
+class TestChatSettle(BaseEdgeTest):
+ """A restart leaves a chat unknown here; settling it never stops a run."""
+
+ async def activity(self, user):
+ async with self.client.post("http://localhost/v1/chat/activity", headers=self.auth(), json={"user": user}) as resp:
+ return await resp.json()
+
+ async def settle(self, user):
+ async with self.client.post("http://localhost/v1/chat/settle", headers=self.auth(), json={"user": user}) as resp:
+ return resp.status, resp.headers.get("Cache-Control"), await resp.json()
+
+ async def test_restart_unknown_chat_becomes_terminal_only_after_ingress_proves_idle(self):
+ # A restarted edge has no record of the turn the restart cut off.
+ self.assertEqual(await self.activity("restarted-chat"), {"state": "unknown"})
+ self.assertEqual(await self.settle("restarted-chat"), (200, "no-store", {"state": "settled"}))
+ self.assertEqual(self.up_runner.app["settle_users"], ["restarted-chat"])
+ self.assertEqual(await self.activity("restarted-chat"), {"state": "terminal"})
+ self.assertEqual(await self.activity("other-chat"), {"state": "unknown"})
+ self.assertEqual(self.up_runner.app["cancel_users"], [])
+ self.assertEqual(self.up_runner.app["chat_requests"], [])
+
+ async def test_stream_in_flight_is_active_without_asking_the_ingress(self):
+ async with self.client.post("http://localhost/v1/chat/completions", headers=self.auth(), json={
+ "model": "pixel/default", "messages": [], "stream": True,
+ "user": "live-chat", "trigger_cancel_wait": True,
+ }) as stream:
+ await asyncio.wait_for(self.up_runner.app["stream_started"].wait(), 1)
+ self.assertEqual(await self.settle("live-chat"), (200, "no-store", {"state": "active"}))
+ self.assertEqual(self.up_runner.app["settle_users"], [])
+ self.up_runner.app["release_stream"].set()
+ await stream.text()
+ self.assertEqual(self.up_runner.app["cancel_users"], [])
+
+ async def test_detached_native_run_stays_active_and_is_never_stopped(self):
+ user = "detached-settle"
+ stream = await self.client.post("http://localhost/v1/chat/completions", headers=self.auth(), json={
+ "model": "pixel/default", "messages": [], "stream": True,
+ "user": user, "trigger_detached_native": True,
+ })
+ await asyncio.wait_for(stream.content.readany(), 2)
+ stream.close()
+ async with async_timeout(2):
+ while self.edge_app[self.pe._CANCEL_EVENTS_KEY].get(user):
+ await asyncio.sleep(0.02)
+ self.assertEqual(await self.settle(user), (200, "no-store", {"state": "active"}))
+ self.assertFalse(self.up_runner.app["native_runs"][user][1].done())
+ self.assertEqual(await self.activity(user), {"state": "unknown"})
+ self.assertEqual(self.up_runner.app["cancel_users"], [])
+
+ async def test_unconfirmed_or_older_ingress_answers_leave_the_chat_unknown(self):
+ for status, receipt in ((200, {"state": "settled", "extra": True}), (200, {"state": "terminal"}),
+ (200, {"aborted": True}), (404, None), (503, None)):
+ self.up_runner.app["settle_reply"].update(status=status, receipt=receipt)
+ code, _cache, body = await self.settle("uncertain-chat")
+ self.assertEqual(code, 502)
+ self.assertEqual(body, {"error": "Portal settle unconfirmed"})
+ self.assertEqual(await self.activity("uncertain-chat"), {"state": "unknown"})
+
+ async def test_settle_requires_exact_authenticated_bounded_request(self):
+ async with self.client.post("http://localhost/v1/chat/settle", json={"user": "safe"}) as resp:
+ self.assertEqual(resp.status, 401)
+ for raw in ('{}', '{"user":"safe","other":"secret"}', '{"user":"../escape"}',
+ '{"user":true}', '{"user":"safe","user":"other"}', '{"user":"' + 'x' * 2048 + '"}'):
+ async with self.client.post("http://localhost/v1/chat/settle", headers={**self.auth(), "Content-Type": "application/json"}, data=raw) as resp:
+ self.assertEqual(resp.status, 400)
+ async with self.client.post("http://localhost/v1/chat/settle?user=other", headers=self.auth(), json={"user": "safe"}) as resp:
+ self.assertEqual(resp.status, 400)
+ async with self.client.post("http://localhost/v1/chat/settle", headers={**self.auth(), "Content-Type": "text/plain"}, data='{"user":"safe"}') as resp:
+ self.assertEqual(resp.status, 415)
+ self.assertEqual(self.up_runner.app["settle_users"], [])
+
+
if __name__ == "__main__":
unittest.main()