ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(portal): a turn cut off by a restart settles by itself, with one Continue when it cannot be proven (#8206)
Mike Bradley09cd7f899d7d
diff --git a/ods/extensions/services/dashboard-api/pixel_chat_results.py b/ods/extensions/services/dashboard-api/pixel_chat_results.pyindex 220d3f172..ebdee0b64 100644--- a/ods/extensions/services/dashboard-api/pixel_chat_results.py+++ b/ods/extensions/services/dashboard-api/pixel_chat_results.py@@ -173,5 +173,12 @@ class ChatResultStore:def has_pending(self, conversation):return self.db.execute("SELECT 1 FROM attempts WHERE owner=? AND chat=? AND state IN ('active','unresolved')", conversation).fetchone() is not None+ def pending_attempts(self, conversation):+ """Keys of this conversation's unfinished attempts, newest first."""+ rows = self.db.execute(+ "SELECT attempt FROM attempts WHERE owner=? AND chat=? AND state IN ('active','unresolved') ORDER BY rowid DESC",+ conversation).fetchall()+ return [(*conversation, row["attempt"]) for row in rows]+def chunks(self, key, after=-1):return self.db.execute("SELECT sequence,data FROM chunks WHERE owner=? AND chat=? AND attempt=? AND sequence>? ORDER BY sequence", (*key, after)).fetchall()diff --git a/ods/extensions/services/dashboard-api/routers/pixel.py b/ods/extensions/services/dashboard-api/routers/pixel.pyindex a4fe37aa1..8b05e86a5 100644--- a/ods/extensions/services/dashboard-api/routers/pixel.py+++ b/ods/extensions/services/dashboard-api/routers/pixel.py@@ -248,6 +248,9 @@ _result_store: ChatResultStore | None = None_result_tasks: dict[tuple[str, str, str], asyncio.Task] = {}_result_preflights: set[tuple[str, str, str]] = set()_result_stops: set[tuple[str, str]] = set()+# Chats whose last turn an activity check is settling right now. Kept apart+# from Stop's fence, so neither can remove the other's.+_result_settles: set[tuple[str, str]] = set()_result_abort_ack: set[tuple[str, str, str]] = set()@@ -366,7 +369,7 @@ async def pixel_chat_result(body: ChatResultRequest, owner: str = Depends(verify# the request identity while its exact cancellation is still pending.return {"state": "active", "events": ""}if row["state"] == "unresolved":- activity = await pixel_chat_activity(ChatCancelRequest(chat_id=body.chat_id))+ activity = await pixel_chat_activity(ChatCancelRequest(chat_id=body.chat_id), owner)if activity["state"] == "terminal":store.finish(key, "interrupted")row = store.get(key)@@ -1001,34 +1004,129 @@ async def pixel_chat_cancel(body: ChatCancelRequest, owner: str = Depends(verifyfinally:_result_abort_ack.discard(identity)_result_stops.discard(identity[:2])- if isinstance(owner, str) and _result_store is not None and _result_store.has_pending((owner_namespace(owner), body.chat_id)):- return {"aborted": False}+ if isinstance(owner, str):+ conversation = (owner_namespace(owner), body.chat_id)+ # A restarted API process has not opened the store yet, but receipts+ # its predecessor left unfinished still hold this chat.+ pending = _chat_results().pending_attempts(conversation)+ if pending:+ # A chat-level request ("Continue in this chat") never stops a turn+ # this process is still streaming; that turn's own Stop names it.+ # An attempt no producer owns any more is resolved by its exact ID.+ if _owns_live_work(conversation):+ return {"aborted": False}+ return await pixel_chat_cancel(ChatCancelRequest(chat_id=body.chat_id, request_id=pending[0][2]), owner)return {"aborted": await _cancel_edge_run(edge_url, key, body.chat_id)}[email protected]("/chat/activity", dependencies=[Depends(verify_api_key)])-async def pixel_chat_activity(body: ChatCancelRequest) -> dict[str, str]:- """One authenticated chat lookup; global activity never supplies its state."""- config = _pixel_config()- if config is None:- return {"state": "unknown"}- edge_url, key = config+def _owns_live_work(conversation) -> bool:+ """True while this process streams, prepares or stops a turn of this chat."""+ return (conversation in _result_stops+ or any(identity[:2] == conversation for identity in _result_preflights)+ or any(identity[:2] == conversation and not task.done() for identity, task in _result_tasks.items()))+++async def _edge_chat_activity(edge_url: str, key: str, chat_id: str) -> str | None:+ """The edge's exact view of one chat, or None when it could not be read."""try:timeout = httpx.Timeout(connect=2.0, read=5.0, write=2.0, pool=2.0)async with httpx.AsyncClient(timeout=timeout, trust_env=False, follow_redirects=False) as client:async with client.stream("POST", f"{edge_url}/v1/chat/activity",- json={"user": body.chat_id},+ json={"user": chat_id},headers=_edge_headers(key, accept="application/json")) as response:if response.status_code != 200 or not response.headers.get("content-type", "").lower().startswith("application/json"):- return {"state": "unknown"}+ return Noneraw = await _bounded_response_bytes(response, 1024)parsed = json.loads(raw)if (isinstance(parsed, dict) and set(parsed) == {"state"}and isinstance(parsed["state"], str) and parsed["state"] in {"active", "terminal", "unknown"}):- return {"state": parsed["state"]}+ return parsed["state"]except (httpx.HTTPError, asyncio.TimeoutError, ValueError, TypeError):pass- return {"state": "unknown"}+ return None+++async def _settle_edge_chat(edge_url: str, key: str, chat_id: str) -> str | None:+ """Ask the ingress, through the edge, to settle a chat it proves idle.++ "settled" means no run of this chat can still be alive and any turn a+ restart cut off is recorded as interrupted; "active" means work may still+ run. Nothing is ever stopped. None means it could not be confirmed, for+ example while Portal is still restarting or on an older edge or ingress.+ """+ try:+ timeout = httpx.Timeout(connect=2.0, read=10.0, write=2.0, pool=2.0)+ async with httpx.AsyncClient(timeout=timeout, trust_env=False, follow_redirects=False) as client:+ async with client.stream("POST", f"{edge_url}/v1/chat/settle",+ json={"user": chat_id},+ headers=_edge_headers(key, accept="application/json")) as response:+ if response.status_code != 200 or not response.headers.get("content-type", "").lower().startswith("application/json"):+ return None+ raw = await _bounded_response_bytes(response, 1024)+ parsed = json.loads(raw)+ if (isinstance(parsed, dict) and set(parsed) == {"state"}+ and isinstance(parsed["state"], str) and parsed["state"] in {"settled", "active"}):+ return parsed["state"]+ except (httpx.HTTPError, asyncio.TimeoutError, ValueError, TypeError):+ pass+ return None++[email protected]("/chat/activity")+async def pixel_chat_activity(body: ChatCancelRequest, owner: str = Depends(verify_api_key)) -> dict[str, str]:+ """One authenticated chat lookup; global activity never supplies its state.++ A turn this process still streams is active. Otherwise, when no stream of+ this chat passes through the edge, the host ingress is asked to settle the+ chat: under its history lock it proves the native session idle before it+ records a turn that a restart cut off as interrupted, and it never stops a+ run. Only that proof turns the chat terminal here, and it also releases+ receipts no producer owns any more, so the chat can take its next message.+ """+ config = _pixel_config()+ if config is None:+ return {"state": "unknown"}+ edge_url, key = config+ conversation = (owner_namespace(owner), body.chat_id) if isinstance(owner, str) else None+ if conversation is not None and _owns_live_work(conversation):+ return {"state": "active"}+ state = await _edge_chat_activity(edge_url, key, body.chat_id)+ if state is None:+ return {"state": "unknown"}+ if state == "active" or conversation is None:+ return {"state": state}+ # A Stop, a turn or another check may have started here during that read.+ # It owns the chat now; settling beside it could race its outcome.+ if _owns_live_work(conversation) or conversation in _result_settles:+ return {"state": "active"}+ # Fence this process's own next attempt so it cannot race the history+ # lock. A Stop keeps its own fence; only this check removes this one.+ _result_settles.add(conversation)+ try:+ settled = await _settle_edge_chat(edge_url, key, body.chat_id)+ if settled == "settled" and _owns_live_work(conversation):+ # A Stop that started meanwhile settles its own receipt.+ settled = "active"+ if settled == "settled":+ store = _chat_results()+ for identity in store.pending_attempts(conversation):+ row = _result_state(store, identity)+ if row is not None and row["state"] == "unresolved":+ # No producer owns this receipt and the ingress proved the+ # chat idle. Keep every saved chunk; never call it complete.+ store.finish(identity, "interrupted")+ finally:+ _result_settles.discard(conversation)+ if settled == "settled":+ return {"state": "terminal"}+ if settled == "active":+ return {"state": "active"}+ # An older edge or ingress cannot settle. Its own memory that the chat+ # ended does not free a chat that a receipt still holds: the next message+ # would only be refused.+ if state == "terminal" and _chat_results().pending_attempts(conversation):+ return {"state": "unknown"}+ return {"state": state}_EXAMPLE_PROJECT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}")@@ -1157,6 +1255,8 @@ async def _retained_chat_stream(request, body, owner):try:if identity[:2] in _result_stops:raise ResultConflict("Stop is still being confirmed")+ if identity[:2] in _result_settles:+ raise ResultConflict("The last turn of this chat is still being checked")created = store.reserve(identity, fingerprint)except ResultConflict as exc:raise HTTPException(status_code=423, detail=str(exc)) from Nonediff --git a/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.py b/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.pynew file mode 100644index 000000000..b6128a40b--- /dev/null+++ b/ods/extensions/services/dashboard-api/tests/test_pixel_restart_recovery.py@@ -0,0 +1,338 @@+"""A turn cut off by a restart heals by itself once nothing can still run.++Seen on the RTX Spark: an update recreated dashboard-api and pixel-edge and+restarted the gateway while "Can you do things on cron timers?" streamed. The+retained receipt stayed unresolved, the restarted edge had no record of the+chat, and every later message was refused while the page polled forever.+"""+import asyncio+import json+import os+from pathlib import Path+import sys+from typing import Any, cast+from unittest.mock import patch++import pytest+from fastapi import HTTPException, Request++os.environ.setdefault("DASHBOARD_API_KEY", "dashboard-test-key")+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))+import pixel_chat_results as receipts+from routers import pixel+import pixel_chat_identity+from test_pixel import EDGE_KEY, ConnectedRequest, FakeClient, FakeResponse++OWNER = "dashboard-test-key"+CHAT = "spark-chat"+CUT_OFF = (receipts.owner_namespace(OWNER), CHAT, "cron-question")+PARTIAL = b'data: {"choices":[{"delta":{"content":"Yes. Portal can run"}}]}\n\n'+++def stores(tmp_path, monkeypatch):+ async def saved_identity(*_args, **_kwargs):+ return {"schemaVersion": 1, "revision": 1, "displayName": "Portal"}+ monkeypatch.setattr(pixel_chat_identity, "async_request_json", saved_identity)+ monkeypatch.setattr(pixel, "_result_tasks", {})+ monkeypatch.setattr(pixel, "_result_preflights", set())+ monkeypatch.setattr(pixel, "_result_stops", set())+ monkeypatch.setattr(pixel, "_result_settles", set())+ monkeypatch.setattr(pixel, "_result_abort_ack", set())+ monkeypatch.setenv("PIXEL_OPENWEBUI_KEY", EDGE_KEY)+ async def ready():+ return None+ monkeypatch.setattr(pixel, "_model_readiness_issue", ready)+ # The API process that was streaming the turn when the update recreated it.+ before = receipts.ChatResultStore(tmp_path / "receipts")+ before.reserve(CUT_OFF, "cron-question-input")+ before.append(CUT_OFF, PARTIAL)+ # The recreated API process: the receipt is unresolved, nobody owns it.+ after = receipts.ChatResultStore(tmp_path / "receipts")+ monkeypatch.setattr(pixel, "_result_store", after)+ yield after+ before.close()+ after.close()+++def edge(monkeypatch):+ """The restarted edge has no record of the chat; the ingress proves it idle."""+ state: dict[str, Any] = {"activity": "unknown", "settle": "settled", "lookups": [], "settles": []}++ async def activity(edge_url, key, chat_id):+ state["lookups"].append(chat_id)+ return state["activity"]++ async def settle(edge_url, key, chat_id):+ state["settles"].append(chat_id)+ # The settle fence is its own; a Stop's fence is never borrowed for it.+ assert (receipts.owner_namespace(OWNER), chat_id) in pixel._result_settles+ assert (receipts.owner_namespace(OWNER), chat_id) not in pixel._result_stops+ return state["settle"]++ async def never_cancel(*_args):+ raise AssertionError("Healing a restart-cut turn must never stop a run")++ monkeypatch.setattr(pixel, "_edge_chat_activity", activity)+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)+ monkeypatch.setattr(pixel, "_cancel_edge_run", never_cancel)+ return state+++def connected() -> Request:+ # The handlers only ask whether the client is still connected.+ return cast(Request, ConnectedRequest())+++def stream(request_id, text="Checking in"):+ return pixel.ChatStreamRequest.model_validate(+ {"chat_id": CHAT, "request_id": request_id, "messages": [{"role": "user", "content": text}]})+++def test_restart_cut_turn_settles_by_itself_and_the_chat_takes_its_next_message(stores, edge):+ assert stores.get(CUT_OFF)["state"] == "unresolved"+ lookup = pixel.ChatResultRequest(chat_id=CHAT, request_id=CUT_OFF[2])+ result = asyncio.run(pixel.pixel_chat_result(lookup, OWNER))+ # No terminal frame: the producer died with the old process. The saved+ # partial answer stays exactly as it was received.+ assert result == {"state": "interrupted", "events": PARTIAL.decode()}+ assert edge["settles"] == [CHAT]+ assert not stores.has_pending(CUT_OFF[:2])+ assert not pixel._result_stops+ assert stores.reserve((*CUT_OFF[:2], "checking-in"), "checking-in-input") is True+++def test_a_refused_follow_up_cannot_strand_the_chat_behind_the_cut_off_turn(stores, edge):+ # "Checking in" was refused while the cut-off turn held the chat, so its+ # request ID has no receipt. Its activity lookup still heals the chat.+ with pytest.raises(HTTPException) as refused:+ asyncio.run(pixel.pixel_chat_stream(connected(), stream("checking-in"), OWNER))+ assert refused.value.status_code == 423+ missing = pixel.ChatResultRequest(chat_id=CHAT, request_id="checking-in")+ assert asyncio.run(pixel.pixel_chat_result(missing, OWNER)) == {"state": "unknown", "events": ""}+ assert asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)) == {"state": "terminal"}+ assert stores.get(CUT_OFF)["state"] == "interrupted"+ assert b"".join(row["data"] for row in stores.chunks(CUT_OFF)) == PARTIAL+ assert not stores.has_pending(CUT_OFF[:2])++[email protected]("activity,settle,expected", [+ ("active", "settled", "active"), # a stream of this chat still passes through the edge+ ("unknown", "active", "active"), # the native session is busy: a run may still be alive+ ("unknown", None, "unknown"), # Portal still restarting, or an older edge or ingress+ ("terminal", None, "unknown"), # an older edge remembers the end, but the receipt holds the chat+ (None, "settled", "unknown"), # the edge could not be read at all+])+def test_work_that_may_still_run_keeps_the_chat_held(stores, edge, activity, settle, expected):+ edge.update(activity=activity, settle=settle)+ result = asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ assert result == {"state": expected}+ assert edge["settles"] == ([CHAT] if activity in {"unknown", "terminal"} else [])+ assert stores.get(CUT_OFF)["state"] == "unresolved"+ assert stores.has_pending(CUT_OFF[:2])+ with pytest.raises(HTTPException) as refused:+ asyncio.run(pixel.pixel_chat_stream(connected(), stream("second-turn"), OWNER))+ assert refused.value.status_code == 423+++def test_an_older_edge_that_remembers_the_end_frees_a_chat_no_receipt_holds(stores, edge):+ stores.finish(CUT_OFF, "interrupted")+ edge.update(activity="terminal", settle=None)+ result = asyncio.run(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ assert result == {"state": "terminal"}+ assert stores.reserve((*CUT_OFF[:2], "next-turn"), "next-input") is True+++def test_a_turn_this_process_still_streams_is_active_without_asking_anyone(stores, edge):+ async def run():+ live = asyncio.create_task(asyncio.sleep(3600))+ pixel._result_tasks[(*CUT_OFF[:2], "live-turn")] = live+ try:+ return await pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)+ finally:+ live.cancel()+ await asyncio.gather(live, return_exceptions=True)+ assert asyncio.run(run()) == {"state": "active"}+ assert edge["lookups"] == [] and edge["settles"] == []+ assert stores.get(CUT_OFF)["state"] == "unresolved"+++def test_settle_fences_a_new_attempt_until_it_answers(stores, edge, monkeypatch):+ async def run():+ entered, release = asyncio.Event(), asyncio.Event()++ async def slow_settle(edge_url, key, chat_id):+ entered.set()+ await release.wait()+ return "settled"++ monkeypatch.setattr(pixel, "_settle_edge_chat", slow_settle)+ healing = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ await entered.wait()+ stores.finish(CUT_OFF, "interrupted") # even with nothing pending, the fence holds+ with pytest.raises(HTTPException) as fenced:+ await pixel.pixel_chat_stream(connected(), stream("during-settle"), OWNER)+ assert fenced.value.status_code == 423+ release.set()+ assert await healing == {"state": "terminal"}+ assert not pixel._result_settles and not pixel._result_stops+ asyncio.run(run())+++def _soon(awaitable):+ """A race that went the wrong way fails here instead of hanging the run."""+ return asyncio.wait_for(awaitable, 5)+++def _blocked(result):+ """A stand-in edge call that waits until the test releases it."""+ entered, release = asyncio.Event(), asyncio.Event()++ async def call(*_args):+ entered.set()+ await release.wait()+ return result+ return call, entered, release+++async def _stop_holds_the_chat(stores):+ """While a Stop's cancel runs, its fence holds: no new turn, result active."""+ assert CUT_OFF[:2] in pixel._result_stops+ lookup = pixel.ChatResultRequest(chat_id=CHAT, request_id=CUT_OFF[2])+ assert await pixel.pixel_chat_result(lookup, OWNER) == {"state": "active", "events": ""}+ with pytest.raises(HTTPException) as refused:+ await pixel.pixel_chat_stream(connected(), stream("during-stop"), OWNER)+ assert refused.value.status_code == 423+ assert stores.get(CUT_OFF)["state"] == "unresolved"+++def test_a_stop_that_starts_during_the_edge_read_is_left_alone(stores, edge, monkeypatch):+ async def run():+ read, reading, answer = _blocked("unknown")+ cancel, cancelling, acknowledge = _blocked(True)+ monkeypatch.setattr(pixel, "_edge_chat_activity", read)+ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)+ check = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ await _soon(reading.wait())+ stop = asyncio.create_task(pixel.pixel_chat_cancel(+ pixel.ChatCancelRequest(chat_id=CHAT, request_id=CUT_OFF[2]), OWNER))+ await _soon(cancelling.wait())+ answer.set()+ # The check finds the Stop and leaves the chat to it: no settle.+ assert await _soon(check) == {"state": "active"}+ assert edge["settles"] == []+ await _stop_holds_the_chat(stores)+ acknowledge.set()+ assert await _soon(stop) == {"aborted": True}+ assert stores.get(CUT_OFF)["state"] == "cancelled"+ assert not pixel._result_stops and not pixel._result_settles+ asyncio.run(run())++[email protected]("ingress", ["settled", "active"])+def test_settle_never_removes_the_fence_of_a_stop_that_started_during_it(stores, edge, monkeypatch, ingress):+ async def run():+ settle, settling, answer = _blocked(ingress)+ cancel, cancelling, acknowledge = _blocked(True)+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)+ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)+ check = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ await _soon(settling.wait())+ # The owner's Stop is not refused because a check is settling.+ stop = asyncio.create_task(pixel.pixel_chat_cancel(+ pixel.ChatCancelRequest(chat_id=CHAT, request_id=CUT_OFF[2]), OWNER))+ await _soon(cancelling.wait())+ answer.set()+ # Whatever the ingress answers, the Stop now owns this receipt.+ assert await _soon(check) == {"state": "active"}+ assert not pixel._result_settles+ await _stop_holds_the_chat(stores)+ acknowledge.set()+ assert await _soon(stop) == {"aborted": True}+ assert stores.get(CUT_OFF)["state"] == "cancelled"+ assert not pixel._result_stops+ asyncio.run(run())+++def test_a_second_check_waits_for_the_settle_already_running(stores, edge, monkeypatch):+ async def run():+ settle, settling, answer = _blocked("settled")+ monkeypatch.setattr(pixel, "_settle_edge_chat", settle)+ first = asyncio.create_task(pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ await _soon(settling.wait())+ assert await pixel.pixel_chat_activity(pixel.ChatCancelRequest(chat_id=CHAT), OWNER) == {"state": "active"}+ assert CUT_OFF[:2] in pixel._result_settles+ answer.set()+ assert await _soon(first) == {"state": "terminal"}+ assert stores.get(CUT_OFF)["state"] == "interrupted"+ assert not pixel._result_settles+ asyncio.run(run())+++def test_continue_in_this_chat_resolves_the_cut_off_turn_by_its_exact_id(stores, monkeypatch):+ calls = []++ async def cancel(edge_url, key, chat_id):+ calls.append(chat_id)+ assert CUT_OFF[:2] in pixel._result_stops+ return True++ monkeypatch.setattr(pixel, "_cancel_edge_run", cancel)+ # The page only knows the refused follow-up's ID, so it asks for the chat.+ result = asyncio.run(pixel.pixel_chat_cancel(pixel.ChatCancelRequest(chat_id=CHAT), OWNER))+ assert result == {"aborted": True}+ assert calls == [CHAT]+ assert stores.get(CUT_OFF)["state"] == "cancelled"+ assert b"".join(row["data"] for row in stores.chunks(CUT_OFF)) == PARTIAL+ assert not stores.has_pending(CUT_OFF[:2])+++def test_continue_never_stops_a_turn_this_process_is_streaming(stores, monkeypatch):+ async def forbidden(*_args):+ raise AssertionError("A chat-level request must not stop a live turn")++ monkeypatch.setattr(pixel, "_cancel_edge_run", forbidden)++ async def run():+ stores.finish(CUT_OFF, "interrupted")+ live = (*CUT_OFF[:2], "live-turn")+ stores.reserve(live, "live-input")+ task = asyncio.create_task(asyncio.sleep(3600))+ pixel._result_tasks[live] = task+ try:+ return await pixel.pixel_chat_cancel(pixel.ChatCancelRequest(chat_id=CHAT), OWNER)+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ assert asyncio.run(run()) == {"aborted": False}++[email protected]("chunks,expected", [+ ([b'{"state":"settled"}'], "settled"),+ ([b'{"state":"active"}'], "active"),+ ([b'{"state":"terminal"}'], None),+ ([b'{"state":"settled","secret":"hidden"}'], None),+ ([b'{"aborted":true}'], None),+ ([b"x" * 1025], None),+])+def test_settle_request_is_exact_and_only_trusts_a_closed_receipt(chunks, expected, monkeypatch):+ monkeypatch.setenv("PIXEL_OPENWEBUI_KEY", EDGE_KEY)+ capture: dict[str, Any] = {}+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(FakeResponse(chunks=chunks), capture)) as client:+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) == expected+ assert capture["method"] == "POST"+ assert capture["url"] == "http://pixel-edge:9595/v1/chat/settle"+ assert capture["json"] == {"user": CHAT}+ assert capture["headers"]["Authorization"] == f"Bearer {EDGE_KEY}"+ assert client.call_args.kwargs["trust_env"] is False+ assert client.call_args.kwargs["follow_redirects"] is False++[email protected]("response", [FakeResponse(status=404), FakeResponse(status=502), FakeResponse(content_type="text/plain")])+def test_an_older_or_failing_edge_never_settles(response, monkeypatch):+ with patch.object(pixel.httpx, "AsyncClient", return_value=FakeClient(response)):+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) is None+ with patch.object(pixel.httpx, "AsyncClient", side_effect=pixel.httpx.ConnectError("private credential")):+ assert asyncio.run(pixel._settle_edge_chat("http://pixel-edge:9595", EDGE_KEY, CHAT)) is None+ assert EDGE_KEY not in json.dumps(response.headers)diff --git a/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx b/ods/extensions/services/dashboard/src/components/PortalResponseError.jsxindex 0e2f5ca9d..7347aac95 100644--- a/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx+++ b/ods/extensions/services/dashboard/src/components/PortalResponseError.jsx@@ -24,7 +24,11 @@ function useModelFailureCause(content) {return cause}+// Replies saved by older versions carry only this marker. Its cause was never+// known, so it is not blamed on the connection.+export const UNDELIVERED_REPLY='This reply did not arrive. Your conversation is preserved; send your message again if you still need it.'+export default function PortalResponseError({content}) {const cause=useModelFailureCause(content)- return <div className="portal-response-error" role="status"><CircleAlert size={13}/><PortalStreamingText instant>{content==='Request failed'?'The response could not be received. Your conversation is preserved; check the connection before continuing.':content}</PortalStreamingText>{cause && <p className="portal-response-cause">{cause}</p>} <HelpLink label="Still stuck? Get help on Discord"/></div>+ return <div className="portal-response-error" role="status"><CircleAlert size={13}/><PortalStreamingText instant>{content==='Request failed'?UNDELIVERED_REPLY:content}</PortalStreamingText>{cause && <p className="portal-response-cause">{cause}</p>} <HelpLink label="Still stuck? Get help on Discord"/></div>}diff --git a/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx b/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsxindex 8a58f67ac..ef5bb3ee8 100644--- a/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx+++ b/ods/extensions/services/dashboard/src/components/PortalResponseError.test.jsx@@ -1,10 +1,11 @@import {render,screen} from '@testing-library/react'-import PortalResponseError from './PortalResponseError'+import PortalResponseError, {UNDELIVERED_REPLY} from './PortalResponseError'-it('explains a generic transport failure without claiming that the work succeeded',()=>{+it('explains an older undelivered reply without claiming success or blaming the connection',()=>{render(<PortalResponseError content="Request failed"/>);- expect(screen.getByRole('status')).toHaveTextContent('The response could not be received.')- expect(screen.getByRole('status')).toHaveTextContent('check the connection before continuing')+ expect(screen.getByRole('status')).toHaveTextContent(UNDELIVERED_REPLY)+ expect(screen.getByRole('status')).not.toHaveTextContent(/connection|could not be received/)+ expect(screen.getByRole('link',{name:/Get help on Discord/})).toBeVisible()expect(screen.queryByText('Request failed')).toBeNull()})it('names the cause of a generic failure when the latest model call failed',async()=>{diff --git a/ods/extensions/services/dashboard/src/components/portal-agent-experience.css b/ods/extensions/services/dashboard/src/components/portal-agent-experience.cssindex caffa7610..48ba41cdf 100644--- a/ods/extensions/services/dashboard/src/components/portal-agent-experience.css+++ b/ods/extensions/services/dashboard/src/components/portal-agent-experience.css@@ -113,3 +113,7 @@.portal-context-status > span { flex:1; min-width:180px; }.portal-context-status > button { padding:3px 5px; background:transparent; color:rgb(var(--theme-text-secondary)); text-decoration:underline; text-underline-offset:3px; }.portal-context-status > button:focus-visible { outline:1px solid currentColor; outline-offset:2px; }+/* The one action that settles a held chat reads as a button, not a link. */+.portal-context-status > button.portal-context-action { display:inline-flex; align-items:center; min-height:30px; padding:5px 11px; border:1px solid rgb(var(--theme-text) / .16); border-radius:8px; color:rgb(var(--theme-text)); background:rgb(var(--theme-text) / .06); font-weight:500; text-decoration:none; }+.portal-context-status > button.portal-context-action:hover:not(:disabled) { background:rgb(var(--theme-text) / .11); }+.portal-context-status > button.portal-context-action:disabled { opacity:.5; cursor:not-allowed; }diff --git a/ods/extensions/services/dashboard/src/lib/pixelMascotState.js b/ods/extensions/services/dashboard/src/lib/pixelMascotState.jsindex b9bcffc48..b47ccd878 100644--- a/ods/extensions/services/dashboard/src/lib/pixelMascotState.js+++ b/ods/extensions/services/dashboard/src/lib/pixelMascotState.js@@ -15,5 +15,7 @@ export function pixelReplyPose(message, active) {if (message.status === 'stopped' || message.goalState === 'cancelled') return 'idle'// A restored/abandoned streaming placeholder is not proof of ongoing work.if (message.status === 'streaming') return active ? (message.task?.state === 'running' && message.task?.calls > 0 ? 'working' : 'thinking') : 'waiting'+ // A reply whose answer stopped arriving waits until Portal settles it.+ if (message.status === 'unconfirmed') return 'waiting'return message.content ? 'done' : 'idle'}diff --git a/ods/extensions/services/dashboard/src/lib/portalContext.js b/ods/extensions/services/dashboard/src/lib/portalContext.jsindex a6fcef18d..42c7dd56d 100644--- a/ods/extensions/services/dashboard/src/lib/portalContext.js+++ b/ods/extensions/services/dashboard/src/lib/portalContext.js@@ -25,6 +25,11 @@ const token=(value,maximum=100_000_000)=>Number.isSafeInteger(value) && value>=0const text=(value,max)=>typeof value==='string' && value.length>0 && value.length<=max && !/[\u0000-\u001f\u007f]/.test(value)const routeFingerprint=value=>typeof value==='string' && value.length===64 && /^[a-f0-9]{64}$/.test(value)const contextRetryDelays=[2000,5000,10000]+// An unconfirmed last turn is checked again on a growing interval, never every+// second. Each check asks the server to settle the chat if it proves idle.+const historyRecheckDelays=[1000,2000,4000,8000,15000,30000,60000]+export const CONTINUE_FAILED='Portal could not confirm this chat is free yet. Try again in a moment.'+export const STOP_UNCONFIRMED='Portal could not confirm the stop yet. Try Stop again in a moment.'export function parseConversationContext(value) {if(!value || value.schemaVersion!==1 || !['ready','missing','busy','unavailable'].includes(value.status))return nullconst {context,compaction,history,model}=value@@ -54,8 +59,12 @@ function requestId() {}/** Compaction belongs to the server session. The browser retains only its request- * identity; neither the transcript nor an inferred model summary is replaced. */-export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdentity,capacity,initialRequestId,blocked,onPendingChange}) {+ * identity; neither the transcript nor an inferred model summary is replaced.+ * `blocked` refuses every mutation. `held` means the page is still settling its+ * own interrupted turn: compaction and automatic checks wait for it, but the+ * owner's explicit Continue stays usable, because that is when it is needed. */+export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdentity,capacity,initialRequestId,blocked:blockedMutation,held=false,onPendingChange}) {+ const blocked=Boolean(blockedMutation || held)const acceptedModel=useRef(null)const runtime=useRef({chatId,identity:{model:'',source:'',provider:'',routeFingerprint:''},epoch:0})const typed=runtimeIdentity!==undefined@@ -79,14 +88,15 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdenif(!sameChat)acceptedModel.current=null}const runtimeKey=typed?`identity-${runtime.current.epoch}`:legacyRuntimeKey- const [view,setView]=useState({chatId,runtimeKey,context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,resolving:false,recoveryNotice:'',readUnavailable:false})+ const [view,setView]=useState({chatId,runtimeKey,context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false,readUnavailable:false})const current=useRef({}),pending=useRef(initialRequestId || null),requests=useRef(new Set()),query=useRef(null),mutation=useRef(false)- const refreshQueued=useRef(false)+ const refreshQueued=useRef(false),historyAttempts=useRef(0),historyPaused=useRef(false)const generation=useRef(0),lastQuery=useRef(0),priorRuntime=useRef(runtimeKey)const priorChat=useRef(chatId),lastMeasurement=useRef(null),invalidatedMeasurement=useRef(null)const lastManual=useRef(initialRequestId || null),observedCompaction=useRef(null)const contextRetries=useRef(0),priorAvailability=useRef({chatId,runtimeKey,blocked})- current.current={chatId,runtimeKey,runtimeIdentity:typed?runtime.current.identity:null,capacity,blocked,onPendingChange,historyUnknown:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyUnknown,+ current.current={chatId,runtimeKey,runtimeIdentity:typed?runtime.current.identity:null,capacity,blocked,explicitBlocked:Boolean(blockedMutation),onPendingChange,historyUnknown:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyUnknown,+ historyActive:view.chatId===chatId && view.runtimeKey===runtimeKey && view.historyActive,readUnavailable:view.chatId===chatId && view.runtimeKey===runtimeKey && view.readUnavailable}const publish=useCallback(update=>setView(previous=>({...previous,...update,chatId:current.current.chatId,runtimeKey:current.current.runtimeKey})),[])const persist=useCallback(id=>{current.current.onPendingChange(id,current.current.chatId);pending.current=id},[])@@ -132,7 +142,8 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdenconst historyUnknown=snapshot.history.status==='unknown'const readUnavailable=snapshot.status==='unavailable'if(!readUnavailable)contextRetries.current=0- publish({context,phase,notice,historyUnknown,readUnavailable,observedModel:modelMatches && ['ready','missing','busy'].includes(snapshot.status)?snapshot.model || null:null,...(snapshot.model?{observedCapacity:modelMatches?snapshot.model.contextWindow:null}:{}),...(!historyUnknown?{recoveryNotice:''}:{})})+ if(!historyUnknown)historyAttempts.current=0+ publish({context,phase,notice,historyUnknown,readUnavailable,observedModel:modelMatches && ['ready','missing','busy'].includes(snapshot.status)?snapshot.model || null:null,...(snapshot.model?{observedCapacity:modelMatches?snapshot.model.contextWindow:null}:{}),...(!historyUnknown?{recoveryNotice:'',recoveryFailed:false,historyChecking:true,historyActive:false}:{})})},[persist,publish])const refresh=useCallback(async(force=false)=>{@@ -168,6 +179,36 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdenreturn run},[accept,publish])+ // An unconfirmed last turn usually means a restart cut it off. The server+ // settles the chat only when it proves nothing can still run there, and it+ // never stops work, so this check may run without the owner. Work it finds+ // still running is shown as such and waited for. Only a check that can tell+ // neither asks the owner to act.+ const settleHistory=useCallback(async()=>{+ if(current.current.blocked || mutation.current || pending.current || !current.current.historyUnknown)return refresh(true)+ const at=generation.current,identity=current.current.chatId,controller=new AbortController()+ requests.current.add(controller)+ const timer=setTimeout(()=>controller.abort(),15000)+ let state='unknown'+ try {+ const response=await fetch('/api/pixel/chat/activity',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({chat_id:identity}),signal:controller.signal})+ const value=await response.json()+ if(response.ok && value && Object.keys(value).length===1 && ['active','terminal','unknown'].includes(value.state))state=value.state+ }catch {+ // An unanswered check proves nothing; the owner can still continue.+ }finally {clearTimeout(timer);requests.current.delete(controller)}+ if(at!==generation.current)return+ // A failed Stop or Continue is shown until the chat's state changes kind.+ const changed=(state==='active')!==Boolean(current.current.historyActive)?{recoveryFailed:false,recoveryNotice:''}:{}+ if(state==='active')publish({historyChecking:false,historyActive:true,...changed})+ else if(state==='unknown')publish({historyChecking:false,historyActive:false,...changed})+ await refresh(true)+ // "Nothing runs" settles the history too. If an older Portal could not+ // record that, its history stays unconfirmed, and then the owner can+ // continue: nothing is running.+ if(state==='terminal' && at===generation.current)setView(previous=>previous.historyUnknown?{...previous,historyChecking:false,historyActive:false}:previous)+ },[publish,refresh])+useEffect(()=>{generation.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=false;mutation.current=false;lastQuery.current=0pending.current=initialRequestId || null@@ -176,7 +217,8 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdenelse if(priorRuntime.current && priorRuntime.current!==runtimeKey)invalidatedMeasurement.current=lastMeasurement.currentpriorChat.current=chatIdcontextRetries.current=0- publish({context:null,observedCapacity:null,observedModel:null,phase:pending.current?'checking':'idle',notice:pending.current?'Checking compaction status…':'',historyUnknown:false,resolving:false,recoveryNotice:'',readUnavailable:false})+ historyAttempts.current=0+ publish({context:null,observedCapacity:null,observedModel:null,phase:pending.current?'checking':'idle',notice:pending.current?'Checking compaction status…':'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false,readUnavailable:false})priorRuntime.current=runtimeKeyvoid refresh(true)return ()=>{generation.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=false}@@ -185,13 +227,25 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIdenuseEffect(()=>{if(view.chatId!==chatId || view.runtimeKey!==runtimeKey)return- const observingOperation=pending.current || observedCompaction.current && view.phase==='unknown' || ['running','busy'].includes(view.phase) || view.historyUnknown- const retryRead=!observingOperation && view.readUnavailable && contextRetries.current<contextRetryDelays.length- if(!observingOperation && !retryRead)return- const delay=retryRead?contextRetryDelays[contextRetries.current]:['unknown','unavailable'].includes(view.phase)?5000:2000- const timer=setTimeout(()=>{if(retryRead)contextRetries.current+=1;void refresh(true)},delay)+ const operation=pending.current || observedCompaction.current && view.phase==='unknown' || ['running','busy'].includes(view.phase)+ const historyOnly=!operation && view.historyUnknown+ // While the page works or checks its own interrupted turn, it refreshes+ // this context when it is done; resume checking promptly after that.+ if(historyOnly && blocked){historyPaused.current=true;return}+ if(historyPaused.current){historyPaused.current=false;historyAttempts.current=0}+ const retryRead=!operation && !historyOnly && view.readUnavailable && contextRetries.current<contextRetryDelays.length+ if(!operation && !historyOnly && !retryRead)return+ // Work still running is checked at least every 15 s, so the chat frees+ // itself soon after that work ends.+ const delay=historyOnly?Math.min(historyRecheckDelays[Math.min(historyAttempts.current,historyRecheckDelays.length-1)],view.historyActive?15000:Infinity)+ :retryRead?contextRetryDelays[contextRetries.current]:['unknown','unavailable'].includes(view.phase)?5000:2000+ const timer=setTimeout(()=>{+ if(retryRead)contextRetries.current+=1+ if(historyOnly){historyAttempts.current+=1;void settleHistory()}+ else void refresh(true)+ },delay)return ()=>clearTimeout(timer)- },[view,chatId,runtimeKey,refresh])+ },[view,chatId,runtimeKey,blocked,refresh,settleHistory])useEffect(()=>{const previous=priorAvailability.current@@ -237,31 +291,40 @@ export function usePortalContext({chatId,runtimeKey:legacyRuntimeKey,runtimeIden}finally {clearTimeout(timer);requests.current.delete(controller);if(at===generation.current){mutation.current=false;if(requery)void refresh(true)}}},[accept,persist,publish,refresh])- const resolveInterrupted=useCallback(async()=>{- if(current.current.blocked || !current.current.historyUnknown || mutation.current || pending.current)return false+ // Both owner actions on an unconfirmed history stop whatever runs for this+ // chat. Continue is for a history Portal cannot tell about; Stop is for+ // work the server reports still running. Merely loading a conversation or+ // reading its context never cancels work, and the hold is released only+ // after the server confirms nothing is running for this chat.+ const releaseChat=useCallback(async stop=>{+ if(current.current.explicitBlocked || !current.current.historyUnknown || Boolean(current.current.historyActive)!==stop || mutation.current || pending.current)return falsegeneration.current+=1;requests.current.forEach(controller=>controller.abort());requests.current.clear();query.current=null;refreshQueued.current=falseconst at=generation.current,identity=current.current.chatId,controller=new AbortController()mutation.current=true;requests.current.add(controller)- publish({resolving:true,recoveryNotice:'Confirming the previous turn has stopped…'})+ publish({resolving:true,recoveryFailed:false,recoveryNotice:stop?'Stopping…':'Making sure nothing is still running in this chat…'})const timer=setTimeout(()=>controller.abort(),30000)try {- // This explicit owner action resolves uncertain history. Merely loading- // a conversation or reading its context must never cancel work.const response=await fetch('/api/pixel/chat/cancel',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({chat_id:identity}),signal:controller.signal})const result=await response.json()if(at!==generation.current)return falseif(!response.ok || result?.aborted!==true)throw new Error('stop unconfirmed')- publish({recoveryNotice:'Stop confirmed. Checking conversation context…'})+ publish({recoveryNotice:stop?'Stopping…':'Continuing in this chat…'})return true}catch {- if(at===generation.current)publish({recoveryNotice:'The stop could not be confirmed. Your conversation is preserved; check status or try again.'})+ if(at===generation.current)publish({recoveryFailed:true,recoveryNotice:stop?STOP_UNCONFIRMED:CONTINUE_FAILED})return false}finally {clearTimeout(timer);requests.current.delete(controller)if(at===generation.current){mutation.current=false;publish({resolving:false});void refresh(true)}}},[publish,refresh])+ // Continue stops whatever is left over, so it is never offered, nor sent,+ // while the server reports work still running in this chat.+ const resolveInterrupted=useCallback(()=>releaseChat(false),[releaseChat])+ const stopHistory=useCallback(()=>releaseChat(true),[releaseChat])- const displayed=view.chatId===chatId && view.runtimeKey===runtimeKey?view:{context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,resolving:false,recoveryNotice:''}- return {...displayed,busy:displayed.resolving || ['requesting','checking','running','unknown'].includes(displayed.phase),canResolve:displayed.historyUnknown && !blocked && !pending.current && !displayed.resolving,compact,refresh,resolveInterrupted}+ const displayed=view.chatId===chatId && view.runtimeKey===runtimeKey?view:{context:null,observedCapacity:null,phase:initialRequestId?'checking':'idle',notice:'',historyUnknown:false,historyChecking:true,historyActive:false,resolving:false,recoveryNotice:'',recoveryFailed:false}+ const releasable=displayed.historyUnknown && !blockedMutation && !pending.current && !displayed.resolving+ return {...displayed,busy:displayed.resolving || ['requesting','checking','running','unknown'].includes(displayed.phase),+ canResolve:releasable && !displayed.historyActive,canStop:releasable && displayed.historyActive,compact,refresh,resolveInterrupted,stopHistory}}diff --git a/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx b/ods/extensions/services/dashboard/src/lib/portalContext.test.jsxindex e51e11c9b..32d4a897f 100644--- a/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx+++ b/ods/extensions/services/dashboard/src/lib/portalContext.test.jsx@@ -1,5 +1,5 @@import {act,renderHook,waitFor} from '@testing-library/react'-import {compactCommand,historySnapshot,parseConversationContext,usePortalContext} from './portalContext'+import {compactCommand,historySnapshot,parseConversationContext,STOP_UNCONFIRMED,usePortalContext} from './portalContext'const context=(used=1200)=>({schemaVersion:1,status:'ready',sessionRevision:null,model:{id:'small',provider:'local',contextWindow:4096},context:{used,window:4096,measuredAt:'2026-09-16T12:00:00.000Z'},compaction:{status:'idle',count:0},history:{revision:null,acknowledgedMessages:2}})@@ -131,6 +131,152 @@ it('never cancels from inspection and requires unknown history plus an explicitexpect(result.current.historyUnknown).toBe(false)})+it('settles an unconfirmed last turn through the server by itself, on a growing interval and without a stop',async()=>{+ vi.useFakeTimers()+ try {+ let history='unknown',checks=0+ globalThis.fetch=vi.fn().mockImplementation(async url=>{+ if(url.endsWith('/activity')){checks+=1;if(checks===2)history='ready';return response({state:checks===2?'terminal':'unknown'})}+ return response({...context(),status:history==='unknown'?'unavailable':'ready',history:{...context().history,status:history}})+ })+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))+ await act(async()=>{})+ expect(result.current.historyUnknown).toBe(true)+ // Until a check fails to prove the turn is over, the owner is not asked.+ expect(result.current.historyChecking).toBe(true)+ await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(checks).toBe(1)+ expect(result.current.historyChecking).toBe(false)+ expect(result.current.canResolve).toBe(true)+ await act(async()=>vi.advanceTimersByTimeAsync(1999))+ expect(checks).toBe(1)+ await act(async()=>vi.advanceTimersByTimeAsync(1))+ expect(checks).toBe(2)+ expect(result.current.historyUnknown).toBe(false)+ const calls=fetch.mock.calls.length+ await act(async()=>vi.advanceTimersByTimeAsync(10*60_000))+ expect(fetch).toHaveBeenCalledTimes(calls)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'},{chat_id:'a'}])+ }finally {vi.useRealTimers()}+})++it('checks an unprovable last turn at most every minute instead of every two seconds',async()=>{+ vi.useFakeTimers()+ try {+ globalThis.fetch=vi.fn().mockImplementation(async url=>response(url.endsWith('/activity')?{state:'unknown'}:{...context(),status:'unavailable',history:{...context().history,status:'unknown'}}))+ renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))+ // Step the clock so each check's re-render schedules the next one.+ for(let second=0;second<10*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ // 1, 2, 4, 8, 15, 30 s, then once a minute: 15 checks in ten minutes, not 300.+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).length).toBeLessThanOrEqual(16)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/context')).length).toBeLessThanOrEqual(17)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)+ }finally {vi.useRealTimers()}+})++it('waits for work the server still runs, never offers Continue for it, and frees the chat when it ends',async()=>{+ vi.useFakeTimers()+ try {+ let history='unknown',state='active'+ globalThis.fetch=vi.fn().mockImplementation(async url=>{+ if(url.endsWith('/activity')){if(state==='terminal')history='ready';return response({state})}+ return response({...context(),status:history==='unknown'?'busy':'ready',history:{...context().history,status:history}})+ })+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))+ await act(async()=>{})+ expect(result.current.historyActive).toBe(false)+ await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(result.current.historyUnknown).toBe(true)+ expect(result.current.historyActive).toBe(true)+ // Continue stops whatever is left over, so it is neither offered nor sent.+ expect(result.current.canResolve).toBe(false)+ await act(async()=>expect(await result.current.resolveInterrupted()).toBe(false))+ // Running work is checked at least every 15 s, not once a minute.+ for(let second=0;second<5*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ const checks=fetch.mock.calls.filter(([url])=>url.endsWith('/activity')).length+ expect(checks).toBeGreaterThanOrEqual(20)+ expect(checks).toBeLessThanOrEqual(26)+ expect(result.current.historyActive).toBe(true)+ state='terminal'+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(result.current.historyUnknown).toBe(false)+ expect(result.current.historyActive).toBe(false)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)+ }finally {vi.useRealTimers()}+})++it('stops work the server still runs only on the owner’s Stop, and frees the chat once that is confirmed',async()=>{+ vi.useFakeTimers()+ try {+ let history='unknown',aborted=false+ globalThis.fetch=vi.fn().mockImplementation(async url=>{+ if(url.endsWith('/activity'))return response({state:'active'})+ if(url.endsWith('/cancel')){if(aborted)history='ready';return response({aborted})}+ return response({...context(),status:history==='unknown'?'busy':'ready',history:{...context().history,status:history}})+ })+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))+ await act(async()=>{})+ await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(result.current.historyActive).toBe(true)+ expect(result.current.canStop).toBe(true)+ expect(result.current.canResolve).toBe(false)+ // Nothing is stopped without the owner.+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)+ await act(async()=>expect(await result.current.stopHistory()).toBe(false))+ expect(result.current.recoveryFailed).toBe(true)+ expect(result.current.recoveryNotice).toBe(STOP_UNCONFIRMED)+ expect(STOP_UNCONFIRMED).toBe('Portal could not confirm the stop yet. Try Stop again in a moment.')+ aborted=true+ await act(async()=>expect(await result.current.stopHistory()).toBe(true))+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'},{chat_id:'a'}])+ expect(result.current.historyUnknown).toBe(false)+ expect(result.current.historyActive).toBe(false)+ }finally {vi.useRealTimers()}+})++it('offers Continue when nothing runs but an older Portal could not record that',async()=>{+ vi.useFakeTimers()+ try {+ // An older pixel edge remembers the chat ended but cannot settle its history.+ globalThis.fetch=vi.fn().mockImplementation(async url=>response(url.endsWith('/activity')?{state:'terminal'}:{...context(),status:'unavailable',history:{...context().history,status:'unknown'}}))+ const {result}=renderHook(()=>usePortalContext({chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn()}))+ await act(async()=>{})+ expect(result.current.historyChecking).toBe(true)+ await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(result.current.historyUnknown).toBe(true)+ expect(result.current.historyChecking).toBe(false)+ expect(result.current.historyActive).toBe(false)+ expect(result.current.canResolve).toBe(true)+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel'))).toHaveLength(0)+ }finally {vi.useRealTimers()}+})++it('waits for the page to settle its own turn, but never blocks the owner’s Continue',async()=>{+ vi.useFakeTimers()+ try {+ let history='unknown'+ globalThis.fetch=vi.fn().mockImplementation(async url=>{+ if(url.endsWith('/cancel')){history='ready';return response({aborted:true})}+ return response(url.endsWith('/activity')?{state:'unknown'}:{...context(),status:'unavailable',history:{...context().history,status:history}})+ })+ const props={chatId:'a',runtimeKey:'small',capacity:4096,onPendingChange:vi.fn(),held:true}+ const {result,rerender}=renderHook(value=>usePortalContext(value),{initialProps:props})+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))+ expect(result.current.historyUnknown).toBe(true)+ // The page checks its interrupted turn itself; nothing here doubles it.+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/activity'))).toHaveLength(0)+ expect(await result.current.compact()).toBe(false)+ // The explicit action is exactly what an unconfirmed turn needs.+ expect(result.current.canResolve).toBe(true)+ await act(async()=>expect(await result.current.resolveInterrupted()).toBe(true))+ expect(fetch.mock.calls.filter(([url])=>url.endsWith('/cancel')).map(([,options])=>JSON.parse(options.body))).toEqual([{chat_id:'a'}])+ expect(result.current.historyUnknown).toBe(false)+ rerender({...props,held:false,blocked:true})+ expect(result.current.canResolve).toBe(false)+ }finally {vi.useRealTimers()}+})+it.each(['runtime-restarted','result-unconfirmed'])('distinguishes a stopped runtime from an unconfirmed running compaction (%s)',async reason=>{const id='11111111-2222-4333-8444-555555555555',persist=vi.fn()globalThis.fetch=vi.fn().mockResolvedValue(response({...context(),diff --git a/ods/extensions/services/dashboard/src/lib/portalTaskState.js b/ods/extensions/services/dashboard/src/lib/portalTaskState.jsindex 2f3ba372a..81725d714 100644--- a/ods/extensions/services/dashboard/src/lib/portalTaskState.js+++ b/ods/extensions/services/dashboard/src/lib/portalTaskState.js@@ -17,10 +17,11 @@ export function newTaskBlockedReason({running, stopping, checking, launching, co// The composer stays editable while Portal works. Sending waits, and says why// instead of doing nothing.-export function sendBlockedReason({deleted, running, stopping, waiting, ready}) {+export function sendBlockedReason({deleted, running, stopping, held, waiting, ready}) {if (deleted) return ''if (stopping) return 'Portal is stopping the current task. Your message is kept here; send it when Portal has stopped.'if (running) return 'Portal is finishing the current task. Your message is kept here; send it when Portal is done, or stop the task.'+ if (held) return 'Choose Continue in this chat first. Your message is kept here.'if (waiting || !ready) return 'Portal is not ready for a new message yet. Your message is kept here; send it when Portal is ready.'return ''}diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.jsxindex 6737eede2..bfb1b03ff 100644--- a/ods/extensions/services/dashboard/src/pages/Pixel.jsx+++ b/ods/extensions/services/dashboard/src/pages/Pixel.jsx@@ -32,7 +32,7 @@ import PixelQuestions from '../components/PixelQuestions'import PortalGoalPlan from '../components/PortalGoalPlan'import {goalCommand,continueGoal} from '../lib/portalGoal'import PortalContextRing from '../components/PortalContextRing'-import {compactCommand,CONTEXT_REQUEST_ID,historySnapshot,usePortalContext} from '../lib/portalContext'+import {compactCommand,CONTEXT_REQUEST_ID,CONTINUE_FAILED,historySnapshot,STOP_UNCONFIRMED,usePortalContext} from '../lib/portalContext'import PortalModelSelector from '../components/PortalModelSelector'import PortalRuntimeIdentity from '../components/PortalRuntimeIdentity'import PortalReadiness from '../components/PortalReadiness'@@ -143,6 +143,36 @@ const MODEL_SWITCH_DETAIL = 'Model switch in progress; Portal will be ready whenconst CLEAN_CONTEXT_RECOVERY_REASON = 'operations-unavailable-zero-submissions'const CLEAN_CONTEXT_RECOVERY_NOTICE = 'The first attempt did not reach the Operations Broker, and the host verified that no work was submitted. Retrying once with a clean context…'const CLEAN_CONTEXT_RECOVERY_FAILED = 'Automatic recovery was attempted once, but Portal again did not reach the Operations Broker. The host verified that no Operations work was submitted. Check any other work before continuing; this does not confirm that other tools had no effects.'+// The last turn's answer did not arrive. Calm wording: nothing failed that the+// owner must repair, and the composer is free again. Its work may have run on+// and finished after the cut-off (the gateway can outlive the dashboard API,+// and delegated work outlives both), so nothing offers to send the message+// again in one click: that could repeat every tool action and file write.+export const RESTART_NOTICE = 'Portal restarted before its reply arrived. Your files are saved. Ask Portal what it did before you send your message again.'+export const NO_REPLY_NOTICE = 'Portal finished working on your last request, but its reply couldn’t be shown here. Your files are saved. Ask Portal what it did, or send your next message.'+export const NOT_RECEIVED_NOTICE = 'Portal didn’t receive your last message. It’s back in the message box, ready to send again.'+const CHECKING_TURN = 'Checking on your last request…'+const TURN_STILL_RUNNING = 'Portal is still working on your last request. Its answer will appear here when it’s done. You can stop it.'+// Work still running in this chat whose answer the page cannot receive, for+// example after a restart cut the reply off while delegated work went on.+const TURN_RUNNING_UNSEEN = 'Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.'+// Work the server found running for this chat when the page has no turn of+// its own to follow there.+const CHAT_STILL_RUNNING = 'Portal is still working on your last request. You can stop it, or send your next message when it’s done.'+const TURN_UNCONFIRMED = 'Portal can’t confirm your last request has finished.'+const CONTINUING_TURN = 'Making sure nothing is still running in this chat…'+// A Stop of earlier work that was not acknowledged. The alert goes once that+// turn settles: there is nothing left to stop.+const RESTORED_STOP_UNCONFIRMED = STOP_UNCONFIRMED+const clearRestoredStopError = current => current === RESTORED_STOP_UNCONFIRMED ? '' : current+// A cut-off turn is checked again on a growing interval and the checks stop+// once its state is proven. Work that is still running is checked at most+// every 15 s so the chat frees up soon after it finishes.+const TURN_RECHECK_MS = [1000, 2000, 4000, 8000, 15000, 30000, 60000]+const ACTIVE_RECHECK_MAX_MS = 15000+// A refused message is sent on its own only soon after the owner sent it.+const HELD_SEND_MAX_MS = 2 * 60 * 1000+const HELD_SEND_NOTE = ' Your new message will go out right after.'const STATUS_POLL_MS = 3000const OPS_STATUS_POLL_MS = 3000const OPS_TERMINAL_STATUSES = new Set(['succeeded', 'failed', 'cancelled', 'rejected'])@@ -441,6 +471,25 @@ function stoppedContent(content) {return `${partial}\n\n---\n\n_${STOPPED_NOTICE}_`}+// A reply cut off before its answer arrived keeps the text that did arrive,+// followed by one calm line.+function cutOffContent(content, notice) {+ const partial = typeof content === 'string' ? content.trimEnd() : ''+ if (!partial) return notice+ if (partial.includes(notice)) return partial+ return `${partial}\n\n_${notice}_`+}++// An activity receipt that recorded nothing would only add an empty log to a+// cut-off reply.+function recordedTask(task) {+ return task && (task.events?.length || task.activities?.length || task.goal) ? task : undefined+}++// A reply whose outcome was never confirmed: the answer stopped arriving and+// no receipt has settled it yet. It is not saved as a final outcome.+const unsettledReply = message => message?.role === 'assistant' && !['done', 'error', 'stopped'].includes(message.status)+function messagePublication(message) {const validate = preview => parseVerifiedPreviewFrame({choices:[{finish_reason:'stop'}],pixel:{schemaVersion:1,preview}})if (message?.role !== 'assistant') return {}@@ -633,6 +682,9 @@ export default function Pixel({ systemStatus = null }) {if (status === 'recovery') window.dispatchEvent(new Event(PIXEL_RECOVERY_REQUIRED))}, [status])const [messages, setMessages] = useState(() => initialChat?.messages || [])+ // Checks that settle a cut-off turn later read the conversation as rendered.+ const messagesRef = useRef(messages)+ messagesRef.current = messagesconst [input, setInput] = useState(() => initialChat?.draft || '')const [persistenceError, setPersistenceError] = useState('')const [sending, setSending] = useState(false)@@ -642,9 +694,26 @@ export default function Pixel({ systemStatus = null }) {const [stopError, setStopError] = useState('')// A turn Portal refused because it was busy; its draft is back in the composer.const [refusal, setRefusal] = useState('')+ // "Continue in this chat": '' | 'pending' | 'failed'.+ const [continuing, setContinuing] = useState('')+ // The chat whose owner chose Continue and whose server confirmed that+ // nothing runs there: that confirmation alone releases its hold.+ const continueRequested = useRef(null)+ // A message refused because an earlier turn still held this chat. It is sent+ // as soon as that turn is cleared up, unless the owner changes it first. The+ // ref makes that send happen once, even if a render still shows it pending.+ const [heldSend, setHeldSend] = useState(null)+ const heldSendRef = useRef(null)+ const holdSend = useCallback(value => { heldSendRef.current = value; setHeldSend(value) }, [])+ // Set only while that automatic send starts: it is made once per owner+ // action, never again for its own refusal.+ const automaticSendRef = useRef(false)const [sendHint, setSendHint] = useState(false)const [exampleError, setExampleError] = useState(null)const [restoredActivity, setRestoredActivity] = useState(() => initialChat?.interrupted ? 'checking' : 'idle')+ // Whether the work still running there is this turn's own reply, which the+ // server keeps receiving and the page then shows, or only work in the chat.+ const [restoredReplyLive, setRestoredReplyLive] = useState(false)const [activityRefresh, setActivityRefresh] = useState(0)const [workingElapsedSeconds, setWorkingElapsedSeconds] = useState(0)const [agentRuntime, setAgentRuntime] = useState(null)@@ -753,13 +822,19 @@ export default function Pixel({ systemStatus = null }) {const previewAccess = resolvePreviewAccess(currentPreview)const restoredActive = interrupted && !sending && restoredActivity === 'active'const restoredChecking = interrupted && !sending && restoredActivity === 'checking'+ // Portal could not prove what happened to the last turn. Sending waits for+ // the owner's Continue, which is offered right where they look.+ const restoredUnconfirmed = interrupted && !sending && restoredActivity === 'unknown'const contextCapacity=contextRuntime?.source==='external-host'? Number(contextRuntime.contextLength) || null: Number(contextRuntime?.contextLength || systemStatus?.inference?.contextSize || systemStatus?.model?.contextLength) || nullconst contextControl=usePortalContext({chatId:chatIdRef.current,runtimeIdentity:{model:contextRuntime?.model || activeModel,source:contextRuntime?.source || '',routeFingerprint:contextRuntime?.routeFingerprint},capacity:contextCapacity,initialRequestId:compactionRequestRef.current,- blocked:conversationDeleted || sending || modelSwitching || stopping || teams.busy || (interrupted && restoredActivity!=='terminal') || status!=='available',+ blocked:conversationDeleted || sending || modelSwitching || stopping || teams.busy || status!=='available',+ // The page settles its own interrupted turn first. That holds compaction+ // and automatic checks, never the owner's explicit Continue.+ held:interrupted && restoredActivity!=='terminal',onPendingChange:(id,chatId)=>{if(chatId!==chatIdRef.current)throw new Error('The conversation changed before compaction could be saved.')historySnapshot(messages)@@ -785,10 +860,60 @@ export default function Pixel({ systemStatus = null }) {let controller = nulllet disposed = falselet timer = null+ let attempt = 0+ let rereadReceipt = Boolean(requestId)+ // Each check waits longer than the last, and a proven state ends them.+ // Work still running is checked at least every 15 s.+ const later = state => {+ const delay = TURN_RECHECK_MS[Math.min(attempt, TURN_RECHECK_MS.length - 1)]+ attempt += 1+ timer = globalThis.setTimeout(checkActivity, state === 'active' ? Math.min(delay, ACTIVE_RECHECK_MAX_MS) : delay)+ }+ // No receipt holds this turn and the server proved that nothing still+ // runs for this chat, so no answer will arrive. The work may well be done,+ // so the reply keeps the text that did arrive and says so plainly. After+ // the owner's Continue, which stops anything left over, it says stopped.+ // A message that has no receipt and got nothing back never reached+ // Portal: it goes back to the composer. The composer is free, with+ // whatever the owner had typed.+ function settleWithoutReceipt({stopped = false, received = true} = {}) {+ continueRequested.current = null+ requestIdRef.current = null+ setStopError(clearRestoredStopError)+ const latest = messagesRef.current+ const last = latest.at(-1)+ const user = latest.at(-2)+ // Its pictures are still attached in the composer, so it goes back as+ // a whole and is not left in the conversation to be sent twice.+ const unsent = !stopped && !received && unsettledReply(last) && !last.content?.trim() && !recordedTask(last.task)+ && user?.role === 'user' && typeof user.content === 'string' && Boolean(user.content.trim() || user.images?.length)+ if (unsent) {+ setMessages(latest.slice(0, -2))+ setInput(current => restoreUnsentDraft(current, user.content))+ setRefusal(NOT_RECEIVED_NOTICE)+ } else if (unsettledReply(last)) {+ setMessages(replaceLastAssistant(latest, {content:stopped ? stoppedContent(last.content) : cutOffContent(last.content, NO_REPLY_NOTICE),+ status:'stopped', task:recordedTask(last.task)}))+ }+ setInterrupted(false)+ updateRestoredActivity('idle')+ void contextControl.refresh(true)+ }async function checkActivity() {+ if (continueRequested.current === chatId) {+ // The owner chose Continue and the server confirmed that nothing runs+ // for this chat. That confirmation releases the hold by itself, even+ // while the receipt or the chat lookup cannot be read.+ settleWithoutReceipt({stopped:true})+ return+ }controller = new AbortController()const deadline = globalThis.setTimeout(() => controller.abort(), 15000)let state = 'unknown'+ let receiptPending = false+ // Only a receipt lookup that answered can say this turn has none; one+ // that failed proves nothing, and then the turn is not settled here.+ let noReceipt = falsetry {if (requestId) {const resultResponse = await fetch('/api/pixel/chat/result', {@@ -800,8 +925,9 @@ export default function Pixel({ systemStatus = null }) {if (resultResponse.ok && result && Object.keys(result).sort().join(',') === 'events,state'&& typeof result.events === 'string' && result.events.length <= 8 * 1024 * 1024) {if (result.state === 'active') {+ setRestoredReplyLive(true)updateRestoredActivity('active')- timer = globalThis.setTimeout(checkActivity, 2000)+ later('active')return}if (['complete', 'interrupted', 'cancelled', 'rejected'].includes(result.state)) {@@ -816,7 +942,21 @@ export default function Pixel({ systemStatus = null }) {let chat = loadStoredChat(readConversations().find(item => item.chatId === chatId))if (!chat || chat.chatId !== chatId) throw new Error('Saved conversation is unavailable')const recovered = retainedResult(result.events)- const successful = result.state === 'complete' && recovered.done && !recovered.failed+ // The ingress sends a turn's whole answer in one frame, after it+ // recorded the turn complete. A receipt that lost its producer+ // after saving that text lost only its closing frames: the turn+ // finished, and its answer is shown as such.+ const answered = result.state === 'interrupted' && !recovered.failed && Boolean(recovered.content.trim())+ const successful = (result.state === 'complete' && recovered.done && !recovered.failed) || answered+ // Its closing frame carried the final activity log; the running+ // snapshot saved before it is kept only if it recorded something,+ // and it replaces the page's own snapshot, never adds to it.+ const recoveredTask = answered ? {task:recordedTask(recovered.task)} : recovered.task ? {task:recovered.task} : {}+ // A receipt without its answer lost its producer with the+ // dashboard API process, and the server then proved nothing+ // still runs: Portal restarted while it was answering. Whether+ // its work ran on cannot be told, so it is never re-sent here.+ const cutOff = result.state === 'interrupted' && !recovered.done && !recovered.failed && !answeredif (chat.requestId === requestId && chat.interrupted) {const source = chat.persistenceSnapshotconst publication = successful ? recovered.preview : null@@ -824,11 +964,15 @@ export default function Pixel({ systemStatus = null }) {const writer = createConversationWriter(source)try {let saved = writer.recover({...source, requestId:null, inFlight:false, interrupted:false,- messages:replaceLastAssistant(source.messages, {+ messages:replaceLastAssistant(source.messages, cutOff ? {+ content: cutOffContent(recovered.content, RESTART_NOTICE),+ status: 'stopped',+ task: recordedTask(recovered.task || source.messages.at(-1)?.task),+ } : {content: result.state === 'cancelled' ? stoppedContent(recovered.content): recovered.content || (successful ? 'Completed without a text response.' : 'Portal could not complete the response. Check saved work before continuing.'),status: result.state === 'cancelled' ? 'stopped' : successful ? 'done' : 'error',- ...(recovered.task ? {task:recovered.task} : {}),+ ...recoveredTask,...(successful && recovered.questions ? {questions:recovered.questions} : {}),...(successful && recovered.artifacts ? {artifacts:recovered.artifacts} : {}),...(publication ? {publication,beforePublication:before} : {}),@@ -850,6 +994,10 @@ export default function Pixel({ systemStatus = null }) {}// The sender may already have saved, or another request may now// own this chat. Adopt its authoritative record without writing.+ // The receipt settled this turn, so a Continue still answering+ // for it has nothing left to release, nor a Stop to retry.+ continueRequested.current = null+ setStopError(clearRestoredStopError)conversationWriter.current = createConversationWriter(chat.persistenceSnapshot)hydrating.current = true; hydrationAction.current = null; savedView.current = nullrequestIdRef.current = chat.requestId@@ -866,6 +1014,10 @@ export default function Pixel({ systemStatus = null }) {void contextControl.refresh(true)return}+ // An unresolved receipt: the chat lookup below may prove the turn+ // is over, and that also settles this receipt on the server.+ receiptPending = result.state === 'unresolved'+ noReceipt = result.state === 'unknown'}}const response = await fetch('/api/pixel/chat/activity', {@@ -879,8 +1031,23 @@ export default function Pixel({ systemStatus = null }) {// A failed lookup or edge restart is not evidence that work finished.} finally { globalThis.clearTimeout(deadline) }if (disposed || chatIdRef.current !== chatId) return+ if (state === 'terminal' && receiptPending && rereadReceipt) {+ // The server settled the receipt as it proved the chat idle; read it+ // once more so whatever was saved of the answer is kept.+ rereadReceipt = false+ timer = globalThis.setTimeout(checkActivity, 0)+ return+ }+ if (state === 'terminal' && (!requestId || noReceipt || receiptPending)) {+ settleWithoutReceipt({received:!noReceipt})+ return+ }+ // Nothing runs, but this turn's receipt could not be read: it may still+ // hold the answer. Check again rather than settle without it.+ if (state === 'terminal') state = 'unknown'+ setRestoredReplyLive(false)updateRestoredActivity(state)- if (state !== 'terminal') timer = globalThis.setTimeout(checkActivity, 2000)+ later(state)}checkActivity()return () => {@@ -890,6 +1057,15 @@ export default function Pixel({ systemStatus = null }) {}}, [interrupted, sending, activityRefresh, updateRestoredActivity])+ // Portal coming back after an update or restart is when a cut-off turn can+ // settle: check it then, not at the end of a long interval.+ const priorStatus = useRef(status)+ useEffect(() => {+ const previous = priorStatus.current+ priorStatus.current = status+ if (status === 'available' && !['available', 'loading'].includes(previous) && interrupted && !sending) setActivityRefresh(value => value + 1)+ }, [status, interrupted, sending])+useEffect(() => {let controller = nulllet stopped = false@@ -1097,12 +1273,14 @@ export default function Pixel({ systemStatus = null }) {}, [messages, preview, workspaceOpen, sending, interrupted, input, imageDraftKey, conversationDeleted])const sendMessage = useCallback(async (answerOverride, continuationId = null) => {+ const automatic = automaticSendRef.current+ automaticSendRef.current = falseif (conversationDeleted) returnconst exactDraft = typeof answerOverride === 'string' ? answerOverride : inputconst trimmed = (typeof answerOverride === 'string' ? answerOverride : input).trim()if(compactCommand(trimmed)){await compactConversation();return}if(contextControl.busy || contextControl.historyUnknown)return- if ((!trimmed && !images.items.length) || sending || modelSwitching || abortRef.current || restoredActive || restoredChecking || status !== 'available' || trimmed.length > MAX_INPUT_LEN) return+ if ((!trimmed && !images.items.length) || sending || modelSwitching || abortRef.current || restoredActive || restoredChecking || restoredUnconfirmed || status !== 'available' || trimmed.length > MAX_INPUT_LEN) returnif(images.busy){setStopError('Finish uploading or remove the failed image before sending. Your draft is preserved.');return}const turnImages=typeof answerOverride==='string'?[]:images.receipts.map(({id,sha256})=>({id,sha256}))const needsImageRoute=turnImages.length>0 || hasImageHistory@@ -1151,6 +1329,9 @@ export default function Pixel({ systemStatus = null }) {userMessage,]const visibleConversation = [...messages, userMessage]+ // A refused send must not replace the identity of a turn that still holds+ // this chat; it is the one that can still be recovered.+ const heldRequestId = requestIdRef.currentsetMessages([...visibleConversation, { role: 'assistant', content: '', status: 'streaming', revealResponse:makeChatId() }])if (typeof answerOverride !== 'string') setInput('')setSending(true)@@ -1159,6 +1340,10 @@ export default function Pixel({ systemStatus = null }) {setStopping(false)setStopError('')setRefusal('')+ holdSend(null)+ setContinuing('')+ // A Continue for an earlier turn never releases this one unchecked.+ continueRequested.current = nullsetExampleError(null)// Text typed while this turn runs is a new draft. A turn that does not// start returns its own text to the composer without replacing that draft.@@ -1255,13 +1440,22 @@ export default function Pixel({ systemStatus = null }) {requestIdRef.current = nullthrow new Error('chat-request-rejected')}+ if (response.status === 423) {+ // An earlier turn still holds this chat, most often one a restart+ // cut off. Nothing started. Keep following that turn, never this+ // refused attempt, which has no receipt to recover.+ requestIdRef.current = heldRequestId+ return { kind: 'held' }+ }if (response.status === 429) {// Portal was busy (for example preparing another picture) and// refused this turn before starting it. Nothing ran.requestIdRef.current = nullreturn { kind: 'busy' }}- if (!response.ok) throw new Error('chat unavailable')+ // A gateway error here usually means the dashboard API is restarting;+ // any other refusal is a real failure the owner may need help with.+ if (!response.ok) throw new Error(response.status >= 502 && response.status <= 504 ? 'chat-connection-lost' : 'chat unavailable')reader = response.body?.getReader()if (!reader) throw new Error('stream unavailable')@@ -1369,7 +1563,7 @@ export default function Pixel({ systemStatus = null }) {// An acknowledged Stop or page disposal can close a reader normally.// Its late close must not overwrite the explicit cancellation outcome.if (!isCurrentTurn()) return- if (attempt.receivedError) { setInterrupted(true); return }+ if (attempt.receivedError) { setInterrupted(true); updateRestoredActivity('checking'); return }if (attempt.receivedDone) requestIdRef.current = nullif (attempt.receivedDone) {const previousPublication = [...messages].reverse().find(message => message.publication?.relativeDirectory === attempt.verifiedPreview?.relativeDirectory)?.publication || preview@@ -1389,11 +1583,13 @@ export default function Pixel({ systemStatus = null }) {}))return}- const content = attempt.assistantText- ? `${attempt.assistantText}\n\n_Response interrupted._`- : 'Connection interrupted'+ // The answer stopped arriving before Portal said it was done, for+ // example because Portal restarted. Keep what arrived; the check that+ // follows settles it. The server may hold the whole answer, may still+ // be working, or may prove the turn was cut off.setInterrupted(true)- setMessages(previous => replaceLastAssistant(previous, { content, status: 'error' }))+ updateRestoredActivity('checking')+ setMessages(previous => replaceLastAssistant(previous, { content: attempt.assistantText, status: 'unconfirmed' }))}try {@@ -1417,6 +1613,25 @@ export default function Pixel({ systemStatus = null }) {setRefusal(busyRefusalText(turnImages.length > 0))return}+ if (attempt.kind === 'held') {+ // Not a failure either, and never the connection: this message did+ // not start because an earlier turn still holds the chat. It waits in+ // the composer while that turn is cleared up, then goes out.+ restoreDraft()+ contextStartRef.current = originalContextStart+ setMessages(messages)+ if (automatic) {+ // That automatic send was refused too. It is not repeated: the+ // message waits in the composer, one click from going out.+ setRefusal(busyRefusalText(false))+ return+ }+ if (typeof answerOverride !== 'string' && !turnImages.length) holdSend({chatId:chatIdRef.current, text:trimmed, at:Date.now()})+ setInterrupted(true)+ updateRestoredActivity('checking')+ setActivityRefresh(value => value + 1)+ return+ }if (attempt.kind === 'rejected') {restoreDraft()contextStartRef.current = originalContextStart@@ -1472,7 +1687,8 @@ export default function Pixel({ systemStatus = null }) {updateRestoredActivity('idle')return}- if (attempt.kind === 'busy') {+ if (attempt.kind === 'busy' || attempt.kind === 'held') {+ if (attempt.kind === 'held') requestIdRef.current = nullcontextStartRef.current = originalContextStartsetMessages(messages)restoreDraft()@@ -1522,10 +1738,18 @@ export default function Pixel({ systemStatus = null }) {const storageFailed = conversationChanged || error?.message === 'chat-recovery-storage-unavailable'const historyTooLarge=error?.message==='history-request-too-large'const requestRejected=error?.message==='chat-request-rejected'- setInterrupted(!storageFailed && !historyTooLarge && !requestRejected)- if (storageFailed || historyTooLarge || requestRejected || turnImages.length) restoreDraft()- setMessages(previous => replaceLastAssistant(previous, {- content: requestRejected?'Portal rejected the request format (HTTP 422). No task was started. Your draft is preserved. Refresh Portal before retrying.':historyTooLarge?'The encoded conversation exceeds the 8 MB request limit. No task was started. Export this conversation before starting a new chat.':conversationChanged ? 'This conversation changed in another tab. No task was started. Download a recovery copy, then reload to read the saved version.' : storageFailed ? 'Could not save the request for recovery. No task was started. Check browser storage and try again.' : latestAssistantText || 'Request failed',+ const refused=error?.message==='chat unavailable'+ // A lost connection is not yet a failure: the turn may have started,+ // finished or been cut off by a restart. Keep what arrived and let+ // the check that follows settle it.+ const uncertain=!storageFailed && !historyTooLarge && !requestRejected && !refused+ // A refusal may still have saved a receipt that explains it; check.+ const recover=!storageFailed && !historyTooLarge && !requestRejected+ setInterrupted(recover)+ if (recover) updateRestoredActivity('checking')+ if (!uncertain || turnImages.length) restoreDraft()+ setMessages(previous => replaceLastAssistant(previous, uncertain ? {content:latestAssistantText, status:'unconfirmed'} : {+ content: requestRejected?'Portal rejected the request format (HTTP 422). No task was started. Your draft is preserved. Refresh Portal before retrying.':historyTooLarge?'The encoded conversation exceeds the 8 MB request limit. No task was started. Export this conversation before starting a new chat.':conversationChanged ? 'This conversation changed in another tab. No task was started. Download a recovery copy, then reload to read the saved version.' : storageFailed ? 'Could not save the request for recovery. No task was started. Check browser storage and try again.' : 'Portal could not start this message. Your draft is kept; try sending it again.',status: 'error',}))}@@ -1543,7 +1767,7 @@ export default function Pixel({ systemStatus = null }) {void contextControl.refresh(true)}}- }, [input, messages, preview, workspaceOpen, sending, modelSwitching, status, restoredActive, restoredChecking, updateRestoredActivity, teams.busy, teams.start,compactConversation,contextControl.busy,contextControl.historyUnknown,contextControl.refresh,startExtensionInstallation,startGithubExtensionRequest,images,imageModel,hasImageHistory,conversationDeleted])+ }, [input, messages, preview, workspaceOpen, sending, modelSwitching, status, restoredActive, restoredChecking, restoredUnconfirmed, updateRestoredActivity, teams.busy, teams.start,compactConversation,contextControl.busy,contextControl.historyUnknown,contextControl.refresh,startExtensionInstallation,startGithubExtensionRequest,images,imageModel,hasImageHistory,conversationDeleted])const stopStreaming = useCallback(async () => {const controller = abortRef.current@@ -1596,10 +1820,13 @@ export default function Pixel({ systemStatus = null }) {controller?.abort()abortRef.current = nullrequestIdRef.current = null- setMessages(previous => replaceLastAssistant(previous, {+ // The live stream's own reply is the one stopped, whatever its last+ // frame said. Without one, only a reply still unresolved is: a held+ // chat's Stop ends older work, and its last reply may be finished.+ setMessages(previous => controller || unsettledReply(previous.at(-1)) ? replaceLastAssistant(previous, {content: stoppedContent(controller?.responseText?.() ?? previous.at(-1)?.content),status: 'stopped',- }))+ }) : previous)setSending(false)setInterrupted(false)updateRestoredActivity('terminal')@@ -1608,7 +1835,7 @@ export default function Pixel({ systemStatus = null }) {// without an exact acknowledgement could leave tools or inference active.if (stopRequestRef.current === stopRequest && chatIdRef.current === chatId && abortRef.current === controller) {setStopError(restored- ? 'Stop was not confirmed. This chat may still have work in progress; check its activity or retry the stop request.'+ ? RESTORED_STOP_UNCONFIRMED: 'Stop was not confirmed. Portal is still connected; retry Stop.')if (restored) setActivityRefresh(value => value + 1)}@@ -1622,6 +1849,47 @@ export default function Pixel({ systemStatus = null }) {}}, [stopping, interrupted, updateRestoredActivity])+ // "Continue in this chat" is the one action offered when Portal cannot prove+ // what happened to the last turn. It asks the server to stop anything left+ // over in this chat and releases the hold only after the server confirms+ // that nothing is running there.+ const continueChat = useCallback(async () => {+ if (!interrupted) { void contextControl.resolveInterrupted(); return }+ if (continuing === 'pending' || sending || stopping) return+ const chatId = chatIdRef.current+ const controller = new AbortController()+ const timeout = setTimeout(() => controller.abort(), 30000)+ setContinuing('pending')+ try {+ const response = await fetch('/api/pixel/chat/cancel', {+ method: 'POST', signal: controller.signal, headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify({ chat_id: chatId }),+ })+ const payload = await response.json()+ if (chatIdRef.current !== chatId) return+ if (!response.ok || !payload || Object.keys(payload).length !== 1 || payload.aborted !== true) throw new Error('continue unconfirmed')+ continueRequested.current = chatId+ setContinuing('')+ // The next check settles the turn at once and frees the composer.+ setActivityRefresh(value => value + 1)+ } catch {+ if (chatIdRef.current === chatId) setContinuing('failed')+ } finally {+ clearTimeout(timeout)+ }+ }, [interrupted, continuing, sending, stopping, contextControl.resolveInterrupted, updateRestoredActivity])++ // Settle, then send: the message a held chat refused goes out once the+ // earlier turn is cleared up, while the composer still holds it unchanged.+ useEffect(() => {+ if (!heldSend || heldSendRef.current !== heldSend || interrupted || sending || stopping || status !== 'available'+ || contextControl.historyUnknown || contextControl.busy) return+ holdSend(null)+ if (heldSend.chatId !== chatIdRef.current || Date.now() - heldSend.at > HELD_SEND_MAX_MS || input.trim() !== heldSend.text) return+ automaticSendRef.current = true+ void sendMessage()+ }, [heldSend, interrupted, sending, stopping, status, contextControl.historyUnknown, contextControl.busy, input, sendMessage, holdSend])+// One live task streams into this page; see portalTaskState for why a new// task waits for it. Every New task control shows this reason, never nothing.const newTaskBlocked = newTaskBlockedReason({running:sending || restoredActive, stopping, checking:restoredChecking,@@ -1632,6 +1900,9 @@ export default function Pixel({ systemStatus = null }) {const startNewChat = useCallback((preserveDraft = false) => {if (newTaskBlocked) { setStopError(newTaskBlocked); return }setRefusal('')+ holdSend(null)+ setContinuing('')+ continueRequested.current = nullsetSendHint(false)chatIdRef.current = makeChatId()images.replace(chatIdRef.current)@@ -1649,7 +1920,7 @@ export default function Pixel({ systemStatus = null }) {setExampleError(null)updateRestoredActivity('idle')inputRef.current?.focus?.()- }, [newTaskBlocked, updateRestoredActivity, images.replace, input])+ }, [newTaskBlocked, updateRestoredActivity, images.replace, input, holdSend])useEffect(() => {const remove = async event => {@@ -1771,6 +2042,9 @@ export default function Pixel({ systemStatus = null }) {setInput(chat.draft)setStopError('')setRefusal('')+ holdSend(null)+ setContinuing('')+ continueRequested.current = nullsetSendHint(false)setExampleError(null)setInterrupted(chat.interrupted)@@ -1782,7 +2056,7 @@ export default function Pixel({ systemStatus = null }) {}window.addEventListener(SELECT_EVENT, select)return () => window.removeEventListener(SELECT_EVENT, select)- }, [sending, restoredActive, restoredChecking, stopping, updateRestoredActivity, teams.launching,contextControl.busy,images.replace])+ }, [sending, restoredActive, restoredChecking, stopping, updateRestoredActivity, teams.launching,contextControl.busy,images.replace,holdSend])const inputOver = input.length > MAX_INPUT_LENconst inputEmpty = !(command?.task ?? goalDraft?.task ?? input).trim() && !images.items.length@@ -1791,6 +2065,7 @@ export default function Pixel({ systemStatus = null }) {// editable and is saved with the chat. Only sending waits, and says why.const composerLocked = conversationDeleted || modelSwitching || status !== 'available'const sendBlocked = sendBlockedReason({deleted:conversationDeleted, stopping, running:sending || restoredActive || teams.busy,+ held:restoredUnconfirmed || (contextControl.historyUnknown && !contextControl.historyChecking && !contextControl.historyActive),waiting:restoredChecking || contextControl.busy || contextControl.historyUnknown || modelSwitching, ready:status === 'available'})useEffect(() => { if (!sendBlocked) setSendHint(false) }, [sendBlocked])const composerVisible = !(workspaceExpanded && workspaceOpen && !previewCollapsed)@@ -1844,9 +2119,7 @@ export default function Pixel({ systemStatus = null }) {: restoredActive? 'Working in this chat': restoredChecking- ? 'Checking previous work'- : interrupted && restoredActivity === 'unknown'- ? 'Activity unknown'+ ? 'Checking last request': status === 'available'? (readinessView.attention ? 'Needs attention' : 'Available'): status === 'switching'@@ -1862,6 +2135,27 @@ export default function Pixel({ systemStatus = null }) {: status === 'loading'? 'Connecting...': 'Degraded'+ // One calm line beside the composer while the last turn is settled. Its one+ // action, Continue in this chat, stops whatever is left over, so it appears+ // only after a check found that Portal cannot tell: never while a check runs+ // or while work is still running.+ const turnHold = sending ? null : interrupted ? (+ continuing === 'pending' ? {text:CONTINUING_TURN, busy:true}+ : restoredActive ? {text:restoredReplyLive ? TURN_STILL_RUNNING : TURN_RUNNING_UNSEEN}+ : restoredUnconfirmed ? {text:continuing === 'failed' ? CONTINUE_FAILED : TURN_UNCONFIRMED,+ failed:continuing === 'failed', action:continueChat, disabled:stopping}+ : {text:CHECKING_TURN, busy:true}+ ) : contextControl.historyUnknown ? (+ contextControl.resolving ? {text:contextControl.recoveryNotice || CONTINUING_TURN, busy:true}+ // Here the page follows no turn of its own, so Stop sits in this line.+ : contextControl.historyActive ? {text:contextControl.recoveryFailed ? STOP_UNCONFIRMED : CHAT_STILL_RUNNING,+ busy:!contextControl.recoveryFailed, failed:contextControl.recoveryFailed,+ action:contextControl.stopHistory, label:'Stop', disabled:!contextControl.canStop}+ : contextControl.recoveryFailed ? {text:CONTINUE_FAILED, failed:true, action:continueChat, disabled:!contextControl.canResolve}+ : contextControl.historyChecking ? {text:CHECKING_TURN, busy:true}+ : {text:TURN_UNCONFIRMED, action:continueChat, disabled:!contextControl.canResolve}+ ) : null+ const heldNote = heldSend?.chatId === chatIdRef.current && turnHold ? HELD_SEND_NOTE : ''// Native Windows Core has local model chat but no Pixel host runtime. The// status API distinguishes that deliberate selection from a temporary@@ -1981,27 +2275,6 @@ export default function Pixel({ systemStatus = null }) {onClick={() => startNewChat(true)}>New chat with draft</button>{(images.items.length > 0 || hasImageHistory) && <p>Attachments will not be copied. Reattach the original files in the new chat.</p>}</div>}- {interrupted && !sending && (- <div role="status" className="mx-auto w-full max-w-5xl rounded-xl border border-theme-border bg-theme-text-secondary/10 px-4 py-3 text-sm text-theme-text-secondary">- {restoredActivity === 'active'- ? 'The previous request is still active in this chat. Your saved conversation and preview are preserved. You can stop that work below; its live response cannot be reattached.'- : restoredActivity === 'terminal'- ? 'The previous request is no longer active. Its final response was not recovered; check the saved files and results before continuing.'- : restoredActivity === 'checking'- ? 'Checking whether this chat’s previous request is still active. Your saved conversation and preview are preserved.'- : 'Completion was not confirmed. This chat’s activity is unknown. Your request and partial response are saved; check its results before continuing. You can attempt to stop previous work in this chat without resending it.'}- {restoredActivity === 'unknown' && (- <>- <button type="button" className="ml-2 underline" disabled={stopping} onClick={() => {- updateRestoredActivity('checking')- setActivityRefresh(value => value + 1)- }}>Check activity again</button>- <button type="button" className="ml-2 underline" disabled={stopping}- onClick={stopStreaming}>Try Stop previous work</button>- </>- )}- </div>- )}{status === 'loading' && messages.length === 0 && (<div className="flex h-full flex-col items-center justify-center text-theme-text-muted"><Loader2 className="mb-3 h-8 w-8 animate-spin" />@@ -2094,7 +2367,7 @@ export default function Pixel({ systemStatus = null }) {{message.role === 'assistant' && message.content ? (<>{message.publication && <PixelSnapshotChanges preview={message.publication} before={message.beforePublication} variant="summary" onPreview={()=>openPublication(message.publication,'preview')} onReview={path=>openPublication(message.publication,'review',path)}/>}- {!message.questions && (shownContent || message.status==='streaming') && (message.status==='error' ? <PortalResponseError content={shownContent}/> : <PortalStreamingText key={message.revealResponse || chatIdRef.current} active={message.status==='streaming'} animate={Boolean(message.revealResponse) || message.status==='streaming'} instant={['stopped','recovering'].includes(message.status)} onReveal={chatScroll.onContentResize} components={MARKDOWN_COMPONENTS} artifacts={message.role==='assistant' && message.status==='done' ? message.artifacts : null}>{shownContent}</PortalStreamingText>)}+ {!message.questions && (shownContent || message.status==='streaming') && (message.status==='error' ? <PortalResponseError content={shownContent}/> : <PortalStreamingText key={message.revealResponse || chatIdRef.current} active={message.status==='streaming'} animate={Boolean(message.revealResponse) || message.status==='streaming'} instant={['stopped','recovering','unconfirmed'].includes(message.status)} onReveal={chatScroll.onContentResize} components={MARKDOWN_COMPONENTS} artifacts={message.role==='assistant' && message.status==='done' ? message.artifacts : null}>{shownContent}</PortalStreamingText>)}<OperationsApprovalCard content={message.content} />{!message.questions && message.status!=='streaming' && <PortalResponseActions content={displayedContent}/>}</>@@ -2168,7 +2441,7 @@ export default function Pixel({ systemStatus = null }) {) : (<buttononClick={sendFromComposer}- disabled={isDisabled || inputOver || inputEmpty || images.busy}+ disabled={isDisabled || restoredUnconfirmed || inputOver || inputEmpty || images.busy}className="inline-flex h-11 w-11 items-center justify-center rounded-xl bg-theme-accent text-white transition hover:bg-theme-accent-hover disabled:cursor-not-allowed disabled:opacity-40"title="Send">@@ -2184,10 +2457,11 @@ export default function Pixel({ systemStatus = null }) {<button type="button" onClick={sendFromComposer}>Send again</button></div>}{exampleError && <p role="alert" className="mt-1.5 px-1 text-xs text-theme-text-secondary">{exampleError.text}{exampleError.help && <> <HelpLink /></>}</p>}- {contextControl.historyUnknown && <div className="portal-context-status" role="status">- {contextControl.resolving && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}- <span>{contextControl.recoveryNotice || 'The previous turn could not be confirmed. Resolve it before continuing; your conversation is preserved.'}</span>- <button type="button" disabled={!contextControl.canResolve} onClick={()=>void contextControl.resolveInterrupted()}>Resolve interrupted turn</button>+ {turnHold && <div className="portal-context-status" role="status" aria-label="Last message">+ {turnHold.busy && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}+ <span>{turnHold.text}{heldNote}</span>+ {turnHold.action && <button type="button" className="portal-context-action" disabled={turnHold.disabled} onClick={()=>void turnHold.action()}>{turnHold.label || 'Continue in this chat'}</button>}+ {turnHold.failed && <HelpLink />}</div>}{contextControl.notice && <div className="portal-context-status" role={contextControl.phase==='failed'?'alert':'status'}>{contextControl.busy && contextControl.phase!=='unknown' && <Loader2 size={13} className="animate-spin" aria-hidden="true"/>}@@ -2206,7 +2480,7 @@ export default function Pixel({ systemStatus = null }) {</div></div><div className="mt-1.5 flex items-center justify-between gap-3 px-1 text-[10px] text-theme-text-muted/70">- <span>{stopping ? 'Waiting for exact cancellation acknowledgement' : restoredActive ? 'Earlier work is active in this chat; Stop targets only this chat.' : sending ? `${displayName} is using the active ODS model and tools · ${workingElapsed} elapsed` : sendKey.mode === 'mod-enter' ? 'Ctrl/⌘+Enter to send • Enter for a new line' : 'Enter to send • Shift+Enter for a new line'}</span>+ <span>{stopping ? 'Waiting for exact cancellation acknowledgement' : restoredActive ? 'Stop ends only the work in this chat.' : sending ? `${displayName} is using the active ODS model and tools · ${workingElapsed} elapsed` : sendKey.mode === 'mod-enter' ? 'Ctrl/⌘+Enter to send • Enter for a new line' : 'Enter to send • Shift+Enter for a new line'}</span></div></div>{inputOver && (diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsxindex 4228a61b5..9bd5f89ab 100644--- a/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx+++ b/ods/extensions/services/dashboard/src/pages/Pixel.stale-writer.test.jsx@@ -61,7 +61,7 @@ it('opening a second tab on an active request does not overwrite the streaming sconst before = localStorage.getItem(CHAT_KEY)const observer = render(<Pixel/>)try {- await within(observer.container).findByText(/The previous request is still active in this chat/)+ await within(observer.container).findByText(/Portal is still working on your last request/)expect(localStorage.getItem(CHAT_KEY)).toBe(before)} finally {observer.unmount()@@ -157,7 +157,7 @@ it.each(['ack-first','stream-first'])('recovers observer Stop without stale conf})await deliver(first)const observer = render(<Pixel/>)- await within(observer.container).findByText(/previous request is still active/)+ await within(observer.container).findByText(/Portal is still working on your last request/)// The source advances after observer hydration; Stop must retain that work.await deliver(second)fireEvent.click(within(observer.container).getByTitle('Stop'))diff --git a/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx b/ods/extensions/services/dashboard/src/pages/Pixel.test.jsxindex ad092a212..f8c01629e 100644--- a/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx+++ b/ods/extensions/services/dashboard/src/pages/Pixel.test.jsx@@ -14,6 +14,7 @@ const previewManifests=new Map()// The repository's base ESLint profile does not mark JSX identifiers as uses.// eslint-disable-next-line no-unused-varsimport Pixel, {+ NO_REPLY_NOTICE,OperationsApprovalCard,formatElapsed,isCleanContextRecoveryFrame,@@ -28,6 +29,8 @@ const response = (body, status = 200) => ({status,json: async () => body,})+// The calm line a reply ends with once the server proved its turn is over.+const CUT_OFF = (_content, node) => node?.tagName === 'EM' && node.textContent === NO_REPLY_NOTICEit('resolves old cards to the latest verified version of their own project',()=>{const publication=(letter,directory)=>{@@ -2044,10 +2047,11 @@ describe('Pixel', () => {await act(async () => {})for (let reload = 0; reload < 2; reload += 1) {const restored = render(<Pixel />)- await screen.findByText('Activity unknown')+ await screen.findByText('Portal can’t confirm your last request has finished.')+ expect(screen.getByRole('button', { name: 'Continue in this chat' })).toBeEnabled()expect(screen.getByText('Create a report, then verify it.')).toBeInTheDocument()expect(screen.getByText('Created report.md; checking its contents.')).toBeInTheDocument()- expect(screen.getByText(/Completion was not confirmed/)).toBeInTheDocument()+ expect(screen.queryByText(/Activity unknown|Completion was not confirmed/)).not.toBeInTheDocument()expect(screen.queryByText(/Stopped by you/)).not.toBeInTheDocument()expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()expect(globalThis.fetch.mock.calls.filter(([url]) => url === '/api/pixel/chat/stream')).toHaveLength(1)@@ -2202,9 +2206,10 @@ describe('Pixel', () => {expect(signals).toHaveLength(1)await act(async () => { await vi.advanceTimersByTimeAsync(15000) })expect(signals[0].aborted).toBe(true)- expect(screen.getByRole('button',{name:'Check activity again'})).toBeEnabled()+ expect(screen.getByRole('button',{name:'Continue in this chat'})).toBeEnabled()expect(screen.getByText('Saved partial result')).toBeVisible()- await act(async () => { fireEvent.click(screen.getByRole('button',{name:'Check activity again'})) })+ // The next check starts by itself, with a fresh deadline.+ await act(async () => { await vi.advanceTimersByTimeAsync(1000) })expect(signals).toHaveLength(2)expect(signals[1]).not.toBe(signals[0])expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()@@ -2230,12 +2235,19 @@ describe('Pixel', () => {expect(screen.getByRole('button', {name:'New chat'})).toHaveAttribute('aria-disabled', 'true')expect(screen.getByRole('button', {name:'New chat'})).toHaveAttribute('title', expect.stringContaining('Portal is finishing the current task'))expect(screen.getByText('Saved partial result')).toBeInTheDocument()+ // No receipt brings this turn's answer back here, so the line never promises it.+ expect(screen.getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()+ expect(screen.queryByRole('button', {name:'Continue in this chat'})).not.toBeInTheDocument()state = 'terminal'- await screen.findByText(/previous request is no longer active/, {}, { timeout: 3000 })+ // The server proved the turn is over: the reply settles calmly, it is+ // never claimed as the owner's Stop, and the composer is free again.+ await screen.findByText(CUT_OFF, {}, { timeout: 3000 })expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()- expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()- expect(screen.getByText(/final response was not recovered/)).toBeInTheDocument()+ expect(screen.queryByText(/Stopped by you|no longer active|was not recovered/)).not.toBeInTheDocument()expect(screen.getByText('Saved partial result')).toBeInTheDocument()+ expect(screen.queryByRole('status', { name: 'Last message' })).not.toBeInTheDocument()+ fireEvent.change(screen.getByPlaceholderText('Message Portal...'), { target: { value: 'What did you change?' } })+ expect(screen.getByTitle('Send')).toBeEnabled()expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/stream')).toBe(false)})@@ -2249,7 +2261,7 @@ describe('Pixel', () => {return response({state:chat_id === 'second_interrupted' ? 'active' : 'terminal'})})render(<Pixel />)- await screen.findByText(/previous request is no longer active/)+ await screen.findByText(CUT_OFF)act(()=>window.dispatchEvent(new CustomEvent('ods:pixel-select-conversation',{detail:'second_interrupted'})))await screen.findByText('Working in this chat')expect(fetch).toHaveBeenCalledWith('/api/pixel/chat/activity',expect.objectContaining({body:JSON.stringify({chat_id:'second_interrupted'})}))@@ -2270,11 +2282,11 @@ describe('Pixel', () => {return response({ aborted })})render(<Pixel />)- await screen.findByText('Activity unknown')+ await screen.findByText('Portal can’t confirm your last request has finished.')expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()state = 'active'- fireEvent.click(screen.getByText('Check activity again'))- await screen.findByText('Working in this chat')+ // The next check, on a growing interval, finds the work still running.+ await screen.findByText('Working in this chat', {}, { timeout: 3000 })expect(screen.getByRole('textbox')).toBeDisabled()fireEvent.click(screen.getByTitle('Stop'))await screen.findByRole('alert')@@ -2301,9 +2313,11 @@ describe('Pixel', () => {await screen.findByTitle('Stop')fireEvent.click(screen.getByTitle('Stop'))state = 'terminal'- await screen.findByText(/previous request is no longer active/, {}, { timeout: 3000 })+ await screen.findByText(CUT_OFF, {}, { timeout: 3000 })await act(async () => { finishCancel(response({ aborted: true })) })- expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()+ // The late acknowledgement never rewrites the settled reply as a Stop.+ expect(screen.queryByText(/Stopped by you/)).not.toBeInTheDocument()+ expect(screen.getByText(CUT_OFF)).toBeInTheDocument()expect(screen.getByText('Saved partial result')).toBeInTheDocument()expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()})@@ -2320,15 +2334,15 @@ describe('Pixel', () => {return response({ state: 'active' })})render(<Pixel />)- await screen.findByText('Activity unknown')- fireEvent.click(screen.getByText('Check activity again'))- await screen.findByText('Activity unknown')+ await screen.findByText('Portal can’t confirm your last request has finished.')+ // Neither a failed lookup nor global activity counts as this chat's work.+ await waitFor(() => expect(calls).toBe(2), { timeout: 3000 })+ expect(screen.getByText('Portal can’t confirm your last request has finished.')).toBeVisible()expect(screen.queryByTitle('Stop')).not.toBeInTheDocument()- fireEvent.click(screen.getByText('Check activity again'))- await screen.findByText('Working in this chat')+ await screen.findByText('Working in this chat', {}, { timeout: 4000 })})- it('restored unknown activity permits an explicit exact-chat stop attempt without replay or false stopped claims', async () => {+ it('restored unknown activity permits an explicit exact-chat Continue without replay, and says stopped only once confirmed', async () => {saveInterruptedChat()let aborted = falseglobalThis.fetch.mockImplementation(async (url, options) => {@@ -2339,15 +2353,19 @@ describe('Pixel', () => {return response({ aborted })})render(<Pixel />)- await screen.findByText('Activity unknown')+ await screen.findByText('Portal can’t confirm your last request has finished.')expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/cancel')).toBe(false)- fireEvent.click(screen.getByText('Try Stop previous work'))- await screen.findByRole('alert')+ fireEvent.click(screen.getByRole('button', { name: 'Continue in this chat' }))+ await screen.findByText('Portal could not confirm this chat is free yet. Try again in a moment.')expect(screen.queryByText('Response stopped')).not.toBeInTheDocument()expect(screen.getByText('Saved partial result')).toBeInTheDocument()aborted = true- fireEvent.click(screen.getByText('Try Stop previous work'))+ fireEvent.click(screen.getByRole('button', { name: 'Continue in this chat' }))+ // Continue stops anything left over: only its confirmation says stopped,+ // and the reply never claims Portal finished.await screen.findByText('Response stopped')+ expect(screen.getByText(/Stopped by you/)).toBeInTheDocument()+ expect(screen.queryByText(CUT_OFF)).not.toBeInTheDocument()expect(screen.getByText('Saved partial result')).toBeInTheDocument()expect(globalThis.fetch.mock.calls.some(([url]) => url === '/api/pixel/chat/stream')).toBe(false)})@@ -2688,7 +2706,7 @@ describe('Pixel', () => {expect(screen.queryByText(/private-upstream-secret/)).toBeNull()})- it('marks a stream that closes without DONE as interrupted', async () => {+ it('keeps a stream that closes without DONE calm and checks it instead of calling it failed', async () => {globalThis.fetch.mockResolvedValueOnce(response({ available: true, model: 'pixel/default', detail: 'local' }))@@ -2701,8 +2719,12 @@ describe('Pixel', () => {fireEvent.change(screen.getByPlaceholderText('Message Portal...'), { target: { value: 'test' } })fireEvent.click(screen.getByTitle('Send'))- await waitFor(() => expect(screen.getByText('Response interrupted.')).toBeInTheDocument())+ expect(await screen.findByRole('status', { name: 'Last message' })).toBeVisible()expect(screen.getByText('Partial answer')).toBeInTheDocument()+ expect(screen.getByRole('button', { name: 'Continue in this chat' })).toBeEnabled()+ expect(screen.queryByText(/Response interrupted|Connection interrupted|could not be received|Ended early/)).not.toBeInTheDocument()+ await waitFor(() => expect(JSON.parse(localStorage.getItem('ods.pixel.chat.v1'))).toMatchObject({ interrupted: true }))+ expect(JSON.parse(localStorage.getItem('ods.pixel.chat.v1')).messages.at(-1).status).toBeUndefined()})it('preserves partial answers and ignores late content after an upstream error', async () => {diff --git a/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsxindex bd242152e..de96af374 100644--- a/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx+++ b/ods/extensions/services/dashboard/src/pages/PixelCompaction.test.jsx@@ -262,23 +262,25 @@ it.each([true,false])('resolves uncertain history only after an explicit stop anreturn Promise.resolve(response(url==='/api/pixel/chat/context'?context:{available:true,runtime}))})render(<Pixel/>);await screen.findByText('Available')- const resolveButton=await screen.findByRole('button',{name:'Resolve interrupted turn'})+ const resolveButton=await screen.findByRole('button',{name:'Continue in this chat'})expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)// The draft stays editable; sending waits until the turn is resolved.expect(screen.getByPlaceholderText('Message Portal...')).toBeEnabled()expect(screen.getByTitle('Send')).toBeDisabled()expect(screen.queryByText(/Context compaction failed/)).toBeNull()fireEvent.click(resolveButton)- expect(resolveButton).toBeDisabled()+ expect(await screen.findByText('Making sure nothing is still running in this chat…')).toBeVisible()+ expect(screen.queryByRole('button',{name:'Continue in this chat'})).toBeNull()expect(screen.getByRole('button',{name:'New chat'})).toHaveAttribute('aria-disabled','true')await act(async()=>finish())if(acknowledged) {- await waitFor(()=>expect(screen.queryByRole('button',{name:'Resolve interrupted turn'})).toBeNull())+ await waitFor(()=>expect(screen.queryByRole('status',{name:'Last message'})).toBeNull())expect(screen.getByTitle('Send')).toBeEnabled()}else {- await screen.findByText(/The stop could not be confirmed/)+ await screen.findByText('Portal could not confirm this chat is free yet. Try again in a moment.')+ expect(screen.getByRole('link',{name:/Get help on Discord/})).toBeVisible()expect(screen.getByTitle('Send')).toBeDisabled()- expect(screen.getByRole('button',{name:'Resolve interrupted turn'})).toBeEnabled()+ expect(screen.getByRole('button',{name:'Continue in this chat'})).toBeEnabled()}expect(stored()).toMatchObject({chatId:'context-chat',draft:'Keep the next request',messages:initial})expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)diff --git a/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsxindex 3e07ce2e0..ee82160ef 100644--- a/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx+++ b/ods/extensions/services/dashboard/src/pages/PixelImages.test.jsx@@ -1,5 +1,5 @@/* global atob */-import {act,fireEvent,screen,waitFor} from '@testing-library/react'+import {act,fireEvent,screen,waitFor,within} from '@testing-library/react'import {render} from '../test/test-utils'import Pixel from './Pixel'import {sha256} from '../lib/pixelArtifacts'@@ -218,11 +218,12 @@ it('waits for an observer image upload before adopting terminal request recoveryreturn originalFetch(url,options)})render(<Pixel/>)- await screen.findByRole('button',{name:'Check activity again'})+ await screen.findByRole('button',{name:'Continue in this chat'})fireEvent.change(screen.getByLabelText('Choose images'),{target:{files:[fixtureFile()]}})await waitFor(()=>expect(finishUpload).toBeTypeOf('function'))+ // The next check finds the completed receipt by itself.terminal=true- fireEvent.click(screen.getByRole('button',{name:'Check activity again'}))+ await waitFor(()=>expect(calls('/api/pixel/chat/result').length).toBeGreaterThan(1),{timeout:3000})await act(async()=>{})expect(signal.aborted).toBe(false)expect(screen.getByRole('list',{name:'Attached images'})).toBeVisible()@@ -382,3 +383,27 @@ it('preserves the draft after preflight validation rejection without claiming aexpect(screen.queryByRole('button',{name:'Resolve interrupted turn'})).toBeNull()expect(stored().requestId).toBeNull()})++it('puts an image message that never reached Portal back once, pictures and all, and says so',async()=>{+ // nginx answered 502 while the dashboard API was recreated: no receipt.+ stream=async()=>response({detail:'Bad Gateway'},502)+ const prior=fetchMock.getMockImplementation()+ fetchMock.mockImplementation(async(url,options)=>url==='/api/pixel/chat/result'?response({state:'unknown',events:''})+ :url==='/api/pixel/chat/activity'?response({state:'terminal'}):prior(url,options))+ await ready();await choose()+ fireEvent.change(screen.getByPlaceholderText('Message Portal...'),{target:{value:'What is in this picture?'}})+ fireEvent.click(screen.getByTitle('Send'))+ const note=await screen.findByRole('status',{name:'Message not started'})+ expect(note).toHaveTextContent('Portal didn’t receive your last message. It’s back in the message box, ready to send again.')+ expect(screen.queryByText(/Portal finished working/)).toBeNull()+ expect(screen.getByPlaceholderText('Message Portal...')).toHaveValue('What is in this picture?')+ expect(screen.getByText(/1 × 1 · Ready/)).toBeVisible()+ await waitFor(()=>expect(stored().messages.some(message=>message.content==='What is in this picture?')).toBe(false))+ // Sending it again adds the turn, with its picture, to the history once.+ stream=async()=>completed('A single pixel.')+ fireEvent.click(within(note).getByRole('button',{name:'Send again'}))+ expect(await screen.findByText('A single pixel.')).toBeVisible()+ const sent=JSON.parse(calls('/api/pixel/chat/stream').at(-1)[1].body)+ expect(sent.history_snapshot.messages.filter(message=>message.role==='user')).toEqual([+ {role:'user',content:'What is in this picture?',images:[{id:imageId,sha256:receipt.sha256}]}])+})diff --git a/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsx b/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsxnew file mode 100644index 000000000..2a70eb079--- /dev/null+++ b/ods/extensions/services/dashboard/src/pages/PixelRestartRecovery.test.jsx@@ -0,0 +1,534 @@+// Seen on the RTX Spark: an update recreated the dashboard and its API and+// restarted Portal's gateway while "Can you do things on cron timers?"+// streamed. The chat showed "Ended early", "No tool activity was recorded"+// and "The response could not be received", the next message failed the same+// way, and the page then sat behind "Activity unknown" with a disabled+// "Resolve interrupted turn" while it polled three endpoints every second.+import {act,fireEvent,screen,waitFor,within} from '@testing-library/react'+import {render} from '../test/test-utils'+// The repository's base ESLint profile does not mark JSX identifiers as uses.+// eslint-disable-next-line no-unused-vars+import Pixel, {NO_REPLY_NOTICE, NOT_RECEIVED_NOTICE, RESTART_NOTICE} from './Pixel'+import {CONTINUE_FAILED} from '../lib/portalContext'++const QUESTION='Can you do things on cron timers or scheduled work tasks?'+const PARTIAL='Yes. Portal can run'+const STALE=/Ended early|No tool activity was recorded|could not be received|check the connection|Activity unknown|could not be confirmed|Resolve interrupted turn|Check activity again|Try Stop previous work|Completion was not confirmed/+const STOPPED='Stopped by you. Workspace changes completed before cancellation were preserved.'+const STILL_WORKING='Portal is still working on your last request. You can stop it, or send your next message when it’s done.'+const CONTINUE={name:'Continue in this chat'}+const response=(body,status=200)=>({ok:status>=200 && status<300,status,json:async()=>body})+const encode=value=>new TextEncoder().encode(`data: ${typeof value==='string'?value:JSON.stringify(value)}\n\n`)+const task={schemaVersion:1,runId:'chatcmpl_11111111-2222-4333-8444-555555555555',startedAt:'2026-10-26T01:34:20.000Z',finishedAt:null,state:'running',calls:0,failures:0,blocked:0,truncated:false,activities:[]}+const context=(history='ready')=>({schemaVersion:1,status:history==='unknown'?'unavailable':'ready',sessionRevision:null,context:null,model:null,+ compaction:{status:'idle',count:0},history:{status:history,revision:null,acknowledgedMessages:0}})+// The ingress sends a turn's whole answer in one frame, after it recorded+// the turn complete. Before that, only task activity arrives.+const TASK_ONLY=`data: ${JSON.stringify({object:'ods.task.activity',id:task.runId,pixel_task:task})}\n\n`+// A turn whose connection drops when the dashboard API is recreated, before+// its answer arrived or, with text, right after it.+function cutOffStream(text) {+ const chunks=[encode({object:'ods.task.activity',id:task.runId,pixel_task:task}),...(text?[encode({choices:[{delta:{content:text}}]})]:[])]+ return {ok:true,status:200,headers:new Map([['content-type','text/event-stream']]),body:{getReader:()=>({+ read:async()=>{+ if(chunks.length)return {done:false,value:chunks.shift()}+ throw new TypeError('network error')+ },releaseLock(){},+ })}}+}+const answer=text=>{+ const chunks=[encode({choices:[{delta:{content:text}}]}),encode('[DONE]')]+ return {ok:true,status:200,headers:new Map([['content-type','text/event-stream']]),body:{getReader:()=>({+ read:async()=>chunks.length?{done:false,value:chunks.shift()}:{done:true},releaseLock(){},+ })}}+}+const stored=()=>JSON.parse(localStorage.getItem('ods.pixel.chat.v1'))+const field=()=>screen.getByPlaceholderText('Message Portal...')+const row=()=>screen.queryByRole('status',{name:'Last message'})++let server,fetchMock+const calls=url=>fetchMock.mock.calls.filter(([called])=>called===url)+beforeEach(()=>{+ localStorage.clear()+ server={result:()=>response({state:'unknown',events:''}),activity:()=>response({state:'unknown'}),cancel:()=>response({aborted:true}),+ context:()=>response(context()),streams:[]}+ fetchMock=vi.fn(async(url,options)=>{+ if(url==='/api/pixel/status')return response({available:true})+ if(url==='/api/pixel/chat/context')return server.context(options)+ if(url==='/api/pixel/chat/result')return server.result(JSON.parse(options.body),options)+ if(url==='/api/pixel/chat/activity')return server.activity(JSON.parse(options.body))+ if(url==='/api/pixel/chat/cancel')return server.cancel(JSON.parse(options.body))+ if(url==='/api/pixel/chat/stream')return (server.streams.shift() || (()=>answer('Done')))(JSON.parse(options.body))+ return response({})+ })+ vi.stubGlobal('fetch',fetchMock)+})+afterEach(()=>{vi.useRealTimers();vi.unstubAllGlobals();vi.restoreAllMocks()})++function saveCutOffChat(extra={}) {+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:'checking-in',inFlight:false,interrupted:true,+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:PARTIAL}],...extra}))+}++it('settles a turn a restart cut off by itself, and never offers to repeat it in one click',async()=>{+ let attempt+ server.streams.push(body=>{attempt=body;return cutOffStream()})+ // The restarted server proved nothing still runs and settled the receipt:+ // the producer died with the old process before the answer was sent. The+ // gateway may well have finished the turn meanwhile, files and all.+ server.result=body=>{+ expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id})+ return response({state:'interrupted',events:TASK_ONLY})+ }+ render(<Pixel/>);await screen.findByText('Available')+ fireEvent.change(field(),{target:{value:QUESTION}})+ fireEvent.click(screen.getByTitle('Send'))+ const reply=await screen.findByText(RESTART_NOTICE)+ expect(RESTART_NOTICE).toBe('Portal restarted before its reply arrived. Your files are saved. Ask Portal what it did before you send your message again.')+ const bubble=reply.closest('[data-pixel-response]')+ expect(within(bubble).getByText('Response stopped')).toBeVisible()+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()+ expect(screen.queryByText(STALE)).toBeNull()+ expect(row()).toBeNull()+ // The composer is free, and nothing is put back to be sent again.+ expect(field()).toBeEnabled()+ expect(field()).toHaveValue('')+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,inFlight:false,requestId:null}))+ expect(stored().messages.at(-1)).toMatchObject({status:'stopped',content:RESTART_NOTICE})+ expect(stored().messages.at(-1).task).toBeUndefined()+ expect(stored().draft || '').toBe('')+ // The next message is the owner's own.+ server.streams.push(()=>answer('I created the scheduled task "cron-check".'))+ fireEvent.change(field(),{target:{value:'What did you do?'}})+ fireEvent.click(screen.getByTitle('Send'))+ expect(await screen.findByText('I created the scheduled task "cron-check".')).toBeVisible()+ expect(calls('/api/pixel/chat/stream')).toHaveLength(2)+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('settles a receipt that kept the answer as finished, with nothing to send again',async()=>{+ // The dashboard API was recreated after it saved the answer, before the+ // closing frames. The ingress had recorded the turn complete before sending.+ const ANSWER='Yes. Portal can run scheduled tasks: open Automations and add one.'+ let attempt+ server.streams.push(body=>{attempt=body;return cutOffStream(ANSWER)})+ server.result=body=>{+ expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id})+ return response({state:'interrupted',events:`${TASK_ONLY}data: ${JSON.stringify({choices:[{delta:{content:ANSWER}}]})}\n\n`})+ }+ render(<Pixel/>);await screen.findByText('Available')+ fireEvent.change(field(),{target:{value:QUESTION}})+ fireEvent.click(screen.getByTitle('Send'))+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,inFlight:false,requestId:null}))+ expect(stored().messages.at(-1)).toMatchObject({role:'assistant',status:'done',content:ANSWER})+ const bubble=screen.getByText(ANSWER).closest('[data-pixel-response]')+ expect(within(bubble).queryByText('Response stopped')).toBeNull()+ expect(within(bubble).queryByRole('button',{name:'Send again'})).toBeNull()+ expect(screen.queryByText(RESTART_NOTICE,{exact:false})).toBeNull()+ expect(screen.queryByText(STALE)).toBeNull()+ expect(row()).toBeNull()+ // Nothing is put back to be sent again.+ expect(field()).toHaveValue('')+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('frees the Spark’s window whose own request has no receipt once the server settles the chat',async()=>{+ // The window kept the refused "Checking in" request, which never got a+ // receipt, while an older dead attempt held the chat on the server.+ saveCutOffChat({draft:'checking in'})+ server.result=body=>{expect(body).toEqual({chat_id:'spark_chat',request_id:'checking-in'});return response({state:'unknown',events:''})}+ server.activity=()=>response({state:'terminal'})+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ expect(screen.getByText('Available')).toBeVisible()+ // The work may well be done, so the reply says so plainly and asks for+ // nothing to be repeated; the owner's own draft stays as it was.+ expect(screen.getByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE)).toBeVisible()+ expect(row()).toBeNull()+ expect(field()).toBeEnabled()+ expect(field()).toHaveValue('checking in')+ expect(screen.getByTitle('Send')).toBeEnabled()+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()+ expect(screen.queryByText(STALE)).toBeNull()+ // A proven state ends the checks: nothing is polled any more.+ const checks=calls('/api/pixel/chat/result').length+calls('/api/pixel/chat/activity').length+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))+ expect(calls('/api/pixel/chat/result').length+calls('/api/pixel/chat/activity').length).toBe(checks)+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+ expect(stored()).toMatchObject({interrupted:false,requestId:null})+})++it('puts a message that never reached Portal back in the composer, and says so',async()=>{+ // nginx answered 502 while the dashboard API was recreated: the request+ // never arrived, so no receipt exists and nothing came back.+ let attempt+ server.streams.push(body=>{attempt=body;return response({detail:'Bad Gateway'},502)})+ server.result=body=>{expect(body).toEqual({chat_id:attempt.chat_id,request_id:attempt.request_id});return response({state:'unknown',events:''})}+ server.activity=()=>response({state:'terminal'})+ render(<Pixel/>);await screen.findByText('Available')+ fireEvent.change(field(),{target:{value:QUESTION}})+ fireEvent.click(screen.getByTitle('Send'))+ const note=await screen.findByRole('status',{name:'Message not started'})+ expect(note).toHaveTextContent(NOT_RECEIVED_NOTICE)+ expect(screen.queryByText(NO_REPLY_NOTICE,{exact:false})).toBeNull()+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText(QUESTION)).toBeNull()+ expect(row()).toBeNull()+ expect(field()).toHaveValue(QUESTION)+ expect(screen.getByTitle('Send')).toBeEnabled()+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,requestId:null,draft:QUESTION}))+ expect(stored().messages.some(message=>message.content===QUESTION)).toBe(false)+ // One click sends it as a new turn.+ server.streams.push(()=>answer('Yes: use a scheduled task.'))+ fireEvent.click(within(note).getByRole('button',{name:'Send again'}))+ expect(await screen.findByText('Yes: use a scheduled task.')).toBeVisible()+ const resent=JSON.parse(calls('/api/pixel/chat/stream')[1][1].body)+ expect(resent.messages.filter(message=>message.role==='user')).toEqual([{role:'user',content:QUESTION}])+ expect(resent.request_id).not.toBe(attempt.request_id)+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('never settles a turn whose receipt could not be read, even when nothing runs',async()=>{+ saveCutOffChat()+ server.result=()=>response({detail:'Portal results are unavailable'},503)+ server.activity=()=>response({state:'terminal'})+ render(<Pixel/>);await screen.findByText('Available')+ const hold=await screen.findByRole('status',{name:'Last message'})+ await within(hold).findByText('Portal can’t confirm your last request has finished.')+ expect(screen.queryByText(NO_REPLY_NOTICE,{exact:false})).toBeNull()+ expect(screen.queryByRole('status',{name:'Message not started'})).toBeNull()+ // Once the receipt can be read, it decides what the reply says. Every+ // check before that said "unknown", which proves nothing about whether+ // the turn finished meanwhile, so nothing is put back to send again.+ server.result=()=>response({state:'interrupted',events:TASK_ONLY})+ await screen.findByText(RESTART_NOTICE,{},{timeout:4000})+ expect(screen.queryByRole('button',{name:'Send again'})).toBeNull()+ expect(field()).toHaveValue('')+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('backs off instead of polling every second while a turn cannot be proven',async()=>{+ saveCutOffChat()+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(3*60_000))+ // Main checked /result and /activity every 2 s: about 90 times each here.+ expect(calls('/api/pixel/chat/activity').length).toBeLessThanOrEqual(9)+ expect(calls('/api/pixel/chat/result').length).toBeLessThanOrEqual(9)+ expect(calls('/api/pixel/chat/context').length).toBeLessThanOrEqual(3)+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('keeps one working Continue in this chat while activity stays unknown, and frees the composer when the server confirms',async()=>{+ saveCutOffChat({draft:'checking in'})+ render(<Pixel/>);await screen.findByText('Available')+ const hold=await screen.findByRole('status',{name:'Last message'})+ await within(hold).findByText('Portal can’t confirm your last request has finished.')+ const proceed=within(hold).getByRole('button',{name:'Continue in this chat'})+ expect(proceed).toBeEnabled()+ expect(screen.queryByText(STALE)).toBeNull()+ // The draft stays editable, and sending waits for the owner's choice.+ expect(field()).toHaveValue('checking in')+ expect(screen.getByTitle('Send')).toBeDisabled()+ fireEvent.keyDown(field(),{key:'Enter'})+ expect(screen.getByText('Choose Continue in this chat first. Your message is kept here.')).toBeVisible()+ // The server's history lock is unusable at first (an empty lock file left+ // by a crash), so it cannot confirm: the control stays and links help.+ server.cancel=()=>response({detail:'Portal cancellation is unavailable'},503)+ fireEvent.click(proceed)+ await within(hold).findByText(CONTINUE_FAILED)+ expect(within(hold).getByRole('link',{name:/Get help on Discord/})).toBeVisible()+ expect(within(hold).getByRole('button',{name:'Continue in this chat'})).toBeEnabled()+ expect(screen.getByRole('button',{name:'New chat'})).not.toHaveAttribute('aria-disabled')+ // Once it can, its confirmation alone releases the hold, even though+ // the activity lookup still answers unknown. Continue stops anything left+ // over, so the reply says it was stopped, never that Portal finished.+ server.cancel=body=>{expect(body).toEqual({chat_id:'spark_chat'});return response({aborted:true})}+ fireEvent.click(within(hold).getByRole('button',{name:'Continue in this chat'}))+ await waitFor(()=>expect(row()).toBeNull())+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(2)+ expect(screen.getByText(STOPPED)).toBeVisible()+ expect(screen.queryByText(NO_REPLY_NOTICE)).toBeNull()+ expect(field()).toBeEnabled()+ expect(field()).toHaveValue('checking in')+ expect(screen.getByTitle('Send')).toBeEnabled()+ // The conversation is saved after it renders; wait for that save.+ await waitFor(()=>expect(stored()).toMatchObject({interrupted:false,requestId:null}))+ await waitFor(()=>expect(stored().messages.at(-1)).toMatchObject({status:'stopped',content:`${PARTIAL}\n\n---\n\n_${STOPPED}_`}))+ server.streams.push(()=>answer('I am here.'))+ fireEvent.click(screen.getByTitle('Send'))+ expect(await screen.findByText('I am here.')).toBeVisible()+ expect(calls('/api/pixel/chat/stream')).toHaveLength(1)+})++it('a Continue that the receipt answered first never settles a later turn unchecked',async()=>{+ saveCutOffChat()+ let finishCancel,cancelled=false+ server.cancel=()=>new Promise(resolve=>{finishCancel=resolve})+ server.result=body=>response(body.request_id==='checking-in'+ ?(cancelled?{state:'cancelled',events:TASK_ONLY}:{state:'unknown',events:''})+ :{state:'active',events:''})+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ await act(async()=>{fireEvent.click(within(row()).getByRole('button',CONTINUE))})+ // The server stops the old turn, and the next scheduled check reads that+ // receipt before the owner's Continue hears back.+ cancelled=true+ await act(async()=>vi.advanceTimersByTimeAsync(5000))+ expect(row()).toBeNull()+ await act(async()=>{finishCancel(response({aborted:true}))})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ // A later turn is cut off. It is checked against its own receipt, which+ // is still being written, and its answer can still arrive.+ server.streams.push(()=>cutOffStream())+ fireEvent.change(field(),{target:{value:'Next question'}})+ await act(async()=>{fireEvent.click(screen.getByTitle('Send'))})+ await act(async()=>vi.advanceTimersByTimeAsync(100))+ const next=JSON.parse(calls('/api/pixel/chat/stream')[0][1].body).request_id+ expect(calls('/api/pixel/chat/result').some(([,options])=>JSON.parse(options.body).request_id===next)).toBe(true)+ expect(within(row()).getByText('Portal is still working on your last request. Its answer will appear here when it’s done. You can stop it.')).toBeVisible()+ expect(stored().messages.at(-1).status).not.toBe('stopped')+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)+})++it('offers Continue in this chat only once a check cannot tell, never while that check runs',async()=>{+ saveCutOffChat()+ // The restarting API does not answer; each check gives up after 15 s.+ server.result=(_body,options)=>new Promise((_resolve,reject)=>options.signal.addEventListener('abort',()=>reject(new DOMException('aborted','AbortError'))))+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ // Continue stops anything left over, and this check may still find work running.+ expect(within(row()).getByText('Checking on your last request…')).toBeVisible()+ expect(within(row()).queryByRole('button',CONTINUE)).toBeNull()+ await act(async()=>vi.advanceTimersByTimeAsync(14_000))+ expect(within(row()).queryByRole('button',CONTINUE)).toBeNull()+ await act(async()=>vi.advanceTimersByTimeAsync(1_000))+ expect(within(row()).getByText('Portal can’t confirm your last request has finished.')).toBeVisible()+ expect(within(row()).getByRole('button',CONTINUE)).toBeEnabled()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('never offers Continue while the server finds this turn still running',async()=>{+ saveCutOffChat()+ let state='active'+ server.activity=()=>response({state})+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ // This turn's reply was cut off, so its answer cannot appear here; the line+ // does not promise that it will.+ expect(within(row()).getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()+ expect(screen.getByTitle('Stop')).toBeEnabled()+ for(let second=0;second<2*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()+ // It was checked at least every 15 s, and the end of that work frees the chat.+ expect(calls('/api/pixel/chat/activity').length).toBeGreaterThanOrEqual(8)+ state='terminal'+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(row()).toBeNull()+ expect(screen.getByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE)).toBeVisible()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('never asks to send again when the cut-off turn’s work went on, so nothing is done twice',async()=>{+ // The turn delegated work. A restart cut its reply off while that work+ // went on, and later ended. The settled reply says what it says for any+ // cut-off turn, claiming nothing about what ran.+ saveCutOffChat()+ let running=true+ server.result=()=>response(running?{state:'unresolved',events:''}:{state:'interrupted',events:TASK_ONLY})+ server.activity=()=>response({state:running?'active':'terminal'})+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ expect(within(row()).getByText('Portal is still working on your last request. When it’s done, ask Portal what it did. You can stop it.')).toBeVisible()+ running=false+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(row()).toBeNull()+ const reply=screen.getByText(RESTART_NOTICE)+ const bubble=reply.closest('[data-pixel-response]')+ expect(within(bubble).queryByRole('button',{name:'Send again'})).toBeNull()+ expect(screen.queryByText(/kept working on this/)).toBeNull()+ // The question is not put back to be sent again; the composer is free.+ expect(field()).toHaveValue('')+ expect(field()).toBeEnabled()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+})++it('offers Stop in place for work found running in the chat, and frees the chat once it is stopped',async()=>{+ let history='unknown'+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))+ server.context=()=>response({...context(history),status:history==='unknown'?'busy':'ready'})+ server.activity=()=>response({state:'active'})+ let aborted=false+ server.cancel=body=>{expect(body).toEqual({chat_id:'history_chat'});if(aborted)history='ready';return response({aborted})}+ render(<Pixel/>);await screen.findByText('Available')+ const hold=await screen.findByRole('status',{name:'Last message'})+ await within(hold).findByText(STILL_WORKING,{},{timeout:4000})+ expect(within(hold).queryByRole('button',CONTINUE)).toBeNull()+ // A Stop that is not confirmed says so, links help, and can be tried again.+ fireEvent.click(within(hold).getByRole('button',{name:'Stop'}))+ await within(hold).findByText('Portal could not confirm the stop yet. Try Stop again in a moment.')+ expect(within(hold).getByRole('link',{name:/Get help on Discord/})).toBeVisible()+ aborted=true+ fireEvent.click(within(hold).getByRole('button',{name:'Stop'}))+ await waitFor(()=>expect(row()).toBeNull())+ expect(screen.getByTitle('Send')).toBeEnabled()+ expect(screen.getByText('Earlier answer')).toBeVisible()+ expect(screen.queryByText(/Stopped by you/)).toBeNull()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(2)+})++it('says Portal is still working, with no Continue, when it finds the chat busy, and frees the chat when that work ends',async()=>{+ let history='unknown',state='active'+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))+ server.context=()=>response({...context(history),status:history==='unknown'?'busy':'ready'})+ server.activity=body=>{expect(body).toEqual({chat_id:'history_chat'});if(state==='terminal')history='ready';return response({state})}+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(1_100))+ expect(within(row()).getByText(STILL_WORKING)).toBeVisible()+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()+ expect(screen.getByTitle('Send')).toBeDisabled()+ fireEvent.keyDown(field(),{key:'Enter'})+ expect(screen.getByText('Portal is not ready for a new message yet. Your message is kept here; send it when Portal is ready.')).toBeVisible()+ // It stays that way while the work runs, checked at least every 15 s.+ for(let second=0;second<2*60;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(within(row()).getByText(STILL_WORKING)).toBeVisible()+ expect(screen.queryByRole('button',CONTINUE)).toBeNull()+ expect(calls('/api/pixel/chat/activity').length).toBeGreaterThanOrEqual(8)+ state='terminal'+ for(let second=0;second<15;second++)await act(async()=>vi.advanceTimersByTimeAsync(1000))+ expect(row()).toBeNull()+ expect(screen.getByTitle('Send')).toBeEnabled()+ expect(field()).toHaveValue('checking in')+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+ expect(screen.queryByText(STALE)).toBeNull()+})++it('does not blame the connection when an older turn holds the chat, and sends the new message once it is cleared up',async()=>{+ // The window believes the chat is free, but the server still holds an+ // older dead attempt and refuses the new one before starting it.+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))+ let settled=false+ server.streams.push(()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423))+ server.activity=()=>response({state:settled?'terminal':'unknown'})+ render(<Pixel/>);await screen.findByText('Available')+ fireEvent.change(field(),{target:{value:'Checking in'}})+ fireEvent.click(screen.getByTitle('Send'))+ const hold=await screen.findByRole('status',{name:'Last message'})+ await waitFor(()=>expect(hold).toHaveTextContent('Your new message will go out right after.'))+ expect(await within(hold).findByRole('button',{name:'Continue in this chat'})).toBeEnabled()+ expect(screen.queryByText(STALE)).toBeNull()+ // The refused message is not left in the conversation; it waits in the composer.+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText('Checking in')).toBeNull()+ expect(field()).toHaveValue('Checking in')+ // The server then proves the older attempt is over: the held message goes out.+ settled=true+ server.streams.push(()=>answer('Hi, I am here.'))+ expect(await screen.findByText('Hi, I am here.',{},{timeout:4000})).toBeVisible()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+ const sent=calls('/api/pixel/chat/stream').map(([,options])=>JSON.parse(options.body).messages.at(-1))+ expect(sent).toEqual([{role:'user',content:'Checking in'},{role:'user',content:'Checking in'}])+ expect(screen.getByText('Earlier answer')).toBeVisible()+ expect(screen.queryByText(STALE)).toBeNull()+})++it.each(['without a receipt','from its receipt'])('clears a Stop that could not be confirmed once the turn settles %s',async source=>{+ saveCutOffChat()+ let state='active'+ server.activity=()=>response({state})+ if(source==='from its receipt')server.result=()=>response(state==='active'?{state:'unresolved',events:''}:{state:'interrupted',events:TASK_ONLY})+ server.cancel=()=>response({aborted:false})+ render(<Pixel/>)+ fireEvent.click(await screen.findByTitle('Stop'))+ const failed='Portal could not confirm the stop yet. Try Stop again in a moment.'+ expect(await screen.findByText(failed)).toBeVisible()+ // The work then ends by itself, and the chat frees itself.+ state='terminal'+ if(source==='without a receipt')await screen.findByText((_,node)=>node?.tagName==='EM' && node.textContent===NO_REPLY_NOTICE,{},{timeout:4000})+ else await screen.findByText(RESTART_NOTICE,{},{timeout:4000})+ expect(row()).toBeNull()+ expect(screen.queryByText(failed)).toBeNull()+})++it('a Stop in a held chat never stamps the earlier, finished reply as stopped',async()=>{+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))+ // The send is refused because older work still holds the chat, and the+ // check finds that work running, so Stop is offered.+ server.streams.push(()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423))+ let running=true+ server.activity=()=>response({state:running?'active':'terminal'})+ server.cancel=body=>{expect(body).toEqual({chat_id:'spark_chat'});running=false;return response({aborted:true})}+ render(<Pixel/>);await screen.findByText('Available')+ fireEvent.change(field(),{target:{value:'Checking in'}})+ fireEvent.click(screen.getByTitle('Send'))+ fireEvent.click(await screen.findByTitle('Stop'))+ await waitFor(()=>expect(screen.queryByTitle('Stop')).toBeNull())+ expect(screen.queryByText(/Stopped by you/)).toBeNull()+ expect(screen.getByText('Earlier answer')).toBeVisible()+ await waitFor(()=>expect(stored().messages.find(message=>message.content.startsWith('Earlier answer'))).toMatchObject({content:'Earlier answer',status:'done'}))+})++it('sends a held message by itself once per owner action, never in a loop',async()=>{+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'spark_chat',requestId:null,inFlight:false,interrupted:false,+ messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}]}))+ const held=()=>response({detail:'This conversation has an unresolved attempt; recover or stop it first'},423)+ // The chat looks free each time it is checked, yet every send is refused.+ server.activity=()=>response({state:'terminal'})+ server.streams.push(held,held,held,held)+ vi.useFakeTimers()+ await act(async()=>{render(<Pixel/>)})+ await act(async()=>vi.advanceTimersByTimeAsync(10))+ fireEvent.change(field(),{target:{value:'Checking in'}})+ await act(async()=>{fireEvent.click(screen.getByTitle('Send'))})+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))+ // It went out by itself once more, was refused again, and now waits.+ expect(calls('/api/pixel/chat/stream')).toHaveLength(2)+ const note=screen.getByRole('status',{name:'Message not started'})+ expect(note).toHaveTextContent('Portal is busy with another request, so it did not start this message. Your message is kept; send it again in a moment.')+ expect(field()).toHaveValue('Checking in')+ expect(within(screen.getByRole('region',{name:'Conversation messages'})).queryByText('Checking in')).toBeNull()+ // Send again is the owner's next action, and it too is retried once at most.+ await act(async()=>{fireEvent.click(within(note).getByRole('button',{name:'Send again'}))})+ await act(async()=>vi.advanceTimersByTimeAsync(5*60_000))+ expect(calls('/api/pixel/chat/stream')).toHaveLength(4)+ expect(screen.getByRole('status',{name:'Message not started'})).toBeVisible()+ expect(field()).toHaveValue('Checking in')+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+ expect(screen.queryByText(STALE)).toBeNull()+})++it('heals an unconfirmed history by itself when the server proves the chat idle',async()=>{+ let unknown=true+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))+ server.context=()=>response(context(unknown?'unknown':'ready'))+ server.activity=body=>{expect(body).toEqual({chat_id:'history_chat'});unknown=false;return response({state:'terminal'})}+ render(<Pixel/>);await screen.findByText('Available')+ await screen.findByRole('status',{name:'Last message'})+ await waitFor(()=>expect(row()).toBeNull(),{timeout:4000})+ expect(screen.getByTitle('Send')).toBeEnabled()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(0)+ expect(screen.queryByText(STALE)).toBeNull()+})++it('asks once, in place, when an unconfirmed history cannot be proven idle',async()=>{+ let unknown=true+ localStorage.setItem('ods.pixel.chat.v1',JSON.stringify({schema:1,chatId:'history_chat',messages:[{role:'user',content:QUESTION},{role:'assistant',content:'Earlier answer',status:'done'}],draft:'checking in'}))+ server.context=()=>response(context(unknown?'unknown':'ready'))+ server.cancel=body=>{expect(body).toEqual({chat_id:'history_chat'});unknown=false;return response({aborted:true})}+ render(<Pixel/>);await screen.findByText('Available')+ const hold=await screen.findByRole('status',{name:'Last message'})+ const proceed=await within(hold).findByRole('button',{name:'Continue in this chat'})+ await within(hold).findByText('Portal can’t confirm your last request has finished.',{},{timeout:4000})+ expect(screen.getByTitle('Send')).toBeDisabled()+ fireEvent.click(proceed)+ await waitFor(()=>expect(row()).toBeNull())+ expect(screen.getByTitle('Send')).toBeEnabled()+ expect(calls('/api/pixel/chat/cancel')).toHaveLength(1)+})diff --git a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjsindex d15989879..62eec58db 100644--- a/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs+++ b/ods/extensions/services/pixel-agent/host/pixel_ingress.mjs@@ -60,6 +60,9 @@ const GATEWAY_ABORT_MAX_ATTEMPTS = 30;const DOCKER_TIMEOUT_MS = 10000;const MAX_TOKEN_LEN = 4096;const MAX_CANCEL_BODY = 256;+// Settle reads one native context under the history lock; keep it well inside+// the edge's relay deadline so a slow gateway answers "unconfirmed", not a hang.+const SETTLE_CONTEXT_TIMEOUT_MS = 5000;const MAX_STATUS_INTERVAL_MS = 86400000; // 1 dayconst STATUS_MODE = 0o640; // group-readable, service-owner-writable projectionconst ODS_VERSION_RE = /^(?:unknown|[0-9]+(?:\.[0-9]+){1,3}(?:[-+][A-Za-z0-9.-]+)?)$/;@@ -1984,6 +1987,24 @@ export function createIngressServer({ token, gatewayPort, deps = defaultDeps, hireturn;}+ if (pathname === "/v1/chat/settle") {+ if (req.method !== "POST") {+ sendError(res, 405, "method not allowed");+ return;+ }+ if (req.url !== pathname) {+ sendError(res, 400, "invalid settle request");+ return;+ }+ const contentType = String(req.headers["content-type"] || "").toLowerCase();+ if (contentType.split(";", 1)[0].trim() !== "application/json") {+ sendError(res, 415, "content type must be application/json");+ return;+ }+ void handleSettle(req, res, token, gatewayPort, deps, historyLedger);+ return;+ }+sendError(res, 404, "not found");});}@@ -2046,6 +2067,60 @@ async function handleCancel(req, res, token, gatewayPort, deps, ledger, historyA} catch {sendError(res,503,'cancellation-unconfirmed');}}+// A restart can cut a turn off and leave its history outcome unknown while+// nothing runs any more. Settle proves that without stopping anything: under+// the history lock no turn of this ingress is in flight, and an idle native+// session has no run of this chat. Only then is the unresolved turn recorded+// as interrupted. A held lock or a busy session answers "active" and changes+// nothing; Stop stays the only way to end work that may still be running.+async function handleSettle(req, res, token, gatewayPort, deps, ledger) {+ let parsed;+ try {+ parsed = JSON.parse((await readBody(req, MAX_CANCEL_BODY)).toString("utf8"));+ } catch (error) {+ sendError(res, error instanceof HttpError ? error.status : 400,+ error instanceof HttpError ? error.message : "invalid settle request");+ return;+ }+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed) || Object.keys(parsed).length !== 1+ || typeof parsed.user !== "string" || !/^[A-Za-z0-9_-]{1,128}$/.test(parsed.user)) {+ sendError(res, 400, "invalid settle request");+ return;+ }+ if (!ledger) {+ sendError(res, 503, "history-storage-unavailable");+ return;+ }+ const user = computeSessionUser({ user: parsed.user });+ let release;+ try {+ try {+ release = ledger.lock(user);+ } catch (error) {+ if (error instanceof HistoryError && error.status === 409) {+ sendJson(res, 200, { state: "active" });+ return;+ }+ throw error;+ }+ const native = await nativeContextRequest("context", { user }, token, gatewayPort, deps, undefined, SETTLE_CONTEXT_TIMEOUT_MS);+ if (native.status === "busy") {+ sendJson(res, 200, { state: "active" });+ return;+ }+ if (!["ready", "missing"].includes(native.status)) throw new HistoryError("context-unavailable", 503);+ const current = ledger.read(user);+ if (current && ["pending", "unknown"].includes(current.status) && !ledger.interrupt(user, current.requestId)) {+ throw new HistoryError("history-settle-unconfirmed", 503);+ }+ sendJson(res, 200, { state: "settled" });+ } catch {+ sendError(res, 503, "settle-unconfirmed");+ } finally {+ try { release?.(); } catch { /* The next lock() reclaims a lock it could not remove, once removal works. */ }+ }+}+async function handleImageDelete(req,res,{imageStore,historyLedger,imageRoutes,token,gatewayPort,deps}) {let release;try {diff --git a/ods/extensions/services/pixel-agent/plugin/index.js b/ods/extensions/services/pixel-agent/plugin/index.jsindex b74c44ec6..81e46eafe 100644--- a/ods/extensions/services/pixel-agent/plugin/index.js+++ b/ods/extensions/services/pixel-agent/plugin/index.js@@ -694,8 +694,14 @@ export default definePluginEntry({handler:async (req,res) => {const parsed = await readContextRequest(req, operation === 'compact');if (parsed.status !== 200) {sendJson(res,parsed.status,{error:'invalid context request'});return true;}- const result = operation === 'compact' ? await contextCompaction.compact(parsed.user,parsed.requestId)+ let result = operation === 'compact' ? await contextCompaction.compact(parsed.user,parsed.requestId): contextCompaction.context(parsed.user);+ // The owner session can be idle while children it delegated to still+ // run or will report back into this chat. Then the chat is busy, so+ // restart recovery and a new turn wait for that work instead of+ // treating the turn as over.+ if (operation === 'context' && ['ready','missing'].includes(result?.status)+ && delegationDelivery.pending(parsed.user,key => Boolean(resolveActiveEmbeddedRunSessionId(key)))) result = {...result,status:'busy'};sendJson(res,200,result);return true;}});}diff --git a/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs b/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjsindex 297a0d91d..68e3f6df9 100644--- a/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs+++ b/ods/extensions/services/pixel-agent/plugin/subagent-delivery.mjs@@ -266,6 +266,24 @@ export function createSubagentDelivery({agentId = 'pixel', now = Date.now,const chain=roots.get(runId);return chain && ownerMatches(chain,user) && valid(chain) ? chain.ready?.runId : undefined;}+ // Whether this user's chat may still have delegated work while its owner+ // session is idle: a child that one of its turns started still runs, or a+ // registered child will still report back into the chat. Only an explicit+ // false from isActive proves a child idle. A delegation that can no longer+ // report back (Stop, access change, expiry) is over once its children have+ // ended; valid() records that, so a later announcement is refused as well.+ function pending(user, isActive) {+ if (!USER.test(user ?? '')) return false;+ const running = key => { try { return isActive(key) !== false; } catch { return true; } };+ for (const chain of roots.values()) {+ if (chain.sessionKey !== prefix + user) continue;+ const started = [...spawned.entries()].filter(([,record]) => record.chainId === chain.id || record.cancelledChainIds?.includes(chain.id));+ if ([...chain.children.keys(), ...started.map(([key]) => key)].some(running)) return true;+ if (!chain.delivered && !chain.ready && valid(chain)+ && (pendingSpawn(chain) || [...chain.children.values()].some(child => !child.announced))) return true;+ }+ return false;+ }function hasPendingChildren(context) {const run = owned(context);// Advice must have the same live owner custody as delivery. A prompt,@@ -369,6 +387,6 @@ export function createSubagentDelivery({agentId = 'pixel', now = Date.now,return {block:true,blockReason:'No registered child completion event is pending. Review the current event and the earlier child results supplied in this run, then consolidate the owner response. This yield was not executed.'};}}- return {observe,before,after,nativeSpawn,finalize,end,read,finalRun,cancel,blocked,promptContext,ownerRunFor,ownerRunOf,admission,hasPendingChildren,+ return {observe,before,after,nativeSpawn,finalize,end,read,finalRun,cancel,pending,blocked,promptContext,ownerRunFor,ownerRunOf,admission,hasPendingChildren,invalidate:() => {for (const chain of roots.values()) fail(chain);}};}diff --git a/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjs b/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjsnew file mode 100644index 000000000..1c22d9f12--- /dev/null+++ b/ods/extensions/services/pixel-agent/tests/ingress_settle.test.mjs@@ -0,0 +1,112 @@+// A restart that cuts a turn off leaves its history outcome unknown. Settle+// must prove nothing still runs before recording that turn as interrupted, and+// must never stop work itself: Stop stays the only way to end running work.+import test from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import os from 'node:os';+import path from 'node:path';+import http from 'node:http';+import {createIngressServer,computeSessionUser} from '../host/pixel_ingress.mjs';+import {createChatHistoryLedger} from '../host/chat_history_ledger.mjs';++const u=content=>({role:'user',content});+const rawUser='settle-canary', user=computeSessionUser({user:rawUser});+const runId='chatcmpl_11111111-2222-4333-8444-555555555555';+const nativeState=()=>({schemaVersion:1,status:'ready',sessionExists:true,sessionRevision:'a'.repeat(64),context:{used:100,window:32000,measuredAt:1},model:{id:'test',provider:'local',contextWindow:32000},compaction:{status:'idle',requestId:null,tokensBefore:null,tokensAfter:null,reason:null,count:0}});+const listen=server=>new Promise(resolve=>server.listen(0,'127.0.0.1',()=>resolve(server.address().port)));++async function fixture(t) {+ const dir=fs.mkdtempSync(path.join(os.tmpdir(),'ods-ingress-settle-'));fs.chmodSync(dir,0o700);+ // The turn that a restart cut off: prepared (pending) by the previous ingress process.+ const before=createChatHistoryLedger(dir);+ before.prepare(user,'cut-off',{schemaVersion:1,messages:[u('Can you do things on cron timers?')]},nativeState());+ // The restarted ingress has no record of that turn in flight.+ const ledger=createChatHistoryLedger(dir),calls=[],native=nativeState();+ const gateway=http.createServer(async(req,res)=>{+ if(req.url==='/health') {res.setHeader('content-type','application/json');return res.end('{"ok":true}');}+ let raw='';for await(const part of req) raw+=part;+ const body=JSON.parse(raw||'{}');calls.push({path:req.url,body});res.setHeader('content-type','application/json');+ if(req.url==='/pixel-ods/context') return res.end(JSON.stringify(native));+ if(req.url==='/pixel-ods/abort') return res.end(JSON.stringify({aborted:true}));+ if(req.url==='/pixel-ods/verification') return res.end(JSON.stringify({status:'none'}));+ if(req.url==='/pixel-ods/subagent-delivery') return res.end(JSON.stringify({schemaVersion:1,kind:'ods-subagent-delivery',runId:body.runId,status:'not-delegated'}));+ if(req.url==='/v1/chat/completions') return res.end(JSON.stringify({id:runId,choices:[{finish_reason:'stop',message:{role:'assistant',content:'Yes, I can schedule work.'}}]}));+ res.statusCode=404;res.end('{}');+ });+ const port=await listen(gateway),ingress=createIngressServer({token:'test-token',gatewayPort:port,historyLedger:ledger});+ const ingressPort=await listen(ingress);+ t.after(async()=>{await Promise.all([new Promise(r=>ingress.close(r)),new Promise(r=>gateway.close(r))]);fs.rmSync(dir,{recursive:true,force:true})});+ async function request(route,body,{method='POST',contentType='application/json'}={}) {+ const response=await fetch(`http://127.0.0.1:${ingressPort}${route}`,{method,headers:{'content-type':contentType},...(method==='POST'?{body:typeof body==='string'?body:JSON.stringify(body)}:{})});+ return {status:response.status,value:await response.json()};+ }+ const settle=(body={user:rawUser})=>request('/v1/chat/settle',body);+ const chat=(request_id,messages)=>request('/v1/chat/completions',{user:rawUser,request_id,history_snapshot:{schemaVersion:1,messages},messages:[{role:'system',content:'Trusted identity'},u(messages.at(-1).content)],stream:false});+ const stops=()=>calls.filter(call=>call.path==='/pixel-ods/abort' || call.path==='/v1/chat/completions');+ return {dir,ledger,calls,native,request,settle,chat,stops};+}++test('a turn cut off by a restart settles without stopping anything, and the next turn runs',async t=>{+ const f=await fixture(t);+ assert.equal(f.ledger.projection(user).status,'unknown');+ const refused=await f.chat('checking-in',[u('Can you do things on cron timers?'),u('Checking in')]);+ assert.equal(refused.status,409,'an unresolved turn still refuses a silent resubmission');+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});+ assert.deepEqual(f.stops(),[],'settle never aborts a run or starts one');+ const settled=f.ledger.read(user);+ assert.equal(settled.status,'ready');+ assert.equal(settled.reason,'history-interrupted');+ assert.equal(settled.requests.at(-1).status,'interrupted');+ // Settling twice is harmless: there is nothing left to record.+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});+ const next=await f.chat('checking-in-again',[u('Can you do things on cron timers?'),u('Checking in')]);+ assert.equal(next.status,200);+ assert.deepEqual(f.calls.filter(call=>call.path==='/v1/chat/completions').at(-1).body.messages,[{role:'system',content:'Trusted identity'},u('Checking in')]);+});++test('a busy native session answers active and leaves the unresolved turn alone',async t=>{+ const f=await fixture(t);+ f.native.status='busy';+ assert.deepEqual(await f.settle(),{status:200,value:{state:'active'}});+ assert.equal(f.ledger.projection(user).status,'unknown');+ assert.deepEqual(f.stops(),[]);+});++test('a turn holding the history lock answers active without reading or changing history',async t=>{+ const f=await fixture(t);+ const release=f.ledger.lock(user);+ try {assert.deepEqual(await f.settle(),{status:200,value:{state:'active'}});}+ finally {release();}+ assert.equal(f.calls.filter(call=>call.path==='/pixel-ods/context').length,0);+ assert.equal(f.ledger.projection(user).status,'unknown');+});++test('an unusable history lock answers unconfirmed, never settled and never active forever',async t=>{+ const f=await fixture(t);+ // Something that is not a lock file sits where the lock belongs, as after a+ // crash. Settle cannot take the lock, so it must not claim anything.+ const lockPath=path.join(f.dir,`${user}.lock`);+ fs.mkdirSync(lockPath);+ const answer=await f.settle();+ assert.equal(answer.status,503);+ assert.equal(answer.value.error.message,'settle-unconfirmed');+ assert.equal(f.ledger.projection(user).status,'unknown');+ assert.deepEqual(f.stops(),[]);+ fs.rmdirSync(lockPath);+ assert.deepEqual(await f.settle(),{status:200,value:{state:'settled'}});+});++test('settle fails closed when idle cannot be proven or the request is not exact',async t=>{+ const f=await fixture(t);+ f.native.status='unavailable';+ assert.equal((await f.settle()).status,503);+ assert.equal(f.ledger.projection(user).status,'unknown');+ for(const body of [{},{user:rawUser,request_id:'cut-off'},{user:'../escape'},{user:7},'not json'])+ assert.equal((await f.settle(body)).status,400,JSON.stringify(body));+ assert.equal((await f.request('/v1/chat/settle?user=other',{user:rawUser})).status,400);+ assert.equal((await f.request('/v1/chat/settle',{user:rawUser},{contentType:'text/plain'})).status,415);+ assert.equal((await f.request('/v1/chat/settle',undefined,{method:'GET'})).status,405);+ assert.equal(f.ledger.projection(user).status,'unknown');+ assert.deepEqual(f.stops(),[]);+});diff --git a/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjs b/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjsnew file mode 100644index 000000000..756fdd4fe--- /dev/null+++ b/ods/extensions/services/pixel-agent/tests/subagent_delivery_pending.test.mjs@@ -0,0 +1,156 @@+// Restart recovery (the ingress's /v1/chat/settle) and a new owner turn treat+// an idle owner session as proof that no run of the chat is left. A turn that+// delegated can leave the owner session idle while its children still work or+// will still report back into the chat. These tests pin when the delegation+// registry keeps such a chat busy, and that the plugin's context report says so.+import test from 'node:test';+import assert from 'node:assert/strict';+import fs from 'node:fs';+import vm from 'node:vm';+import {Readable} from 'node:stream';+import {createSubagentDelivery} from '../plugin/subagent-delivery.mjs';+import {readContextRequest} from '../plugin/context-compaction.mjs';++const user='ods-'+'a'.repeat(64), other='ods-'+'b'.repeat(64);+const id='chatcmpl_11111111-2222-4333-8444-555555555555';+const child='agent:pixel:subagent:22222222-2222-4333-8444-555555555555';+const childRun='33333333-2222-4333-8444-555555555555';+const owner={agentId:'pixel',runId:id,sessionId:'owner-session',sessionKey:'agent:pixel:openai-user:'+user,trigger:'user'};+const continuation={...owner,runId:`announce:v1:${child}:${childRun}`,+ inputProvenance:{kind:'inter_session',sourceTool:'subagent_announce',sourceSessionKey:child}};++function fixture() {+ let clock=1, access='current';+ // Native sessions with an active embedded run, as resolveActiveEmbeddedRunSessionId reports them.+ const running=new Set();+ const registry=createSubagentDelivery({now:()=>clock,accessIdentity:()=>access,+ finalText:message=>message.content.filter(block=>block.type==='text').map(block=>block.text).join('\n'),+ resolveOwnerSession:key=>key===owner.sessionKey?{sessionId:owner.sessionId}:null,+ abortSession:async key=>!running.has(key)});+ registry.observe({},owner);+ function spawn({key=child,runId=childRun,accepted=true}={}) {+ const context={...owner,toolName:'sessions_spawn',toolCallId:'spawn-'+key};+ registry.before({params:{runtime:'subagent',mode:'run'}},context);+ registry.nativeSpawn({runId,childSessionKey:key},{runId,childSessionKey:key,requesterSessionKey:owner.sessionKey});+ running.add(key);+ if (accepted) registry.after({result:{details:{status:'accepted',runId,childSessionKey:key}}},context);+ }+ function yieldTurn() {+ const context={...owner,toolName:'sessions_yield',toolCallId:'yield'};+ registry.before({params:{}},context);+ registry.after({result:{details:{status:'yielded'}}},context);+ registry.end({success:true},owner);+ }+ function final(text='Both reviews are done.') {+ registry.finalize({lastAssistantMessage:text},continuation);+ registry.end({success:true,messages:[{role:'assistant',stopReason:'stop',content:[{type:'text',text}]}]},continuation);+ }+ return {registry,running,spawn,yieldTurn,final,+ pending:(who=user,isActive=key=>running.has(key))=>registry.pending(who,isActive),+ tick:()=>{clock+=33*60*1000;},revoke:()=>{access='downgraded';}};+}++test('a delegated turn stays pending while its child runs and until the child reports back', () => {+ const f=fixture();+ f.spawn();f.yieldTurn();+ assert.equal(f.pending(),true,'the child still runs while the owner session waits');+ f.running.delete(child);+ assert.equal(f.pending(),true,'the child ended but its result has not reached the chat yet');+ f.registry.observe({prompt:'Review result: nothing to fix.'},continuation);+ f.final();+ assert.equal(f.pending(),false,'the consolidated answer is ready and nothing runs');+ assert.equal(f.registry.read(user,id).status,'ready');+ assert.equal(f.pending(),false,'a delivered answer is settled');+});++test('a native child that started before its spawn reply arrived keeps the turn pending', () => {+ const f=fixture();+ f.spawn({accepted:false});+ assert.equal(f.pending(),true,'the spawn call is still open and its child already runs');+ f.running.delete(child);+ assert.equal(f.pending(),true,'the spawn call has not answered yet');+});++test('Stop ends the wait; only a child that still runs keeps the chat busy', async () => {+ const f=fixture();+ f.spawn();f.yieldTurn();+ assert.deepEqual(await f.registry.cancel(user),{tracked:true,aborted:false},'the child could not be stopped yet');+ assert.equal(f.pending(),true,'never treat a child that still runs as finished');+ f.running.delete(child);+ assert.equal(f.pending(),false,'a stopped delegation can no longer report back');+});++test('a delegation that can no longer report back is over once its child ends', () => {+ for (const end of ['access change','expiry']) {+ const f=fixture();+ f.spawn();f.yieldTurn();+ if (end==='access change') f.revoke(); else f.tick();+ assert.equal(f.pending(),true,`${end}: the child still runs`);+ f.running.delete(child);+ assert.equal(f.pending(),false,end);+ assert.equal(f.registry.admission(continuation)?.outcome,'block',`${end}: its late result cannot start a turn either`);+ }+});++test('ordinary turns, other chats and malformed users are never pending', () => {+ const f=fixture();+ assert.equal(f.pending(),false,'an owner turn that never delegated');+ f.spawn();f.yieldTurn();+ assert.equal(f.pending(other),false,'another chat');+ for (const malformed of [undefined,'',owner.sessionKey,'ods-'+'A'.repeat(64)]) assert.equal(f.registry.pending(malformed,()=>true),false,String(malformed));+});++test('only an explicit idle answer proves a child has ended', () => {+ const f=fixture();+ f.spawn();f.yieldTurn();f.revoke();+ assert.equal(f.pending(user,()=>{throw new Error('registry unavailable');}),true);+ assert.equal(f.pending(user,()=>undefined),true);+ assert.equal(f.pending(user,()=>false),false);+});++// Values built inside the vm context have that context's prototypes.+const plain=value=>JSON.parse(JSON.stringify(value));++function contextRoute(registry,{active=new Set(),report}) {+ const source=fs.readFileSync(new URL('../plugin/index.js',import.meta.url),'utf8');+ const start=source.indexOf(" for (const operation of ['context','compact']) {");+ const end=source.indexOf(' // The OpenAI-compatible gateway route does not dispatch',start);+ assert.ok(start>=0 && end>start);+ const routes={}, compacted=[];+ vm.runInNewContext(source.slice(start,end),{api:{registerHttpRoute:value=>{routes[value.path]=value;}},+ readContextRequest,delegationDelivery:registry,resolveActiveEmbeddedRunSessionId:key=>active.has(key)?'native-session':undefined,+ contextCompaction:{context:()=>({...report()}),compact:async (who,requestId)=>{compacted.push([who,requestId]);return {...report()};}},+ sendJson:(res,status,value)=>{res.status=status;res.value=value;}});+ const call=async (path,body)=>{+ const req=Readable.from([Buffer.from(JSON.stringify(body))]);+ req.method='POST';req.headers={'content-type':'application/json'};const res={};+ await routes[path].handler(req,res);return res;+ };+ return {context:who=>call('/pixel-ods/context',{user:who}),compact:(who,requestId)=>call('/pixel-ods/compact',{user:who,request_id:requestId}),compacted};+}++test('the plugin reports an idle owner session busy while its delegation is pending', async () => {+ const f=fixture();+ let status='ready';+ const report=()=>({schemaVersion:1,status,sessionExists:status!=='missing',sessionRevision:status==='missing'?null:'r1',context:null,model:null,+ compaction:{status:'idle',count:0}});+ const route=contextRoute(f.registry,{active:f.running,report});+ assert.deepEqual(plain((await route.context(user)).value),report(),'nothing delegated: the report is unchanged');+ f.spawn();f.yieldTurn();+ const busy=await route.context(user);+ assert.equal(busy.status,200);+ assert.deepEqual(plain(busy.value),{...report(),status:'busy'},'only the status changes');+ status='missing';+ assert.equal((await route.context(user)).value.status,'busy');+ status='unavailable';+ assert.equal((await route.context(user)).value.status,'unavailable','an unavailable report stays unavailable');+ status='ready';+ assert.equal((await route.context(other)).value.status,'ready','another chat is not affected');+ assert.equal((await route.compact(user,'manual-1')).value.status,'ready','a compaction receipt is passed through as it is');+ assert.deepEqual(route.compacted,[[user,'manual-1']]);+ f.running.delete(child);+ assert.equal((await route.context(user)).value.status,'busy','the child result has not reached the chat yet');+ f.registry.observe({prompt:'Review result: nothing to fix.'},continuation);+ f.final();+ assert.equal((await route.context(user)).value.status,'ready');+});diff --git a/ods/extensions/services/pixel-edge/pixel_edge.py b/ods/extensions/services/pixel-edge/pixel_edge.pyindex 713150de5..4fc237a37 100644--- a/ods/extensions/services/pixel-edge/pixel_edge.py+++ b/ods/extensions/services/pixel-edge/pixel_edge.py@@ -997,6 +997,58 @@ async def handle_chat_activity(request: web.Request):return web.json_response({"state": state}, headers={"Cache-Control": "no-store"})+async def handle_chat_settle(request: web.Request):+ """Ask the ingress to settle a chat it proves idle; never stop a run.++ A restart can leave a turn's outcome unknown while nothing runs any more.+ The ingress proves that under its history lock and records the turn as+ interrupted. A stream still passing through this edge, a held lock or a+ busy native session answers "active" and nothing changes.+ """+ fail = _check_auth(request)+ if fail is not None:+ return fail+ if request.content_type != "application/json":+ return web.json_response({"error": "Content-Type must be application/json"}, status=415)+ if request.query_string or (request.content_length and request.content_length > _MAX_CANCEL_BODY):+ return web.json_response({"error": "invalid settle request"}, status=400)+ try:+ raw = await request.read()+ if len(raw) > _MAX_CANCEL_BODY:+ raise ValueError("request too large")+ data = strict_json(raw)+ if (not isinstance(data, dict) or set(data) != {"user"}+ or not isinstance(data["user"], str) or not _SAFE_CHAT_ID.fullmatch(data["user"])):+ raise ValueError("invalid settle request")+ except (ValueError, UnicodeDecodeError, TypeError):+ return web.json_response({"error": "invalid settle request"}, status=400)+ chat_id = data["user"]+ if request.app[_CANCEL_EVENTS_KEY].get(chat_id):+ return web.json_response({"state": "active"}, headers={"Cache-Control": "no-store"})+ try:+ connector = UnixConnector(path=_SOCKET_PATH)+ # The ingress bounds its own native read at 5 s under the history lock.+ timeout = ClientTimeout(total=8, sock_connect=2, sock_read=7)+ async with ClientSession(connector=connector, timeout=timeout) as session:+ async with session.post(+ "http://pixel-upstream/v1/chat/settle",+ json={"user": chat_id},+ headers={"Content-Type": "application/json", "Accept": "application/json"},+ ) as resp:+ if resp.status != 200 or "application/json" not in resp.headers.get("Content-Type", "").lower():+ raise ValueError("settle unconfirmed")+ result = strict_json(await _read_bounded(resp.content, _MAX_CANCEL_RESPONSE_BYTES))+ if (not isinstance(result, dict) or set(result) != {"state"}+ or result["state"] not in ("settled", "active")):+ raise ValueError("invalid settle receipt")+ except (ConnectionError, OSError, asyncio.TimeoutError, ClientError, ValueError, TypeError,+ UnicodeError, RecursionError):+ return web.json_response({"error": "Portal settle unconfirmed"}, status=502)+ if result["state"] == "settled" and not request.app[_CANCEL_EVENTS_KEY].get(chat_id):+ _remember_chat_activity(request.app, chat_id, "terminal")+ return web.json_response({"state": result["state"]}, headers={"Cache-Control": "no-store"})++async def _context_upstream(data, *, compact=False):connector = UnixConnector(path=_SOCKET_PATH)timeout = ClientTimeout(total=18, sock_connect=2, sock_read=16)@@ -1970,6 +2022,7 @@ def create_app() -> web.Application:app.router.add_post("/v1/chat/completions", handle_chat_completions)app.router.add_post("/v1/chat/cancel", handle_chat_cancel)app.router.add_post("/v1/chat/activity", handle_chat_activity)+ app.router.add_post("/v1/chat/settle", handle_chat_settle)app.router.add_post("/v1/chat/context", handle_chat_context)app.router.add_post("/v1/chat/images-delete", handle_chat_images_delete)app.router.add_post("/v1/chat/compact", handle_chat_context)diff --git a/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py b/ods/extensions/services/pixel-edge/tests/test_pixel_edge.pyindex e9c4bd466..4a06f83af 100644--- a/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py+++ b/ods/extensions/services/pixel-edge/tests/test_pixel_edge.py@@ -183,6 +183,19 @@ async def _upstream_health(_request):return web.json_response({"status": "ok"})+async def _upstream_settle(request):+ data = await request.json()+ request.app["settle_users"].append(data.get("user"))+ native = request.app["native_runs"].get(data.get("user"))+ if native is not None and not native[1].done():+ # The ingress reads its native session: a detached run is still busy.+ return web.json_response({"state": "active"})+ reply = request.app["settle_reply"]+ if reply["status"] != 200:+ return web.json_response({"error": "not found"}, status=reply["status"])+ return web.json_response(reply["receipt"])++async def _upstream_cancel(request):data = await request.json()request.app["cancel_users"].append(data.get("user"))@@ -244,6 +257,8 @@ async def _start_upstream():app = web.Application(client_max_size=8 * 1024 * 1024 + 1)app["chat_requests"] = []app["cancel_users"] = []+ app["settle_users"] = []+ app["settle_reply"] = {"status": 200, "receipt": {"state": "settled"}}app["native_runs"] = {}app["stream_started"] = asyncio.Event()app["release_stream"] = asyncio.Event()@@ -252,6 +267,7 @@ async def _start_upstream():app["preview_paths"] = []app.router.add_post("/v1/chat/completions", _upstream_chat)app.router.add_post("/v1/chat/cancel", _upstream_cancel)+ app.router.add_post("/v1/chat/settle", _upstream_settle)app.router.add_get("/v1/models", _upstream_models)app.router.add_get("/health", _upstream_health)app.router.add_get("/{site_id}/{tail:.*}", _upstream_preview)@@ -2188,6 +2204,78 @@ class TestChatActivity(BaseEdgeTest):self.assertEqual(await self.activity(user), {"state": "terminal"})+class TestChatSettle(BaseEdgeTest):+ """A restart leaves a chat unknown here; settling it never stops a run."""++ async def activity(self, user):+ async with self.client.post("http://localhost/v1/chat/activity", headers=self.auth(), json={"user": user}) as resp:+ return await resp.json()++ async def settle(self, user):+ async with self.client.post("http://localhost/v1/chat/settle", headers=self.auth(), json={"user": user}) as resp:+ return resp.status, resp.headers.get("Cache-Control"), await resp.json()++ async def test_restart_unknown_chat_becomes_terminal_only_after_ingress_proves_idle(self):+ # A restarted edge has no record of the turn the restart cut off.+ self.assertEqual(await self.activity("restarted-chat"), {"state": "unknown"})+ self.assertEqual(await self.settle("restarted-chat"), (200, "no-store", {"state": "settled"}))+ self.assertEqual(self.up_runner.app["settle_users"], ["restarted-chat"])+ self.assertEqual(await self.activity("restarted-chat"), {"state": "terminal"})+ self.assertEqual(await self.activity("other-chat"), {"state": "unknown"})+ self.assertEqual(self.up_runner.app["cancel_users"], [])+ self.assertEqual(self.up_runner.app["chat_requests"], [])++ async def test_stream_in_flight_is_active_without_asking_the_ingress(self):+ async with self.client.post("http://localhost/v1/chat/completions", headers=self.auth(), json={+ "model": "pixel/default", "messages": [], "stream": True,+ "user": "live-chat", "trigger_cancel_wait": True,+ }) as stream:+ await asyncio.wait_for(self.up_runner.app["stream_started"].wait(), 1)+ self.assertEqual(await self.settle("live-chat"), (200, "no-store", {"state": "active"}))+ self.assertEqual(self.up_runner.app["settle_users"], [])+ self.up_runner.app["release_stream"].set()+ await stream.text()+ self.assertEqual(self.up_runner.app["cancel_users"], [])++ async def test_detached_native_run_stays_active_and_is_never_stopped(self):+ user = "detached-settle"+ stream = await self.client.post("http://localhost/v1/chat/completions", headers=self.auth(), json={+ "model": "pixel/default", "messages": [], "stream": True,+ "user": user, "trigger_detached_native": True,+ })+ await asyncio.wait_for(stream.content.readany(), 2)+ stream.close()+ async with async_timeout(2):+ while self.edge_app[self.pe._CANCEL_EVENTS_KEY].get(user):+ await asyncio.sleep(0.02)+ self.assertEqual(await self.settle(user), (200, "no-store", {"state": "active"}))+ self.assertFalse(self.up_runner.app["native_runs"][user][1].done())+ self.assertEqual(await self.activity(user), {"state": "unknown"})+ self.assertEqual(self.up_runner.app["cancel_users"], [])++ async def test_unconfirmed_or_older_ingress_answers_leave_the_chat_unknown(self):+ for status, receipt in ((200, {"state": "settled", "extra": True}), (200, {"state": "terminal"}),+ (200, {"aborted": True}), (404, None), (503, None)):+ self.up_runner.app["settle_reply"].update(status=status, receipt=receipt)+ code, _cache, body = await self.settle("uncertain-chat")+ self.assertEqual(code, 502)+ self.assertEqual(body, {"error": "Portal settle unconfirmed"})+ self.assertEqual(await self.activity("uncertain-chat"), {"state": "unknown"})++ async def test_settle_requires_exact_authenticated_bounded_request(self):+ async with self.client.post("http://localhost/v1/chat/settle", json={"user": "safe"}) as resp:+ self.assertEqual(resp.status, 401)+ for raw in ('{}', '{"user":"safe","other":"secret"}', '{"user":"../escape"}',+ '{"user":true}', '{"user":"safe","user":"other"}', '{"user":"' + 'x' * 2048 + '"}'):+ async with self.client.post("http://localhost/v1/chat/settle", headers={**self.auth(), "Content-Type": "application/json"}, data=raw) as resp:+ self.assertEqual(resp.status, 400)+ async with self.client.post("http://localhost/v1/chat/settle?user=other", headers=self.auth(), json={"user": "safe"}) as resp:+ self.assertEqual(resp.status, 400)+ async with self.client.post("http://localhost/v1/chat/settle", headers={**self.auth(), "Content-Type": "text/plain"}, data='{"user":"safe"}') as resp:+ self.assertEqual(resp.status, 415)+ self.assertEqual(self.up_runner.app["settle_users"], [])++if __name__ == "__main__":unittest.main()