Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(agent-chat): preserve completion across hosted CLI recovery

rubenluetke10-beep9161cb44cacf
diff --git a/jarvis/agent_chat/runner_cli.py b/jarvis/agent_chat/runner_cli.py
index 3bfe5de22c..f04850851c 100644
--- a/jarvis/agent_chat/runner_cli.py
+++ b/jarvis/agent_chat/runner_cli.py
@@ -3671,6 +3671,8 @@ async def _drive_cli(
assert proc.stdout is not None
while True:
raw = await proc.stdout.readline()
+ if hosted and getattr(proc, "handed_over", False):
+ return
if not raw:
return
# A line this turn handled before an app restart: rebuild state only.
@@ -3915,12 +3917,21 @@ async def _drive_cli(
feeder = asyncio.create_task(_feed_stdin())
watcher = asyncio.create_task(_watch_cancel())
try:
- if getattr(handle, "goal_turn", False):
+ # EOF is not process exit: a CLI may close both pipes and keep
+ # running. The deadline must also cover waiting for the process.
+ deadline = None if getattr(handle, "goal_turn", False) else timeout_s
+ async with asyncio.timeout(deadline):
await asyncio.gather(pump, drain, feeder)
- else:
- await asyncio.wait_for(asyncio.gather(pump, drain, feeder), timeout=timeout_s)
- await proc.wait()
+ await proc.wait()
except TimeoutError:
+ log.warning(
+ "agent chat %s: %s timed out (result=%s, returncode=%s, stdout_done=%s)",
+ handle.turn_id,
+ runner,
+ getattr(state, "saw_result", False),
+ proc.returncode,
+ pump.done(),
+ )
_kill(proc)
status = "error"
error_text = f"{runner} did not finish within {int(_TURN_TIMEOUT_S)} s."
@@ -3983,6 +3994,9 @@ async def _drive_cli(
elif plan.shape == "codex" and state.failed_tools:
status = "error"
error_text = "Unresolved tool failure: " + ", ".join(sorted(state.failed_tools))
+ elif runner in {"claude-cli", "glm-cli"} and not state.saw_result:
+ status = "error"
+ error_text = f"{runner} exited without a terminal result; its output may be incomplete."
usage = dict(state.usage)
cost_usd = getattr(state, "cost_usd", None)
diff --git a/jarvis/agent_chat/service.py b/jarvis/agent_chat/service.py
index 266b081481..e74c35a5c4 100644
--- a/jarvis/agent_chat/service.py
+++ b/jarvis/agent_chat/service.py
@@ -845,6 +845,11 @@ class AgentChatService:
self._publish_event(session_id, event)
def _publish_event(self, session_id: str, event: dict[str, Any]) -> None:
+ if event.get("kind") == "turn_finished":
+ turn_id = str((event.get("payload") or {}).get("turn_id") or "")
+ if turn_id and self.store.turn_terminal(session_id, turn_id) is not None:
+ log.warning("agent chat: ignored duplicate completion for turn %s", turn_id)
+ return
stored = self.store.append_event(session_id, event)
if event.get("kind") == "turn_finished":
self._announce_jarvis_turn(session_id, event)
diff --git a/jarvis/agent_chat/store.py b/jarvis/agent_chat/store.py
index 1d420bd04f..7237386110 100644
--- a/jarvis/agent_chat/store.py
+++ b/jarvis/agent_chat/store.py
@@ -621,12 +621,12 @@ class AgentChatStore:
]
def turn_terminal(self, session_id: str, turn_id: str) -> dict[str, Any] | None:
- """Read one durable terminal event for an exact owned turn without loading history."""
+ """Read the first durable completion; stale readers cannot revise its outcome."""
with self._lock:
row = self._conn.execute(
"SELECT seq, ts_ms, kind, payload FROM agent_chat_events "
"WHERE session_id = ? AND kind = 'turn_finished' "
- "AND json_extract(payload, '$.turn_id') = ? ORDER BY seq DESC LIMIT 1",
+ "AND json_extract(payload, '$.turn_id') = ? ORDER BY seq ASC LIMIT 1",
(session_id, turn_id),
).fetchone()
if row is None:
diff --git a/jarvis/agent_chat/turn_host_client.py b/jarvis/agent_chat/turn_host_client.py
index 0183241bc4..eb850d4edc 100644
--- a/jarvis/agent_chat/turn_host_client.py
+++ b/jarvis/agent_chat/turn_host_client.py
@@ -248,6 +248,8 @@ class HostedCli:
def release(self) -> None:
"""The turn is over and handled: the host (or the spool) may drop it."""
+ if self.detached:
+ return
if self._spool_path is not None:
with contextlib.suppress(OSError):
self._spool_path.unlink()
@@ -377,6 +379,12 @@ class TurnHostClient:
return cli
def _register(self, cli: HostedCli) -> None:
+ previous = self._procs.get(cli.host_id)
+ if previous is not None and previous is not cli:
+ # A second recovery in this app used to silently strand the old
+ # reader until its one-hour timeout, even after the CLI exited.
+ # Wake it with the same ownership signal as a cross-app handover.
+ previous._on_handed_over()
self._procs[cli.host_id] = cli
for frame in self._early.pop(cli.host_id, []):
self._deliver(cli, frame)
diff --git a/jarvis/ui/web/agent_chat_routes.py b/jarvis/ui/web/agent_chat_routes.py
index f9b4e7d8c8..a4ad8275f5 100644
--- a/jarvis/ui/web/agent_chat_routes.py
+++ b/jarvis/ui/web/agent_chat_routes.py
@@ -35,6 +35,7 @@ from __future__ import annotations
import asyncio
import logging
import os
+import threading
import time
from pathlib import Path
from typing import Any, Final, Literal
@@ -286,21 +287,31 @@ class PickFolderBody(BaseModel):
# ------------------------------------------------------------------ helpers
+# Sync HTTP routes run in worker threads while boot and WebSockets use the
+# event loop. Construction opens SQLite and schedules turn recovery, so the
+# first requests must not create competing owners of the same hosted turns.
+_SERVICE_BUILD_LOCK = threading.Lock()
+
+
def _service_from_state(state: Any) -> AgentChatService | None:
"""The service, built on first use from ``app.state.agent_chat_factory``."""
svc = getattr(state, "agent_chat", None)
if svc is not None:
return svc
- factory = getattr(state, "agent_chat_factory", None)
- if factory is None:
- return None
- try:
- svc = factory()
- except Exception as exc: # noqa: BLE001 — surfaces as 503 with the reason in the log
- log.warning("agent chat: service could not be built: %s", exc)
- return None
- state.agent_chat = svc
- return svc
+ with _SERVICE_BUILD_LOCK:
+ svc = getattr(state, "agent_chat", None)
+ if svc is not None:
+ return svc
+ factory = getattr(state, "agent_chat_factory", None)
+ if factory is None:
+ return None
+ try:
+ svc = factory()
+ except Exception as exc: # noqa: BLE001 — surfaces as 503 with the reason in the log
+ log.warning("agent chat: service could not be built: %s", exc)
+ return None
+ state.agent_chat = svc
+ return svc
#: How long after the server is up the boot reattach waits, so it never sits
diff --git a/jarvis/ui/web/frontend/src/components/agentchat/reduce.test.ts b/jarvis/ui/web/frontend/src/components/agentchat/reduce.test.ts
index 60a4ddddbc..4ce88a0478 100644
--- a/jarvis/ui/web/frontend/src/components/agentchat/reduce.test.ts
+++ b/jarvis/ui/web/frontend/src/components/agentchat/reduce.test.ts
@@ -17,6 +17,29 @@ function ev(kind: string, payload: Record<string, unknown>, persisted = true): A
}
describe("agent-chat reduce", () => {
+ it.each(["done", "error", "cancelled"])("preserves the first %s completion through live delivery and replay", (status) => {
+ const start = ev("turn_started", { turn_id: "settled" });
+ const answer = ev("assistant_text", { turn_id: "settled", text: "Fix committed locally." });
+ const finish = ev("turn_finished", { turn_id: "settled", status, duration_ms: 1993344, usage: { output_tokens: 12 }, cost_usd: 0.01 });
+ const late = ev("turn_finished", { turn_id: "settled", status: "error", duration_ms: 3604992, usage: {}, error: "claude-cli did not finish within 3600 s." });
+ const events = [start, answer, finish, late];
+ const live = events.reduce(reduceEvent, EMPTY_TIMELINE);
+ const replay = reduceEvents(EMPTY_TIMELINE, events);
+ expect(live).toEqual(replay);
+ expect(live.items[0]).toMatchObject({ status, error: null, durationMs: 1993344, usage: { output_tokens: 12 }, costUsd: 0.01 });
+ expect(live.lastSeq).toBe(late.seq);
+ });
+
+ it("shows a real timeout even when partial text sounds final", () => {
+ const partial = reduceEvents(EMPTY_TIMELINE, [
+ ev("turn_started", { turn_id: "partial" }),
+ ev("assistant_text", { turn_id: "partial", text: "Fix committed locally." }),
+ ]);
+ expect(runningTurn(partial)?.id).toBe("partial");
+ const failed = reduceEvent(partial, ev("turn_finished", { turn_id: "partial", status: "error", error: "claude-cli did not finish within 3600 s." }));
+ expect(failed.items[0]).toMatchObject({ status: "error", error: "claude-cli did not finish within 3600 s." });
+ });
+
it("folds a full turn: user line, deltas into one text block, tool call + result, finish", () => {
const tl = reduceEvents(EMPTY_TIMELINE, [
ev("user_message", { text: "hi" }),
diff --git a/jarvis/ui/web/frontend/src/components/agentchat/reduce.ts b/jarvis/ui/web/frontend/src/components/agentchat/reduce.ts
index 368fc0903c..88daf716c0 100644
--- a/jarvis/ui/web/frontend/src/components/agentchat/reduce.ts
+++ b/jarvis/ui/web/frontend/src/components/agentchat/reduce.ts
@@ -1047,6 +1047,11 @@ export function reduceEvent(tl: Timeline, ev: AgentChatEvent): Timeline {
}
case "turn_finished": {
+ // Older event logs can contain a stale reader's timeout after the real
+ // process completed. Only a terminal event settles a turn, and it does
+ // so once; streamed text alone never suppresses a genuine timeout.
+ const existing = base.items[findTurn(base.items, turnId)];
+ if (existing?.type === "turn" && existing.durationMs !== null) return base;
const status = str(p.status, "done") as TurnStatus;
const finished = updateTurn(base, turnId, (turn) => ({
...turn,
diff --git a/tests/unit/agent_chat/test_cli_lifecycle.py b/tests/unit/agent_chat/test_cli_lifecycle.py
new file mode 100644
index 0000000000..d5bfb23e98
--- /dev/null
+++ b/tests/unit/agent_chat/test_cli_lifecycle.py
@@ -0,0 +1,153 @@
+"""Completion requires a terminal result and a reaped process, not final-sounding text."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+from jarvis.agent_chat import runner_cli
+from jarvis.agent_chat.events import make_event
+from jarvis.agent_chat.runner_api import TurnHandle
+from jarvis.agent_chat.service import AgentChatService
+from jarvis.agent_chat.store import AgentChatStore
+from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+
+
+ ("result", "ending", "expected", "error"),
+ [
+ (True, "pass", "done", None),
+ (True, "sys.stdin.read()", "done", None),
+ (True, "sys.exit(7)", "error", "exited with code 7"),
+ (False, "pass", "error", "without a terminal result"),
+ (False, "time.sleep(60)", "error", "did not finish within"),
+ (True, "time.sleep(60)", "error", "did not finish within"),
+ (True, "os.close(1); os.close(2); time.sleep(60)", "error", "did not finish within"),
+ ],
+ ids=[
+ "success",
+ "stdin-eof",
+ "nonzero",
+ "partial-exit",
+ "partial-stuck",
+ "result-stuck",
+ "pipes-closed-stuck",
+ ],
+)
+async def test_claude_process_completion(tmp_path: Path, result, ending, expected, error):
+ store = AgentChatStore(":memory:")
+ session = store.create_session(
+ provider="claude-api", model="fake", effort="", cwd=str(tmp_path)
+ )
+ events = []
+
+ async def emit(event):
+ events.append(event)
+
+ async def approve(*_args):
+ pytest.fail("No approval should be requested")
+
+ handle = TurnHandle(session, "turn", emit, approve, asyncio.Event())
+ answer = {
+ "type": "assistant",
+ "message": {"id": "m", "content": [{"type": "text", "text": "Fix committed locally."}]},
+ }
+ lines = [answer]
+ if result:
+ lines.append(
+ {
+ "type": "result",
+ "subtype": "success",
+ "is_error": False,
+ "usage": {"output_tokens": 12},
+ }
+ )
+ source = (
+ "import sys, time, os\n"
+ + "\n".join(f"print({json.dumps(line)!r}, flush=True)" for line in lines)
+ + "\n"
+ + ending
+ )
+ proc = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-u",
+ "-c",
+ source,
+ stdin=asyncio.subprocess.PIPE,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+ try:
+ outcome = await asyncio.wait_for(
+ runner_cli._drive_cli(
+ handle,
+ proc,
+ runner_cli.CliPlan(
+ argv=[],
+ env={},
+ stdin_text="prompt\n",
+ shape="claude",
+ vendor_session=None,
+ keep_stdin=True,
+ ),
+ "claude-cli",
+ cwd=tmp_path,
+ started_at=time.time(),
+ placement=None,
+ tree=None,
+ bridge=None,
+ timeout_s=1.0,
+ ),
+ timeout=4.0,
+ )
+ assert outcome.status == expected
+ assert (error in outcome.error) if error else outcome.error is None
+ assert proc.returncode is not None
+ assert [e["payload"]["text"] for e in events if e["kind"] == "assistant_text"] == [
+ "Fix committed locally."
+ ]
+ if result:
+ assert outcome.usage["output_tokens"] == 12
+ finally:
+ if proc.returncode is None:
+ proc.kill()
+ await asyncio.wait_for(proc.wait(), 3)
+ store.close()
+
+
[email protected]("status", ["done", "error", "cancelled"])
+async def test_a_turn_publishes_its_terminal_status_only_once(tmp_path, status):
+ store = AgentChatStore(":memory:")
+ service = AgentChatService(store)
+ session = store.create_session(provider="openai", model="fake", effort="", cwd=str(tmp_path))
+ seen = []
+ service._event_listeners.append(lambda _sid, event: seen.append(event))
+ try:
+ first = make_event(
+ "turn_finished", {"turn_id": "t", "status": status, "usage": {"output_tokens": 12}}
+ )
+ late = make_event(
+ "turn_finished",
+ {
+ "turn_id": "t",
+ "status": "error",
+ "error": "claude-cli did not finish within 3600 s.",
+ },
+ )
+ await service._emit(session.session_id, first)
+ await service._emit(session.session_id, late)
+ assert len(seen) == 1
+ assert len(store.list_events(session.session_id)) == 1
+ # Old logs already contain duplicates. Reading their canonical outcome
+ # must agree with replay in the UI, without rewriting the audit trail.
+ store.append_event(session.session_id, late)
+ terminal = store.turn_terminal(session.session_id, "t")
+ assert terminal["payload"] == first["payload"]
+ finally:
+ store.close()
diff --git a/tests/unit/agent_chat/test_mars_turn_terminal.py b/tests/unit/agent_chat/test_mars_turn_terminal.py
index 88a54c64f0..2d681d7c52 100644
--- a/tests/unit/agent_chat/test_mars_turn_terminal.py
+++ b/tests/unit/agent_chat/test_mars_turn_terminal.py
@@ -51,9 +51,9 @@ def test_terminal_lookup_is_exactly_session_and_turn_scoped(store):
assert store.turn_terminal("society:one", "missing-turn") is None
-def test_latest_durable_terminal_wins_without_reading_other_event_kinds(store):
- _terminal(store, "society:one", "target", "done")
- latest = _terminal(store, "society:one", "target", "cancelled")
+def test_stale_cancellation_cannot_replace_a_durable_completion(store):
+ first = _terminal(store, "society:one", "target", "done")
+ _terminal(store, "society:one", "target", "cancelled")
store.append_event(
"society:one",
make_event(
@@ -62,9 +62,9 @@ def test_latest_durable_terminal_wins_without_reading_other_event_kinds(store):
),
)
record = store.turn_terminal("society:one", "target")
- assert record["seq"] == latest["seq"]
+ assert record["seq"] == first["seq"]
assert record["kind"] == "turn_finished"
- assert record["payload"]["status"] == "cancelled"
+ assert record["payload"]["status"] == "done"
def test_turn_reference_is_a_bound_sql_value(store):
diff --git a/tests/unit/agent_chat/test_turn_host.py b/tests/unit/agent_chat/test_turn_host.py
index d2817c9423..7555ad8272 100644
--- a/tests/unit/agent_chat/test_turn_host.py
+++ b/tests/unit/agent_chat/test_turn_host.py
@@ -559,3 +559,135 @@ async def test_the_host_closes_stdin_after_the_result_when_no_app_is_attached(
assert host._turns[cli.host_id].exit_code == 0
finally:
await _stop(host, task)
+
+
+def test_concurrent_first_requests_build_only_one_chat_service() -> None:
+ from concurrent.futures import ThreadPoolExecutor
+ from threading import Barrier
+ from types import SimpleNamespace
+
+ from jarvis.ui.web.agent_chat_routes import _service_from_state
+
+ entrants = Barrier(3)
+ built = []
+
+ def factory():
+ service = object()
+ built.append(service)
+ time.sleep(0.1) # SQLite initialization releases the GIL in production.
+ return service
+
+ state = SimpleNamespace(agent_chat=None, agent_chat_factory=factory)
+
+ def first_request():
+ entrants.wait(timeout=3)
+ return _service_from_state(state)
+
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ services = list(pool.map(lambda _: first_request(), range(3)))
+ assert len(built) == 1
+ assert all(svc is state.agent_chat for svc in services)
+
+
+async def test_reattaching_twice_hands_over_the_old_reader_without_killing_the_cli(tmp_path):
+ host, task, port = await _start_host(tmp_path)
+ client = await _client(port)
+ try:
+ spawned = await client.spawn(
+ [sys.executable, "-c", _CHILD],
+ cwd=str(tmp_path),
+ env=None,
+ stdin="hi\n",
+ keep_stdin=True,
+ meta={"turn_id": "duplicate", "keep_stdin": True},
+ )
+ client.detach()
+ client = await _client(port)
+ first = await client.attach(spawned.host_id)
+ assert first is not None
+ assert await first.stdout.readline() == b"got hi\n"
+ second = await client.attach(first.host_id)
+ assert second is not None
+ assert first.handed_over and first.detached
+ assert await asyncio.wait_for(first.wait(), 1) == turn_host_client.HOST_LOST_CODE
+ first.kill()
+ first.release() # a stale owner must not release the current reader
+ assert client.live() == [second]
+ assert second.stdin is not None
+ second.stdin.write(b"go\n")
+ assert [await second.stdout.readline() for _ in range(4)] == [
+ b"got hi\n",
+ b"two\n",
+ b"three\n",
+ b"",
+ ]
+ assert await asyncio.wait_for(second.wait(), 3) == 3
+ second.release()
+ finally:
+ client.detach()
+ await _stop(host, task)
+
+
+async def test_duplicate_recovery_publishes_one_success_and_no_late_timeout(tmp_path):
+ from jarvis.agent_chat.turn_host_client import TurnHandedOver
+
+ host, task, port = await _start_host(tmp_path)
+ client = await _client(port)
+ store = AgentChatStore(":memory:")
+ session = store.create_session(
+ provider="claude-api", model="fake", effort="", cwd=str(tmp_path)
+ )
+ events = []
+ readers = []
+
+ async def emit(event):
+ events.append(event)
+
+ async def approve(*_args):
+ pytest.fail("No approval expected")
+
+ try:
+ source = "import sys, json\nsys.stdin.readline()\nsys.stdin.readline()\n" + "\n".join(
+ f"print({json.dumps(line)!r}, flush=True)" for line in _CLAUDE_LINES
+ )
+ spawned = await client.spawn(
+ [sys.executable, "-u", "-c", source],
+ cwd=str(tmp_path),
+ env=None,
+ stdin="prompt\n",
+ keep_stdin=True,
+ meta={
+ "turn_id": "duplicate",
+ "runner": "claude-cli",
+ "shape": "claude",
+ "keep_stdin": True,
+ "started_at": time.time(),
+ },
+ )
+ client.detach()
+ client = await _client(port)
+ for _ in range(3):
+ proc = await client.attach(spawned.host_id)
+ assert proc is not None
+ handle = runner_api.TurnHandle(session, "duplicate", emit, approve, asyncio.Event())
+ readers.append(asyncio.create_task(resume_hosted_cli_turn(handle, proc)))
+ await asyncio.sleep(0)
+ for reader in readers[:-1]:
+ with pytest.raises(TurnHandedOver):
+ await asyncio.wait_for(reader, 2)
+ assert proc.stdin is not None
+ proc.stdin.write(b"go\n")
+ await asyncio.wait_for(readers[-1], 3)
+ finishes = [e["payload"] for e in events if e["kind"] == "turn_finished"]
+ assert len(finishes) == 1
+ assert finishes[0]["status"] == "done"
+ assert finishes[0]["usage"]["output_tokens"] == 5
+ assert proc.returncode == 0
+ finally:
+ for reader in readers:
+ if not reader.done():
+ reader.cancel()
+ await asyncio.gather(*readers, return_exceptions=True)
+ client.detach()
+ await _stop(host, task)
+ store.close()