Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(agentic-ide): thread turns survive app restarts in a turn host

rubenluetke10-beep79999d39b413
diff --git a/docs/agentic-ide-threads.md b/docs/agentic-ide-threads.md
index 670d0ee519..44b732ea8a 100644
--- a/docs/agentic-ide-threads.md
+++ b/docs/agentic-ide-threads.md
@@ -70,6 +70,31 @@ what to change. A card nobody answered closes when the person sends another
message instead. Cards are rebuilt from the session's event log, so they
survive reopening the thread and restarting the app.
+## Restarts do not stop a thread
+
+A thread's coding CLI does not run as a child of the app. It runs in the
+turn host (`jarvis/agent_chat/turn_host.py`), a small background process
+that the app starts on first use and talks to over a loopback socket. The
+terminal grid keeps its agents in the PTY host the same way. The host keeps
+every line the CLI prints and streams them to the app.
+
+- **Restart, update, crash:** the CLI keeps working. The next app start
+ attaches again. It feeds the lines the old app already handled back into
+ the translator without showing them twice, then carries the turn on live:
+ new output, approval cards, the real ending.
+- **Quit:** the turn stays open while the CLI works. A turn that ends while
+ no app is attached is written to a spool folder, and the next start shows
+ its answer.
+- **Stop** ends the CLI as before.
+- **Another app process takes over** (a second start of the same instance):
+ the old process hands its turns over without ending them.
+
+Only thread turns run there. A turn running as Jarvis, a goal turn and a
+helper turn without tools need state inside the app and stay its children.
+A frozen build cannot start the host and keeps the old behaviour. So does
+any process that is not the desktop app or the web launcher, such as tests
+and scripts. Logs: `logs/turn_host.log` in the Jarvis data folder.
+
## Platforms
Nothing in the thread layout is OS-specific: paths are compared
diff --git a/jarvis/agent_chat/runner_cli.py b/jarvis/agent_chat/runner_cli.py
index b8b6cbc726..f70a57c17c 100644
--- a/jarvis/agent_chat/runner_cli.py
+++ b/jarvis/agent_chat/runner_cli.py
@@ -2957,6 +2957,67 @@ async def run_cli_turn(
return outcome.vendor_session
+async def resume_hosted_cli_turn(handle: TurnHandle, proc: Any) -> str | None:
+ """Carry on a thread turn whose CLI kept running in the turn host.
+
+ The app restarted while the turn ran; the CLI did not. ``proc`` is the
+ reattached :class:`~jarvis.agent_chat.turn_host_client.HostedCli` (or a
+ spooled one that ended meanwhile). Its ``meta`` holds what the plan said
+ at spawn; the lines the old app already handled rebuild the translator,
+ every later one is emitted, and the turn ends with its real outcome.
+ Returns the vendor session id to persist, like :func:`run_cli_turn`.
+ """
+ meta = dict(getattr(proc, "meta", {}) or {})
+ runner = str(meta.get("runner") or "")
+ shape = str(meta.get("shape") or "text")
+ started_at = float(meta.get("started_at") or time.time())
+ codex_home = str(meta.get("codex_home") or "")
+ plan = CliPlan(
+ argv=[],
+ env={"CODEX_HOME": codex_home} if codex_home else {},
+ stdin_text=None,
+ shape=shape if shape in _SHAPES else "text",
+ vendor_session=str(meta.get("vendor_session") or "") or None,
+ keep_stdin=bool(meta.get("keep_stdin")),
+ # Only its presence matters from here on: control requests are answered.
+ control_init="" if meta.get("control") else None,
+ discover=_kimi_session_after if meta.get("discover") else None,
+ )
+ elapsed = max(0.0, time.time() - started_at)
+ log.info(
+ "agent chat %s: carrying on a %s turn the turn host kept running (%.0f s old)",
+ handle.turn_id,
+ runner,
+ elapsed,
+ )
+ outcome = await _drive_cli(
+ handle,
+ proc,
+ plan,
+ runner,
+ cwd=Path(str(meta.get("cwd") or handle.session.cwd or Path.home())),
+ started_at=started_at,
+ placement=None,
+ tree=None,
+ bridge=None,
+ timeout_s=max(60.0, _TURN_TIMEOUT_S - elapsed),
+ )
+ await handle.emit(
+ make_event(
+ "turn_finished",
+ {
+ "turn_id": handle.turn_id,
+ "status": outcome.status,
+ "duration_ms": int((time.time() - started_at) * 1000),
+ "usage": outcome.usage,
+ "error": outcome.error,
+ "cost_usd": outcome.cost_usd,
+ },
+ )
+ )
+ return outcome.vendor_session
+
+
def _agy_effective_effort(model: str, effort: str) -> str:
"""The ``--effort`` value agy is launched with for ``model`` + ``effort``."""
try:
@@ -2977,14 +3038,9 @@ async def _run_cli_once(
bridge: Any | None = None,
) -> _Outcome:
session = handle.session
- chat_ref = approval_ref(session.session_id)
cwd = _resolved_cwd(session.cwd or Path.home())
effort = normalize_effort(session.provider, session.effort)
planner = _PLANNERS[runner]
- status = "done"
- error_text: str | None = None
- usage: dict[str, int] = {}
- cost_usd: float | None = None
vendor_session: str | None = None
from jarvis.society.remote import placement_for_session
@@ -3122,27 +3178,135 @@ async def _run_cli_once(
except RemoteCliUnavailable as exc:
return _Outcome("error", str(exc), {}, None, None)
else:
- try:
- proc = await asyncio.create_subprocess_exec(
- *plan.argv,
+ hosted = None
+ if tree is None and _survives_restart(handle, identity):
+ from jarvis.agent_chat import turn_host_client
+
+ hosted = await turn_host_client.spawn(
+ plan.argv,
cwd=str(cwd),
env=plan.env,
- stdin=asyncio.subprocess.PIPE
- if plan.stdin_text is not None
- else asyncio.subprocess.DEVNULL,
- stdout=asyncio.subprocess.PIPE,
- stderr=asyncio.subprocess.PIPE,
- creationflags=NO_WINDOW_CREATIONFLAGS,
- start_new_session=tree is not None and os.name != "nt",
- limit=_READLINE_LIMIT,
+ # The host writes the handshake and the prompt itself, so a
+ # restart between spawn and feed cannot leave the CLI waiting.
+ stdin=(
+ (plan.control_init or "") + plan.stdin_text
+ if plan.stdin_text is not None
+ else None
+ ),
+ keep_stdin=plan.keep_stdin,
+ meta=_hosted_meta(handle, runner, plan, cwd, started_at),
)
- if tree is not None:
- tree.assign(proc.pid)
- except (OSError, ValueError) as exc:
- if tree is not None:
- tree.close()
- return _Outcome("error", f"Could not start {runner}: {exc}", {}, None, None)
+ if hosted is not None:
+ proc = hosted
+ else:
+ proc = await _spawn_child(plan, cwd, tree, runner)
+ if isinstance(proc, _Outcome):
+ return proc
+ return await _drive_cli(
+ handle,
+ proc,
+ plan,
+ runner,
+ cwd=cwd,
+ started_at=started_at,
+ placement=placement,
+ tree=tree,
+ bridge=bridge,
+ )
+
+
+def _survives_restart(handle: TurnHandle, identity: Any) -> bool:
+ """Whether this turn's CLI runs in the turn host and outlives an app restart.
+
+ Threads in the Agentic IDE (surface ``agent``) only: their CLI is a plain
+ coding agent whose whole turn is its own process. A turn running AS
+ Jarvis (``identity``), a goal or a tool-less helper turn depends on state
+ that lives in this process (MCP tool context, approval bridge, goal
+ control) and would lose it across a restart, so it stays a child here.
+ """
+ session = handle.session
+ return (
+ getattr(session, "surface", "") == "agent"
+ and identity is None
+ and not getattr(handle, "goal_turn", False)
+ and not getattr(handle, "tools_disabled", False)
+ and not getattr(handle, "gateway_only", False)
+ )
+
+
+def _hosted_meta(
+ handle: TurnHandle, runner: str, plan: CliPlan, cwd: Path, started_at: float
+) -> dict[str, Any]:
+ """What a hosted turn needs to be carried on after an app restart."""
+ return {
+ "session_id": handle.session.session_id,
+ "turn_id": handle.turn_id,
+ "runner": runner,
+ "shape": plan.shape,
+ "keep_stdin": plan.keep_stdin,
+ "control": plan.control_init is not None,
+ "vendor_session": plan.vendor_session,
+ "discover": plan.discover is not None,
+ "codex_home": plan.env.get("CODEX_HOME", ""),
+ "cwd": str(cwd),
+ "started_at": started_at,
+ }
+
+
+async def _spawn_child(
+ plan: CliPlan, cwd: Path, tree: Any, runner: str
+) -> asyncio.subprocess.Process | _Outcome:
+ """Start the CLI as this process's own child (no turn host)."""
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ *plan.argv,
+ cwd=str(cwd),
+ env=plan.env,
+ stdin=asyncio.subprocess.PIPE
+ if plan.stdin_text is not None
+ else asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ start_new_session=tree is not None and os.name != "nt",
+ limit=_READLINE_LIMIT,
+ )
+ if tree is not None:
+ tree.assign(proc.pid)
+ except (OSError, ValueError) as exc:
+ if tree is not None:
+ tree.close()
+ return _Outcome("error", f"Could not start {runner}: {exc}", {}, None, None)
+ return proc
+
+async def _drive_cli(
+ handle: TurnHandle,
+ proc: Any,
+ plan: CliPlan,
+ runner: str,
+ *,
+ cwd: Path,
+ started_at: float,
+ placement: Any,
+ tree: Any,
+ bridge: Any | None,
+ timeout_s: float = _TURN_TIMEOUT_S,
+) -> _Outcome:
+ """Read a started CLI to its end: translate, answer, and judge the outcome.
+
+ ``proc`` is a child process, a remote one, or a CLI in the turn host
+ (``turn_host_client.HostedCli``). A hosted CLI reattached after an app
+ restart replays the lines this turn already handled first
+ (``stdout.replaying``): they rebuild the translator's state and emit
+ nothing, so the timeline carries on from the first line it has not seen.
+ """
+ session = handle.session
+ chat_ref = approval_ref(session.session_id)
+ status = "done"
+ error_text: str | None = None
+ vendor_session = plan.vendor_session
+ hosted = bool(getattr(proc, "hosted", False))
make_state, translate = _SHAPES[plan.shape]
state: Any = make_state(handle.turn_id, vendor_session)
# Lines that are not the CLI's JSON — a banner, a warning, the whole
@@ -3163,8 +3327,10 @@ async def _run_cli_once(
stderr_tail.append(text)
del stderr_tail[:-40]
- async def _say_plain(line: str) -> None:
+ async def _say_plain(line: str, *, replay: bool = False) -> None:
plain.append(line + "\n")
+ if replay:
+ return
await handle.emit(
make_event(
"text_delta",
@@ -3178,19 +3344,25 @@ async def _run_cli_once(
raw = await proc.stdout.readline()
if not raw:
return
+ # A line this turn handled before an app restart: rebuild state only.
+ replay = hosted and bool(getattr(proc.stdout, "replaying", False))
line = raw.decode("utf-8", errors="replace").strip()
if not line:
continue
if plan.shape == "text":
- await _say_plain(line)
+ await _say_plain(line, replay=replay)
continue
try:
obj = json.loads(line)
except ValueError:
- await _say_plain(line)
+ await _say_plain(line, replay=replay)
continue
if not isinstance(obj, dict):
continue
+ if replay:
+ if obj.get("type") not in ("control_request", "control_response"):
+ translate(obj, state)
+ continue
if plan.control_init is not None:
if obj.get("type") == "control_request":
await _answer_control_request(obj)
@@ -3343,7 +3515,8 @@ async def _run_cli_once(
return build
async def _feed_stdin() -> None:
- if proc.stdin is None:
+ if proc.stdin is None or hosted:
+ # The turn host wrote the handshake and the prompt at spawn.
return
try:
# The control-protocol handshake goes FIRST, on the same stdin, so
@@ -3376,14 +3549,19 @@ async def _run_cli_once(
if getattr(handle, "goal_turn", False):
await asyncio.gather(pump, drain, feeder)
else:
- await asyncio.wait_for(asyncio.gather(pump, drain, feeder), timeout=_TURN_TIMEOUT_S)
+ await asyncio.wait_for(asyncio.gather(pump, drain, feeder), timeout=timeout_s)
await proc.wait()
except TimeoutError:
_kill(proc)
status = "error"
error_text = f"{runner} did not finish within {int(_TURN_TIMEOUT_S)} s."
except asyncio.CancelledError:
- _kill(proc)
+ if hosted and not handle.cancel.is_set():
+ # The app is going away, nobody pressed Stop: the CLI keeps working
+ # in the turn host and the next app start carries the turn on.
+ proc.detach()
+ else:
+ _kill(proc)
raise
finally:
watcher.cancel()
@@ -3393,13 +3571,23 @@ async def _run_cli_once(
await asyncio.gather(watcher, pump, drain, feeder, return_exceptions=True)
if tree is not None:
tree.close()
- if proc.returncode is None:
+ if proc.returncode is None and not getattr(proc, "detached", False):
_kill(proc)
try:
await asyncio.wait_for(proc.wait(), timeout=5)
except TimeoutError:
log.warning("Agent CLI did not reap after cancellation")
+ if hosted and getattr(proc, "handed_over", False):
+ from jarvis.agent_chat.turn_host_client import TurnHandedOver
+
+ raise TurnHandedOver(handle.turn_id)
+ if hosted:
+ # Handled to the end: the host (or its spool file) may drop the lines.
+ proc.release()
+ if getattr(proc, "host_lost", False) and not handle.cancel.is_set():
+ status = "error"
+ error_text = "The process that ran this agent stopped unexpectedly."
if handle.cancel.is_set():
status = "cancelled"
elif status == "done":
diff --git a/jarvis/agent_chat/service.py b/jarvis/agent_chat/service.py
index f0343f2a48..28919e168f 100644
--- a/jarvis/agent_chat/service.py
+++ b/jarvis/agent_chat/service.py
@@ -61,6 +61,7 @@ from jarvis.agent_chat.store import (
AgentChatStore,
)
from jarvis.agent_chat.surface_kits import kit_for
+from jarvis.agent_chat.turn_host_client import TurnHandedOver
from jarvis.core.protocols import ChatCompletion, ChatTurn, current_chat_turn
from jarvis.society.delivery import IncomingMessage
@@ -73,6 +74,26 @@ _ORPHANED_TURN_ERROR: Final = (
)
Subscriber = asyncio.Queue[dict[str, Any]]
+
+
+def _orphan_event(turn_id: str, started_ms: int, last_ms: int) -> dict[str, Any]:
+ """The ``turn_finished`` that closes a turn a restart cut off."""
+ return {
+ **make_event(
+ "turn_finished",
+ {
+ "turn_id": turn_id,
+ "status": "error",
+ "duration_ms": max(0, last_ms - started_ms),
+ "usage": {},
+ "error": _ORPHANED_TURN_ERROR,
+ "cost_usd": None,
+ },
+ ),
+ "ts_ms": last_ms,
+ }
+
+
DECISIONS: tuple[str, ...] = ("allow", "allow_always", "deny")
@@ -221,7 +242,7 @@ def _current_task() -> asyncio.Task[Any] | None:
class _Running:
- __slots__ = ("asks", "task", "cancel", "turn_id", "setup_task", "ready")
+ __slots__ = ("asks", "task", "cancel", "turn_id", "setup_task", "ready", "detached")
def __init__(self, turn_id: str, cancel: asyncio.Event) -> None:
self.turn_id = turn_id
@@ -231,6 +252,9 @@ class _Running:
self.ready = asyncio.Event()
#: Question cards this turn has shown (MAX_ASKS_PER_TURN).
self.asks = 0
+ #: The app is shutting down and this turn's CLI keeps running in the
+ #: turn host: the task ends without closing the turn.
+ self.detached = False
class AgentChatService:
@@ -305,34 +329,212 @@ class AgentChatService:
last moment the session heard anything, so the thread list keeps its
order. A CLI that outlived its parent is not hunted down here: a stop
by folder would also hit the person's own terminals in that folder.
+
+ A thread turn (surface ``agent``) is the exception: its CLI runs in
+ the turn host (``turn_host_client``) and may still be working. While
+ such a host or its spool exists, those turns are held as running and
+ handed to :meth:`_reattach_hosted`, which carries on every one the
+ host still knows and seals the rest exactly like here.
"""
try:
orphans = self.store.open_turns()
except sqlite3.Error:
log.warning("agent chat: could not look for turns a restart left open", exc_info=True)
return
+ try:
+ loop: asyncio.AbstractEventLoop | None = asyncio.get_running_loop()
+ except RuntimeError:
+ loop = None
+ hosted_possible: bool | None = None
+ deferred: list[tuple[str, str, int, int]] = []
+ sealed = 0
for session_id, turn_id, started_ms, last_ms in orphans:
if not turn_id:
continue
- self.store.append_event(
- session_id,
- {
- **make_event(
+ if loop is not None and self._may_be_hosted(session_id):
+ if hosted_possible is None:
+ from jarvis.agent_chat import turn_host_client
+
+ hosted_possible = turn_host_client.may_hold_turns()
+ if hosted_possible:
+ deferred.append((session_id, turn_id, started_ms, last_ms))
+ continue
+ self.store.append_event(session_id, _orphan_event(turn_id, started_ms, last_ms))
+ sealed += 1
+ if sealed:
+ log.info("agent chat: closed %d turn(s) a restart left open", sealed)
+ if deferred and loop is not None:
+ for session_id, turn_id, _started, _last in deferred:
+ # Held as running until the host answers: a message sent now
+ # must not start a second CLI on the same conversation.
+ held = _Running(turn_id, asyncio.Event())
+ held.setup_task = None
+ self._running[session_id] = held
+ self._reattach_task: asyncio.Task[None] | None = loop.create_task(
+ self._reattach_hosted(deferred), name="agent-chat-reattach"
+ )
+
+ def _may_be_hosted(self, session_id: str) -> bool:
+ session = self.store.get_session(session_id)
+ return session is not None and session.surface == "agent"
+
+ async def _reattach_hosted(self, orphans: list[tuple[str, str, int, int]]) -> None:
+ """Carry on every open thread turn whose CLI the turn host still holds."""
+ try:
+ await self._reattach_hosted_turns(orphans)
+ except Exception: # noqa: BLE001 — a held thread must never stay on Working
+ log.exception("agent chat: reattaching thread turns failed")
+ for session_id, turn_id, started_ms, last_ms in orphans:
+ held = self._running.get(session_id)
+ if held is None or held.turn_id != turn_id or held.task is not None:
+ continue
+ self._running.pop(session_id, None)
+ held.ready.set()
+ if self.store.turn_terminal(session_id, turn_id) is None:
+ self._publish_event(session_id, _orphan_event(turn_id, started_ms, last_ms))
+
+ async def _reattach_hosted_turns(self, orphans: list[tuple[str, str, int, int]]) -> None:
+ from jarvis.agent_chat import turn_host_client
+
+ client = None
+ for delay in (0.0, 5.0, 15.0, 30.0):
+ if delay:
+ await asyncio.sleep(delay)
+ client = await turn_host_client.get_client(start=False)
+ if client is not None or not turn_host_client.may_hold_turns():
+ break
+ by_turn: dict[str, Any] = {}
+ if client is not None:
+ for host_id, info in list(client.turns.items()):
+ meta = info.get("meta") or {}
+ by_turn[str(meta.get("turn_id") or "")] = ("host", host_id)
+ for record in turn_host_client.read_spool():
+ meta = record.get("meta") or {}
+ by_turn.setdefault(str(meta.get("turn_id") or ""), ("spool", record))
+ open_ids = {turn_id for _sid, turn_id, _s, _l in orphans}
+ resumed = 0
+ for session_id, turn_id, started_ms, last_ms in orphans:
+ held = self._running.get(session_id)
+ source = by_turn.get(turn_id)
+ proc = None
+ try:
+ if source is not None and source[0] == "host" and client is not None:
+ proc = await client.attach(source[1])
+ elif source is not None and source[0] == "spool":
+ proc = turn_host_client.spooled_cli(source[1])
+ except (ConnectionError, OSError, ValueError, KeyError):
+ log.warning("agent chat: could not reattach turn %s", turn_id, exc_info=True)
+ proc = None
+ if proc is None or held is None or held.turn_id != turn_id:
+ if held is not None and held.turn_id == turn_id and held.task is None:
+ self._running.pop(session_id, None)
+ held.ready.set()
+ if self.store.turn_terminal(session_id, turn_id) is None:
+ self._publish_event(session_id, _orphan_event(turn_id, started_ms, last_ms))
+ continue
+ self._resume_hosted_turn(session_id, held, proc)
+ resumed += 1
+ # A CLI the host holds for a turn nobody has open any more (closed by
+ # an older build, deleted thread) has no reader: end it.
+ for turn_id, source in by_turn.items():
+ if not turn_id or turn_id in open_ids:
+ continue
+ try:
+ if source[0] == "host" and client is not None:
+ stray = await client.attach(source[1])
+ if stray is not None:
+ stray.kill()
+ stray.release()
+ elif source[0] == "spool":
+ turn_host_client.spooled_cli(source[1]).release()
+ except (ConnectionError, OSError, ValueError, KeyError):
+ log.warning("agent chat: stray hosted turn %s not ended", turn_id, exc_info=True)
+ if resumed:
+ log.info("agent chat: carried on %d thread turn(s) across a restart", resumed)
+
+ def _resume_hosted_turn(self, session_id: str, run: _Running, proc: Any) -> None:
+ """Run the rest of a reattached thread turn as this session's task."""
+ from jarvis.agent_chat.runner_cli import resume_hosted_cli_turn
+
+ session = self.store.get_session(session_id)
+ assert session is not None
+ turn_id = run.turn_id
+ runner = str((getattr(proc, "meta", {}) or {}).get("runner") or "")
+ kit = kit_for(session.surface)
+ handle = TurnHandle(
+ session=session,
+ turn_id=turn_id,
+ emit=lambda ev: self._emit(session_id, ev),
+ request_approval=lambda call_id, name, args, summary: self._ask(
+ session_id, turn_id, call_id, name, args, summary
+ ),
+ cancel=run.cancel,
+ history=self.store.list_events(session_id),
+ assistant_name=self._assistant_name(),
+ bus=self._bus(),
+ surface=session.surface,
+ stance=session.permission_mode if kit.uses_stance else "",
+ control_service=self,
+ )
+
+ async def _carry_on() -> None:
+ finished = False
+ try:
+ vendor = await resume_hosted_cli_turn(handle, proc)
+ finished = True
+ if vendor and vendor != session.vendor_session:
+ self.store.update_session(session_id, vendor_session=vendor)
+ except asyncio.CancelledError:
+ if not run.detached:
+ await self._emit(
+ session_id,
+ make_event(
+ "turn_finished",
+ {
+ "turn_id": turn_id,
+ "status": "cancelled",
+ "duration_ms": 0,
+ "usage": {},
+ "error": None,
+ },
+ ),
+ )
+ raise
+ except TurnHandedOver:
+ run.detached = True
+ log.info("agent chat turn %s handed over to another app process", turn_id)
+ except Exception as exc: # noqa: BLE001 — a runner bug must not leave the UI spinning
+ log.exception("agent chat: reattached turn %s crashed", turn_id)
+ await self._emit(
+ session_id,
+ make_event(
"turn_finished",
{
"turn_id": turn_id,
"status": "error",
- "duration_ms": max(0, last_ms - started_ms),
+ "duration_ms": 0,
"usage": {},
- "error": _ORPHANED_TURN_ERROR,
- "cost_usd": None,
+ "error": f"{type(exc).__name__}: {exc}",
},
),
- "ts_ms": last_ms,
- },
- )
- if orphans:
- log.info("agent chat: closed %d turn(s) a restart left open", len(orphans))
+ )
+ finally:
+ if self._running.get(session_id) is run:
+ self._running.pop(session_id, None)
+ for aid in self.pending_approvals(session_id):
+ fut = self._approvals.pop(aid, None)
+ self._approval_session.pop(aid, None)
+ if fut is not None and not fut.done():
+ fut.set_result("cancel")
+ self._cancel_questions(session_id)
+ if finished and kit.turn_prompts:
+ try:
+ await self._open_turn_prompts(session_id, turn_id, runner)
+ except Exception: # noqa: BLE001 — a missing card leaves the reply readable
+ log.exception("agent chat: end-of-turn card failed for %s", turn_id)
+
+ run.task = asyncio.create_task(_carry_on(), name=f"agent-chat-{turn_id[:8]}")
+ run.ready.set()
def _retire_cli_seats(self) -> None:
"""Move chats off a CLI seat their surface no longer offers.
@@ -1138,6 +1340,10 @@ class AgentChatService:
if completion is not None:
await completion.publish()
except asyncio.CancelledError:
+ if run.detached:
+ # The CLI keeps working in the turn host; the next app
+ # start carries this turn on, so it stays open.
+ raise
await self._emit(
session_id,
make_event(
@@ -1156,6 +1362,11 @@ class AgentChatService:
),
)
raise
+ except TurnHandedOver:
+ # Another app process attached to the turn host and carries
+ # this turn on; it stays open for that process to finish.
+ run.detached = True
+ log.info("agent chat turn %s handed over to another app process", turn_id)
except Exception as exc: # noqa: BLE001 — a runner bug must not leave the UI spinning
log.exception("agent chat turn %s crashed", turn_id)
await self._emit(
@@ -1190,7 +1401,7 @@ class AgentChatService:
),
)
current_chat_turn.reset(origin_token)
- if kit.turn_completed is not None:
+ if kit.turn_completed is not None and not run.detached:
try:
first_seq = max(
(int(e.get("seq") or 0) for e in history), default=history_start
@@ -1212,6 +1423,7 @@ class AgentChatService:
and origin.direct_user
and control_runner is None
and not native_goal
+ and not run.detached
):
try:
await self._open_turn_prompts(session_id, turn_id, runner)
@@ -1457,8 +1669,26 @@ class AgentChatService:
return True
async def cancel_all(self) -> None:
+ """App shutdown: stop every turn — except thread turns the turn host keeps.
+
+ Those CLIs go on working in the host; their tasks end without closing
+ the turn, and the next app start carries them on (``_reattach_hosted``).
+ """
if hasattr(self, "_controls"):
await self._controls.close()
+ from jarvis.agent_chat import turn_host_client
+
+ hosted = turn_host_client.live_turn_ids()
+ detached: list[asyncio.Task[None]] = []
+ for run in list(self._running.values()):
+ if run.turn_id in hosted and run.task is not None and not run.task.done():
+ run.detached = True
+ run.task.cancel()
+ detached.append(run.task)
+ if detached:
+ await asyncio.gather(*detached, return_exceptions=True)
+ log.info("agent chat: %d thread turn(s) keep running in the turn host", len(detached))
+ turn_host_client.detach_all()
for sid in list(dict.fromkeys([*self._running, *self._preparing])):
await self.cancel(sid)
diff --git a/jarvis/agent_chat/turn_host.py b/jarvis/agent_chat/turn_host.py
new file mode 100644
index 0000000000..34ccbea2e5
--- /dev/null
+++ b/jarvis/agent_chat/turn_host.py
@@ -0,0 +1,614 @@
+"""The turn host — a small process that owns the coding CLIs of thread turns.
+
+Run as ``python -m jarvis.agent_chat.turn_host --state <file> --spool <dir>``.
+Started on demand by :mod:`jarvis.agent_chat.turn_host_client`, never by hand.
+
+## Why a thread's CLI lives in its own process
+
+A thread in the Agentic IDE (agent-chat surface ``agent``) is answered by a
+coding CLI in print mode (``runner_cli``). That CLI used to be a child of the
+app: a restart to load a fix, an update or a crash closed its pipes and the
+turn ended as "Jarvis restarted while this turn was running" — while the
+agent was in the middle of its work.
+
+The terminal grid solved the same problem with the PTY host
+(:mod:`jarvis.terminal.pty_host`): a long-lived server owns the processes and
+the app is a client that attaches to them. This is the same design for pipe
+processes. The host starts the CLI, keeps every line it prints, and streams
+them to the attached app. When the app goes away the CLI keeps working and
+its lines keep collecting here; the next app start attaches again, replays
+what it already handled into a fresh translator (without emitting it twice)
+and carries the turn on from the first line it has not seen.
+
+Everything that is policy — prompts, translation, approvals, the event log —
+stays in the app. The host knows turn ids, argv, the lines and an opaque
+``meta`` dict the client attached at spawn.
+
+## Lifetime
+
+The kill-on-close container (a Job Object on Windows, process groups on
+POSIX) belongs to THIS process, so the CLIs live exactly as long as the host
+does. With no CLI running and no client attached for :data:`IDLE_EXIT_S` the
+host exits. Turns that ended while nobody was attached are written to the
+spool folder first (one JSON file each), so the next app start can still
+collect their answer.
+
+## Wire protocol
+
+Newline-delimited JSON over one loopback TCP connection, authenticated by a
+random token the client handed over in the environment. One client at a time;
+a new authenticated client replaces the old one (an app restart).
+
+Requests carry ``rid`` and are answered ``{"rid", "ok", ...}``. Unprompted
+events carry ``ev``: ``line`` (one stdout line, numbered by ``seq`` from 1),
+``err`` (one stderr line), ``exit``, and ``replaced`` (another client took
+over; the old one hands its turns over without ending them). A client acknowledges every stdout
+line it has fully handled (``ack``); after a reconnect, ``attach`` answers
+with the acknowledged ``seq`` and then replays all lines in order before
+streaming live ones, so translator state can be rebuilt exactly.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import hmac
+import json
+import os
+import sys
+import time
+import uuid
+from collections import deque
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from loguru import logger
+
+#: Bumped only for an incompatible change; a client leaves a host speaking
+#: another version alone (it may hold turns from before an update).
+PROTOCOL_VERSION = 1
+
+#: Environment variable carrying the auth token. Never on the command line.
+TOKEN_ENV = "JARVIS_TURN_HOST_TOKEN" # noqa: S105 - a variable name, not a secret
+
+#: With no CLI running and no client attached for this long, the host exits.
+IDLE_EXIT_S = 60.0
+
+_IDLE_POLL_S = 5.0
+
+#: Largest frame a client may send (a spawn carries the prompt and the env).
+MAX_FRAME_BYTES = 64 * 1024 * 1024
+
+#: Longest stdout line read from a CLI (a large tool result is one JSON line).
+LINE_LIMIT = 16 * 1024 * 1024
+
+#: Total stdout a turn may keep for replay. Lines past it still stream live
+#: but are not kept, so a runaway CLI cannot exhaust memory.
+MAX_KEPT_BYTES = 256 * 1024 * 1024
+
+_STDERR_TAIL = 40
+
+
+@dataclass(slots=True)
+class HostedTurn:
+ """One CLI process the host owns."""
+
+ turn_id: str
+ proc: asyncio.subprocess.Process
+ meta: dict[str, Any]
+ started_at: float
+ lines: list[str] = field(default_factory=list)
+ kept_bytes: int = 0
+ truncated: bool = False
+ stderr_tail: deque[str] = field(default_factory=lambda: deque(maxlen=_STDERR_TAIL))
+ acked: int = 0
+ exit_code: int | None = None
+ streaming: bool = False
+ pumps: list[asyncio.Task[None]] = field(default_factory=list)
+
+ @property
+ def alive(self) -> bool:
+ return self.exit_code is None
+
+ def summary(self) -> dict[str, Any]:
+ return {
+ "id": self.turn_id,
+ "pid": self.proc.pid,
+ "meta": self.meta,
+ "started_at": self.started_at,
+ "alive": self.alive,
+ "code": self.exit_code,
+ "lines": len(self.lines),
+ "acked": self.acked,
+ }
+
+ def spool_record(self) -> dict[str, Any]:
+ return {
+ **self.summary(),
+ "stdout": self.lines,
+ "stderr": list(self.stderr_tail),
+ "truncated": self.truncated,
+ }
+
+
+@dataclass
+class _Client:
+ reader: asyncio.StreamReader
+ writer: asyncio.StreamWriter
+ send_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
+ closed: bool = False
+
+ async def send(self, frame: dict[str, Any]) -> None:
+ if self.closed:
+ return
+ data = (json.dumps(frame, ensure_ascii=False) + "\n").encode("utf-8")
+ async with self.send_lock:
+ if self.closed:
+ return
+ try:
+ self.writer.write(data)
+ await self.writer.drain()
+ except (ConnectionError, OSError, RuntimeError) as exc:
+ # The client went away mid-send; its turns keep running and
+ # keep collecting lines for the next client.
+ logger.info("Turn host: client connection lost while sending: {}", exc)
+ self.closed = True
+
+ def close(self) -> None:
+ self.closed = True
+ try:
+ self.writer.close()
+ except Exception as exc: # noqa: BLE001 - already closed is the goal
+ logger.debug("Turn host: closing a client socket failed: {}", exc)
+
+
+class TurnHost:
+ """The server: CLI processes, their kept output, and one client."""
+
+ def __init__(
+ self, token: str, spool_dir: Path, *, idle_exit_s: float = IDLE_EXIT_S
+ ) -> None:
+ from jarvis.core.process_tree import make_process_tree
+
+ self._token = token
+ self._spool_dir = spool_dir
+ self._turns: dict[str, HostedTurn] = {}
+ self._client: _Client | None = None
+ self._idle_exit_s = idle_exit_s
+ self._idle_since: float | None = time.monotonic()
+ self._stop = asyncio.Event()
+ # One kill-on-close container for every CLI: if the host dies, its
+ # CLIs die with it instead of running on with nobody to read them.
+ self._tree = make_process_tree("jarvis-turn-host")
+
+ # ------------------------------------------------------------ lifecycle
+ async def serve(self, state_path: Path) -> None:
+ server = await asyncio.start_server(
+ self._on_connect, host="127.0.0.1", port=0, limit=MAX_FRAME_BYTES
+ )
+ port = int(server.sockets[0].getsockname()[1])
+ _write_state(
+ state_path,
+ {
+ "pid": os.getpid(),
+ "port": port,
+ "token": self._token,
+ "proto": PROTOCOL_VERSION,
+ "started_at": time.time(),
+ "boot_time": _boot_time(),
+ },
+ )
+ logger.info("Turn host listening on 127.0.0.1:{} (pid {})", port, os.getpid())
+ idle = asyncio.create_task(self._idle_watch(), name="turn-host-idle")
+ try:
+ async with server:
+ await self._stop.wait()
+ finally:
+ idle.cancel()
+ self._spool_unclaimed()
+ _remove_state(state_path, os.getpid())
+ for turn in self._turns.values():
+ _kill(turn.proc)
+ self._tree.close()
+ logger.info("Turn host stopped")
+
+ async def _idle_watch(self) -> None:
+ while not self._stop.is_set():
+ await asyncio.sleep(_IDLE_POLL_S)
+ running = any(turn.alive for turn in self._turns.values())
+ attached = self._client is not None and not self._client.closed
+ now = time.monotonic()
+ if running or attached:
+ self._idle_since = None
+ continue
+ if self._idle_since is None:
+ self._idle_since = now
+ elif now - self._idle_since >= self._idle_exit_s:
+ logger.info("Turn host idle for {:.0f} s — exiting", now - self._idle_since)
+ self._stop.set()
+
+ def _spool_unclaimed(self) -> None:
+ """Write every ended turn nobody collected to disk for the next app start."""
+ for turn in list(self._turns.values()):
+ if turn.alive:
+ continue
+ try:
+ self._spool_dir.mkdir(parents=True, exist_ok=True)
+ target = self._spool_dir / f"{turn.turn_id}.json"
+ tmp = target.with_name(target.name + ".tmp")
+ tmp.write_text(json.dumps(turn.spool_record(), ensure_ascii=False), "utf-8")
+ os.replace(tmp, target)
+ logger.info("Turn host: spooled ended turn {} for the next app start", turn.turn_id)
+ except OSError as exc:
+ logger.warning("Turn host: could not spool turn {}: {}", turn.turn_id, exc)
+
+ # ----------------------------------------------------------- connection
+ async def _on_connect(
+ self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter
+ ) -> None:
+ client = _Client(reader, writer)
+ try:
+ hello = await asyncio.wait_for(_read_frame(reader), timeout=10.0)
+ except (TimeoutError, ValueError, ConnectionError, OSError) as exc:
+ logger.info("Turn host: connection dropped before hello: {}", exc)
+ client.close()
+ return
+ if (
+ not isinstance(hello, dict)
+ or hello.get("op") != "hello"
+ or not hmac.compare_digest(str(hello.get("token", "")), self._token)
+ ):
+ logger.warning("Turn host: refused a connection with a bad hello")
+ client.close()
+ return
+ if hello.get("proto") != PROTOCOL_VERSION:
+ await client.send(
+ {
+ "rid": hello.get("rid"),
+ "ok": False,
+ "error": "protocol",
+ "proto": PROTOCOL_VERSION,
+ }
+ )
+ client.close()
+ return
+ previous = self._client
+ if previous is not None and not previous.closed:
+ logger.info("Turn host: a new client replaces the attached one")
+ # Told first, so the old app hands its turns over instead of
+ # reporting them as failed when its connection drops.
+ await previous.send({"ev": "replaced"})
+ previous.close()
+ for turn in self._turns.values():
+ turn.streaming = False
+ self._client = client
+ await client.send(
+ {
+ "rid": hello.get("rid"),
+ "ok": True,
+ "pid": os.getpid(),
+ "proto": PROTOCOL_VERSION,
+ "turns": [turn.summary() for turn in self._turns.values()],
+ }
+ )
+ try:
+ while not client.closed:
+ try:
+ frame = await _read_frame(reader)
+ except (ValueError, ConnectionError, OSError) as exc:
+ logger.info("Turn host: client disconnected: {}", exc)
+ break
+ if frame is None:
+ break
+ await self._handle(client, frame)
+ finally:
+ client.close()
+ if self._client is client:
+ self._client = None
+ for turn in self._turns.values():
+ turn.streaming = False
+ logger.info(
+ "Turn host: client detached — {} CLI(s) keep running",
+ sum(1 for turn in self._turns.values() if turn.alive),
+ )
+
+ async def _handle(self, client: _Client, frame: dict[str, Any]) -> None:
+ op = frame.get("op")
+ rid = frame.get("rid")
+ turn = self._turns.get(str(frame.get("id", "")))
+ reply: dict[str, Any] | None
+ try:
+ if op == "spawn":
+ reply = await self._spawn(frame)
+ elif op == "attach":
+ # Answers itself: the reply must precede the replayed lines.
+ await self._attach(client, rid, turn)
+ return
+ elif op == "write":
+ if turn is not None:
+ await _write_stdin(turn, str(frame.get("d", "")))
+ reply = {}
+ elif op == "close_stdin":
+ if turn is not None:
+ _close_stdin(turn)
+ reply = {}
+ elif op == "kill":
+ if turn is not None:
+ _kill(turn.proc)
+ reply = {}
+ elif op == "ack":
+ if turn is not None:
+ turn.acked = max(turn.acked, int(frame.get("seq") or 0))
+ reply = None
+ elif op == "release":
+ if turn is not None and not turn.alive:
+ self._turns.pop(turn.turn_id, None)
+ reply = {}
+ elif op == "list":
+ reply = {"turns": [t.summary() for t in self._turns.values()]}
+ elif op == "ping":
+ reply = {}
+ else:
+ reply = {"ok": False, "error": f"unknown op {op!r}"}
+ except Exception as exc: # noqa: BLE001 - one bad request must not end the host
+ logger.warning("Turn host: {} failed: {}", op, exc)
+ reply = {"ok": False, "error": str(exc)}
+ if rid is not None and reply is not None:
+ reply.setdefault("ok", True)
+ reply["rid"] = rid
+ await client.send(reply)
+
+ # ------------------------------------------------------------ requests
+ async def _spawn(self, frame: dict[str, Any]) -> dict[str, Any]:
+ from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+
+ argv = [str(part) for part in frame.get("argv") or ()]
+ if not argv:
+ return {"ok": False, "error": "empty argv"}
+ env = frame.get("env")
+ stdin_data = frame.get("stdin")
+ proc = await asyncio.create_subprocess_exec(
+ *argv,
+ cwd=frame.get("cwd") or None,
+ env=dict(env) if isinstance(env, dict) else None,
+ stdin=asyncio.subprocess.PIPE if stdin_data is not None else asyncio.subprocess.DEVNULL,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ # Its own process group on POSIX: the container reaps by group,
+ # and the host's own group must never be the one it signals.
+ start_new_session=os.name != "nt",
+ limit=LINE_LIMIT,
+ )
+ try:
+ self._tree.assign(proc.pid)
+ except Exception as exc: # noqa: BLE001 - containment is best effort, the turn runs
+ logger.debug("Turn host: could not contain pid {}: {}", proc.pid, exc)
+ turn = HostedTurn(
+ turn_id=uuid.uuid4().hex,
+ proc=proc,
+ meta=dict(frame.get("meta") or {}),
+ started_at=time.time(),
+ # The spawning client reads from the first line on.
+ streaming=True,
+ )
+ self._turns[turn.turn_id] = turn
+ if stdin_data is not None:
+ await _write_stdin(turn, str(stdin_data))
+ if not frame.get("keep_stdin"):
+ _close_stdin(turn)
+ turn.pumps = [
+ asyncio.create_task(self._pump_stdout(turn), name=f"turn-out-{turn.turn_id[:8]}"),
+ asyncio.create_task(self._pump_stderr(turn), name=f"turn-err-{turn.turn_id[:8]}"),
+ ]
+ asyncio.create_task(self._reap(turn), name=f"turn-reap-{turn.turn_id[:8]}")
+ logger.info("Turn host: started turn {} (pid {})", turn.turn_id, proc.pid)
+ return {"id": turn.turn_id, "pid": proc.pid}
+
+ async def _attach(self, client: _Client, rid: Any, turn: HostedTurn | None) -> None:
+ if turn is None:
+ await client.send({"rid": rid, "ok": True, "known": False})
+ return
+ turn.streaming = False
+ await client.send(
+ {
+ "rid": rid,
+ "ok": True,
+ "known": True,
+ "pid": turn.proc.pid,
+ "meta": turn.meta,
+ "acked": turn.acked,
+ "stderr": list(turn.stderr_tail),
+ "truncated": turn.truncated,
+ }
+ )
+ # Replay every kept line in order, then switch to live streaming with
+ # no await in between: a line the pump appends while a send is in
+ # flight is picked up by the next round of this loop, never lost.
+ sent = 0
+ while sent < len(turn.lines):
+ seq = sent + 1
+ await client.send(
+ {"ev": "line", "id": turn.turn_id, "seq": seq, "d": turn.lines[sent]}
+ )
+ sent = seq
+ if client.closed:
+ return
+ turn.streaming = True
+ if not turn.alive:
+ await client.send({"ev": "exit", "id": turn.turn_id, "code": turn.exit_code})
+
+ # --------------------------------------------------------------- pumps
+ async def _pump_stdout(self, turn: HostedTurn) -> None:
+ stream = turn.proc.stdout
+ assert stream is not None
+ while True:
+ try:
+ raw = await stream.readline()
+ except (ValueError, asyncio.LimitOverrunError):
+ # A line past LINE_LIMIT: drop it, keep reading.
+ logger.warning("Turn host: dropped an over-long line of turn {}", turn.turn_id)
+ continue
+ if not raw:
+ return
+ text = raw.decode("utf-8", errors="replace").rstrip("\r\n")
+ if turn.kept_bytes + len(raw) > MAX_KEPT_BYTES:
+ turn.truncated = True
+ continue
+ turn.lines.append(text)
+ turn.kept_bytes += len(raw)
+ client = self._client
+ if turn.streaming and client is not None and not client.closed:
+ await client.send(
+ {"ev": "line", "id": turn.turn_id, "seq": len(turn.lines), "d": text}
+ )
+
+ async def _pump_stderr(self, turn: HostedTurn) -> None:
+ stream = turn.proc.stderr
+ assert stream is not None
+ while True:
+ try:
+ raw = await stream.readline()
+ except (ValueError, asyncio.LimitOverrunError):
+ continue
+ if not raw:
+ return
+ text = raw.decode("utf-8", errors="replace").rstrip()
+ if not text:
+ continue
+ turn.stderr_tail.append(text)
+ client = self._client
+ if turn.streaming and client is not None and not client.closed:
+ await client.send({"ev": "err", "id": turn.turn_id, "d": text})
+
+ async def _reap(self, turn: HostedTurn) -> None:
+ code = await turn.proc.wait()
+ # Every line first: the exit must never overtake the answer.
+ await asyncio.gather(*turn.pumps, return_exceptions=True)
+ turn.exit_code = int(code)
+ logger.info("Turn host: turn {} ended (exit {})", turn.turn_id, code)
+ client = self._client
+ if turn.streaming and client is not None and not client.closed:
+ await client.send({"ev": "exit", "id": turn.turn_id, "code": turn.exit_code})
+
+
+# ---------------------------------------------------------------- helpers
+async def _write_stdin(turn: HostedTurn, text: str) -> None:
+ stdin = turn.proc.stdin
+ if stdin is None or stdin.is_closing():
+ return
+ try:
+ stdin.write(text.encode("utf-8"))
+ await stdin.drain()
+ except (BrokenPipeError, ConnectionResetError, OSError) as exc:
+ logger.debug("Turn host: stdin write for {} failed: {}", turn.turn_id, exc)
+
+
+def _close_stdin(turn: HostedTurn) -> None:
+ stdin = turn.proc.stdin
+ if stdin is None or stdin.is_closing():
+ return
+ try:
+ stdin.close()
+ except OSError as exc:
+ logger.debug("Turn host: stdin close for {} failed: {}", turn.turn_id, exc)
+
+
+def _kill(proc: asyncio.subprocess.Process) -> None:
+ if proc.returncode is not None:
+ return
+ try:
+ proc.kill()
+ except (ProcessLookupError, OSError) as exc:
+ logger.debug("Turn host: killing pid {} failed: {}", proc.pid, exc)
+
+
+async def _read_frame(reader: asyncio.StreamReader) -> dict[str, Any] | None:
+ line = await reader.readline()
+ if not line:
+ return None
+ frame = json.loads(line.decode("utf-8"))
+ if not isinstance(frame, dict):
+ raise ValueError("frame is not an object")
+ return frame
+
+
+def _write_state(path: Path, data: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ tmp = path.with_name(path.name + f".{os.getpid()}.tmp")
+ tmp.write_text(json.dumps(data), encoding="utf-8")
+ try:
+ os.chmod(tmp, 0o600)
+ except OSError as exc:
+ # Windows keeps per-user ACLs on the data dir; chmod is a POSIX nicety.
+ logger.debug("Turn host: could not restrict the state file: {}", exc)
+ os.replace(tmp, path)
+
+
+def _remove_state(path: Path, pid: int) -> None:
+ try:
+ current = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError): # gone or rewritten by a newer host: not ours
+ return
+ if isinstance(current, dict) and current.get("pid") == pid:
+ try:
+ path.unlink()
+ except OSError as exc:
+ logger.debug("Turn host: state file not removed: {}", exc)
+
+
+def _boot_time() -> float:
+ try:
+ import psutil
+
+ return float(psutil.boot_time())
+ except Exception as exc: # noqa: BLE001 - optional evidence, never fatal
+ logger.debug("Turn host: boot time unavailable: {}", exc)
+ return 0.0
+
+
+def _configure_logging(log_path: Path | None) -> None:
+ logger.remove()
+ if log_path is None:
+ return
+ log_path.parent.mkdir(parents=True, exist_ok=True)
+ logger.add(str(log_path), level="INFO", rotation="2 MB", retention=2, encoding="utf-8")
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(prog="jarvis.agent_chat.turn_host")
+ parser.add_argument("--state", required=True, help="Where to publish the port and token.")
+ parser.add_argument("--spool", required=True, help="Where ended, uncollected turns go.")
+ parser.add_argument("--log", default=None, help="Log file (none when omitted).")
+ parser.add_argument("--idle-exit", type=float, default=IDLE_EXIT_S)
+ parser.add_argument("--token-file", default=None, help="Read the token from this file.")
+ args = parser.parse_args(argv)
+
+ token = os.environ.pop(TOKEN_ENV, "")
+ if not token and args.token_file:
+ token_path = Path(args.token_file)
+ try:
+ token = token_path.read_text(encoding="utf-8").strip()
+ except OSError as exc:
+ print(f"Turn host: token file unreadable: {exc}", file=sys.stderr)
+ finally:
+ try:
+ token_path.unlink()
+ except OSError as exc: # already gone is fine; anything else is reported
+ print(f"Turn host: token file not removed: {exc}", file=sys.stderr)
+ if not token:
+ print("Turn host: no token — refusing to start.", file=sys.stderr)
+ return 2
+ _configure_logging(Path(args.log) if args.log else None)
+ host = TurnHost(token, Path(args.spool), idle_exit_s=args.idle_exit)
+ try:
+ asyncio.run(host.serve(Path(args.state)))
+ except KeyboardInterrupt:
+ # Ctrl+C is the normal way to stop the host by hand.
+ return 0
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/jarvis/agent_chat/turn_host_client.py b/jarvis/agent_chat/turn_host_client.py
new file mode 100644
index 0000000000..d075c16240
--- /dev/null
+++ b/jarvis/agent_chat/turn_host_client.py
@@ -0,0 +1,706 @@
+"""The app's side of the turn host — CLI processes that outlive the app.
+
+:mod:`jarvis.agent_chat.turn_host` explains why a thread's coding CLI runs in a
+process of its own. This module is how ``runner_cli`` talks to it:
+
+* :func:`spawn` starts a CLI in the host and hands back a :class:`HostedCli`,
+ which answers the same calls ``runner_cli`` makes on an
+ ``asyncio.subprocess.Process`` (``stdout.readline``, ``stderr.readline``,
+ ``stdin.write``/``drain``/``close``, ``wait``, ``kill``, ``returncode``,
+ ``pid``) — so the stream translation, approvals and outcome are the exact
+ code a child process gets.
+* Reading the next stdout line acknowledges the previous one: by then the
+ runner has emitted everything it made of it. After an app restart
+ :func:`attach` replays every line again, and ``stdout.replaying`` says which
+ ones were already handled — the runner rebuilds its translator state from
+ them without emitting a second copy.
+* :meth:`HostedCli.detach` lets go of a CLI without ending it (app shutdown).
+ ``kill`` ends it (Stop).
+
+Nothing here raises when the host is missing: :func:`spawn` returns ``None``
+and the caller starts the CLI as its own child, as before. A turn that dies
+with the app is worse than one that survives it, and far better than none.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import json
+import logging
+import secrets
+import sys
+import time
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from .turn_host import MAX_FRAME_BYTES, PROTOCOL_VERSION, TOKEN_ENV
+
+log = logging.getLogger(__name__)
+
+#: The host's ``python -m`` module — also how a live host is recognised.
+MODULE = "jarvis.agent_chat.turn_host"
+
+REQUEST_TIMEOUT_S = 30.0
+START_TIMEOUT_S = 15.0
+#: How long a host that is alive but silent gets before this start gives up
+#: on it (its turns are then collected on a later attempt).
+LIVE_HOST_PATIENCE_S = 10.0
+
+#: The exit code a turn reports when the host itself went away under it.
+HOST_LOST_CODE = -1
+
+
+class TurnHandedOver(RuntimeError):
+ """Another app process attached to the turn host and carries this turn on.
+
+ The runner raises it instead of judging an outcome: the turn is neither
+ done nor failed here, and must stay open for the process that took over.
+ """
+
+
+# --------------------------------------------------------------- locations
+def _suffix() -> str:
+ from jarvis.core.instance import current_instance
+
+ return current_instance().state_file_suffix
+
+
+def _state_path() -> Path:
+ from jarvis.core.paths import user_data_dir
+
+ return user_data_dir() / "agent_chat" / f"turn_host{_suffix()}.json"
+
+
+def spool_dir() -> Path:
+ from jarvis.core.paths import user_data_dir
+
+ return user_data_dir() / "agent_chat" / f"turn_host{_suffix()}_spool"
+
+
+def _log_path() -> Path:
+ from jarvis.core.paths import user_data_dir
+
+ return user_data_dir() / "logs" / f"turn_host{_suffix()}.log"
+
+
+def host_available() -> bool:
+ """Does this process keep thread CLIs in the turn host?
+
+ Only a real app process does: the desktop shell and the web launcher turn
+ on ``agentic_ide.host_mode`` (the same switch that keeps the terminal
+ grid's agents in the PTY host). A test, a script or the CLI building an
+ agent-chat service never starts or attaches to the user's host. A frozen
+ build has no ``python -m`` to start one with.
+ """
+ from jarvis.agentic_ide import host_mode
+
+ return host_mode.enabled() and not getattr(sys, "frozen", False)
+
+
+def _read_state(path: Path) -> dict[str, Any] | None:
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError): # no readable state file means no host
+ return None
+ return data if isinstance(data, dict) else None
+
+
+def _host_alive(state: dict[str, Any] | None) -> bool:
+ from jarvis.terminal.pty_host_client import host_is_alive
+
+ return host_is_alive(state, module=MODULE)
+
+
+def may_hold_turns() -> bool:
+ """Is there anything to collect: a live host, or a turn it spooled to disk?
+
+ Synchronous and cheap (a file read and a process lookup), so a service can
+ ask it while it decides which open turns a restart really orphaned.
+ """
+ if not host_available():
+ return False
+ try:
+ if any(spool_dir().glob("*.json")):
+ return True
+ except OSError as exc:
+ log.debug("turn host: spool unreadable: %s", exc)
+ return _host_alive(_read_state(_state_path()))
+
+
+# ---------------------------------------------------------- the process API
+class _LineStream:
+ """``readline`` over lines the host sends, numbered from 1."""
+
+ def __init__(self, before_read: Callable[[], None] | None = None) -> None:
+ self._queue: asyncio.Queue[tuple[int, bytes] | None] = asyncio.Queue()
+ self._before_read = before_read
+ self._eof = False
+ #: Lines up to this ``seq`` were handled before an app restart.
+ self.replay_upto = 0
+ #: Whether the line ``readline`` returned last is such a replay.
+ self.replaying = False
+ self.last_seq = 0
+
+ def feed(self, seq: int, text: str) -> None:
+ if not self._eof:
+ self._queue.put_nowait((seq, (text + "\n").encode("utf-8")))
+
+ def feed_eof(self) -> None:
+ if not self._eof:
+ self._eof = True
+ self._queue.put_nowait(None)
+
+ async def readline(self) -> bytes:
+ if self._before_read is not None:
+ self._before_read()
+ item = await self._queue.get()
+ if item is None:
+ self._queue.put_nowait(None) # every later read sees the end too
+ self.replaying = False
+ return b""
+ seq, data = item
+ self.last_seq = seq
+ self.replaying = 0 < seq <= self.replay_upto
+ return data
+
+
+class _HostStdin:
+ """The CLI's stdin, written through the host."""
+
+ def __init__(self, cli: HostedCli) -> None:
+ self._cli = cli
+ self._closed = False
+
+ def write(self, data: bytes) -> None:
+ if not self._closed:
+ self._cli._send({"op": "write", "d": data.decode("utf-8", errors="replace")})
+
+ async def drain(self) -> None:
+ await asyncio.sleep(0)
+
+ def is_closing(self) -> bool:
+ return self._closed or self._cli.returncode is not None
+
+ def close(self) -> None:
+ if not self._closed:
+ self._closed = True
+ self._cli._send({"op": "close_stdin"})
+
+
+class HostedCli:
+ """One CLI in the turn host, shaped like ``asyncio.subprocess.Process``."""
+
+ hosted = True
+
+ def __init__(
+ self,
+ client: TurnHostClient | None,
+ host_id: str,
+ pid: int,
+ meta: dict[str, Any],
+ *,
+ has_stdin: bool,
+ spool_path: Path | None = None,
+ ) -> None:
+ self._client = client
+ self.host_id = host_id
+ self.pid = pid
+ self.meta = meta
+ self.stdout = _LineStream(self._ack_previous)
+ self.stderr = _LineStream()
+ self.stdin: _HostStdin | None = _HostStdin(self) if has_stdin else None
+ self._exit: asyncio.Future[int] = asyncio.get_running_loop().create_future()
+ self._fed = 0
+ self._acked = 0
+ self.detached = False
+ self.host_lost = False
+ #: Another app process took the host over and carries this turn on.
+ self.handed_over = False
+ self._spool_path = spool_path
+
+ # ------------------------------------------------------- process calls
+ @property
+ def returncode(self) -> int | None:
+ return self._exit.result() if self._exit.done() else None
+
+ async def wait(self) -> int:
+ return await asyncio.shield(self._exit)
+
+ def kill(self) -> None:
+ if self.returncode is None:
+ self._send({"op": "kill"})
+
+ def detach(self) -> None:
+ """Stop reading without ending the CLI: it keeps running in the host."""
+ if self.detached or self.returncode is not None:
+ return
+ self.detached = True
+ if self._client is not None:
+ self._client._forget(self.host_id)
+ self.stdout.feed_eof()
+ self.stderr.feed_eof()
+
+ def release(self) -> None:
+ """The turn is over and handled: the host (or the spool) may drop it."""
+ if self._spool_path is not None:
+ with contextlib.suppress(OSError):
+ self._spool_path.unlink()
+ return
+ if self._client is not None:
+ self._send({"op": "release"})
+ self._client._forget(self.host_id)
+
+ # ----------------------------------------------------------- internals
+ def _send(self, frame: dict[str, Any]) -> None:
+ if self._client is not None and not self.detached:
+ self._client._send({**frame, "id": self.host_id})
+
+ def _ack_previous(self) -> None:
+ seq = self.stdout.last_seq
+ if seq > self._acked:
+ self._acked = seq
+ self._send({"op": "ack", "seq": seq})
+
+ def _on_line(self, seq: int, text: str) -> None:
+ if seq <= self._fed:
+ return
+ self._fed = seq
+ self.stdout.feed(seq, text)
+
+ def _on_err(self, text: str) -> None:
+ self.stderr.feed(0, text)
+
+ def _on_exit(self, code: int) -> None:
+ self.stdout.feed_eof()
+ self.stderr.feed_eof()
+ if not self._exit.done():
+ self._exit.set_result(int(code))
+
+ def _on_lost(self) -> None:
+ self.host_lost = True
+ self._on_exit(HOST_LOST_CODE)
+
+ def _on_handed_over(self) -> None:
+ self.handed_over = True
+ self.detached = True
+ self._on_exit(HOST_LOST_CODE)
+
+
+# ------------------------------------------------------------ the connection
+class TurnHostClient:
+ """One authenticated connection to the turn host."""
+
+ def __init__(
+ self,
+ reader: asyncio.StreamReader,
+ writer: asyncio.StreamWriter,
+ turns: list[dict[str, Any]],
+ host_pid: int,
+ ) -> None:
+ self._reader = reader
+ self._writer = writer
+ self._loop = asyncio.get_running_loop()
+ self._procs: dict[str, HostedCli] = {}
+ # Events for an id whose spawn reply has not been handled yet.
+ self._early: dict[str, list[dict[str, Any]]] = {}
+ self._futures: dict[int, asyncio.Future[dict[str, Any]]] = {}
+ self._next_rid = 1
+ self._closed = False
+ self.host_pid = host_pid
+ self.turns = {str(t.get("id")): t for t in turns if isinstance(t, dict) and t.get("id")}
+ self._reader_task = self._loop.create_task(self._read_loop(), name="turn-host-client")
+
+ @property
+ def connected(self) -> bool:
+ return not self._closed
+
+ async def spawn(
+ self,
+ argv: list[str],
+ *,
+ cwd: str,
+ env: dict[str, str] | None,
+ stdin: str | None,
+ keep_stdin: bool,
+ meta: dict[str, Any],
+ ) -> HostedCli:
+ reply = await self._request(
+ {
+ "op": "spawn",
+ "argv": list(argv),
+ "cwd": cwd,
+ "env": dict(env) if env is not None else None,
+ "stdin": stdin,
+ "keep_stdin": keep_stdin,
+ "meta": meta,
+ }
+ )
+ if not reply.get("ok"):
+ raise RuntimeError(str(reply.get("error") or "The turn host refused the spawn."))
+ cli = HostedCli(
+ self, str(reply["id"]), int(reply.get("pid") or 0), meta, has_stdin=stdin is not None
+ )
+ self._register(cli)
+ return cli
+
+ async def attach(self, host_id: str) -> HostedCli | None:
+ """Take over a turn that was running (or ended) before this client attached."""
+ info = self.turns.get(host_id) or {}
+ meta = dict(info.get("meta") or {})
+ cli = HostedCli(
+ self,
+ host_id,
+ int(info.get("pid") or 0),
+ meta,
+ has_stdin=bool(meta.get("keep_stdin")),
+ )
+ # Registered BEFORE the request: the replayed lines follow the reply
+ # immediately and must find their reader.
+ self._register(cli)
+ reply = await self._request({"op": "attach", "id": host_id})
+ if not reply.get("ok") or not reply.get("known"):
+ self._forget(host_id)
+ return None
+ acked = int(reply.get("acked") or 0)
+ cli.stdout.replay_upto = acked
+ cli._acked = acked
+ for line in reply.get("stderr") or ():
+ cli._on_err(str(line))
+ return cli
+
+ def _register(self, cli: HostedCli) -> None:
+ self._procs[cli.host_id] = cli
+ for frame in self._early.pop(cli.host_id, []):
+ self._deliver(cli, frame)
+
+ def _forget(self, host_id: str) -> None:
+ self._procs.pop(host_id, None)
+ self.turns.pop(host_id, None)
+
+ def live(self) -> list[HostedCli]:
+ return [cli for cli in self._procs.values() if cli.returncode is None]
+
+ def _send(self, frame: dict[str, Any]) -> bool:
+ if self._closed:
+ return False
+ data = (json.dumps(frame, ensure_ascii=False) + "\n").encode("utf-8")
+ try:
+ self._writer.write(data)
+ except (ConnectionError, OSError, RuntimeError) as exc:
+ log.warning("turn host: send failed: %s", exc)
+ self._lost(exc)
+ return False
+ return True
+
+ async def _request(self, frame: dict[str, Any]) -> dict[str, Any]:
+ if self._closed:
+ raise ConnectionError("The turn host is not connected.")
+ rid = self._next_rid
+ self._next_rid += 1
+ future: asyncio.Future[dict[str, Any]] = self._loop.create_future()
+ self._futures[rid] = future
+ if not self._send({**frame, "rid": rid}):
+ self._futures.pop(rid, None)
+ raise ConnectionError("The turn host is not connected.")
+ try:
+ await self._writer.drain()
+ return await asyncio.wait_for(future, timeout=REQUEST_TIMEOUT_S)
+ except TimeoutError as exc:
+ raise ConnectionError("The turn host did not answer in time.") from exc
+ finally:
+ self._futures.pop(rid, None)
+
+ async def _read_loop(self) -> None:
+ error: BaseException | None = None
+ try:
+ while True:
+ line = await self._reader.readline()
+ if not line:
+ break
+ try:
+ frame = json.loads(line.decode("utf-8"))
+ except ValueError as exc:
+ log.warning("turn host: unreadable frame dropped: %s", exc)
+ continue
+ if isinstance(frame, dict):
+ self._dispatch(frame)
+ except asyncio.CancelledError:
+ raise
+ except (ConnectionError, OSError, ValueError) as exc:
+ # ANY read error ends the loop (AP-20).
+ error = exc
+ self._lost(error)
+
+ def _dispatch(self, frame: dict[str, Any]) -> None:
+ rid = frame.get("rid")
+ if rid is not None:
+ future = self._futures.get(int(rid))
+ if future is not None and not future.done():
+ future.set_result(frame)
+ return
+ if frame.get("ev") == "replaced":
+ self._handed_over()
+ return
+ host_id = str(frame.get("id", ""))
+ cli = self._procs.get(host_id)
+ if cli is None:
+ early = self._early.setdefault(host_id, [])
+ if len(early) < 100_000:
+ early.append(frame)
+ return
+ self._deliver(cli, frame)
+
+ @staticmethod
+ def _deliver(cli: HostedCli, frame: dict[str, Any]) -> None:
+ event = frame.get("ev")
+ if event == "line":
+ cli._on_line(int(frame.get("seq") or 0), str(frame.get("d", "")))
+ elif event == "err":
+ cli._on_err(str(frame.get("d", "")))
+ elif event == "exit":
+ code = frame.get("code")
+ cli._on_exit(int(code) if code is not None else HOST_LOST_CODE)
+
+ def _lost(self, error: BaseException | None) -> None:
+ was_open = not self._closed
+ self._closed = True
+ for future in self._futures.values():
+ if not future.done():
+ future.set_exception(ConnectionError("The turn host went away."))
+ self._futures.clear()
+ if was_open:
+ log.warning(
+ "turn host connection lost (%s) — %d turn(s) reported as ended",
+ error or "end of stream",
+ len(self._procs),
+ )
+ for cli in list(self._procs.values()):
+ cli._on_lost()
+ self._procs.clear()
+ self._early.clear()
+
+ def _handed_over(self) -> None:
+ log.info(
+ "turn host: another app process took over — handing %d turn(s) to it",
+ len(self._procs),
+ )
+ procs = list(self._procs.values())
+ self._procs.clear()
+ self._closed = True
+ for future in self._futures.values():
+ if not future.done():
+ future.set_exception(ConnectionError("Another app took the turn host over."))
+ self._futures.clear()
+ for cli in procs:
+ cli._on_handed_over()
+
+ def detach(self) -> None:
+ if self._closed:
+ return
+ self._closed = True
+ with contextlib.suppress(Exception):
+ self._writer.close()
+ self._reader_task.cancel()
+
+
+# --------------------------------------------------------------- discovery
+_client: TurnHostClient | None = None
+_lock: asyncio.Lock | None = None
+_lock_loop: asyncio.AbstractEventLoop | None = None
+
+
+def _connect_lock() -> asyncio.Lock:
+ global _lock, _lock_loop
+ loop = asyncio.get_running_loop()
+ if _lock is None or _lock_loop is not loop:
+ _lock, _lock_loop = asyncio.Lock(), loop
+ return _lock
+
+
+async def _open(*, start: bool) -> TurnHostClient | None:
+ from jarvis.terminal.pty_host_client import _handshake, _start_host
+
+ state_path = _state_path()
+ state = await asyncio.to_thread(_read_state, state_path)
+ alive = await asyncio.to_thread(_host_alive, state)
+ if alive and state is not None:
+ if state.get("proto") != PROTOCOL_VERSION:
+ # A host from another build still holding its turns: never start a
+ # second one beside it. Turns run in-process until it exits.
+ log.info("turn host: a host speaking protocol %s is running", state.get("proto"))
+ return None
+ deadline = time.monotonic() + LIVE_HOST_PATIENCE_S
+ delay = 0.25
+ while True:
+ found = await _handshake(
+ int(state["port"]),
+ str(state["token"]),
+ 3.0,
+ proto=PROTOCOL_VERSION,
+ limit=MAX_FRAME_BYTES,
+ )
+ if found is not None:
+ return _client_from(*found)
+ if time.monotonic() >= deadline or not await asyncio.to_thread(_host_alive, state):
+ break
+ await asyncio.sleep(delay)
+ delay = min(delay * 2, 2.0)
+ if await asyncio.to_thread(_host_alive, state):
+ log.warning("turn host (pid %s) is running but not answering", state.get("pid"))
+ return None
+ if not start:
+ return None
+ token = secrets.token_urlsafe(32)
+ started = await asyncio.to_thread(
+ _start_host,
+ state_path,
+ token,
+ module=MODULE,
+ token_env=TOKEN_ENV,
+ log_path=_log_path(),
+ extra_args=("--spool", str(spool_dir())),
+ )
+ if not started:
+ return None
+ deadline = time.monotonic() + START_TIMEOUT_S
+ while time.monotonic() < deadline:
+ await asyncio.sleep(0.1)
+ state = await asyncio.to_thread(_read_state, state_path)
+ if not state or state.get("token") != token or not state.get("port"):
+ continue
+ found = await _handshake(
+ int(state["port"]), token, 3.0, proto=PROTOCOL_VERSION, limit=MAX_FRAME_BYTES
+ )
+ if found is not None:
+ return _client_from(*found)
+ log.warning("turn host did not come up within %.0f s", START_TIMEOUT_S)
+ return None
+
+
+def _client_from(
+ reader: asyncio.StreamReader, writer: asyncio.StreamWriter, reply: dict[str, Any]
+) -> TurnHostClient:
+ turns = [t for t in reply.get("turns") or () if isinstance(t, dict)]
+ client = TurnHostClient(reader, writer, turns, int(reply.get("pid") or 0))
+ log.info("turn host attached (pid %s) — %d turn(s) held", client.host_pid, len(turns))
+ return client
+
+
+async def get_client(*, start: bool = True) -> TurnHostClient | None:
+ """The connected client, attaching to (or starting) the host on first use."""
+ global _client
+ if not host_available():
+ return None
+ if _client is not None and _client.connected:
+ return _client
+ async with _connect_lock():
+ if _client is not None and _client.connected:
+ return _client
+ try:
+ _client = await _open(start=start)
+ except Exception: # noqa: BLE001 - no host means the CLI runs in-process
+ log.warning("turn host: could not attach", exc_info=True)
+ _client = None
+ return _client
+
+
+async def spawn(
+ argv: list[str],
+ *,
+ cwd: str,
+ env: dict[str, str] | None,
+ stdin: str | None,
+ keep_stdin: bool,
+ meta: dict[str, Any],
+) -> HostedCli | None:
+ """Start ``argv`` in the turn host, or ``None`` when no host can take it."""
+ client = await get_client(start=True)
+ if client is None:
+ return None
+ try:
+ return await client.spawn(
+ argv, cwd=cwd, env=env, stdin=stdin, keep_stdin=keep_stdin, meta=meta
+ )
+ except (ConnectionError, RuntimeError, OSError) as exc:
+ log.warning("turn host: spawn refused (%s) — the CLI runs in-process", exc)
+ return None
+
+
+def live_turn_ids() -> set[str]:
+ """Chat turn ids whose CLI runs in the host and is still being read."""
+ client = _client
+ if client is None or not client.connected:
+ return set()
+ return {str(cli.meta.get("turn_id") or "") for cli in client.live()} - {""}
+
+
+def detach_all() -> None:
+ """App shutdown: let go of every hosted CLI and the connection. Nothing ends."""
+ global _client
+ client = _client
+ if client is None:
+ return
+ for cli in client.live():
+ cli.detach()
+ client.detach()
+ _client = None
+
+
+def read_spool() -> list[dict[str, Any]]:
+ """Turns the host wrote to disk because they ended while no app was attached."""
+ records: list[dict[str, Any]] = []
+ try:
+ paths = sorted(spool_dir().glob("*.json"))
+ except OSError:
+ return records
+ for path in paths:
+ try:
+ record = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, ValueError) as exc:
+ log.warning("turn host: spooled turn %s unreadable: %s", path.name, exc)
+ continue
+ if isinstance(record, dict):
+ record["spool_path"] = str(path)
+ records.append(record)
+ return records
+
+
+def spooled_cli(record: dict[str, Any]) -> HostedCli:
+ """A spooled turn, read back as an ended process with all its lines."""
+ cli = HostedCli(
+ None,
+ str(record.get("id") or ""),
+ int(record.get("pid") or 0),
+ dict(record.get("meta") or {}),
+ has_stdin=False,
+ spool_path=Path(str(record["spool_path"])),
+ )
+ acked = int(record.get("acked") or 0)
+ cli.stdout.replay_upto = acked
+ cli._acked = acked
+ for seq, line in enumerate(record.get("stdout") or (), start=1):
+ cli._on_line(seq, str(line))
+ for line in record.get("stderr") or ():
+ cli._on_err(str(line))
+ code = record.get("code")
+ cli._on_exit(int(code) if code is not None else HOST_LOST_CODE)
+ return cli
+
+
+__all__ = [
+ "HOST_LOST_CODE",
+ "HostedCli",
+ "TurnHandedOver",
+ "TurnHostClient",
+ "detach_all",
+ "get_client",
+ "host_available",
+ "live_turn_ids",
+ "may_hold_turns",
+ "read_spool",
+ "spawn",
+ "spool_dir",
+ "spooled_cli",
+]
diff --git a/jarvis/terminal/pty_host_client.py b/jarvis/terminal/pty_host_client.py
index 077b6ff5e8..4ad50d2af5 100644
--- a/jarvis/terminal/pty_host_client.py
+++ b/jarvis/terminal/pty_host_client.py
@@ -477,7 +477,9 @@ def _current_boot_time() -> float:
return 0.0
-def host_is_alive(state: dict[str, Any] | None) -> bool:
+def host_is_alive(
+ state: dict[str, Any] | None, *, module: str = "jarvis.terminal.pty_host"
+) -> bool:
"""Is the host this state file describes still running, in THIS boot?
Three facts, all checked: the process exists, it is a PTY host (a pid is
@@ -485,6 +487,9 @@ def host_is_alive(state: dict[str, Any] | None) -> bool:
state file from before a reboot therefore reads as dead however its pid
happens to be reused now — which is what makes "the machine restarted"
a measurement instead of a guess.
+
+ ``module`` names the host's ``python -m`` module: the turn host
+ (``jarvis.agent_chat.turn_host``) is told apart from this one the same way.
"""
if not state or not state.get("pid"):
return False
@@ -497,7 +502,7 @@ def host_is_alive(state: dict[str, Any] | None) -> bool:
except Exception as exc: # noqa: BLE001 - gone, not ours, or unreadable: not alive
logger.debug("PTY host pid {} is not a live host: {}", state.get("pid"), exc)
return False
- if "jarvis.terminal.pty_host" not in cmdline:
+ if module not in cmdline:
return False
boot = _current_boot_time()
recorded = float(state.get("boot_time") or 0.0)
@@ -515,17 +520,22 @@ def _read_state(path: Path) -> dict[str, Any] | None:
async def _handshake(
- port: int, token: str, wait_s: float
+ port: int,
+ token: str,
+ wait_s: float,
+ *,
+ proto: int = PROTOCOL_VERSION,
+ limit: int = MAX_FRAME_BYTES,
) -> tuple[asyncio.StreamReader, asyncio.StreamWriter, dict[str, Any]] | None:
try:
reader, writer = await asyncio.wait_for(
- asyncio.open_connection("127.0.0.1", port, limit=MAX_FRAME_BYTES), timeout=wait_s
+ asyncio.open_connection("127.0.0.1", port, limit=limit), timeout=wait_s
)
except (OSError, TimeoutError) as exc:
logger.debug("PTY host: no answer on port {}: {}", port, exc)
return None
try:
- hello = {"op": "hello", "token": token, "proto": PROTOCOL_VERSION, "rid": 0}
+ hello = {"op": "hello", "token": token, "proto": proto, "rid": 0}
writer.write((json.dumps(hello) + "\n").encode("utf-8"))
await writer.drain()
line = await asyncio.wait_for(reader.readline(), timeout=wait_s)
@@ -573,14 +583,23 @@ def host_available() -> bool:
return bool(detect_capabilities().has_pty)
-def _start_host(state_path: Path, token: str) -> bool:
+def _start_host(
+ state_path: Path,
+ token: str,
+ *,
+ module: str = "jarvis.terminal.pty_host",
+ token_env: str = TOKEN_ENV,
+ log_path: Path | None = None,
+ extra_args: tuple[str, ...] = (),
+) -> bool:
+ """Start a detached host process; the turn host reuses this with its own module."""
import jarvis
from jarvis.ui.relauncher import spawn_detached
# The host must import THIS jarvis, whichever interpreter path found it.
package_root = str(Path(jarvis.__file__).resolve().parent.parent)
env = dict(os.environ)
- env[TOKEN_ENV] = token
+ env[token_env] = token
env["PYTHONPATH"] = os.pathsep.join(
part for part in (package_root, env.get("PYTHONPATH", "")) if part
)
@@ -588,11 +607,12 @@ def _start_host(state_path: Path, token: str) -> bool:
argv = [
sys.executable,
"-m",
- "jarvis.terminal.pty_host",
+ module,
"--state",
str(state_path),
"--log",
- str(_log_path()),
+ str(log_path or _log_path()),
+ *extra_args,
]
if sys.platform == "win32":
return _start_host_windows(argv, package_root, env, token, state_path)
diff --git a/tests/unit/agent_chat/test_turn_host.py b/tests/unit/agent_chat/test_turn_host.py
new file mode 100644
index 0000000000..b3bf61c8a5
--- /dev/null
+++ b/tests/unit/agent_chat/test_turn_host.py
@@ -0,0 +1,329 @@
+"""The turn host — thread CLIs that outlive the app, and carrying their turns on."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from jarvis.agent_chat import runner_api, turn_host, turn_host_client
+from jarvis.agent_chat.runner_cli import resume_hosted_cli_turn
+from jarvis.agent_chat.service import AgentChatService
+from jarvis.agent_chat.store import AgentChatStore
+from jarvis.agentic_ide import host_mode
+from jarvis.terminal.pty_host_client import _handshake
+
+TOKEN = "test-token" # noqa: S105 - a fixture value
+
+# Prints what it got on stdin, then a second line, waits for "go", prints a
+# third line and exits 3 — enough to tell replayed, live and written lines apart.
+_CHILD = (
+ "import sys\n"
+ "print('got ' + sys.stdin.readline().strip(), flush=True)\n"
+ "print('two', flush=True)\n"
+ "sys.stdin.readline()\n"
+ "print('three', flush=True)\n"
+ "sys.exit(3)\n"
+)
+
+
+async def _start_host(
+ tmp_path: Path, *, idle_exit_s: float = 60.0
+) -> tuple[turn_host.TurnHost, asyncio.Task[None], int]:
+ host = turn_host.TurnHost(TOKEN, tmp_path / "spool", idle_exit_s=idle_exit_s)
+ state = tmp_path / "state.json"
+ task = asyncio.create_task(host.serve(state))
+ for _ in range(200):
+ if state.exists():
+ break
+ await asyncio.sleep(0.02)
+ return host, task, int(json.loads(state.read_text(encoding="utf-8"))["port"])
+
+
+async def _client(port: int) -> turn_host_client.TurnHostClient:
+ found = await _handshake(
+ port,
+ TOKEN,
+ 3.0,
+ proto=turn_host.PROTOCOL_VERSION,
+ limit=turn_host.MAX_FRAME_BYTES,
+ )
+ assert found is not None
+ return turn_host_client._client_from(*found)
+
+
+async def _stop(host: turn_host.TurnHost, task: asyncio.Task[None]) -> None:
+ host._stop.set()
+ await asyncio.wait_for(task, timeout=10)
+
+
+async def test_a_cli_keeps_running_when_the_app_detaches_and_replays_to_the_next(
+ tmp_path: Path,
+) -> None:
+ host, task, port = await _start_host(tmp_path)
+ try:
+ first = await _client(port)
+ cli = await first.spawn(
+ [sys.executable, "-c", _CHILD],
+ cwd=str(tmp_path),
+ env=None,
+ stdin="hello\n",
+ keep_stdin=True,
+ meta={"turn_id": "t1", "keep_stdin": True},
+ )
+ assert await cli.stdout.readline() == b"got hello\n"
+ assert await cli.stdout.readline() == b"two\n" # acknowledges line 1
+ await asyncio.sleep(0.2)
+ # The app quits: nothing ends, the CLI waits for its "go".
+ first.detach()
+ await asyncio.sleep(0.2)
+
+ second = await _client(port)
+ assert [t["meta"]["turn_id"] for t in second.turns.values()] == ["t1"]
+ again = await second.attach(cli.host_id)
+ assert again is not None
+ assert await again.stdout.readline() == b"got hello\n"
+ assert again.stdout.replaying # handled before the restart: state only
+ assert await again.stdout.readline() == b"two\n"
+ assert not again.stdout.replaying # never acknowledged: handled again
+ assert again.stdin is not None
+ again.stdin.write(b"go\n")
+ await again.stdin.drain()
+ assert await again.stdout.readline() == b"three\n"
+ assert await again.stdout.readline() == b""
+ assert await asyncio.wait_for(again.wait(), timeout=10) == 3
+ again.release()
+ await asyncio.sleep(0.2)
+ assert host._turns == {}
+ second.detach()
+ finally:
+ await _stop(host, task)
+
+
+async def test_a_second_app_process_takes_the_turn_over_without_ending_it(
+ tmp_path: Path,
+) -> None:
+ host, task, port = await _start_host(tmp_path)
+ try:
+ first = await _client(port)
+ cli = await first.spawn(
+ [sys.executable, "-c", _CHILD],
+ cwd=str(tmp_path),
+ env=None,
+ stdin="hi\n",
+ keep_stdin=True,
+ meta={"turn_id": "t4", "keep_stdin": True},
+ )
+ assert await cli.stdout.readline() == b"got hi\n"
+
+ second = await _client(port)
+ await asyncio.wait_for(cli.wait(), timeout=5)
+ assert cli.handed_over and cli.detached # not an ending: no kill, no outcome
+ again = await second.attach(cli.host_id)
+ assert again is not None
+ assert again.stdin is not None
+ again.stdin.write(b"go\n")
+ lines = [await again.stdout.readline() for _ in range(4)]
+ assert lines == [b"got hi\n", b"two\n", b"three\n", b""]
+ assert await asyncio.wait_for(again.wait(), timeout=10) == 3
+ second.detach()
+ finally:
+ await _stop(host, task)
+
+
+async def test_a_turn_that_ends_with_no_app_attached_is_spooled_for_the_next_start(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(turn_host, "_IDLE_POLL_S", 0.05)
+ host, task, port = await _start_host(tmp_path, idle_exit_s=0.2)
+ client = await _client(port)
+ await client.spawn(
+ [sys.executable, "-c", "print('a', flush=True)"],
+ cwd=str(tmp_path),
+ env=None,
+ stdin=None,
+ keep_stdin=False,
+ meta={"turn_id": "t2"},
+ )
+ client.detach()
+ await asyncio.wait_for(task, timeout=15) # idle: spools and exits on its own
+ del host
+
+ monkeypatch.setattr(turn_host_client, "spool_dir", lambda: tmp_path / "spool")
+ [record] = turn_host_client.read_spool()
+ assert record["meta"] == {"turn_id": "t2"}
+ cli = turn_host_client.spooled_cli(record)
+ assert await cli.stdout.readline() == b"a\n"
+ assert not cli.stdout.replaying
+ assert await cli.stdout.readline() == b""
+ assert await cli.wait() == 0
+ cli.release()
+ assert turn_host_client.read_spool() == []
+
+
+_CLAUDE_LINES: list[dict[str, Any]] = [
+ {"type": "system", "subtype": "init", "session_id": "sess-1"},
+ {"type": "assistant", "message": {"id": "m1", "content": [{"type": "text", "text": "hello"}]}},
+ {
+ "type": "assistant",
+ "message": {
+ "id": "m1",
+ "content": [
+ {"type": "tool_use", "id": "tu1", "name": "Bash", "input": {"command": "ls"}}
+ ],
+ },
+ },
+ {
+ "type": "user",
+ "message": {
+ "content": [
+ {"type": "tool_result", "tool_use_id": "tu1", "content": "a b", "is_error": False}
+ ]
+ },
+ },
+ {"type": "assistant", "message": {"id": "m2", "content": [{"type": "text", "text": "done"}]}},
+ {
+ "type": "result",
+ "subtype": "success",
+ "is_error": False,
+ "total_cost_usd": 0.01,
+ "usage": {"output_tokens": 5},
+ },
+]
+
+
+def _spool_record(tmp_path: Path, *, session_id: str, turn_id: str, acked: int) -> dict[str, Any]:
+ path = tmp_path / "spool" / f"host-{turn_id}.json"
+ path.parent.mkdir(parents=True, exist_ok=True)
+ record = {
+ "id": f"host-{turn_id}",
+ "pid": 1,
+ "meta": {
+ "session_id": session_id,
+ "turn_id": turn_id,
+ "runner": "claude-cli",
+ "shape": "claude",
+ "keep_stdin": True,
+ "control": True,
+ "vendor_session": "sess-1",
+ "discover": False,
+ "codex_home": "",
+ "cwd": str(tmp_path),
+ "started_at": time.time() - 5,
+ },
+ "alive": False,
+ "code": 0,
+ "acked": acked,
+ "stdout": [json.dumps(line) for line in _CLAUDE_LINES],
+ "stderr": [],
+ "truncated": False,
+ }
+ path.write_text(json.dumps(record), encoding="utf-8")
+ return {**record, "spool_path": str(path)}
+
+
+async def test_a_carried_on_turn_emits_only_what_the_old_app_never_saw(tmp_path: Path) -> None:
+ store = AgentChatStore(":memory:")
+ try:
+ session = store.create_session(
+ provider="claude-api", model="m", effort="medium", cwd=str(tmp_path), surface="agent"
+ )
+ events: list[dict[str, Any]] = []
+
+ async def emit(event: dict[str, Any]) -> None:
+ events.append(event)
+
+ async def no_approval(*_args: Any) -> str:
+ pytest.fail("a replayed turn must not ask again")
+
+ handle = runner_api.TurnHandle(
+ session=session,
+ turn_id="t3",
+ emit=emit,
+ request_approval=no_approval,
+ cancel=asyncio.Event(),
+ )
+ # The old app handled the init line, "hello" and the tool call.
+ record = _spool_record(tmp_path, session_id=session.session_id, turn_id="t3", acked=3)
+ vendor = await resume_hosted_cli_turn(handle, turn_host_client.spooled_cli(record))
+
+ kinds = [e["kind"] for e in events]
+ texts = [e["payload"]["text"] for e in events if e["kind"] == "assistant_text"]
+ assert texts == ["done"]
+ assert "tool_call" not in kinds
+ assert "tool_result" in kinds
+ assert kinds[-1] == "turn_finished"
+ assert events[-1]["payload"]["status"] == "done"
+ assert events[-1]["payload"]["duration_ms"] >= 5000
+ assert vendor == "sess-1"
+ assert not await asyncio.to_thread(Path(record["spool_path"]).exists)
+ finally:
+ store.close()
+
+
+def app_host_mode() -> Any:
+ host_mode.enable()
+ yield
+ host_mode.reset()
+
+
+async def test_a_restarted_app_carries_on_the_thread_turn_the_host_kept(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, app_host_mode: Any
+) -> None:
+ monkeypatch.setattr(turn_host_client, "spool_dir", lambda: tmp_path / "spool")
+ monkeypatch.setattr(turn_host_client, "_host_alive", lambda state: False)
+
+ async def no_live_host(*, start: bool = True) -> None:
+ return None
+
+ monkeypatch.setattr(turn_host_client, "get_client", no_live_host)
+ db = tmp_path / "agent_chat.db"
+ store = AgentChatStore(db)
+ kept = store.create_session(
+ provider="claude-api", model="m", effort="medium", cwd=str(tmp_path), surface="agent"
+ )
+ lost = store.create_session(
+ provider="claude-api", model="m", effort="medium", cwd=str(tmp_path), surface="agent"
+ )
+ for sid in (kept.session_id, lost.session_id):
+ store.append_event(
+ sid, {"kind": "turn_started", "ts_ms": 1_000, "payload": {"turn_id": f"t-{sid}"}}
+ )
+ _spool_record(tmp_path, session_id=kept.session_id, turn_id=f"t-{kept.session_id}", acked=0)
+
+ svc = AgentChatService(AgentChatStore(db))
+ # Held as running until the host was asked: no second CLI on the conversation.
+ assert svc.is_running(kept.session_id) and svc.is_running(lost.session_id)
+ await svc._reattach_task
+ run = svc._running.get(kept.session_id)
+ if run is not None and run.task is not None:
+ await asyncio.wait_for(run.task, timeout=10)
+
+ finished = store.list_events(kept.session_id)[-1]
+ assert finished["kind"] == "turn_finished"
+ assert finished["payload"]["status"] == "done"
+ texts = [
+ e["payload"]["text"]
+ for e in store.list_events(kept.session_id)
+ if e["kind"] == "assistant_text"
+ ]
+ assert texts == ["hello", "done"]
+ assert store.get_session(kept.session_id).vendor_session == "sess-1" # type: ignore[union-attr]
+
+ sealed = store.list_events(lost.session_id)[-1]
+ assert sealed["kind"] == "turn_finished"
+ assert sealed["payload"]["status"] == "error"
+ assert not svc.is_running(kept.session_id) and not svc.is_running(lost.session_id)
+ assert store.open_turns() == []
+
+
+def test_tests_and_scripts_never_reach_the_users_turn_host() -> None:
+ host_mode.reset()
+ assert not turn_host_client.host_available()
+ assert not turn_host_client.may_hold_turns()