Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(ide): read-only screen feed of coding panes for the office monitors

rubenluetke10-beep71880305e41d
diff --git a/jarvis/agentic_ide/screen_feed.py b/jarvis/agentic_ide/screen_feed.py
new file mode 100644
index 0000000000..83ce74472d
--- /dev/null
+++ b/jarvis/agentic_ide/screen_feed.py
@@ -0,0 +1,162 @@
+"""Read-only snapshots of pane screens — what the office's monitors show.
+
+The coding floor of the agent office draws every seated coding agent's terminal
+on its desk monitor. That picture comes from here: the replayed
+:class:`~jarvis.agentic_ide.screen.ScreenBuffer` every pane keeps in its
+transcript, copied out as plain rows.
+
+Strictly a READER. Nothing here resizes, attaches to or writes to a PTY — a
+monitor is a bystander, and a bystander that nudged the geometry would repaint
+the agent's real pane under the person working in it.
+
+Thread safety: the buffer has no lock of its own. Every write to it (the PTY
+output callbacks, adoption, resizes) runs on the event loop, so the route that
+calls :func:`collect_screens` is ``async`` and copies the rows on that same loop
+without awaiting in between — the copy can never interleave with a half-applied
+chunk.
+
+Remote panes (an agent running on a connected computer) stream their output
+through the same callbacks, so they carry the same local mirror. A pane that
+has never produced output simply comes back with no rows.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any, Protocol
+
+#: Panes per request — the office polls only the handful of monitors near its
+#: camera, so anything more is a caller bug, not a bigger office.
+MAX_PANES = 8
+#: Rows per screen. The BOTTOM rows are kept: that is where an agent's prompt,
+#: spinner and newest output live.
+MAX_ROWS = 60
+#: Characters per row; a monitor texture has no use for more.
+MAX_COLS = 240
+
+
+class _Screen(Protocol):
+ cols: int
+ rows: int
+
+ def display(self) -> list[str]: ...
+
+ @property
+ def visible_cursor(self) -> tuple[int, int] | None: ...
+
+
+class _Transcript(Protocol):
+ @property
+ def screen(self) -> _Screen: ...
+
+
+class _Terminal(Protocol):
+ key: str
+ name: str
+ last_output_at: float | None
+ started_at: float | None
+
+ @property
+ def transcript(self) -> _Transcript: ...
+
+
+class _Session(Protocol):
+ id: str
+
+ @property
+ def terminals(self) -> list[Any]: ...
+
+
+class _Registry(Protocol):
+ def get(self, workspace_id: str | None) -> Any: ...
+
+
+def parse_pane_refs(values: Iterable[str]) -> list[tuple[str, str]]:
+ """``"<workspace_id>:<key>"`` strings as pairs — deduplicated, capped.
+
+ Malformed entries (no colon, an empty half) are dropped rather than
+ rejected: an unknown pane is simply omitted from the answer, and a garbled
+ one is just a pane nobody knows.
+ """
+ out: list[tuple[str, str]] = []
+ seen: set[tuple[str, str]] = set()
+ for raw in values:
+ workspace_id, sep, key = (raw or "").strip().partition(":")
+ if not sep or not workspace_id or not key:
+ continue
+ pair = (workspace_id, key)
+ if pair in seen:
+ continue
+ seen.add(pair)
+ out.append(pair)
+ if len(out) >= MAX_PANES:
+ break
+ return out
+
+
+def _find(session: _Session, key: str) -> Any | None:
+ """The pane filed under ``key`` — the key first, then its call-sign.
+
+ Exact matches only. ``Session.find`` also resolves spoken phrases, which is
+ right for a voice command and wrong for a list row: a near miss would put
+ another agent's screen on this agent's monitor.
+ """
+ for term in session.terminals:
+ if term.key == key:
+ return term
+ folded = key.casefold()
+ for term in session.terminals:
+ if term.name.casefold() == folded:
+ return term
+ return None
+
+
+def pane_screen(workspace_id: str, term: _Terminal) -> dict[str, Any]:
+ """One pane's visible screen, trimmed and capped."""
+ screen = term.transcript.screen
+ rows = screen.display()
+ cursor = screen.visible_cursor
+ offset = max(0, len(rows) - MAX_ROWS)
+ lines = [row[:MAX_COLS] for row in rows[offset:]]
+ if cursor is not None:
+ row, col = cursor[0] - offset, cursor[1]
+ # A cursor outside the rows handed out has no cell to be drawn in.
+ cursor = (row, col) if 0 <= row < MAX_ROWS and 0 <= col < MAX_COLS else None
+ return {
+ "workspace_id": workspace_id,
+ "key": term.key,
+ "name": term.name,
+ "cols": min(screen.cols, MAX_COLS),
+ "rows": min(screen.rows, MAX_ROWS),
+ "lines": lines,
+ "cursor": list(cursor) if cursor is not None else None,
+ # Changes exactly when the screen can have: a monitor redraws on it.
+ "at": float(term.last_output_at or term.started_at or 0.0),
+ }
+
+
+def collect_screens(registry: _Registry, refs: Iterable[tuple[str, str]]) -> list[dict[str, Any]]:
+ """Screens for the requested panes, in request order; unknown ones omitted."""
+ out: list[dict[str, Any]] = []
+ sessions: dict[str, Any | None] = {}
+ for workspace_id, key in list(refs)[:MAX_PANES]:
+ if workspace_id not in sessions:
+ sessions[workspace_id] = registry.get(workspace_id)
+ session = sessions[workspace_id]
+ if session is None:
+ continue
+ term = _find(session, key)
+ if term is None:
+ continue
+ out.append(pane_screen(workspace_id, term))
+ return out
+
+
+__all__ = [
+ "MAX_COLS",
+ "MAX_PANES",
+ "MAX_ROWS",
+ "collect_screens",
+ "pane_screen",
+ "parse_pane_refs",
+]
diff --git a/jarvis/ui/web/agentic_ide_routes.py b/jarvis/ui/web/agentic_ide_routes.py
index 505941b45e..c5325b2afa 100644
--- a/jarvis/ui/web/agentic_ide_routes.py
+++ b/jarvis/ui/web/agentic_ide_routes.py
@@ -37,6 +37,7 @@ Endpoints (prefix ``/api/agentic-ide``):
* ``POST /fanout`` → run ONE task across several agents
(open the panes, divide the work, brief each one, report who was reached)
* ``GET /terminals/{name}/report`` → what one named terminal is doing
+* ``GET /screens`` → read-only screen snapshots of up to 8 panes
* ``PATCH /terminals/{terminal}`` → give that pane another call-sign
* ``POST /terminals/{name}/archive`` → hide or restore a chat in the session list
* ``GET /terminals/{name}/prompts`` → every prompt handed to that pane
@@ -62,13 +63,14 @@ import re
import time
from dataclasses import asdict
from pathlib import Path
-from typing import Any, Literal, get_args
+from typing import Annotated, Any, Literal, get_args
from fastapi import (
APIRouter,
File,
Form,
HTTPException,
+ Query,
Request,
UploadFile,
WebSocket,
@@ -90,6 +92,7 @@ from jarvis.agentic_ide import (
recap_engine,
recents,
resume_store,
+ screen_feed,
workspace_catalog,
)
from jarvis.agentic_ide.activity import has_work_behind_it
@@ -3321,6 +3324,23 @@ def terminal_report(name: str, lines: int = 40) -> dict:
raise HTTPException(status_code=404, detail=str(exc)) from exc
[email protected]("/screens", summary="Read-only screen snapshots of several panes")
+async def pane_screens(
+ pane: Annotated[
+ list[str] | None,
+ Query(description="`<workspace_id>:<key>`, repeatable; at most 8 are read."),
+ ] = None,
+) -> dict:
+ """The visible rows of each requested pane — what the office's monitors draw.
+
+ Unknown panes are omitted rather than failing the whole poll. Async on
+ purpose: the screen buffers are written on the event loop, so reading them
+ there needs no lock (see :mod:`jarvis.agentic_ide.screen_feed`).
+ """
+ refs = screen_feed.parse_pane_refs(pane or [])
+ return {"screens": screen_feed.collect_screens(get_registry(), refs)}
+
+
@router.get(
"/terminals/{name}/conversation",
summary="One terminal's conversation, as the CLI itself recorded it",
diff --git a/jarvis/ui/web/frontend/src/lib/paneScreensApi.ts b/jarvis/ui/web/frontend/src/lib/paneScreensApi.ts
new file mode 100644
index 0000000000..9df9ddc92a
--- /dev/null
+++ b/jarvis/ui/web/frontend/src/lib/paneScreensApi.ts
@@ -0,0 +1,39 @@
+// Read-only screen snapshots of Agentic-IDE panes — what the office's desk
+// monitors draw. Plain same-origin fetch like agenticIdeApi, `no-store` because
+// WebView2 would otherwise serve a stale screen from cache.
+
+export interface PaneScreen {
+ workspace_id: string;
+ key: string;
+ name: string;
+ cols: number;
+ rows: number;
+ /** Visible rows, right-trimmed; at most 60 rows of 240 characters. */
+ lines: string[];
+ /** `[row, col]` into `lines`, or null when the agent hides its cursor. */
+ cursor: [number, number] | null;
+ /** Last output time (epoch seconds) — changes exactly when the screen can. */
+ at: number;
+}
+
+/** The server reads at most this many panes per call. */
+export const MAX_PANE_SCREENS = 8;
+
+/**
+ * Screens for the given panes. Unknown panes are simply missing from the
+ * answer; more than {@link MAX_PANE_SCREENS} are cut here rather than by the
+ * server.
+ */
+export async function fetchPaneScreens(
+ panes: { workspaceId: string; key: string }[],
+): Promise<PaneScreen[]> {
+ if (panes.length === 0) return [];
+ const query = new URLSearchParams();
+ for (const pane of panes.slice(0, MAX_PANE_SCREENS)) {
+ query.append("pane", `${pane.workspaceId}:${pane.key}`);
+ }
+ const res = await fetch(`/api/agentic-ide/screens?${query}`, { cache: "no-store" });
+ if (!res.ok) throw new Error(`pane screens failed: HTTP ${res.status}`);
+ const body = (await res.json()) as { screens?: PaneScreen[] };
+ return Array.isArray(body.screens) ? body.screens : [];
+}
diff --git a/tests/unit/agentic_ide/test_screen_feed.py b/tests/unit/agentic_ide/test_screen_feed.py
new file mode 100644
index 0000000000..71f0681e52
--- /dev/null
+++ b/tests/unit/agentic_ide/test_screen_feed.py
@@ -0,0 +1,156 @@
+"""`GET /api/agentic-ide/screens` — the office monitors' read-only screen feed.
+
+Pinned here: panes are found by exact key (never a fuzzy call-sign match that
+could put another agent's screen on a monitor), unknown panes are omitted, the
+caps hold, and reading a screen never touches the pane's geometry.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from jarvis.agentic_ide import screen_feed
+from jarvis.agentic_ide import session as ide
+from jarvis.agentic_ide.transcript import Transcript
+from jarvis.ui.web import agentic_ide_routes
+
+# --------------------------------------------------------------------- fakes
+
+
+@dataclass
+class _FakeTerm:
+ key: str
+ name: str
+ transcript: Transcript = field(default_factory=Transcript)
+ last_output_at: float | None = None
+ started_at: float | None = None
+
+
+@dataclass
+class _FakeSession:
+ id: str
+ terminals: list[_FakeTerm]
+
+
+@dataclass
+class _FakeRegistry:
+ sessions: dict[str, _FakeSession]
+ asked: list[str | None] = field(default_factory=list)
+
+ def get(self, workspace_id: str | None) -> _FakeSession | None:
+ self.asked.append(workspace_id)
+ return self.sessions.get(workspace_id or "")
+
+
+# ---------------------------------------------------------------- unit tests
+
+
+def test_parse_drops_garbage_dedups_and_caps() -> None:
+ refs = screen_feed.parse_pane_refs(
+ ["ws1:t1", "ws1:t1", "nocolon", ":t2", "ws1:", *[f"ws2:t{i}" for i in range(20)]]
+ )
+ assert refs[0] == ("ws1", "t1")
+ assert len(refs) == screen_feed.MAX_PANES
+ assert ("ws1", "") not in refs
+
+
+def test_screen_rows_cursor_and_timestamp() -> None:
+ term = _FakeTerm(key="t1", name="T1", last_output_at=1234.5)
+ term.transcript.resize(40, 10)
+ term.transcript.feed("hello\r\nworld ")
+
+ shot = screen_feed.pane_screen("ws1", term)
+
+ assert shot["lines"] == ["hello", "world"]
+ assert shot["cols"] == 40 and shot["rows"] == 10
+ assert shot["cursor"] == [1, 8]
+ assert shot["at"] == 1234.5
+ assert (shot["workspace_id"], shot["key"], shot["name"]) == ("ws1", "t1", "T1")
+
+
+def test_caps_keep_the_bottom_rows_and_shift_the_cursor() -> None:
+ term = _FakeTerm(key="t1", name="T1")
+ term.transcript.resize(300, 80)
+ term.transcript.feed("\r\n".join(f"row{i}".ljust(290, "x") for i in range(80)))
+
+ shot = screen_feed.pane_screen("ws1", term)
+
+ assert len(shot["lines"]) == screen_feed.MAX_ROWS
+ assert shot["lines"][-1].startswith("row79")
+ assert all(len(line) <= screen_feed.MAX_COLS for line in shot["lines"])
+ assert shot["cols"] == screen_feed.MAX_COLS and shot["rows"] == screen_feed.MAX_ROWS
+ # The cursor sits past column 240 on the last row: no cell to draw it in.
+ assert shot["cursor"] is None
+
+
+def test_reading_never_changes_the_geometry() -> None:
+ term = _FakeTerm(key="t1", name="T1")
+ term.transcript.resize(50, 12)
+ term.transcript.feed("x")
+ before = (term.transcript.screen.cols, term.transcript.screen.rows)
+
+ screen_feed.pane_screen("ws1", term)
+
+ assert (term.transcript.screen.cols, term.transcript.screen.rows) == before
+
+
+def test_collect_matches_exactly_and_omits_unknown() -> None:
+ t1 = _FakeTerm(key="t1", name="Frontend")
+ t2 = _FakeTerm(key="t2", name="T2")
+ registry = _FakeRegistry({"ws1": _FakeSession("ws1", [t1, t2])})
+
+ shots = screen_feed.collect_screens(
+ registry,
+ [("ws1", "t2"), ("ws1", "frontend"), ("ws1", "terminal two"), ("gone", "t1")],
+ )
+
+ assert [(s["key"], s["name"]) for s in shots] == [("t2", "T2"), ("t1", "Frontend")]
+ # One lookup per workspace, never the implicit "active" one.
+ assert registry.asked == ["ws1", "gone"]
+
+
+# ---------------------------------------------------------------- route test
+
+
+def client() -> TestClient:
+ ide.reset_registry()
+ app = FastAPI()
+ app.include_router(agentic_ide_routes.router)
+ with TestClient(app) as test_client:
+ yield test_client
+ ide.reset_registry()
+
+
+def test_route_serves_the_contract_shape(client, tmp_path) -> None:
+ registry = ide.get_registry()
+ term = ide.Terminal(key="t1", name="T1", agent="claude", display_name="Claude Code", index=0)
+ term.transcript.resize(30, 6)
+ term.transcript.feed("$ pytest\r\nok")
+ session = ide.Session(
+ id="ide_test",
+ folder=str(tmp_path),
+ name="Test",
+ profile=ide.probe_project(tmp_path),
+ terminals=[term],
+ created_at=0.0,
+ )
+ registry._sessions[session.id] = session # noqa: SLF001 - no spawn in a unit test
+
+ body = client.get(
+ "/api/agentic-ide/screens", params=[("pane", "ide_test:t1"), ("pane", "ide_test:t9")]
+ ).json()
+
+ assert list(body) == ["screens"]
+ (shot,) = body["screens"]
+ assert set(shot) == {"workspace_id", "key", "name", "cols", "rows", "lines", "cursor", "at"}
+ assert shot["lines"] == ["$ pytest", "ok"]
+ assert shot["cursor"] == [1, 2]
+
+
+def test_route_without_panes_is_empty(client) -> None:
+ assert client.get("/api/agentic-ide/screens").json() == {"screens": []}