Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(ide): read-only screen feed of coding panes for the office monitors
rubenluetke10-beep71880305e41d
diff --git a/jarvis/agentic_ide/screen_feed.py b/jarvis/agentic_ide/screen_feed.pynew file mode 100644index 0000000000..83ce74472d--- /dev/null+++ b/jarvis/agentic_ide/screen_feed.py@@ -0,0 +1,162 @@+"""Read-only snapshots of pane screens — what the office's monitors show.++The coding floor of the agent office draws every seated coding agent's terminal+on its desk monitor. That picture comes from here: the replayed+:class:`~jarvis.agentic_ide.screen.ScreenBuffer` every pane keeps in its+transcript, copied out as plain rows.++Strictly a READER. Nothing here resizes, attaches to or writes to a PTY — a+monitor is a bystander, and a bystander that nudged the geometry would repaint+the agent's real pane under the person working in it.++Thread safety: the buffer has no lock of its own. Every write to it (the PTY+output callbacks, adoption, resizes) runs on the event loop, so the route that+calls :func:`collect_screens` is ``async`` and copies the rows on that same loop+without awaiting in between — the copy can never interleave with a half-applied+chunk.++Remote panes (an agent running on a connected computer) stream their output+through the same callbacks, so they carry the same local mirror. A pane that+has never produced output simply comes back with no rows.+"""++from __future__ import annotations++from collections.abc import Iterable+from typing import Any, Protocol++#: Panes per request — the office polls only the handful of monitors near its+#: camera, so anything more is a caller bug, not a bigger office.+MAX_PANES = 8+#: Rows per screen. The BOTTOM rows are kept: that is where an agent's prompt,+#: spinner and newest output live.+MAX_ROWS = 60+#: Characters per row; a monitor texture has no use for more.+MAX_COLS = 240+++class _Screen(Protocol):+ cols: int+ rows: int++ def display(self) -> list[str]: ...++ @property+ def visible_cursor(self) -> tuple[int, int] | None: ...+++class _Transcript(Protocol):+ @property+ def screen(self) -> _Screen: ...+++class _Terminal(Protocol):+ key: str+ name: str+ last_output_at: float | None+ started_at: float | None++ @property+ def transcript(self) -> _Transcript: ...+++class _Session(Protocol):+ id: str++ @property+ def terminals(self) -> list[Any]: ...+++class _Registry(Protocol):+ def get(self, workspace_id: str | None) -> Any: ...+++def parse_pane_refs(values: Iterable[str]) -> list[tuple[str, str]]:+ """``"<workspace_id>:<key>"`` strings as pairs — deduplicated, capped.++ Malformed entries (no colon, an empty half) are dropped rather than+ rejected: an unknown pane is simply omitted from the answer, and a garbled+ one is just a pane nobody knows.+ """+ out: list[tuple[str, str]] = []+ seen: set[tuple[str, str]] = set()+ for raw in values:+ workspace_id, sep, key = (raw or "").strip().partition(":")+ if not sep or not workspace_id or not key:+ continue+ pair = (workspace_id, key)+ if pair in seen:+ continue+ seen.add(pair)+ out.append(pair)+ if len(out) >= MAX_PANES:+ break+ return out+++def _find(session: _Session, key: str) -> Any | None:+ """The pane filed under ``key`` — the key first, then its call-sign.++ Exact matches only. ``Session.find`` also resolves spoken phrases, which is+ right for a voice command and wrong for a list row: a near miss would put+ another agent's screen on this agent's monitor.+ """+ for term in session.terminals:+ if term.key == key:+ return term+ folded = key.casefold()+ for term in session.terminals:+ if term.name.casefold() == folded:+ return term+ return None+++def pane_screen(workspace_id: str, term: _Terminal) -> dict[str, Any]:+ """One pane's visible screen, trimmed and capped."""+ screen = term.transcript.screen+ rows = screen.display()+ cursor = screen.visible_cursor+ offset = max(0, len(rows) - MAX_ROWS)+ lines = [row[:MAX_COLS] for row in rows[offset:]]+ if cursor is not None:+ row, col = cursor[0] - offset, cursor[1]+ # A cursor outside the rows handed out has no cell to be drawn in.+ cursor = (row, col) if 0 <= row < MAX_ROWS and 0 <= col < MAX_COLS else None+ return {+ "workspace_id": workspace_id,+ "key": term.key,+ "name": term.name,+ "cols": min(screen.cols, MAX_COLS),+ "rows": min(screen.rows, MAX_ROWS),+ "lines": lines,+ "cursor": list(cursor) if cursor is not None else None,+ # Changes exactly when the screen can have: a monitor redraws on it.+ "at": float(term.last_output_at or term.started_at or 0.0),+ }+++def collect_screens(registry: _Registry, refs: Iterable[tuple[str, str]]) -> list[dict[str, Any]]:+ """Screens for the requested panes, in request order; unknown ones omitted."""+ out: list[dict[str, Any]] = []+ sessions: dict[str, Any | None] = {}+ for workspace_id, key in list(refs)[:MAX_PANES]:+ if workspace_id not in sessions:+ sessions[workspace_id] = registry.get(workspace_id)+ session = sessions[workspace_id]+ if session is None:+ continue+ term = _find(session, key)+ if term is None:+ continue+ out.append(pane_screen(workspace_id, term))+ return out+++__all__ = [+ "MAX_COLS",+ "MAX_PANES",+ "MAX_ROWS",+ "collect_screens",+ "pane_screen",+ "parse_pane_refs",+]diff --git a/jarvis/ui/web/agentic_ide_routes.py b/jarvis/ui/web/agentic_ide_routes.pyindex 505941b45e..c5325b2afa 100644--- a/jarvis/ui/web/agentic_ide_routes.py+++ b/jarvis/ui/web/agentic_ide_routes.py@@ -37,6 +37,7 @@ Endpoints (prefix ``/api/agentic-ide``):* ``POST /fanout`` → run ONE task across several agents(open the panes, divide the work, brief each one, report who was reached)* ``GET /terminals/{name}/report`` → what one named terminal is doing+* ``GET /screens`` → read-only screen snapshots of up to 8 panes* ``PATCH /terminals/{terminal}`` → give that pane another call-sign* ``POST /terminals/{name}/archive`` → hide or restore a chat in the session list* ``GET /terminals/{name}/prompts`` → every prompt handed to that pane@@ -62,13 +63,14 @@ import reimport timefrom dataclasses import asdictfrom pathlib import Path-from typing import Any, Literal, get_args+from typing import Annotated, Any, Literal, get_argsfrom fastapi import (APIRouter,File,Form,HTTPException,+ Query,Request,UploadFile,WebSocket,@@ -90,6 +92,7 @@ from jarvis.agentic_ide import (recap_engine,recents,resume_store,+ screen_feed,workspace_catalog,)from jarvis.agentic_ide.activity import has_work_behind_it@@ -3321,6 +3324,23 @@ def terminal_report(name: str, lines: int = 40) -> dict:raise HTTPException(status_code=404, detail=str(exc)) from exc[email protected]("/screens", summary="Read-only screen snapshots of several panes")+async def pane_screens(+ pane: Annotated[+ list[str] | None,+ Query(description="`<workspace_id>:<key>`, repeatable; at most 8 are read."),+ ] = None,+) -> dict:+ """The visible rows of each requested pane — what the office's monitors draw.++ Unknown panes are omitted rather than failing the whole poll. Async on+ purpose: the screen buffers are written on the event loop, so reading them+ there needs no lock (see :mod:`jarvis.agentic_ide.screen_feed`).+ """+ refs = screen_feed.parse_pane_refs(pane or [])+ return {"screens": screen_feed.collect_screens(get_registry(), refs)}++@router.get("/terminals/{name}/conversation",summary="One terminal's conversation, as the CLI itself recorded it",diff --git a/jarvis/ui/web/frontend/src/lib/paneScreensApi.ts b/jarvis/ui/web/frontend/src/lib/paneScreensApi.tsnew file mode 100644index 0000000000..9df9ddc92a--- /dev/null+++ b/jarvis/ui/web/frontend/src/lib/paneScreensApi.ts@@ -0,0 +1,39 @@+// Read-only screen snapshots of Agentic-IDE panes — what the office's desk+// monitors draw. Plain same-origin fetch like agenticIdeApi, `no-store` because+// WebView2 would otherwise serve a stale screen from cache.++export interface PaneScreen {+ workspace_id: string;+ key: string;+ name: string;+ cols: number;+ rows: number;+ /** Visible rows, right-trimmed; at most 60 rows of 240 characters. */+ lines: string[];+ /** `[row, col]` into `lines`, or null when the agent hides its cursor. */+ cursor: [number, number] | null;+ /** Last output time (epoch seconds) — changes exactly when the screen can. */+ at: number;+}++/** The server reads at most this many panes per call. */+export const MAX_PANE_SCREENS = 8;++/**+ * Screens for the given panes. Unknown panes are simply missing from the+ * answer; more than {@link MAX_PANE_SCREENS} are cut here rather than by the+ * server.+ */+export async function fetchPaneScreens(+ panes: { workspaceId: string; key: string }[],+): Promise<PaneScreen[]> {+ if (panes.length === 0) return [];+ const query = new URLSearchParams();+ for (const pane of panes.slice(0, MAX_PANE_SCREENS)) {+ query.append("pane", `${pane.workspaceId}:${pane.key}`);+ }+ const res = await fetch(`/api/agentic-ide/screens?${query}`, { cache: "no-store" });+ if (!res.ok) throw new Error(`pane screens failed: HTTP ${res.status}`);+ const body = (await res.json()) as { screens?: PaneScreen[] };+ return Array.isArray(body.screens) ? body.screens : [];+}diff --git a/tests/unit/agentic_ide/test_screen_feed.py b/tests/unit/agentic_ide/test_screen_feed.pynew file mode 100644index 0000000000..71f0681e52--- /dev/null+++ b/tests/unit/agentic_ide/test_screen_feed.py@@ -0,0 +1,156 @@+"""`GET /api/agentic-ide/screens` — the office monitors' read-only screen feed.++Pinned here: panes are found by exact key (never a fuzzy call-sign match that+could put another agent's screen on a monitor), unknown panes are omitted, the+caps hold, and reading a screen never touches the pane's geometry.+"""++from __future__ import annotations++from dataclasses import dataclass, field++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient++from jarvis.agentic_ide import screen_feed+from jarvis.agentic_ide import session as ide+from jarvis.agentic_ide.transcript import Transcript+from jarvis.ui.web import agentic_ide_routes++# --------------------------------------------------------------------- fakes+++@dataclass+class _FakeTerm:+ key: str+ name: str+ transcript: Transcript = field(default_factory=Transcript)+ last_output_at: float | None = None+ started_at: float | None = None+++@dataclass+class _FakeSession:+ id: str+ terminals: list[_FakeTerm]+++@dataclass+class _FakeRegistry:+ sessions: dict[str, _FakeSession]+ asked: list[str | None] = field(default_factory=list)++ def get(self, workspace_id: str | None) -> _FakeSession | None:+ self.asked.append(workspace_id)+ return self.sessions.get(workspace_id or "")+++# ---------------------------------------------------------------- unit tests+++def test_parse_drops_garbage_dedups_and_caps() -> None:+ refs = screen_feed.parse_pane_refs(+ ["ws1:t1", "ws1:t1", "nocolon", ":t2", "ws1:", *[f"ws2:t{i}" for i in range(20)]]+ )+ assert refs[0] == ("ws1", "t1")+ assert len(refs) == screen_feed.MAX_PANES+ assert ("ws1", "") not in refs+++def test_screen_rows_cursor_and_timestamp() -> None:+ term = _FakeTerm(key="t1", name="T1", last_output_at=1234.5)+ term.transcript.resize(40, 10)+ term.transcript.feed("hello\r\nworld ")++ shot = screen_feed.pane_screen("ws1", term)++ assert shot["lines"] == ["hello", "world"]+ assert shot["cols"] == 40 and shot["rows"] == 10+ assert shot["cursor"] == [1, 8]+ assert shot["at"] == 1234.5+ assert (shot["workspace_id"], shot["key"], shot["name"]) == ("ws1", "t1", "T1")+++def test_caps_keep_the_bottom_rows_and_shift_the_cursor() -> None:+ term = _FakeTerm(key="t1", name="T1")+ term.transcript.resize(300, 80)+ term.transcript.feed("\r\n".join(f"row{i}".ljust(290, "x") for i in range(80)))++ shot = screen_feed.pane_screen("ws1", term)++ assert len(shot["lines"]) == screen_feed.MAX_ROWS+ assert shot["lines"][-1].startswith("row79")+ assert all(len(line) <= screen_feed.MAX_COLS for line in shot["lines"])+ assert shot["cols"] == screen_feed.MAX_COLS and shot["rows"] == screen_feed.MAX_ROWS+ # The cursor sits past column 240 on the last row: no cell to draw it in.+ assert shot["cursor"] is None+++def test_reading_never_changes_the_geometry() -> None:+ term = _FakeTerm(key="t1", name="T1")+ term.transcript.resize(50, 12)+ term.transcript.feed("x")+ before = (term.transcript.screen.cols, term.transcript.screen.rows)++ screen_feed.pane_screen("ws1", term)++ assert (term.transcript.screen.cols, term.transcript.screen.rows) == before+++def test_collect_matches_exactly_and_omits_unknown() -> None:+ t1 = _FakeTerm(key="t1", name="Frontend")+ t2 = _FakeTerm(key="t2", name="T2")+ registry = _FakeRegistry({"ws1": _FakeSession("ws1", [t1, t2])})++ shots = screen_feed.collect_screens(+ registry,+ [("ws1", "t2"), ("ws1", "frontend"), ("ws1", "terminal two"), ("gone", "t1")],+ )++ assert [(s["key"], s["name"]) for s in shots] == [("t2", "T2"), ("t1", "Frontend")]+ # One lookup per workspace, never the implicit "active" one.+ assert registry.asked == ["ws1", "gone"]+++# ---------------------------------------------------------------- route test+++def client() -> TestClient:+ ide.reset_registry()+ app = FastAPI()+ app.include_router(agentic_ide_routes.router)+ with TestClient(app) as test_client:+ yield test_client+ ide.reset_registry()+++def test_route_serves_the_contract_shape(client, tmp_path) -> None:+ registry = ide.get_registry()+ term = ide.Terminal(key="t1", name="T1", agent="claude", display_name="Claude Code", index=0)+ term.transcript.resize(30, 6)+ term.transcript.feed("$ pytest\r\nok")+ session = ide.Session(+ id="ide_test",+ folder=str(tmp_path),+ name="Test",+ profile=ide.probe_project(tmp_path),+ terminals=[term],+ created_at=0.0,+ )+ registry._sessions[session.id] = session # noqa: SLF001 - no spawn in a unit test++ body = client.get(+ "/api/agentic-ide/screens", params=[("pane", "ide_test:t1"), ("pane", "ide_test:t9")]+ ).json()++ assert list(body) == ["screens"]+ (shot,) = body["screens"]+ assert set(shot) == {"workspace_id", "key", "name", "cols", "rows", "lines", "cursor", "at"}+ assert shot["lines"] == ["$ pytest", "ok"]+ assert shot["cursor"] == [1, 2]+++def test_route_without_panes_is_empty(client) -> None:+ assert client.get("/api/agentic-ide/screens").json() == {"screens": []}