Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(voice): show exact content on screen, and open one thing in the app (#641)
Ruben Lütkeb9d709ecdb40
diff --git a/jarvis/brain/tool_gateway.py b/jarvis/brain/tool_gateway.pyindex abe8045a54..8413e80f62 100644--- a/jarvis/brain/tool_gateway.py+++ b/jarvis/brain/tool_gateway.py@@ -166,6 +166,10 @@ class BrainSupervisorToolGateway:# runs, into the MEMORY.md every later conversation reads. Voice-only# like the tools above: the router stays a pure dispatcher (ADR-0011).tools["remember"] = RememberTool()+ # A voice carries a takeaway, not a table: exact content goes on screen.+ from jarvis.plugins.tool.show_on_screen import ShowOnScreenTool++ tools[ShowOnScreenTool.name] = ShowOnScreenTool()if self._workspace_tool is not None:# Live delegates coding to one addressed service. The old prompt# tools silently choose an ambient pane and cannot safely coexist.diff --git a/jarvis/core/events.py b/jarvis/core/events.pyindex 5b0c79015c..c3cf122212 100644--- a/jarvis/core/events.py+++ b/jarvis/core/events.py@@ -129,6 +129,20 @@ class VoiceTranscriptUpdated(Event):revision: int = 1+@dataclass(frozen=True, slots=True)+class VoiceContentShown(Event):+ """Content put on screen during a spoken conversation instead of read out.++ ``markdown`` is shown as a card in the conversation the user is looking at+ (``lib/homeTranscript.ts``). ``session_id`` names the live call it belongs+ to; empty when no live call owns the turn.+ """++ session_id: str = ""+ title: str = ""+ markdown: str = ""++@dataclass(frozen=True, slots=True)class ReasoningSummaryUpdated(Event):"""Provider-exposed reasoning summary, never private reasoning or audio."""@@ -871,8 +885,14 @@ class NavigateSidebar(Event):id is a graceful no-op there. ``section`` mirrors the frontend``SECTION_IDS`` (``store/events.ts``) — kept in sync via the navigate tool'sparity test.++ ``target`` narrows the destination inside the section: ``"agent"`` opens+ the chat of the agent ``target_id``, ``"chat"`` the conversation+ ``target_id``. Empty for a plain section switch."""section: str = ""+ target: str = ""+ target_id: str = ""@dataclass(frozen=True, slots=True)diff --git a/jarvis/live/config.py b/jarvis/live/config.pyindex 0cbb0a8d66..fff7a7c6a6 100644--- a/jarvis/live/config.py+++ b/jarvis/live/config.py@@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, Fieldfrom jarvis.core.agent_brief import AGENT_BRIEF_RULEfrom jarvis.cu.direct import COMPUTER_CONTROL_RULES-from jarvis.live.coordination import WORK_STATUS_RULE, conversation_rules+from jarvis.live.coordination import ON_SCREEN_RULE, WORK_STATUS_RULE, conversation_rulesfrom jarvis.live.product import PRODUCT_BRIEFfrom jarvis.live.recovery import HISTORY_CONTEXT_RULE@@ -122,6 +122,7 @@ class LiveConfig(BaseModel):"workspace-orchestrate create in the named or visible workspace, with cli, ""count and the task as prompt; never an existing agent and never spawn_worker. "+ WORK_STATUS_RULE+ + ON_SCREEN_RULE+ AGENT_BRIEF_RULE+ " "+ (COMPUTER_CONTROL_RULES if computer else "")diff --git a/jarvis/live/coordination.py b/jarvis/live/coordination.pyindex a6d31e5fd4..4c8dace8d5 100644--- a/jarvis/live/coordination.py+++ b/jarvis/live/coordination.py@@ -27,6 +27,16 @@ WORK_STATUS_RULE: Final[str] = ()+#: For the model that executes tools: what goes on screen instead of into speech.+ON_SCREEN_RULE: Final[str] = (+ "When an answer has more structure than a sentence carries (a table, code, a diff, "+ "several links or steps), call show_on_screen with it as Markdown and say only the "+ "takeaway and that the details are on screen. To take the user somewhere in the app "+ "use navigate: a section, a named agent's chat, a conversation by its title, or a "+ "coding workspace. "+)++def conversation_rules(executor: str) -> str:"""The speaking model's rules; ``executor`` is what carries out work for it."""return (@@ -59,4 +69,4 @@ def conversation_rules(executor: str) -> str:)-__all__ = ["WORK_STATUS_RULE", "conversation_rules"]+__all__ = ["ON_SCREEN_RULE", "WORK_STATUS_RULE", "conversation_rules"]diff --git a/jarvis/live/native.py b/jarvis/live/native.pyindex 4c3fbbb2ab..541c210439 100644--- a/jarvis/live/native.py+++ b/jarvis/live/native.py@@ -18,7 +18,7 @@ from jarvis.core.paths import user_data_dirfrom jarvis.core.runtime_refs import get_supervisor_tool_gatewayfrom jarvis.core.tool_budget import VOICE_TOOL_BUDGET_Sfrom jarvis.cu.direct import COMPUTER_CONTROL_RULES-from jarvis.live.coordination import WORK_STATUS_RULE, conversation_rules+from jarvis.live.coordination import ON_SCREEN_RULE, WORK_STATUS_RULE, conversation_rulesfrom jarvis.live.product import PRODUCT_BRIEFfrom jarvis.live.runtime import claim, register, unregisterfrom jarvis.live.session import LiveVoiceSession, _identity@@ -178,6 +178,7 @@ class NativeLiveVoiceSession(LiveVoiceSession):"Do not switch the UI to address another workspace. Reuse request_id on ""retries and never replay uncertain delivery. "+ WORK_STATUS_RULE+ + ON_SCREEN_RULE+ AGENT_BRIEF_RULE),history=tuple(diff --git a/jarvis/live/runtime.py b/jarvis/live/runtime.pyindex 8ab9b7c194..582ba20c10 100644--- a/jarvis/live/runtime.py+++ b/jarvis/live/runtime.py@@ -117,6 +117,19 @@ async def on_session_loop(session: Any, operation: Any, **kwargs: Any) -> Any:return await asyncio.wrap_future(future)+async def show_content(session_id: str, *, title: str, markdown: str) -> bool:+ """Show content in the live call ``session_id``; False when it is not running."""+ session = _active.get(session_id)+ show = getattr(session, "show_content", None)+ if session is None or show is None:+ return False+ try:+ await on_session_loop(session, show, title=title, markdown=markdown)+ except RuntimeError: # the call ended between the lookup and the delivery+ return False+ return True++def owns_microphone(*, except_session_id: str | None = None) -> bool:"""A pending or active browser call must not be replaced by a wake start."""return any(sid != except_session_id for sid in (*_opening, *_active))diff --git a/jarvis/live/session.py b/jarvis/live/session.pyindex fe9edad15f..79fd897432 100644--- a/jarvis/live/session.py+++ b/jarvis/live/session.py@@ -1860,6 +1860,32 @@ class LiveVoiceSession:})+ async def show_content(self, *, title: str, markdown: str) -> None:+ """Put content on the user's screen and keep it in this call's record.++ It is stored as one assistant passage right after the newest words, so+ the reasoning backend sees it as part of the conversation and the+ archived call still shows it. Nothing here is spoken.+ """+ from jarvis.core.events import VoiceContentShown++ if self._ledger is not None:+ # Every live engine keeps its own clock; the record is the common one.+ said = await asyncio.to_thread(self._ledger.transcript, self.session_id)+ at = max((int(row["end_ms"]) for row in said), default=-1) + 1+ body = f"**{title}**\n\n{markdown}" if title else markdown+ fragment = TranscriptFragment(+ self.session_id, f"shown-{uuid4().hex}", "assistant", f"\n\n{body}\n\n", at, at+ )+ await asyncio.to_thread(self._ledger.transcript_snapshot, fragment)+ if self._bus is not None:+ await self._bus.publish(+ VoiceContentShown(+ source_layer="live.session", session_id=self.session_id,+ title=title, markdown=markdown,+ )+ )+async def deliver_announcement(self, text: str, *, report: str | None = None, **kwargs: Any) -> bool:diff --git a/jarvis/plugins/tool/navigate.py b/jarvis/plugins/tool/navigate.pyindex a5d0df3400..127730cf2a 100644--- a/jarvis/plugins/tool/navigate.py+++ b/jarvis/plugins/tool/navigate.py@@ -14,6 +14,9 @@ a parity test (tests/unit/plugins/tool/test_navigate.py) guards against drift.from __future__ import annotations+import asyncio+import logging+from collections.abc import Awaitable, Callablefrom typing import Anyfrom jarvis.core.events import NavigateSidebar@@ -303,8 +306,84 @@ def _resolve(raw: str) -> str | None:return None+log = logging.getLogger(__name__)++# What a target lookup answers: ``found`` with an id and a name, ``ask`` with+# the candidates that fit, ``unknown`` with what exists, or ``unavailable``.+Lookup = dict[str, Any]+#: How many conversations a title is matched against, newest first.+_CHAT_SEARCH_LIMIT = 200+++def _find_agent(spoken: str) -> Lookup:+ """The named agent a spoken reference means, from the live roster."""+ try:+ from jarvis.society.agent_names import display_name, lookup_in+ from jarvis.society.runtime import current_runtime++ runtime = current_runtime()+ if runtime is None:+ return {"status": "unavailable"}+ found = lookup_in(runtime.roster.snapshot(), spoken, surface="navigate")+ except Exception: # noqa: BLE001 - no agent team on this install: say so, do not crash+ log.warning("navigate: agent lookup failed", exc_info=True)+ return {"status": "unavailable"}+ if found.decision == "act" and found.agent is not None:+ return {"status": "found", "id": found.agent.agent_id, "name": display_name(found.agent)}+ if found.decision == "ask" and found.candidates:+ return {"status": "ask", "candidates": [display_name(a) for a in found.candidates]}+ return {"status": "unknown", "available": found.available_names()}+++async def _find_chat(spoken: str) -> Lookup:+ """The conversation with this id or title among the assistant's own chats."""+ from jarvis.core.runtime_refs import get_web_app++ service = getattr(getattr(get_web_app(), "state", None), "agent_chat", None)+ store = getattr(service, "store", None)+ if store is None:+ return {"status": "unavailable"}+ sessions = await asyncio.to_thread(store.list_sessions, _CHAT_SEARCH_LIMIT, surface="jarvis")+ wanted = " ".join(spoken.split()).casefold()+ exact = [+ s for s in sessions+ if s.session_id == spoken.strip() or " ".join(s.title.split()).casefold() == wanted+ ]+ matches = exact or [s for s in sessions if wanted and wanted in s.title.casefold()]+ if len(matches) == 1 or exact:+ # Several chats can carry one title; the newest is the one meant.+ return {"status": "found", "id": matches[0].session_id, "name": matches[0].title}+ if matches:+ return {"status": "ask", "candidates": [s.title for s in matches[:8]]}+ return {"status": "unknown", "available": [s.title for s in sessions[:8] if s.title]}+++async def _show_workspace(spoken: str) -> Lookup:+ """Bring a coding workspace on screen through the workspace service."""+ try:+ from jarvis.agentic_ide.orchestration import get_orchestrator++ result = await get_orchestrator().run({"action": "show", "workspace": spoken})+ except Exception: # noqa: BLE001 - no coding surface on this install: say so+ log.warning("navigate: workspace lookup failed", exc_info=True)+ return {"status": "unavailable"}+ if result.get("status") == "shown":+ return {"status": "found", "id": "", "name": str(result.get("workspace") or spoken)}+ candidates = [+ str(item.get("name") or item.get("workspace") or "")+ for item in result.get("candidates") or []+ if isinstance(item, dict)+ ]+ return {+ "status": "ask" if candidates else "unknown",+ "candidates": candidates,+ "available": candidates,+ "reason": str(result.get("reason") or ""),+ }++class NavigateTool:- """Switch the desktop app's active sidebar section."""+ """Switch the desktop app's active section, or open one thing inside it."""name: str = "navigate"risk_tier: str = "safe"@@ -312,8 +391,10 @@ class NavigateTool:"Open/switch the desktop app to a sidebar section (the UI screen the user ""sees). Use when the user asks to show or open a section — 'zeig die ""Socials', 'open settings', 'show the agents', 'geh zu den Aufgaben'. Pass "- "the section id in 'section'. Do NOT use it for anything other than moving "- "the UI."+ "the section id in 'section'. To open one thing instead, pass exactly one of "+ "agent (a named agent's chat), chat (a conversation, by its title) or "+ "workspace (a coding workspace) and no section. Do NOT use it for anything "+ "other than moving the UI.")schema: dict[str, Any] = {"type": "object",@@ -324,19 +405,54 @@ class NavigateTool:"description": ("The sidebar section id to open (e.g. 'socials', 'settings', 'agents')."),- }+ },+ "agent": {+ "type": "string",+ "description": "A named agent: opens the chat with that agent.",+ },+ "chat": {+ "type": "string",+ "description": "A conversation's title (or id): opens that conversation.",+ },+ "workspace": {+ "type": "string",+ "description": "A coding workspace's name: brings it on screen.",+ },},- "required": ["section"],}- def __init__(self, bus: Any) -> None:+ def __init__(+ self,+ bus: Any,+ *,+ find_agent: Callable[[str], Lookup] = _find_agent,+ find_chat: Callable[[str], Awaitable[Lookup]] = _find_chat,+ show_workspace: Callable[[str], Awaitable[Lookup]] = _show_workspace,+ ) -> None:self._bus = bus+ self._find_agent = find_agent+ self._find_chat = find_chat+ self._show_workspace = show_workspace@classmethoddef known_sections(cls) -> set[str]:return set(KNOWN)async def execute(self, args: dict[str, Any], ctx: Any) -> ToolResult:+ targets = {+ kind: " ".join(str(args.get(kind) or "").split())+ for kind in ("agent", "chat", "workspace")+ }+ named = {kind: value for kind, value in targets.items() if value}+ if len(named) > 1 or (named and str(args.get("section") or "").strip()):+ return ToolResult(+ success=False,+ output={"requested": dict(args)},+ error="Pass one destination: a section, or one of agent, chat, workspace.",+ )+ if named:+ kind, spoken = next(iter(named.items()))+ return await self._open(kind, spoken)section = _resolve(str(args.get("section", "")))if section is None:return ToolResult(@@ -354,3 +470,43 @@ class NavigateTool:success=True,output={"section": section, "summary": f"Opened the {section} section."},)++ async def _open(self, kind: str, spoken: str) -> ToolResult:+ """Resolve one named destination and move the UI to it, or say why not."""+ if kind == "agent":+ found = self._find_agent(spoken)+ elif kind == "chat":+ found = await self._find_chat(spoken)+ else:+ found = await self._show_workspace(spoken)+ status = found.get("status")+ if status != "found":+ if status == "ask":+ error = (+ f"More than one {kind} fits {spoken!r}: "+ + ", ".join(found.get("candidates") or [])+ + ". Ask the user which one."+ )+ elif status == "unknown":+ known = ", ".join(found.get("available") or []) or "none"+ detail = f" {found['reason']}" if found.get("reason") else ""+ error = f"No {kind} is called {spoken!r}. Known: {known}.{detail}"+ else:+ error = f"This app cannot open a {kind} right now."+ return ToolResult(success=False, output={"requested": spoken, **found}, error=error)+ if kind == "workspace":+ # The workspace service already brought it on screen; show its section.+ event = NavigateSidebar(section="agentic-ide", source_layer="brain.tool.navigate")+ else:+ event = NavigateSidebar(+ section="chats", target=kind, target_id=str(found["id"]),+ source_layer="brain.tool.navigate",+ )+ await self._bus.publish(event)+ return ToolResult(+ success=True,+ output={+ "section": event.section, kind: found["name"],+ "summary": f"Opened the {kind} {found['name']}.",+ },+ )diff --git a/jarvis/plugins/tool/show_on_screen.py b/jarvis/plugins/tool/show_on_screen.pynew file mode 100644index 0000000000..b623646bb5--- /dev/null+++ b/jarvis/plugins/tool/show_on_screen.py@@ -0,0 +1,91 @@+"""show_on_screen tool: put exact content in front of the user during a call.++A voice says a takeaway well and a table badly. This tool shows Markdown (a+table, a code block, a diff, a list of links) in the conversation the user is+looking at, so the spoken answer can stay one sentence. Inside a live call the+content also becomes part of that call's record, so it is still there when the+conversation is opened again later.++Risk tier: safe - it only displays what the model was about to say anyway.+"""++from __future__ import annotations++from typing import Any++from jarvis.core.events import VoiceContentShown+from jarvis.core.protocols import ExecutionContext, ToolResult++#: A screenful or two; longer material belongs in a file or an artifact.+MAX_MARKDOWN_CHARS = 20_000+_MAX_TITLE_CHARS = 120+++def _app_bus() -> Any | None:+ from jarvis.core.runtime_refs import get_brain_manager, get_web_app++ bus = getattr(getattr(get_web_app(), "state", None), "bus", None)+ if bus is not None:+ return bus+ return getattr(get_brain_manager(), "_bus", None)+++class ShowOnScreenTool:+ name: str = "show_on_screen"+ risk_tier: str = "safe"+ description: str = (+ "Show exact content on the user's screen instead of reading it out: a table, a "+ "code block, a diff, a list of links or steps, as Markdown. Use it whenever the "+ "answer has more structure than a sentence carries, then say only the takeaway "+ "and that the details are on screen. Never use it for the spoken answer itself "+ "or for something the user did not ask to see."+ )+ schema: dict[str, Any] = {+ "type": "object",+ "additionalProperties": False,+ "properties": {+ "markdown": {+ "type": "string",+ "description": "The content, as GitHub-flavoured Markdown.",+ },+ "title": {+ "type": "string",+ "description": "A short heading for it, in the user's language.",+ },+ },+ "required": ["markdown"],+ }++ async def execute(self, args: dict[str, Any], ctx: ExecutionContext) -> ToolResult:+ markdown = str(args.get("markdown") or "").strip()+ if not markdown:+ return ToolResult(success=False, output=None, error="markdown is missing")+ if len(markdown) > MAX_MARKDOWN_CHARS:+ return ToolResult(+ success=False,+ output=None,+ error=(+ f"Too long to show ({len(markdown)} characters, at most "+ f"{MAX_MARKDOWN_CHARS}). Shorten it or write it to a file."+ ),+ )+ title = " ".join(str(args.get("title") or "").split())[:_MAX_TITLE_CHARS]+ session_id = str((getattr(ctx, "config", None) or {}).get("live_session_id") or "")+ if session_id:+ from jarvis.live.runtime import show_content++ if await show_content(session_id, title=title, markdown=markdown):+ return ToolResult(success=True, output={"shown": True, "kept_in_call": True})+ # No live call owns this turn: show it, without a call record to keep it in.+ bus = _app_bus()+ if bus is None:+ return ToolResult(+ success=False, output=None, error="No screen is connected to show this on."+ )+ await bus.publish(+ VoiceContentShown(+ trace_id=ctx.trace_id, source_layer="brain.tool.show_on_screen",+ session_id="", title=title, markdown=markdown,+ )+ )+ return ToolResult(success=True, output={"shown": True, "kept_in_call": False})diff --git a/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx b/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsxindex 17ade43551..3e66660f28 100644--- a/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx+++ b/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx@@ -1,4 +1,11 @@-import { act, cleanup, fireEvent, render as rtlRender, screen } from "@testing-library/react";+import {+ act,+ cleanup,+ fireEvent,+ render as rtlRender,+ screen,+ waitFor,+} from "@testing-library/react";import { QueryClient, QueryClientProvider } from "@tanstack/react-query";import { afterEach, describe, expect, it, vi } from "vitest";@@ -235,3 +242,34 @@ describe("VoiceStage helpers", () => {expect(greetingKey(21)).toBe("home.greeting_evening");});});++describe("VoiceStage content shown on screen", () => {+ afterEach(() => {+ cleanup();+ useHomeStore.getState().resetTranscript();+ });++ it("draws a card with its heading and renders the table instead of raw Markdown", async () => {+ const card: TranscriptLine = {+ id: "shown-1",+ who: "assistant",+ text: "Changed files",+ ts: 2,+ kind: "shown",+ markdown: "| file | lines |\n|---|---|\n| app.py | 12 |",+ };+ useHomeStore.setState({+ transcript: [said("user", "which files changed", 1), card, said("assistant", "Two files.", 3)],+ });+ render();+ const shown = screen.getByTestId("transcript-shown");+ expect(shown.textContent).toContain("Changed files");+ // The content is readable at once, as text, while the renderer loads.+ expect(shown.textContent).toContain("app.py");+ await waitFor(() => expect(shown.querySelector("table")).not.toBeNull(), { timeout: 15_000 });+ expect(shown.textContent).toContain("app.py");+ expect(shown.textContent).not.toContain("|---|");+ // The spoken lines around it stay ordinary lines.+ expect(screen.getAllByTestId("transcript-line")).toHaveLength(2);+ }, 20_000);+});diff --git a/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx b/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsxindex 468fc701f2..0b84e58089 100644--- a/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx+++ b/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx@@ -1,4 +1,4 @@-import { useLayoutEffect, useMemo } from "react";+import { Suspense, lazy, useLayoutEffect, useMemo } from "react";import { useEventStore, type VoiceState } from "@/store/events";import { useHomeStore } from "@/store/home";@@ -17,6 +17,13 @@ import { VoiceGlow } from "@/components/home/VoiceGlow";import { useSpokenCursor } from "@/components/home/useSpokenCursor";import { TurnSteps, traceWorthShowing } from "@/components/home/TurnSteps";import { traceModel } from "@/lib/thinkingSteps";+import { SHOWN_KIND } from "@/lib/homeTranscript";++const ChatMarkdown = lazy(() =>+ import("@/components/agentchat/ChatMarkdown").then((module) => ({+ default: module.ChatMarkdown,+ })),+);/*** The live voice conversation shares the chat column and scroll behavior.@@ -110,6 +117,8 @@ export function VoiceStage({ onExit }: { onExit?: () => void } = {}) {defaultOpen={m.live}/></div>+ ) : m.kind === SHOWN_KIND && m.markdown ? (+ <ShownContent key={m.id} title={m.text} markdown={m.markdown} />) : (<TranscriptLinekey={m.id}@@ -204,6 +213,29 @@ export function TranscriptLine({);}+/**+ * Content the assistant put on screen instead of reading it out: a table,+ * code, links. Drawn as a card on the assistant's side of the lane, wide+ * enough for a table, with the same Markdown the chat uses.+ */+export function ShownContent({ title, markdown }: { title: string; markdown: string }) {+ return (+ <div className="flex justify-start" data-testid="transcript-shown" data-who="assistant">+ <div className="w-full min-w-0 rounded-2xl border border-border bg-card px-4 py-3 text-card-foreground">+ {title && <div className="mb-2 text-sm font-medium text-foreground">{title}</div>}+ {/* The Markdown renderer is loaded when the first card appears, so the+ front page does not carry it at startup. Until then: the raw text. */}+ <Suspense fallback={<pre className="whitespace-pre-wrap text-sm">{markdown}</pre>}>+ <ChatMarkdown+ text={markdown}+ className="prose prose-sm max-w-none overflow-x-auto dark:prose-invert"+ />+ </Suspense>+ </div>+ </div>+ );+}+/** The voice composer's line: what to do, or what is happening. */export function hintFor({connected,diff --git a/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx b/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsxindex d8b3a2002e..fc080b6e26 100644--- a/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx+++ b/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx@@ -17,6 +17,7 @@ import {} from "@/lib/voiceInputLevel";import { useEventStore } from "@/store/events";import { useHomeStore } from "@/store/home";+import { useAgentChatStore } from "@/store/agentChat";import { SPEAKER_MUTE_EVENT } from "@/lib/speakerMute";/** Minimal mock for WebSocket (mirrors __tests__/ws.test.ts). */@@ -388,6 +389,54 @@ describe("useWebSocket VoiceBootStatus handling", () => {expect(useEventStore.getState().visualStage).toBeNull();});+ it("opens a named agent's chat on the front page", async () => {+ useEventStore.setState({ solo: false, activeSection: "settings", toasts: [] });+ useHomeStore.setState({ agentChatId: null });+ render(<Harness />);+ await Promise.resolve();++ MockWebSocket.last!.deliver(+ envelope("NavigateSidebar", { section: "chats", target: "agent", target_id: "agent-7" }),+ );++ expect(useEventStore.getState().activeSection).toBe("chats");+ expect(useHomeStore.getState().agentChatId).toBe("agent-7");+ });++ it("opens one conversation and leaves an agent's chat for it", async () => {+ const openSession = vi.fn();+ const original = useAgentChatStore.getState().openSession;+ useAgentChatStore.setState({ openSession });+ useEventStore.setState({ solo: false, activeSection: "settings", toasts: [] });+ useHomeStore.setState({ agentChatId: "agent-7", jarvisCardMode: "voice" });+ render(<Harness />);+ await Promise.resolve();++ MockWebSocket.last!.deliver(+ envelope("NavigateSidebar", { section: "chats", target: "chat", target_id: "s-1" }),+ );++ expect(openSession).toHaveBeenCalledWith("s-1");+ expect(useHomeStore.getState().agentChatId).toBeNull();+ expect(useHomeStore.getState().jarvisCardMode).toBe("chat");+ expect(useEventStore.getState().activeSection).toBe("chats");+ useAgentChatStore.setState({ openSession: original });+ });++ it("a detached window keeps its view when a target is opened elsewhere", async () => {+ useEventStore.setState({ solo: true, activeSection: "settings", toasts: [] });+ useHomeStore.setState({ agentChatId: null });+ render(<Harness />);+ await Promise.resolve();++ MockWebSocket.last!.deliver(+ envelope("NavigateSidebar", { section: "chats", target: "agent", target_id: "agent-7" }),+ );++ expect(useEventStore.getState().activeSection).toBe("settings");+ expect(useHomeStore.getState().agentChatId).toBeNull();+ });+it("invalidates every documentation query after a registry reload", async () => {const invalidate = vi.spyOn(queryClient, "invalidateQueries");render(<Harness />);diff --git a/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts b/jarvis/ui/web/frontend/src/hooks/useWebSocket.tsindex 8564b0e9a9..6f3edf13a9 100644--- a/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts+++ b/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts@@ -30,6 +30,7 @@ import { handleAppshotEditRequest } from "@/store/appshotEditor";import { useDeckStore } from "@/store/deck";import { MEMORY_WRITE_EVENT, useMemoryWrites } from "@/store/memoryWrites";import { useHomeStore } from "@/store/home";+import { useAgentChatStore } from "@/store/agentChat";import { PANE_ACTIVITY_EVENT } from "@/store/workspacePanes";import { WSAudioLevel, WSEventEnvelope, WSWelcome } from "@/schema/ws";import { useI18nStore, hydrateUiLanguage, hydrateReplyLanguage, isUiLanguage, translate, fill } from "@/i18n";@@ -466,7 +467,7 @@ export function useWebSocket(): void {}if (env.event_name === "NavigateSidebar") {- const p = env.payload as { section?: string };+ const p = env.payload as { section?: string; target?: string; target_id?: string };const solo = useEventStore.getState().solo;// A retired id ("outputs") still arrives from older deep links and// spoken aliases; it lands on the section that holds its content now.@@ -487,6 +488,19 @@ export function useWebSocket(): void {// "go to settings" targets the main window, and following it here// would blank the very view the user split off to keep.setActiveSection(section);+ // One thing inside the section: an agent's chat or a conversation.+ if (section === "chats" && p.target_id) {+ if (p.target === "agent") {+ useHomeStore.getState().openAgentChat(p.target_id);+ } else if (p.target === "chat") {+ const events = useEventStore.getState();+ events.setActiveConversation("text", null);+ events.setMessages([]);+ useHomeStore.getState().openAgentChat(null);+ useAgentChatStore.getState().openSession(p.target_id);+ useHomeStore.getState().setJarvisCardMode("chat");+ }+ }}// The Settings "Autopilot Toasts" switch owns this note.if (!solo && isAutopilotToastsEnabled()) {diff --git a/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts b/jarvis/ui/web/frontend/src/lib/homeTranscript.test.tsindex 3ea306db17..08e21e5887 100644--- a/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts+++ b/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts@@ -293,3 +293,39 @@ describe("transcriptFromMessages with stored traces", () => {}});});++describe("homeTranscript reducer — content shown on screen", () => {+ const table = "| file | lines |\n|---|---|\n| app.py | 12 |";++ it("adds a card line with its heading and never melts speech into it", () => {+ const lines = run([+ ["TranscriptFinal", { transcript: { text: "which files changed" } }, 1000],+ ["VoiceContentShown", { session_id: "c1", title: " Changed files ", markdown: `\n${table}\n` }, 2000],+ ["SpeechSpoken", { text: "Two files changed, the list is on screen." }, 2500],+ ]);+ expect(lines).toHaveLength(3);+ const card = lines[1];+ expect(card.who).toBe("assistant");+ if (card.who === "steps") throw new Error("a card is a text line");+ expect(card.kind).toBe("shown");+ expect(card.text).toBe("Changed files");+ expect(card.markdown).toBe(table);+ expect(lines[2]).toMatchObject({ who: "assistant", text: "Two files changed, the list is on screen." });+ });++ it("shows the same card once when the event is delivered twice", () => {+ const once = run([["VoiceContentShown", { markdown: table }, 1000]]);+ const twice = reduceTranscript(once, "VoiceContentShown", { markdown: table }, 1500);+ expect(twice).toBe(once);+ const later = reduceTranscript(once, "VoiceContentShown", { markdown: table }, 60_000);+ expect(later).toHaveLength(2);+ });++ it("ignores a card without content and keeps a heading from blocking spoken words", () => {+ const start = run([["VoiceContentShown", { title: "Summary", markdown: table }, 1000]]);+ expect(reduceTranscript(start, "VoiceContentShown", { title: "x", markdown: " " }, 1100)).toBe(start);+ // Saying the heading out loud right after is still a line of its own.+ const spoken = reduceTranscript(start, "SpeechSpoken", { text: "Summary" }, 1200);+ expect(spoken).toHaveLength(2);+ });+});diff --git a/jarvis/ui/web/frontend/src/lib/homeTranscript.ts b/jarvis/ui/web/frontend/src/lib/homeTranscript.tsindex af033eb8a5..bb33de9294 100644--- a/jarvis/ui/web/frontend/src/lib/homeTranscript.ts+++ b/jarvis/ui/web/frontend/src/lib/homeTranscript.ts@@ -61,6 +61,12 @@ export interface TranscriptTextLine extends TranscriptLineBase {*/kind?: string;revision?: number;+ /**+ * Content the assistant put on screen instead of reading it out+ * (`VoiceContentShown`): a table, code, links. The line's kind is+ * "shown" and `text` is its optional heading; it is never spoken.+ */+ markdown?: string;}/** The reasoning steps of one turn — tools, workers, the brain call. */@@ -129,6 +135,9 @@ const MID_TURN_SPOKEN_KINDS: ReadonlySet<string> = new Set(["subagent",]);+/** The kind of a line that shows content on screen (see `markdown`). */+export const SHOWN_KIND = "shown";+let counter = 0;function nextId(ts: number): string {counter = (counter + 1) % 1_000_000;@@ -152,6 +161,7 @@ function isDuplicate(for (let i = lines.length - 1; i >= 0; i--) {const line = lines[i];if (ts - line.ts > DUPLICATE_WITHIN_MS) return false;+ if (line.who !== "steps" && line.kind === SHOWN_KIND) continue;if (line.who === who && (line.text === text || line.text.endsWith(text))) return true;}return false;@@ -427,6 +437,7 @@ export function reduceTranscript(if (last &&last.who === "assistant" &&+ last.kind !== SHOWN_KIND &&tsMs - last.ts < JOIN_WITHIN_MS &&(!last.kind || !kind || last.kind === kind)) {@@ -446,6 +457,20 @@ export function reduceTranscript(if (!text || isDuplicate(lines, role, text, tsMs)) return lines;return push(role === "user" ? closeOpenTurn(lines, tsMs) : lines, role, text, tsMs);}+ case "VoiceContentShown": {+ const markdown = typeof p.markdown === "string" ? p.markdown.trim() : "";+ if (!markdown) return lines;+ // The same card delivered twice (a reconnect replays recent events).+ for (let i = lines.length - 1; i >= 0; i--) {+ const line = lines[i];+ if (tsMs - line.ts > DUPLICATE_WITHIN_MS) break;+ if (line.who === "assistant" && line.markdown === markdown) return lines;+ }+ return cap([...lines, {+ id: nextId(tsMs), ts: tsMs, who: "assistant", text: clean(p.title),+ kind: SHOWN_KIND, markdown,+ }]);+ }case "VoiceTurnCompleted":return closeOpenTurn(lines, tsMs);case "SystemStateChanged": {diff --git a/jarvis/ui/web/frontend/src/store/home.ts b/jarvis/ui/web/frontend/src/store/home.tsindex e78ada883a..4b1afc6b63 100644--- a/jarvis/ui/web/frontend/src/store/home.ts+++ b/jarvis/ui/web/frontend/src/store/home.ts@@ -126,6 +126,11 @@ export const useHomeStore = create<HomeStore>((set, get) => ({useEventStore.getState().setTranscription("", true);set({ liveReply: "", liveSessionId: sessionId || null });}+ if (name === "VoiceContentShown") {+ // A card belongs to the call it was shown in, never to another one.+ const sessionId = (payload as { session_id?: string } | null)?.session_id;+ if (sessionId && get().liveSessionId && sessionId !== get().liveSessionId) return;+ }if (name === "VoiceSessionEnded") {const sessionId = (payload as { session_id?: string } | null)?.session_id;if (sessionId && get().liveSessionId && sessionId !== get().liveSessionId) return;diff --git a/tests/contract/test_workspace_orchestration.py b/tests/contract/test_workspace_orchestration.pyindex 8fd0dcf0c1..f0b4dbf57c 100644--- a/tests/contract/test_workspace_orchestration.py+++ b/tests/contract/test_workspace_orchestration.py@@ -64,6 +64,12 @@ class Sessions:@pytest.fixturedef rig(tmp_path, monkeypatch):+ # The panes below are stamped idle once, at setup. A slow runner can take+ # longer than the stamp's freshness window to reach the step under test,+ # and the pane then reads as unknown instead of idle; keep it fresh here.+ from jarvis.agentic_ide import activity++ monkeypatch.setattr(activity, "STAMP_FRESH_S", 3600.0)monkeypatch.setattr(library, "_root", lambda: tmp_path / "library")monkeypatch.setattr(resume_store, "_store_path", lambda: tmp_path / "snapshot.json")registry = Registry(pty_manager=FakePtyManager())diff --git a/tests/unit/plugins/tool/test_navigate.py b/tests/unit/plugins/tool/test_navigate.pyindex 9627eef30e..f739f807cb 100644--- a/tests/unit/plugins/tool/test_navigate.py+++ b/tests/unit/plugins/tool/test_navigate.py@@ -129,3 +129,82 @@ def test_known_sections_match_frontend_section_ids() -> None:ts_ids = set(re.findall(r'"([a-z0-9_-]+)"', block.group(1)))assert ts_ids, "no ids parsed from SECTION_IDS"assert NavigateTool.known_sections() == ts_ids+++# --- one thing inside a section ---------------------------------------------------+++def _targets(bus: RecordingBus, **lookups) -> NavigateTool:+ async def chat(spoken: str) -> dict:+ return lookups.get("chat", {"status": "unavailable"})++ async def workspace(spoken: str) -> dict:+ return lookups.get("workspace", {"status": "unavailable"})++ return NavigateTool(+ bus=bus,+ find_agent=lambda spoken: lookups.get("agent", {"status": "unavailable"}),+ find_chat=chat,+ show_workspace=workspace,+ )+++async def test_a_named_agent_opens_that_agents_chat(bus: RecordingBus) -> None:+ tool = _targets(bus, agent={"status": "found", "id": "agent-7", "name": "Scout"})+ res = await tool.execute({"agent": " scout "}, None)+ assert res.success and res.output["agent"] == "Scout"+ (event,) = bus.events+ assert isinstance(event, NavigateSidebar)+ assert (event.section, event.target, event.target_id) == ("chats", "agent", "agent-7")+++async def test_a_conversation_title_opens_that_conversation(bus: RecordingBus) -> None:+ tool = _targets(bus, chat={"status": "found", "id": "s-1", "name": "Trip to Lisbon"})+ res = await tool.execute({"chat": "trip to lisbon"}, None)+ assert res.success+ (event,) = bus.events+ assert (event.section, event.target, event.target_id) == ("chats", "chat", "s-1")+++async def test_a_workspace_is_shown_in_the_coding_section(bus: RecordingBus) -> None:+ tool = _targets(bus, workspace={"status": "found", "id": "", "name": "Installer"})+ res = await tool.execute({"workspace": "installer"}, None)+ assert res.success and res.output["workspace"] == "Installer"+ (event,) = bus.events+ assert (event.section, event.target, event.target_id) == ("agentic-ide", "", "")+++ ("lookup", "says"),+ [+ ({"status": "ask", "candidates": ["Scout", "Scout 2"]}, "More than one agent"),+ ({"status": "unknown", "available": ["Archivist"]}, "Known: Archivist"),+ ({"status": "unavailable"}, "cannot open a agent right now"),+ ],+)+async def test_an_unresolved_target_moves_nothing_and_says_why(+ bus: RecordingBus, lookup: dict, says: str+) -> None:+ res = await _targets(bus, agent=lookup).execute({"agent": "scout"}, None)+ assert res.success is False and says in res.error+ assert bus.events == []+++async def test_two_destinations_at_once_are_refused(bus: RecordingBus) -> None:+ tool = _targets(bus, agent={"status": "found", "id": "a", "name": "Scout"})+ for args in ({"agent": "scout", "section": "settings"}, {"agent": "scout", "chat": "x"}):+ res = await tool.execute(args, None)+ assert res.success is False and "one destination" in res.error+ assert bus.events == []+++async def test_a_plain_section_switch_carries_no_target(tool: NavigateTool, bus) -> None:+ await tool.execute({"section": "settings"}, None)+ (event,) = bus.events+ assert (event.target, event.target_id) == ("", "")diff --git a/tests/unit/plugins/tool/test_show_on_screen.py b/tests/unit/plugins/tool/test_show_on_screen.pynew file mode 100644index 0000000000..a52523e18e--- /dev/null+++ b/tests/unit/plugins/tool/test_show_on_screen.py@@ -0,0 +1,130 @@+"""Exact content goes on screen; the voice keeps the takeaway."""++from __future__ import annotations++from types import SimpleNamespace+from uuid import uuid4++import pytest++from jarvis.core.events import VoiceContentShown+from jarvis.live import runtime+from jarvis.plugins.tool import show_on_screen+from jarvis.plugins.tool.show_on_screen import MAX_MARKDOWN_CHARS, ShowOnScreenTool++_TABLE = "| file | lines |\n|---|---|\n| app.py | 12 |"+++class RecordingBus:+ def __init__(self) -> None:+ self.events: list[object] = []++ async def publish(self, event: object) -> None:+ self.events.append(event)+++def _ctx(session_id: str = "") -> SimpleNamespace:+ config = {"live_session_id": session_id} if session_id else {}+ return SimpleNamespace(trace_id=uuid4(), user_utterance="show me", config=config)+++async def test_in_a_live_call_the_content_is_handed_to_that_call(monkeypatch) -> None:+ shown = []++ async def show(session_id: str, *, title: str, markdown: str) -> bool:+ shown.append((session_id, title, markdown))+ return True++ monkeypatch.setattr(runtime, "show_content", show)+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: pytest.fail("the call owns it"))+ res = await ShowOnScreenTool().execute(+ {"markdown": f" {_TABLE}\n", "title": " Changed files "}, _ctx("call-1")+ )+ assert res.success and res.output == {"shown": True, "kept_in_call": True}+ assert shown == [("call-1", "Changed files", _TABLE)]++[email protected]("session_id", ["", "ended-call"])+async def test_without_a_running_call_it_is_still_shown(monkeypatch, session_id) -> None:+ bus = RecordingBus()++ async def gone(session_id: str, *, title: str, markdown: str) -> bool:+ return False++ monkeypatch.setattr(runtime, "show_content", gone)+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: bus)+ res = await ShowOnScreenTool().execute({"markdown": _TABLE}, _ctx(session_id))+ assert res.success and res.output["kept_in_call"] is False+ (event,) = bus.events+ assert isinstance(event, VoiceContentShown)+ assert (event.session_id, event.title, event.markdown) == ("", "", _TABLE)+++async def test_nothing_to_show_or_nowhere_to_show_it_is_an_error(monkeypatch) -> None:+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: None)+ tool = ShowOnScreenTool()+ assert (await tool.execute({"markdown": " "}, _ctx())).error == "markdown is missing"+ too_long = await tool.execute({"markdown": "x" * (MAX_MARKDOWN_CHARS + 1)}, _ctx())+ assert too_long.success is False and "Too long to show" in too_long.error+ nowhere = await tool.execute({"markdown": _TABLE}, _ctx())+ assert nowhere.success is False and "No screen" in nowhere.error+++async def test_the_runtime_reports_a_call_that_is_not_running() -> None:+ assert await runtime.show_content("no-such-call", title="", markdown=_TABLE) is False+++# --- inside the call: kept in its record, announced to the screen ------------------+++async def test_a_call_keeps_shown_content_in_its_record_after_the_last_words(tmp_path) -> None:+ from jarvis.live.session import LiveVoiceSession+ from jarvis.live.state import LiveLedger, TranscriptFragment++ ledger = LiveLedger(tmp_path / "live.sqlite3")+ bus = RecordingBus()+ call = SimpleNamespace(session_id="call-1", _ledger=ledger, _bus=bus)+ try:+ ledger.append(TranscriptFragment("call-1", "e1", "user", "Which files changed?", 100, 900))+ await LiveVoiceSession.show_content(call, title="Changed files", markdown=_TABLE)+ rows = ledger.transcript("call-1")+ finally:+ ledger.close()+ assert [row["role"] for row in rows] == ["user", "assistant"]+ assert rows[1]["start_ms"] == 901 # right after what was said last+ assert rows[1]["delta"] == f"\n\n**Changed files**\n\n{_TABLE}\n\n"+ (event,) = bus.events+ assert isinstance(event, VoiceContentShown)+ assert (event.session_id, event.title, event.markdown) == ("call-1", "Changed files", _TABLE)+++async def test_a_title_or_id_finds_one_conversation(monkeypatch) -> None:+ from jarvis.core import runtime_refs+ from jarvis.plugins.tool import navigate++ class Store:+ def list_sessions(self, limit: int, *, surface: str):+ assert surface == "jarvis"+ return [+ SimpleNamespace(session_id="s-2", title="Trip to Lisbon"),+ SimpleNamespace(session_id="s-1", title="Trip to Lisbon"),+ SimpleNamespace(session_id="s-0", title="Tax questions"),+ ]++ app = SimpleNamespace(state=SimpleNamespace(agent_chat=SimpleNamespace(store=Store())))+ monkeypatch.setattr(runtime_refs, "get_web_app", lambda: app)+ newest = await navigate._find_chat("trip to lisbon")+ assert newest == {"status": "found", "id": "s-2", "name": "Trip to Lisbon"}+ assert (await navigate._find_chat("s-0"))["id"] == "s-0"+ assert (await navigate._find_chat("tax questions"))["id"] == "s-0"+ assert (await navigate._find_chat("t"))["status"] == "ask"+ unknown = await navigate._find_chat("no such chat")+ assert unknown["status"] == "unknown" and "Trip to Lisbon" in unknown["available"]+ monkeypatch.setattr(runtime_refs, "get_web_app", lambda: None)+ assert (await navigate._find_chat("anything"))["status"] == "unavailable"