Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(voice): show exact content on screen, and open one thing in the app (#641)

Ruben Lütkeb9d709ecdb40
diff --git a/jarvis/brain/tool_gateway.py b/jarvis/brain/tool_gateway.py
index abe8045a54..8413e80f62 100644
--- a/jarvis/brain/tool_gateway.py
+++ b/jarvis/brain/tool_gateway.py
@@ -166,6 +166,10 @@ class BrainSupervisorToolGateway:
# runs, into the MEMORY.md every later conversation reads. Voice-only
# like the tools above: the router stays a pure dispatcher (ADR-0011).
tools["remember"] = RememberTool()
+ # A voice carries a takeaway, not a table: exact content goes on screen.
+ from jarvis.plugins.tool.show_on_screen import ShowOnScreenTool
+
+ tools[ShowOnScreenTool.name] = ShowOnScreenTool()
if self._workspace_tool is not None:
# Live delegates coding to one addressed service. The old prompt
# tools silently choose an ambient pane and cannot safely coexist.
diff --git a/jarvis/core/events.py b/jarvis/core/events.py
index 5b0c79015c..c3cf122212 100644
--- a/jarvis/core/events.py
+++ b/jarvis/core/events.py
@@ -129,6 +129,20 @@ class VoiceTranscriptUpdated(Event):
revision: int = 1
+@dataclass(frozen=True, slots=True)
+class VoiceContentShown(Event):
+ """Content put on screen during a spoken conversation instead of read out.
+
+ ``markdown`` is shown as a card in the conversation the user is looking at
+ (``lib/homeTranscript.ts``). ``session_id`` names the live call it belongs
+ to; empty when no live call owns the turn.
+ """
+
+ session_id: str = ""
+ title: str = ""
+ markdown: str = ""
+
+
@dataclass(frozen=True, slots=True)
class ReasoningSummaryUpdated(Event):
"""Provider-exposed reasoning summary, never private reasoning or audio."""
@@ -871,8 +885,14 @@ class NavigateSidebar(Event):
id is a graceful no-op there. ``section`` mirrors the frontend
``SECTION_IDS`` (``store/events.ts``) — kept in sync via the navigate tool's
parity test.
+
+ ``target`` narrows the destination inside the section: ``"agent"`` opens
+ the chat of the agent ``target_id``, ``"chat"`` the conversation
+ ``target_id``. Empty for a plain section switch.
"""
section: str = ""
+ target: str = ""
+ target_id: str = ""
@dataclass(frozen=True, slots=True)
diff --git a/jarvis/live/config.py b/jarvis/live/config.py
index 0cbb0a8d66..fff7a7c6a6 100644
--- a/jarvis/live/config.py
+++ b/jarvis/live/config.py
@@ -8,7 +8,7 @@ from pydantic import BaseModel, ConfigDict, Field
from jarvis.core.agent_brief import AGENT_BRIEF_RULE
from jarvis.cu.direct import COMPUTER_CONTROL_RULES
-from jarvis.live.coordination import WORK_STATUS_RULE, conversation_rules
+from jarvis.live.coordination import ON_SCREEN_RULE, WORK_STATUS_RULE, conversation_rules
from jarvis.live.product import PRODUCT_BRIEF
from jarvis.live.recovery import HISTORY_CONTEXT_RULE
@@ -122,6 +122,7 @@ class LiveConfig(BaseModel):
"workspace-orchestrate create in the named or visible workspace, with cli, "
"count and the task as prompt; never an existing agent and never spawn_worker. "
+ WORK_STATUS_RULE
+ + ON_SCREEN_RULE
+ AGENT_BRIEF_RULE
+ " "
+ (COMPUTER_CONTROL_RULES if computer else "")
diff --git a/jarvis/live/coordination.py b/jarvis/live/coordination.py
index a6d31e5fd4..4c8dace8d5 100644
--- a/jarvis/live/coordination.py
+++ b/jarvis/live/coordination.py
@@ -27,6 +27,16 @@ WORK_STATUS_RULE: Final[str] = (
)
+#: For the model that executes tools: what goes on screen instead of into speech.
+ON_SCREEN_RULE: Final[str] = (
+ "When an answer has more structure than a sentence carries (a table, code, a diff, "
+ "several links or steps), call show_on_screen with it as Markdown and say only the "
+ "takeaway and that the details are on screen. To take the user somewhere in the app "
+ "use navigate: a section, a named agent's chat, a conversation by its title, or a "
+ "coding workspace. "
+)
+
+
def conversation_rules(executor: str) -> str:
"""The speaking model's rules; ``executor`` is what carries out work for it."""
return (
@@ -59,4 +69,4 @@ def conversation_rules(executor: str) -> str:
)
-__all__ = ["WORK_STATUS_RULE", "conversation_rules"]
+__all__ = ["ON_SCREEN_RULE", "WORK_STATUS_RULE", "conversation_rules"]
diff --git a/jarvis/live/native.py b/jarvis/live/native.py
index 4c3fbbb2ab..541c210439 100644
--- a/jarvis/live/native.py
+++ b/jarvis/live/native.py
@@ -18,7 +18,7 @@ from jarvis.core.paths import user_data_dir
from jarvis.core.runtime_refs import get_supervisor_tool_gateway
from jarvis.core.tool_budget import VOICE_TOOL_BUDGET_S
from jarvis.cu.direct import COMPUTER_CONTROL_RULES
-from jarvis.live.coordination import WORK_STATUS_RULE, conversation_rules
+from jarvis.live.coordination import ON_SCREEN_RULE, WORK_STATUS_RULE, conversation_rules
from jarvis.live.product import PRODUCT_BRIEF
from jarvis.live.runtime import claim, register, unregister
from jarvis.live.session import LiveVoiceSession, _identity
@@ -178,6 +178,7 @@ class NativeLiveVoiceSession(LiveVoiceSession):
"Do not switch the UI to address another workspace. Reuse request_id on "
"retries and never replay uncertain delivery. "
+ WORK_STATUS_RULE
+ + ON_SCREEN_RULE
+ AGENT_BRIEF_RULE
),
history=tuple(
diff --git a/jarvis/live/runtime.py b/jarvis/live/runtime.py
index 8ab9b7c194..582ba20c10 100644
--- a/jarvis/live/runtime.py
+++ b/jarvis/live/runtime.py
@@ -117,6 +117,19 @@ async def on_session_loop(session: Any, operation: Any, **kwargs: Any) -> Any:
return await asyncio.wrap_future(future)
+async def show_content(session_id: str, *, title: str, markdown: str) -> bool:
+ """Show content in the live call ``session_id``; False when it is not running."""
+ session = _active.get(session_id)
+ show = getattr(session, "show_content", None)
+ if session is None or show is None:
+ return False
+ try:
+ await on_session_loop(session, show, title=title, markdown=markdown)
+ except RuntimeError: # the call ended between the lookup and the delivery
+ return False
+ return True
+
+
def owns_microphone(*, except_session_id: str | None = None) -> bool:
"""A pending or active browser call must not be replaced by a wake start."""
return any(sid != except_session_id for sid in (*_opening, *_active))
diff --git a/jarvis/live/session.py b/jarvis/live/session.py
index fe9edad15f..79fd897432 100644
--- a/jarvis/live/session.py
+++ b/jarvis/live/session.py
@@ -1860,6 +1860,32 @@ class LiveVoiceSession:
}
)
+ async def show_content(self, *, title: str, markdown: str) -> None:
+ """Put content on the user's screen and keep it in this call's record.
+
+ It is stored as one assistant passage right after the newest words, so
+ the reasoning backend sees it as part of the conversation and the
+ archived call still shows it. Nothing here is spoken.
+ """
+ from jarvis.core.events import VoiceContentShown
+
+ if self._ledger is not None:
+ # Every live engine keeps its own clock; the record is the common one.
+ said = await asyncio.to_thread(self._ledger.transcript, self.session_id)
+ at = max((int(row["end_ms"]) for row in said), default=-1) + 1
+ body = f"**{title}**\n\n{markdown}" if title else markdown
+ fragment = TranscriptFragment(
+ self.session_id, f"shown-{uuid4().hex}", "assistant", f"\n\n{body}\n\n", at, at
+ )
+ await asyncio.to_thread(self._ledger.transcript_snapshot, fragment)
+ if self._bus is not None:
+ await self._bus.publish(
+ VoiceContentShown(
+ source_layer="live.session", session_id=self.session_id,
+ title=title, markdown=markdown,
+ )
+ )
+
async def deliver_announcement(
self, text: str, *, report: str | None = None, **kwargs: Any
) -> bool:
diff --git a/jarvis/plugins/tool/navigate.py b/jarvis/plugins/tool/navigate.py
index a5d0df3400..127730cf2a 100644
--- a/jarvis/plugins/tool/navigate.py
+++ b/jarvis/plugins/tool/navigate.py
@@ -14,6 +14,9 @@ a parity test (tests/unit/plugins/tool/test_navigate.py) guards against drift.
from __future__ import annotations
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
from typing import Any
from jarvis.core.events import NavigateSidebar
@@ -303,8 +306,84 @@ def _resolve(raw: str) -> str | None:
return None
+log = logging.getLogger(__name__)
+
+# What a target lookup answers: ``found`` with an id and a name, ``ask`` with
+# the candidates that fit, ``unknown`` with what exists, or ``unavailable``.
+Lookup = dict[str, Any]
+#: How many conversations a title is matched against, newest first.
+_CHAT_SEARCH_LIMIT = 200
+
+
+def _find_agent(spoken: str) -> Lookup:
+ """The named agent a spoken reference means, from the live roster."""
+ try:
+ from jarvis.society.agent_names import display_name, lookup_in
+ from jarvis.society.runtime import current_runtime
+
+ runtime = current_runtime()
+ if runtime is None:
+ return {"status": "unavailable"}
+ found = lookup_in(runtime.roster.snapshot(), spoken, surface="navigate")
+ except Exception: # noqa: BLE001 - no agent team on this install: say so, do not crash
+ log.warning("navigate: agent lookup failed", exc_info=True)
+ return {"status": "unavailable"}
+ if found.decision == "act" and found.agent is not None:
+ return {"status": "found", "id": found.agent.agent_id, "name": display_name(found.agent)}
+ if found.decision == "ask" and found.candidates:
+ return {"status": "ask", "candidates": [display_name(a) for a in found.candidates]}
+ return {"status": "unknown", "available": found.available_names()}
+
+
+async def _find_chat(spoken: str) -> Lookup:
+ """The conversation with this id or title among the assistant's own chats."""
+ from jarvis.core.runtime_refs import get_web_app
+
+ service = getattr(getattr(get_web_app(), "state", None), "agent_chat", None)
+ store = getattr(service, "store", None)
+ if store is None:
+ return {"status": "unavailable"}
+ sessions = await asyncio.to_thread(store.list_sessions, _CHAT_SEARCH_LIMIT, surface="jarvis")
+ wanted = " ".join(spoken.split()).casefold()
+ exact = [
+ s for s in sessions
+ if s.session_id == spoken.strip() or " ".join(s.title.split()).casefold() == wanted
+ ]
+ matches = exact or [s for s in sessions if wanted and wanted in s.title.casefold()]
+ if len(matches) == 1 or exact:
+ # Several chats can carry one title; the newest is the one meant.
+ return {"status": "found", "id": matches[0].session_id, "name": matches[0].title}
+ if matches:
+ return {"status": "ask", "candidates": [s.title for s in matches[:8]]}
+ return {"status": "unknown", "available": [s.title for s in sessions[:8] if s.title]}
+
+
+async def _show_workspace(spoken: str) -> Lookup:
+ """Bring a coding workspace on screen through the workspace service."""
+ try:
+ from jarvis.agentic_ide.orchestration import get_orchestrator
+
+ result = await get_orchestrator().run({"action": "show", "workspace": spoken})
+ except Exception: # noqa: BLE001 - no coding surface on this install: say so
+ log.warning("navigate: workspace lookup failed", exc_info=True)
+ return {"status": "unavailable"}
+ if result.get("status") == "shown":
+ return {"status": "found", "id": "", "name": str(result.get("workspace") or spoken)}
+ candidates = [
+ str(item.get("name") or item.get("workspace") or "")
+ for item in result.get("candidates") or []
+ if isinstance(item, dict)
+ ]
+ return {
+ "status": "ask" if candidates else "unknown",
+ "candidates": candidates,
+ "available": candidates,
+ "reason": str(result.get("reason") or ""),
+ }
+
+
class NavigateTool:
- """Switch the desktop app's active sidebar section."""
+ """Switch the desktop app's active section, or open one thing inside it."""
name: str = "navigate"
risk_tier: str = "safe"
@@ -312,8 +391,10 @@ class NavigateTool:
"Open/switch the desktop app to a sidebar section (the UI screen the user "
"sees). Use when the user asks to show or open a section — 'zeig die "
"Socials', 'open settings', 'show the agents', 'geh zu den Aufgaben'. Pass "
- "the section id in 'section'. Do NOT use it for anything other than moving "
- "the UI."
+ "the section id in 'section'. To open one thing instead, pass exactly one of "
+ "agent (a named agent's chat), chat (a conversation, by its title) or "
+ "workspace (a coding workspace) and no section. Do NOT use it for anything "
+ "other than moving the UI."
)
schema: dict[str, Any] = {
"type": "object",
@@ -324,19 +405,54 @@ class NavigateTool:
"description": (
"The sidebar section id to open (e.g. 'socials', 'settings', 'agents')."
),
- }
+ },
+ "agent": {
+ "type": "string",
+ "description": "A named agent: opens the chat with that agent.",
+ },
+ "chat": {
+ "type": "string",
+ "description": "A conversation's title (or id): opens that conversation.",
+ },
+ "workspace": {
+ "type": "string",
+ "description": "A coding workspace's name: brings it on screen.",
+ },
},
- "required": ["section"],
}
- def __init__(self, bus: Any) -> None:
+ def __init__(
+ self,
+ bus: Any,
+ *,
+ find_agent: Callable[[str], Lookup] = _find_agent,
+ find_chat: Callable[[str], Awaitable[Lookup]] = _find_chat,
+ show_workspace: Callable[[str], Awaitable[Lookup]] = _show_workspace,
+ ) -> None:
self._bus = bus
+ self._find_agent = find_agent
+ self._find_chat = find_chat
+ self._show_workspace = show_workspace
@classmethod
def known_sections(cls) -> set[str]:
return set(KNOWN)
async def execute(self, args: dict[str, Any], ctx: Any) -> ToolResult:
+ targets = {
+ kind: " ".join(str(args.get(kind) or "").split())
+ for kind in ("agent", "chat", "workspace")
+ }
+ named = {kind: value for kind, value in targets.items() if value}
+ if len(named) > 1 or (named and str(args.get("section") or "").strip()):
+ return ToolResult(
+ success=False,
+ output={"requested": dict(args)},
+ error="Pass one destination: a section, or one of agent, chat, workspace.",
+ )
+ if named:
+ kind, spoken = next(iter(named.items()))
+ return await self._open(kind, spoken)
section = _resolve(str(args.get("section", "")))
if section is None:
return ToolResult(
@@ -354,3 +470,43 @@ class NavigateTool:
success=True,
output={"section": section, "summary": f"Opened the {section} section."},
)
+
+ async def _open(self, kind: str, spoken: str) -> ToolResult:
+ """Resolve one named destination and move the UI to it, or say why not."""
+ if kind == "agent":
+ found = self._find_agent(spoken)
+ elif kind == "chat":
+ found = await self._find_chat(spoken)
+ else:
+ found = await self._show_workspace(spoken)
+ status = found.get("status")
+ if status != "found":
+ if status == "ask":
+ error = (
+ f"More than one {kind} fits {spoken!r}: "
+ + ", ".join(found.get("candidates") or [])
+ + ". Ask the user which one."
+ )
+ elif status == "unknown":
+ known = ", ".join(found.get("available") or []) or "none"
+ detail = f" {found['reason']}" if found.get("reason") else ""
+ error = f"No {kind} is called {spoken!r}. Known: {known}.{detail}"
+ else:
+ error = f"This app cannot open a {kind} right now."
+ return ToolResult(success=False, output={"requested": spoken, **found}, error=error)
+ if kind == "workspace":
+ # The workspace service already brought it on screen; show its section.
+ event = NavigateSidebar(section="agentic-ide", source_layer="brain.tool.navigate")
+ else:
+ event = NavigateSidebar(
+ section="chats", target=kind, target_id=str(found["id"]),
+ source_layer="brain.tool.navigate",
+ )
+ await self._bus.publish(event)
+ return ToolResult(
+ success=True,
+ output={
+ "section": event.section, kind: found["name"],
+ "summary": f"Opened the {kind} {found['name']}.",
+ },
+ )
diff --git a/jarvis/plugins/tool/show_on_screen.py b/jarvis/plugins/tool/show_on_screen.py
new file mode 100644
index 0000000000..b623646bb5
--- /dev/null
+++ b/jarvis/plugins/tool/show_on_screen.py
@@ -0,0 +1,91 @@
+"""show_on_screen tool: put exact content in front of the user during a call.
+
+A voice says a takeaway well and a table badly. This tool shows Markdown (a
+table, a code block, a diff, a list of links) in the conversation the user is
+looking at, so the spoken answer can stay one sentence. Inside a live call the
+content also becomes part of that call's record, so it is still there when the
+conversation is opened again later.
+
+Risk tier: safe - it only displays what the model was about to say anyway.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from jarvis.core.events import VoiceContentShown
+from jarvis.core.protocols import ExecutionContext, ToolResult
+
+#: A screenful or two; longer material belongs in a file or an artifact.
+MAX_MARKDOWN_CHARS = 20_000
+_MAX_TITLE_CHARS = 120
+
+
+def _app_bus() -> Any | None:
+ from jarvis.core.runtime_refs import get_brain_manager, get_web_app
+
+ bus = getattr(getattr(get_web_app(), "state", None), "bus", None)
+ if bus is not None:
+ return bus
+ return getattr(get_brain_manager(), "_bus", None)
+
+
+class ShowOnScreenTool:
+ name: str = "show_on_screen"
+ risk_tier: str = "safe"
+ description: str = (
+ "Show exact content on the user's screen instead of reading it out: a table, a "
+ "code block, a diff, a list of links or steps, as Markdown. Use it whenever the "
+ "answer has more structure than a sentence carries, then say only the takeaway "
+ "and that the details are on screen. Never use it for the spoken answer itself "
+ "or for something the user did not ask to see."
+ )
+ schema: dict[str, Any] = {
+ "type": "object",
+ "additionalProperties": False,
+ "properties": {
+ "markdown": {
+ "type": "string",
+ "description": "The content, as GitHub-flavoured Markdown.",
+ },
+ "title": {
+ "type": "string",
+ "description": "A short heading for it, in the user's language.",
+ },
+ },
+ "required": ["markdown"],
+ }
+
+ async def execute(self, args: dict[str, Any], ctx: ExecutionContext) -> ToolResult:
+ markdown = str(args.get("markdown") or "").strip()
+ if not markdown:
+ return ToolResult(success=False, output=None, error="markdown is missing")
+ if len(markdown) > MAX_MARKDOWN_CHARS:
+ return ToolResult(
+ success=False,
+ output=None,
+ error=(
+ f"Too long to show ({len(markdown)} characters, at most "
+ f"{MAX_MARKDOWN_CHARS}). Shorten it or write it to a file."
+ ),
+ )
+ title = " ".join(str(args.get("title") or "").split())[:_MAX_TITLE_CHARS]
+ session_id = str((getattr(ctx, "config", None) or {}).get("live_session_id") or "")
+ if session_id:
+ from jarvis.live.runtime import show_content
+
+ if await show_content(session_id, title=title, markdown=markdown):
+ return ToolResult(success=True, output={"shown": True, "kept_in_call": True})
+ # No live call owns this turn: show it, without a call record to keep it in.
+ bus = _app_bus()
+ if bus is None:
+ return ToolResult(
+ success=False, output=None, error="No screen is connected to show this on."
+ )
+ await bus.publish(
+ VoiceContentShown(
+ trace_id=ctx.trace_id, source_layer="brain.tool.show_on_screen",
+ session_id="", title=title, markdown=markdown,
+ )
+ )
+ return ToolResult(success=True, output={"shown": True, "kept_in_call": False})
diff --git a/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx b/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx
index 17ade43551..3e66660f28 100644
--- a/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx
+++ b/jarvis/ui/web/frontend/src/components/home/VoiceStage.test.tsx
@@ -1,4 +1,11 @@
-import { act, cleanup, fireEvent, render as rtlRender, screen } from "@testing-library/react";
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render as rtlRender,
+ screen,
+ waitFor,
+} from "@testing-library/react";
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -235,3 +242,34 @@ describe("VoiceStage helpers", () => {
expect(greetingKey(21)).toBe("home.greeting_evening");
});
});
+
+describe("VoiceStage content shown on screen", () => {
+ afterEach(() => {
+ cleanup();
+ useHomeStore.getState().resetTranscript();
+ });
+
+ it("draws a card with its heading and renders the table instead of raw Markdown", async () => {
+ const card: TranscriptLine = {
+ id: "shown-1",
+ who: "assistant",
+ text: "Changed files",
+ ts: 2,
+ kind: "shown",
+ markdown: "| file | lines |\n|---|---|\n| app.py | 12 |",
+ };
+ useHomeStore.setState({
+ transcript: [said("user", "which files changed", 1), card, said("assistant", "Two files.", 3)],
+ });
+ render();
+ const shown = screen.getByTestId("transcript-shown");
+ expect(shown.textContent).toContain("Changed files");
+ // The content is readable at once, as text, while the renderer loads.
+ expect(shown.textContent).toContain("app.py");
+ await waitFor(() => expect(shown.querySelector("table")).not.toBeNull(), { timeout: 15_000 });
+ expect(shown.textContent).toContain("app.py");
+ expect(shown.textContent).not.toContain("|---|");
+ // The spoken lines around it stay ordinary lines.
+ expect(screen.getAllByTestId("transcript-line")).toHaveLength(2);
+ }, 20_000);
+});
diff --git a/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx b/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx
index 468fc701f2..0b84e58089 100644
--- a/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx
+++ b/jarvis/ui/web/frontend/src/components/home/VoiceStage.tsx
@@ -1,4 +1,4 @@
-import { useLayoutEffect, useMemo } from "react";
+import { Suspense, lazy, useLayoutEffect, useMemo } from "react";
import { useEventStore, type VoiceState } from "@/store/events";
import { useHomeStore } from "@/store/home";
@@ -17,6 +17,13 @@ import { VoiceGlow } from "@/components/home/VoiceGlow";
import { useSpokenCursor } from "@/components/home/useSpokenCursor";
import { TurnSteps, traceWorthShowing } from "@/components/home/TurnSteps";
import { traceModel } from "@/lib/thinkingSteps";
+import { SHOWN_KIND } from "@/lib/homeTranscript";
+
+const ChatMarkdown = lazy(() =>
+ import("@/components/agentchat/ChatMarkdown").then((module) => ({
+ default: module.ChatMarkdown,
+ })),
+);
/**
* The live voice conversation shares the chat column and scroll behavior.
@@ -110,6 +117,8 @@ export function VoiceStage({ onExit }: { onExit?: () => void } = {}) {
defaultOpen={m.live}
/>
</div>
+ ) : m.kind === SHOWN_KIND && m.markdown ? (
+ <ShownContent key={m.id} title={m.text} markdown={m.markdown} />
) : (
<TranscriptLine
key={m.id}
@@ -204,6 +213,29 @@ export function TranscriptLine({
);
}
+/**
+ * Content the assistant put on screen instead of reading it out: a table,
+ * code, links. Drawn as a card on the assistant's side of the lane, wide
+ * enough for a table, with the same Markdown the chat uses.
+ */
+export function ShownContent({ title, markdown }: { title: string; markdown: string }) {
+ return (
+ <div className="flex justify-start" data-testid="transcript-shown" data-who="assistant">
+ <div className="w-full min-w-0 rounded-2xl border border-border bg-card px-4 py-3 text-card-foreground">
+ {title && <div className="mb-2 text-sm font-medium text-foreground">{title}</div>}
+ {/* The Markdown renderer is loaded when the first card appears, so the
+ front page does not carry it at startup. Until then: the raw text. */}
+ <Suspense fallback={<pre className="whitespace-pre-wrap text-sm">{markdown}</pre>}>
+ <ChatMarkdown
+ text={markdown}
+ className="prose prose-sm max-w-none overflow-x-auto dark:prose-invert"
+ />
+ </Suspense>
+ </div>
+ </div>
+ );
+}
+
/** The voice composer's line: what to do, or what is happening. */
export function hintFor({
connected,
diff --git a/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx b/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx
index d8b3a2002e..fc080b6e26 100644
--- a/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx
+++ b/jarvis/ui/web/frontend/src/hooks/useWebSocket.test.tsx
@@ -17,6 +17,7 @@ import {
} from "@/lib/voiceInputLevel";
import { useEventStore } from "@/store/events";
import { useHomeStore } from "@/store/home";
+import { useAgentChatStore } from "@/store/agentChat";
import { SPEAKER_MUTE_EVENT } from "@/lib/speakerMute";
/** Minimal mock for WebSocket (mirrors __tests__/ws.test.ts). */
@@ -388,6 +389,54 @@ describe("useWebSocket VoiceBootStatus handling", () => {
expect(useEventStore.getState().visualStage).toBeNull();
});
+ it("opens a named agent's chat on the front page", async () => {
+ useEventStore.setState({ solo: false, activeSection: "settings", toasts: [] });
+ useHomeStore.setState({ agentChatId: null });
+ render(<Harness />);
+ await Promise.resolve();
+
+ MockWebSocket.last!.deliver(
+ envelope("NavigateSidebar", { section: "chats", target: "agent", target_id: "agent-7" }),
+ );
+
+ expect(useEventStore.getState().activeSection).toBe("chats");
+ expect(useHomeStore.getState().agentChatId).toBe("agent-7");
+ });
+
+ it("opens one conversation and leaves an agent's chat for it", async () => {
+ const openSession = vi.fn();
+ const original = useAgentChatStore.getState().openSession;
+ useAgentChatStore.setState({ openSession });
+ useEventStore.setState({ solo: false, activeSection: "settings", toasts: [] });
+ useHomeStore.setState({ agentChatId: "agent-7", jarvisCardMode: "voice" });
+ render(<Harness />);
+ await Promise.resolve();
+
+ MockWebSocket.last!.deliver(
+ envelope("NavigateSidebar", { section: "chats", target: "chat", target_id: "s-1" }),
+ );
+
+ expect(openSession).toHaveBeenCalledWith("s-1");
+ expect(useHomeStore.getState().agentChatId).toBeNull();
+ expect(useHomeStore.getState().jarvisCardMode).toBe("chat");
+ expect(useEventStore.getState().activeSection).toBe("chats");
+ useAgentChatStore.setState({ openSession: original });
+ });
+
+ it("a detached window keeps its view when a target is opened elsewhere", async () => {
+ useEventStore.setState({ solo: true, activeSection: "settings", toasts: [] });
+ useHomeStore.setState({ agentChatId: null });
+ render(<Harness />);
+ await Promise.resolve();
+
+ MockWebSocket.last!.deliver(
+ envelope("NavigateSidebar", { section: "chats", target: "agent", target_id: "agent-7" }),
+ );
+
+ expect(useEventStore.getState().activeSection).toBe("settings");
+ expect(useHomeStore.getState().agentChatId).toBeNull();
+ });
+
it("invalidates every documentation query after a registry reload", async () => {
const invalidate = vi.spyOn(queryClient, "invalidateQueries");
render(<Harness />);
diff --git a/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts b/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts
index 8564b0e9a9..6f3edf13a9 100644
--- a/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts
+++ b/jarvis/ui/web/frontend/src/hooks/useWebSocket.ts
@@ -30,6 +30,7 @@ import { handleAppshotEditRequest } from "@/store/appshotEditor";
import { useDeckStore } from "@/store/deck";
import { MEMORY_WRITE_EVENT, useMemoryWrites } from "@/store/memoryWrites";
import { useHomeStore } from "@/store/home";
+import { useAgentChatStore } from "@/store/agentChat";
import { PANE_ACTIVITY_EVENT } from "@/store/workspacePanes";
import { WSAudioLevel, WSEventEnvelope, WSWelcome } from "@/schema/ws";
import { useI18nStore, hydrateUiLanguage, hydrateReplyLanguage, isUiLanguage, translate, fill } from "@/i18n";
@@ -466,7 +467,7 @@ export function useWebSocket(): void {
}
if (env.event_name === "NavigateSidebar") {
- const p = env.payload as { section?: string };
+ const p = env.payload as { section?: string; target?: string; target_id?: string };
const solo = useEventStore.getState().solo;
// A retired id ("outputs") still arrives from older deep links and
// spoken aliases; it lands on the section that holds its content now.
@@ -487,6 +488,19 @@ export function useWebSocket(): void {
// "go to settings" targets the main window, and following it here
// would blank the very view the user split off to keep.
setActiveSection(section);
+ // One thing inside the section: an agent's chat or a conversation.
+ if (section === "chats" && p.target_id) {
+ if (p.target === "agent") {
+ useHomeStore.getState().openAgentChat(p.target_id);
+ } else if (p.target === "chat") {
+ const events = useEventStore.getState();
+ events.setActiveConversation("text", null);
+ events.setMessages([]);
+ useHomeStore.getState().openAgentChat(null);
+ useAgentChatStore.getState().openSession(p.target_id);
+ useHomeStore.getState().setJarvisCardMode("chat");
+ }
+ }
}
// The Settings "Autopilot Toasts" switch owns this note.
if (!solo && isAutopilotToastsEnabled()) {
diff --git a/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts b/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts
index 3ea306db17..08e21e5887 100644
--- a/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts
+++ b/jarvis/ui/web/frontend/src/lib/homeTranscript.test.ts
@@ -293,3 +293,39 @@ describe("transcriptFromMessages with stored traces", () => {
}
});
});
+
+describe("homeTranscript reducer — content shown on screen", () => {
+ const table = "| file | lines |\n|---|---|\n| app.py | 12 |";
+
+ it("adds a card line with its heading and never melts speech into it", () => {
+ const lines = run([
+ ["TranscriptFinal", { transcript: { text: "which files changed" } }, 1000],
+ ["VoiceContentShown", { session_id: "c1", title: " Changed files ", markdown: `\n${table}\n` }, 2000],
+ ["SpeechSpoken", { text: "Two files changed, the list is on screen." }, 2500],
+ ]);
+ expect(lines).toHaveLength(3);
+ const card = lines[1];
+ expect(card.who).toBe("assistant");
+ if (card.who === "steps") throw new Error("a card is a text line");
+ expect(card.kind).toBe("shown");
+ expect(card.text).toBe("Changed files");
+ expect(card.markdown).toBe(table);
+ expect(lines[2]).toMatchObject({ who: "assistant", text: "Two files changed, the list is on screen." });
+ });
+
+ it("shows the same card once when the event is delivered twice", () => {
+ const once = run([["VoiceContentShown", { markdown: table }, 1000]]);
+ const twice = reduceTranscript(once, "VoiceContentShown", { markdown: table }, 1500);
+ expect(twice).toBe(once);
+ const later = reduceTranscript(once, "VoiceContentShown", { markdown: table }, 60_000);
+ expect(later).toHaveLength(2);
+ });
+
+ it("ignores a card without content and keeps a heading from blocking spoken words", () => {
+ const start = run([["VoiceContentShown", { title: "Summary", markdown: table }, 1000]]);
+ expect(reduceTranscript(start, "VoiceContentShown", { title: "x", markdown: " " }, 1100)).toBe(start);
+ // Saying the heading out loud right after is still a line of its own.
+ const spoken = reduceTranscript(start, "SpeechSpoken", { text: "Summary" }, 1200);
+ expect(spoken).toHaveLength(2);
+ });
+});
diff --git a/jarvis/ui/web/frontend/src/lib/homeTranscript.ts b/jarvis/ui/web/frontend/src/lib/homeTranscript.ts
index af033eb8a5..bb33de9294 100644
--- a/jarvis/ui/web/frontend/src/lib/homeTranscript.ts
+++ b/jarvis/ui/web/frontend/src/lib/homeTranscript.ts
@@ -61,6 +61,12 @@ export interface TranscriptTextLine extends TranscriptLineBase {
*/
kind?: string;
revision?: number;
+ /**
+ * Content the assistant put on screen instead of reading it out
+ * (`VoiceContentShown`): a table, code, links. The line's kind is
+ * "shown" and `text` is its optional heading; it is never spoken.
+ */
+ markdown?: string;
}
/** The reasoning steps of one turn — tools, workers, the brain call. */
@@ -129,6 +135,9 @@ const MID_TURN_SPOKEN_KINDS: ReadonlySet<string> = new Set([
"subagent",
]);
+/** The kind of a line that shows content on screen (see `markdown`). */
+export const SHOWN_KIND = "shown";
+
let counter = 0;
function nextId(ts: number): string {
counter = (counter + 1) % 1_000_000;
@@ -152,6 +161,7 @@ function isDuplicate(
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i];
if (ts - line.ts > DUPLICATE_WITHIN_MS) return false;
+ if (line.who !== "steps" && line.kind === SHOWN_KIND) continue;
if (line.who === who && (line.text === text || line.text.endsWith(text))) return true;
}
return false;
@@ -427,6 +437,7 @@ export function reduceTranscript(
if (
last &&
last.who === "assistant" &&
+ last.kind !== SHOWN_KIND &&
tsMs - last.ts < JOIN_WITHIN_MS &&
(!last.kind || !kind || last.kind === kind)
) {
@@ -446,6 +457,20 @@ export function reduceTranscript(
if (!text || isDuplicate(lines, role, text, tsMs)) return lines;
return push(role === "user" ? closeOpenTurn(lines, tsMs) : lines, role, text, tsMs);
}
+ case "VoiceContentShown": {
+ const markdown = typeof p.markdown === "string" ? p.markdown.trim() : "";
+ if (!markdown) return lines;
+ // The same card delivered twice (a reconnect replays recent events).
+ for (let i = lines.length - 1; i >= 0; i--) {
+ const line = lines[i];
+ if (tsMs - line.ts > DUPLICATE_WITHIN_MS) break;
+ if (line.who === "assistant" && line.markdown === markdown) return lines;
+ }
+ return cap([...lines, {
+ id: nextId(tsMs), ts: tsMs, who: "assistant", text: clean(p.title),
+ kind: SHOWN_KIND, markdown,
+ }]);
+ }
case "VoiceTurnCompleted":
return closeOpenTurn(lines, tsMs);
case "SystemStateChanged": {
diff --git a/jarvis/ui/web/frontend/src/store/home.ts b/jarvis/ui/web/frontend/src/store/home.ts
index e78ada883a..4b1afc6b63 100644
--- a/jarvis/ui/web/frontend/src/store/home.ts
+++ b/jarvis/ui/web/frontend/src/store/home.ts
@@ -126,6 +126,11 @@ export const useHomeStore = create<HomeStore>((set, get) => ({
useEventStore.getState().setTranscription("", true);
set({ liveReply: "", liveSessionId: sessionId || null });
}
+ if (name === "VoiceContentShown") {
+ // A card belongs to the call it was shown in, never to another one.
+ const sessionId = (payload as { session_id?: string } | null)?.session_id;
+ if (sessionId && get().liveSessionId && sessionId !== get().liveSessionId) return;
+ }
if (name === "VoiceSessionEnded") {
const sessionId = (payload as { session_id?: string } | null)?.session_id;
if (sessionId && get().liveSessionId && sessionId !== get().liveSessionId) return;
diff --git a/tests/contract/test_workspace_orchestration.py b/tests/contract/test_workspace_orchestration.py
index 8fd0dcf0c1..f0b4dbf57c 100644
--- a/tests/contract/test_workspace_orchestration.py
+++ b/tests/contract/test_workspace_orchestration.py
@@ -64,6 +64,12 @@ class Sessions:
@pytest.fixture
def rig(tmp_path, monkeypatch):
+ # The panes below are stamped idle once, at setup. A slow runner can take
+ # longer than the stamp's freshness window to reach the step under test,
+ # and the pane then reads as unknown instead of idle; keep it fresh here.
+ from jarvis.agentic_ide import activity
+
+ monkeypatch.setattr(activity, "STAMP_FRESH_S", 3600.0)
monkeypatch.setattr(library, "_root", lambda: tmp_path / "library")
monkeypatch.setattr(resume_store, "_store_path", lambda: tmp_path / "snapshot.json")
registry = Registry(pty_manager=FakePtyManager())
diff --git a/tests/unit/plugins/tool/test_navigate.py b/tests/unit/plugins/tool/test_navigate.py
index 9627eef30e..f739f807cb 100644
--- a/tests/unit/plugins/tool/test_navigate.py
+++ b/tests/unit/plugins/tool/test_navigate.py
@@ -129,3 +129,82 @@ def test_known_sections_match_frontend_section_ids() -> None:
ts_ids = set(re.findall(r'"([a-z0-9_-]+)"', block.group(1)))
assert ts_ids, "no ids parsed from SECTION_IDS"
assert NavigateTool.known_sections() == ts_ids
+
+
+# --- one thing inside a section ---------------------------------------------------
+
+
+def _targets(bus: RecordingBus, **lookups) -> NavigateTool:
+ async def chat(spoken: str) -> dict:
+ return lookups.get("chat", {"status": "unavailable"})
+
+ async def workspace(spoken: str) -> dict:
+ return lookups.get("workspace", {"status": "unavailable"})
+
+ return NavigateTool(
+ bus=bus,
+ find_agent=lambda spoken: lookups.get("agent", {"status": "unavailable"}),
+ find_chat=chat,
+ show_workspace=workspace,
+ )
+
+
+async def test_a_named_agent_opens_that_agents_chat(bus: RecordingBus) -> None:
+ tool = _targets(bus, agent={"status": "found", "id": "agent-7", "name": "Scout"})
+ res = await tool.execute({"agent": " scout "}, None)
+ assert res.success and res.output["agent"] == "Scout"
+ (event,) = bus.events
+ assert isinstance(event, NavigateSidebar)
+ assert (event.section, event.target, event.target_id) == ("chats", "agent", "agent-7")
+
+
+async def test_a_conversation_title_opens_that_conversation(bus: RecordingBus) -> None:
+ tool = _targets(bus, chat={"status": "found", "id": "s-1", "name": "Trip to Lisbon"})
+ res = await tool.execute({"chat": "trip to lisbon"}, None)
+ assert res.success
+ (event,) = bus.events
+ assert (event.section, event.target, event.target_id) == ("chats", "chat", "s-1")
+
+
+async def test_a_workspace_is_shown_in_the_coding_section(bus: RecordingBus) -> None:
+ tool = _targets(bus, workspace={"status": "found", "id": "", "name": "Installer"})
+ res = await tool.execute({"workspace": "installer"}, None)
+ assert res.success and res.output["workspace"] == "Installer"
+ (event,) = bus.events
+ assert (event.section, event.target, event.target_id) == ("agentic-ide", "", "")
+
+
+ ("lookup", "says"),
+ [
+ ({"status": "ask", "candidates": ["Scout", "Scout 2"]}, "More than one agent"),
+ ({"status": "unknown", "available": ["Archivist"]}, "Known: Archivist"),
+ ({"status": "unavailable"}, "cannot open a agent right now"),
+ ],
+)
+async def test_an_unresolved_target_moves_nothing_and_says_why(
+ bus: RecordingBus, lookup: dict, says: str
+) -> None:
+ res = await _targets(bus, agent=lookup).execute({"agent": "scout"}, None)
+ assert res.success is False and says in res.error
+ assert bus.events == []
+
+
+async def test_two_destinations_at_once_are_refused(bus: RecordingBus) -> None:
+ tool = _targets(bus, agent={"status": "found", "id": "a", "name": "Scout"})
+ for args in ({"agent": "scout", "section": "settings"}, {"agent": "scout", "chat": "x"}):
+ res = await tool.execute(args, None)
+ assert res.success is False and "one destination" in res.error
+ assert bus.events == []
+
+
+async def test_a_plain_section_switch_carries_no_target(tool: NavigateTool, bus) -> None:
+ await tool.execute({"section": "settings"}, None)
+ (event,) = bus.events
+ assert (event.target, event.target_id) == ("", "")
diff --git a/tests/unit/plugins/tool/test_show_on_screen.py b/tests/unit/plugins/tool/test_show_on_screen.py
new file mode 100644
index 0000000000..a52523e18e
--- /dev/null
+++ b/tests/unit/plugins/tool/test_show_on_screen.py
@@ -0,0 +1,130 @@
+"""Exact content goes on screen; the voice keeps the takeaway."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+
+from jarvis.core.events import VoiceContentShown
+from jarvis.live import runtime
+from jarvis.plugins.tool import show_on_screen
+from jarvis.plugins.tool.show_on_screen import MAX_MARKDOWN_CHARS, ShowOnScreenTool
+
+_TABLE = "| file | lines |\n|---|---|\n| app.py | 12 |"
+
+
+class RecordingBus:
+ def __init__(self) -> None:
+ self.events: list[object] = []
+
+ async def publish(self, event: object) -> None:
+ self.events.append(event)
+
+
+def _ctx(session_id: str = "") -> SimpleNamespace:
+ config = {"live_session_id": session_id} if session_id else {}
+ return SimpleNamespace(trace_id=uuid4(), user_utterance="show me", config=config)
+
+
+async def test_in_a_live_call_the_content_is_handed_to_that_call(monkeypatch) -> None:
+ shown = []
+
+ async def show(session_id: str, *, title: str, markdown: str) -> bool:
+ shown.append((session_id, title, markdown))
+ return True
+
+ monkeypatch.setattr(runtime, "show_content", show)
+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: pytest.fail("the call owns it"))
+ res = await ShowOnScreenTool().execute(
+ {"markdown": f" {_TABLE}\n", "title": " Changed files "}, _ctx("call-1")
+ )
+ assert res.success and res.output == {"shown": True, "kept_in_call": True}
+ assert shown == [("call-1", "Changed files", _TABLE)]
+
+
[email protected]("session_id", ["", "ended-call"])
+async def test_without_a_running_call_it_is_still_shown(monkeypatch, session_id) -> None:
+ bus = RecordingBus()
+
+ async def gone(session_id: str, *, title: str, markdown: str) -> bool:
+ return False
+
+ monkeypatch.setattr(runtime, "show_content", gone)
+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: bus)
+ res = await ShowOnScreenTool().execute({"markdown": _TABLE}, _ctx(session_id))
+ assert res.success and res.output["kept_in_call"] is False
+ (event,) = bus.events
+ assert isinstance(event, VoiceContentShown)
+ assert (event.session_id, event.title, event.markdown) == ("", "", _TABLE)
+
+
+async def test_nothing_to_show_or_nowhere_to_show_it_is_an_error(monkeypatch) -> None:
+ monkeypatch.setattr(show_on_screen, "_app_bus", lambda: None)
+ tool = ShowOnScreenTool()
+ assert (await tool.execute({"markdown": " "}, _ctx())).error == "markdown is missing"
+ too_long = await tool.execute({"markdown": "x" * (MAX_MARKDOWN_CHARS + 1)}, _ctx())
+ assert too_long.success is False and "Too long to show" in too_long.error
+ nowhere = await tool.execute({"markdown": _TABLE}, _ctx())
+ assert nowhere.success is False and "No screen" in nowhere.error
+
+
+async def test_the_runtime_reports_a_call_that_is_not_running() -> None:
+ assert await runtime.show_content("no-such-call", title="", markdown=_TABLE) is False
+
+
+# --- inside the call: kept in its record, announced to the screen ------------------
+
+
+async def test_a_call_keeps_shown_content_in_its_record_after_the_last_words(tmp_path) -> None:
+ from jarvis.live.session import LiveVoiceSession
+ from jarvis.live.state import LiveLedger, TranscriptFragment
+
+ ledger = LiveLedger(tmp_path / "live.sqlite3")
+ bus = RecordingBus()
+ call = SimpleNamespace(session_id="call-1", _ledger=ledger, _bus=bus)
+ try:
+ ledger.append(TranscriptFragment("call-1", "e1", "user", "Which files changed?", 100, 900))
+ await LiveVoiceSession.show_content(call, title="Changed files", markdown=_TABLE)
+ rows = ledger.transcript("call-1")
+ finally:
+ ledger.close()
+ assert [row["role"] for row in rows] == ["user", "assistant"]
+ assert rows[1]["start_ms"] == 901 # right after what was said last
+ assert rows[1]["delta"] == f"\n\n**Changed files**\n\n{_TABLE}\n\n"
+ (event,) = bus.events
+ assert isinstance(event, VoiceContentShown)
+ assert (event.session_id, event.title, event.markdown) == ("call-1", "Changed files", _TABLE)
+
+
+async def test_a_title_or_id_finds_one_conversation(monkeypatch) -> None:
+ from jarvis.core import runtime_refs
+ from jarvis.plugins.tool import navigate
+
+ class Store:
+ def list_sessions(self, limit: int, *, surface: str):
+ assert surface == "jarvis"
+ return [
+ SimpleNamespace(session_id="s-2", title="Trip to Lisbon"),
+ SimpleNamespace(session_id="s-1", title="Trip to Lisbon"),
+ SimpleNamespace(session_id="s-0", title="Tax questions"),
+ ]
+
+ app = SimpleNamespace(state=SimpleNamespace(agent_chat=SimpleNamespace(store=Store())))
+ monkeypatch.setattr(runtime_refs, "get_web_app", lambda: app)
+ newest = await navigate._find_chat("trip to lisbon")
+ assert newest == {"status": "found", "id": "s-2", "name": "Trip to Lisbon"}
+ assert (await navigate._find_chat("s-0"))["id"] == "s-0"
+ assert (await navigate._find_chat("tax questions"))["id"] == "s-0"
+ assert (await navigate._find_chat("t"))["status"] == "ask"
+ unknown = await navigate._find_chat("no such chat")
+ assert unknown["status"] == "unknown" and "Trip to Lisbon" in unknown["available"]
+ monkeypatch.setattr(runtime_refs, "get_web_app", lambda: None)
+ assert (await navigate._find_chat("anything"))["status"] == "unavailable"