Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(appshot): let take_appshot capture the whole screen on an explicit request
rubenluetke10-beep2484f11bd4e7
diff --git a/docs/appshots.md b/docs/appshots.mdindex a85edcb519..f5eb85dfae 100644--- a/docs/appshots.md+++ b/docs/appshots.md@@ -11,9 +11,14 @@ text — and hands it to the conversation as context. Settings live under|---|---|| **Window shortcut** — both Alt keys at once by default (both Option keys on a Mac) | The front window is captured and delivered per **Appshot destination**. || **Area shortcut** — Alt+Win+A by default (Option+Command+A on a Mac) | Every screen dims; drag a rectangle and exactly that part is captured and delivered the same way. Esc or a right-click cancels and sends nothing. |-| **Voice or chat** — "take an appshot", "mach einen Appshot", "haz un appshot" | The turn that asked captures the front window and answers with it. |+| **Voice or chat** — "take an appshot", "mach einen Appshot", "haz un appshot" | The turn that asked captures the front window and answers with it. Only an explicit request for the whole screen ("an appshot of my full screen") captures the monitor the cursor is on instead — the live model's `take_appshot` passes `scope: "screen"` for it. || **Try it** buttons on the Appshots page | "Take appshot in 3 s" waits three seconds so you can switch windows; "Select area" opens the area picker at once. Both then behave like the shortcuts. |+"Front window" means the app window you work in. Jarvis's own floating+overlays (the mascot, the bar) take the focus when you click them, so the+capture looks past them to the app window underneath (BUG-228); with no app+window there, it takes the whole screen instead.+## Selecting an areaThe area picker is a short-lived PySide6 process (`python -mdiff --git a/jarvis/plugins/tool/appshot.py b/jarvis/plugins/tool/appshot.pyindex e5d8300970..72fe32d855 100644--- a/jarvis/plugins/tool/appshot.py+++ b/jarvis/plugins/tool/appshot.py@@ -5,10 +5,29 @@ spoken "take an appshot" reaches the live model as words. This tool gives itthe same capture the global shortcut takes — the front window, privacyfiltered, with the shutter effect and sound — and returns the picture as animage input for the answer.++``scope="screen"`` is for an explicit request for the whole screen ("an+appshot of my full screen"): it takes the monitor the cursor is on, through+the same service, instead of narrowing to one window. The default stays the+front window, so nothing wider than asked for is captured."""from __future__ import annotations+import asyncio++#: Scope → the capture it takes. ``window`` is the front app window (Jarvis's+#: own floating overlays are looked past); ``screen`` is the whole monitor.+SCOPES = ("window", "screen")++#: Trusted framing in front of the untrusted screen evidence of a full-screen+#: appshot (the window appshot carries the service's own preamble).+_SCREEN_PREAMBLE = (+ "APPSHOT: the user deliberately asked for their whole screen to give you "+ "context. Use it for their request. If they only asked you to take an "+ "appshot, confirm it in one short sentence and ask what they want to know."+)+def _app_bus():from jarvis.core.runtime_refs import get_brain_manager, get_web_app@@ -20,41 +39,111 @@ def _app_bus():return getattr(get_brain_manager(), "_bus", None)+def _load_config():+ from jarvis.core.config import load_config # noqa: PLC0415++ return load_config()++class AppshotTool:name = "take_appshot"description = (- "Take an appshot: capture the user's front window once, with the usual "- "privacy filter, so you can see what they are working on. Use when the "- "user asks for an appshot or asks you to look at their window or screen."+ "Take an appshot: capture the user's front app window once, with the "+ "usual privacy filter, so you can see what they are working on. Use when "+ "the user asks for an appshot or asks you to look at their window. Set "+ "scope to 'screen' only when they explicitly ask for the whole or full "+ "screen; otherwise leave it at 'window'.")risk_tier = "monitor"- schema = {"type": "object", "properties": {}, "additionalProperties": False}+ schema = {+ "type": "object",+ "properties": {+ "scope": {+ "type": "string",+ "enum": list(SCOPES),+ "description": (+ "'window' (default): the front app window. 'screen': the whole "+ "monitor, only when the user explicitly asks for the full screen."+ ),+ }+ },+ "additionalProperties": False,+ }async def execute(self, args, ctx):- import base64+ from jarvis.core.protocols import ToolResult++ scope = str((args or {}).get("scope") or "window").strip().lower()+ if scope not in SCOPES:+ return ToolResult(+ False, None, f"Unknown appshot scope {scope!r}; use 'window' or 'screen'."+ )+ trace_id = getattr(ctx, "trace_id", None)+ if scope == "screen":+ return await _screen_appshot(trace_id)from jarvis.appshot.service import take_appshot- from jarvis.core.protocols import ToolResult- del argsresult = await take_appshot(trigger="tool",bus=_app_bus(),deliver=False,- trace_id=getattr(ctx, "trace_id", None),+ trace_id=trace_id,)if not result.ok or result.shot is None:return ToolResult(False, None, result.message or "The appshot could not be taken.")- shot = result.shot+ return _picture(result.shot)+++async def _screen_appshot(trace_id):+ """The whole monitor the cursor is on, recorded as an appshot."""+ import uuid+ from dataclasses import replace++ from jarvis.appshot.service import record_turn_capture, shot_from_context+ from jarvis.core.protocols import ToolResult+ from jarvis.screen_context.models import IntentVerdict, VisualIntent+ from jarvis.screen_context.turn import get_service, model_note++ config = await asyncio.to_thread(_load_config)+ if not config.screen_context.enabled:return ToolResult(- True,- {- "description": f"Appshot of the {shot.label}, {shot.width}x{shot.height}.",- "app": shot.app_name,- "evidence": shot.note,- "_image": {- "mime": shot.mime,- "data": base64.b64encode(shot.image).decode("ascii"),- },- },+ False,+ None,+ "Appshots are switched off. Turn them on under Settings > Appshots.",)+ bus = _app_bus()+ service = get_service(bus=bus)+ outcome = await service.capture(+ verdict=IntentVerdict(intent=VisualIntent.SCREEN, evidence=("appshot-screen",)),+ trace_id=trace_id or uuid.uuid4(),+ )+ context = service.consume(outcome.handle_id) if outcome.handle_id else None+ if outcome.status != "captured" or context is None:+ return ToolResult(False, None, outcome.message or "The appshot could not be taken.")+ await record_turn_capture(context, bus=bus, trigger="tool")+ # The shared appshot preamble speaks of the front window; this one is not.+ shot = replace(+ shot_from_context(context, trigger="tool"),+ note=f"{_SCREEN_PREAMBLE}\n{model_note(context)}",+ )+ return _picture(shot)+++def _picture(shot):+ import base64++ from jarvis.core.protocols import ToolResult++ return ToolResult(+ True,+ {+ "description": f"Appshot of the {shot.label}, {shot.width}x{shot.height}.",+ "app": shot.app_name,+ "evidence": shot.note,+ "_image": {+ "mime": shot.mime,+ "data": base64.b64encode(shot.image).decode("ascii"),+ },+ },+ )diff --git a/tests/unit/appshot/test_appshot_tool.py b/tests/unit/appshot/test_appshot_tool.pynew file mode 100644index 0000000000..7f476d7336--- /dev/null+++ b/tests/unit/appshot/test_appshot_tool.py@@ -0,0 +1,193 @@+"""``take_appshot`` takes exactly what was asked for: the front window by+default, the whole screen only on an explicit ``scope="screen"``.++A voice call once answered "take an appshot of my full screen" with a window+capture, because the tool had no way to ask for more than the front window.+"""++from __future__ import annotations++import base64+import uuid+from types import SimpleNamespace++import pytest++from jarvis.appshot import service as appshot_service+from jarvis.appshot.store import get_store+from jarvis.plugins.tool.appshot import AppshotTool+from jarvis.screen_context.models import (+ CaptureTarget,+ ScreenContext,+ TargetKind,+ TargetReason,+ VisualIntent,+ WindowFacts,+)+from jarvis.screen_context.service import CaptureOutcome+++def _context(intent: VisualIntent) -> ScreenContext:+ if intent is VisualIntent.WINDOW:+ target = CaptureTarget(+ kind=TargetKind.WINDOW,+ bbox=(0, 0, 1280, 720),+ reason=TargetReason.FOCUSED_WINDOW,+ window=WindowFacts(app_name="Editor", title="notes.md"),+ )+ size = (1280, 720)+ else:+ target = CaptureTarget(+ kind=TargetKind.MONITOR,+ bbox=(0, 0, 2560, 1440),+ reason=TargetReason.CURSOR_MONITOR,+ monitor_name="1",+ )+ size = (2560, 1440)+ return ScreenContext(+ image=b"jpeg-bytes",+ mime="image/jpeg",+ size=size,+ target=target,+ ui_text="",+ captured_at_ns=1,+ )+++class FakeService:+ """Captures what the verdict scoped, like the real targeting does."""++ def __init__(self) -> None:+ self.verdicts: list = []+ self.handles: dict = {}+ self.refuse = False++ async def capture(self, *, verdict=None, trace_id=None, **_kwargs):+ self.verdicts.append(verdict)+ if self.refuse:+ return CaptureOutcome(+ status="refused",+ verdict=verdict,+ reason_kind="policy",+ message="Blocked by the privacy filter.",+ )+ self.handles["h1"] = _context(verdict.intent)+ return CaptureOutcome(+ status="captured", verdict=verdict, context=self.handles["h1"], handle_id="h1"+ )++ def consume(self, handle_id):+ return self.handles.pop(handle_id, None)+++class Bus:+ def __init__(self) -> None:+ self.events: list = []++ async def publish(self, event) -> None:+ self.events.append(event)+++def _config(*, enabled: bool = True):+ return SimpleNamespace(+ screen_context=SimpleNamespace(enabled=enabled, deck_preview_s=120.0, ttl_s=60.0),+ appshot=SimpleNamespace(target="auto"),+ )+++def wired(monkeypatch):+ import jarvis.plugins.tool.appshot as appshot_tool+ import jarvis.screen_context.turn as turn++ service = FakeService()+ bus = Bus()+ state = {"config": _config()}+ monkeypatch.setattr(turn, "get_service", lambda bus=None: service)+ monkeypatch.setattr(appshot_tool, "_app_bus", lambda: bus)+ monkeypatch.setattr(appshot_tool, "_load_config", lambda: state["config"])+ monkeypatch.setattr(appshot_service, "_load_config", lambda: state["config"])+ get_store().clear()+ yield service, bus, state+ get_store().clear()+++def _ctx():+ return SimpleNamespace(trace_id=uuid.uuid4())+++async def test_default_is_the_front_window_only(wired) -> None:+ service, bus, _state = wired++ result = await AppshotTool().execute({}, _ctx())++ assert result.success+ assert [v.intent for v in service.verdicts] == [VisualIntent.WINDOW]+ assert result.output["description"] == "Appshot of the active window, 1280x720."+ assert bus.events[-1].delivered_to == "turn"+++async def test_explicit_window_scope_is_the_front_window(wired) -> None:+ service, _bus, _state = wired++ result = await AppshotTool().execute({"scope": "window"}, _ctx())++ assert result.success+ assert [v.intent for v in service.verdicts] == [VisualIntent.WINDOW]+++async def test_screen_scope_takes_the_whole_monitor(wired) -> None:+ service, bus, _state = wired++ result = await AppshotTool().execute({"scope": "screen"}, _ctx())++ assert result.success+ assert [v.intent for v in service.verdicts] == [VisualIntent.SCREEN]+ assert result.output["description"] == "Appshot of the monitor 1, 2560x1440."+ assert base64.b64decode(result.output["_image"]["data"]) == b"jpeg-bytes"+ assert "whole screen" in result.output["evidence"]+ assert "front window" not in result.output["evidence"], "it was not the front window"+ assert service.handles == {}, "the capture handle is consumed at once"+ receipt = bus.events[-1]+ assert receipt.trigger == "tool" and receipt.delivered_to == "turn"+ assert get_store().latest().label == "monitor 1"+++async def test_screen_scope_respects_the_off_switch(wired) -> None:+ service, bus, state = wired+ state["config"] = _config(enabled=False)++ result = await AppshotTool().execute({"scope": "screen"}, _ctx())++ assert not result.success+ assert "switched off" in result.error+ assert service.verdicts == [], "nothing is captured while appshots are off"+ assert bus.events == []+++async def test_a_refused_screen_appshot_records_nothing(wired) -> None:+ service, bus, _state = wired+ service.refuse = True++ result = await AppshotTool().execute({"scope": "screen"}, _ctx())++ assert not result.success+ assert result.error == "Blocked by the privacy filter."+ assert bus.events == []+ assert get_store().latest() is None+++async def test_an_unknown_scope_captures_nothing(wired) -> None:+ service, _bus, _state = wired++ result = await AppshotTool().execute({"scope": "everything"}, _ctx())++ assert not result.success+ assert service.verdicts == []+++def test_schema_offers_window_and_screen_and_nothing_wider() -> None:+ schema = AppshotTool.schema+ assert schema["additionalProperties"] is False+ assert schema["properties"]["scope"]["enum"] == ["window", "screen"]+ assert "scope" not in schema.get("required", []), "the window stays the default"