Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(appshot): take an area appshot at the shortcut press, not after selecting
rubenluetke10-beepbb164fb02ff5
diff --git a/docs/appshots.md b/docs/appshots.mdindex fb068c70fc..357cded857 100644--- a/docs/appshots.md+++ b/docs/appshots.md@@ -50,6 +50,10 @@ selection, so nothing stays resident. The picker provides:- every screen is frozen under a light dim the moment the picker opens, sonothing moves under the selection (where a frozen frame cannot be grabbed,a dim layer over the live desktop is used instead);+- the appshot is that same instant: every screen is grabbed at the shortcut+ press, before the picker opens, and the finished area is cut out of that+ frame (at full depth on an HDR monitor), so a video that plays on while+ the area is selected does not change the picture;- the pointer is a thin crosshair with a small ring, and two small numbersbeside it show the position — while dragging, the width and height in realpixels. No lens, no boxed labels, no banner;diff --git a/jarvis/appshot/service.py b/jarvis/appshot/service.pyindex d1dad2cca3..bbbe9422db 100644--- a/jarvis/appshot/service.py+++ b/jarvis/appshot/service.py@@ -119,6 +119,10 @@ async def take_appshot(issue = await check() if check is not None else Noneif issue is not None:return AppshotResult(status="refused", reason_code=issue[0], message=issue[1])+ # The picture is the screen at the press, as the picker shows it+ # frozen — not whatever a video has played on to by the time the+ # user finishes selecting.+ frozen = await service.freeze_screens()picked = await _pick_area(service, _language(config))if isinstance(picked, AppshotResult):return picked@@ -134,6 +138,7 @@ async def take_appshot(trace_id=trace_id or uuid.uuid4(),region=bbox,master=True,+ frozen=frozen,)finally:shutter_markup.reset(token)diff --git a/jarvis/screen_context/service.py b/jarvis/screen_context/service.pyindex 3de3ee99b2..172b898a56 100644--- a/jarvis/screen_context/service.py+++ b/jarvis/screen_context/service.py@@ -32,7 +32,7 @@ import mathimport secretsimport threadingimport time-from dataclasses import dataclass, replace+from dataclasses import dataclass, field, replacefrom typing import Any, Literalfrom uuid import UUID, uuid4@@ -162,6 +162,80 @@ class _Handle:expires_at_ns: int+@dataclass(frozen=True, slots=True)+class _FrozenMonitor:+ #: The monitor in capture coordinates.+ bbox: tuple[int, int, int, int]+ #: The frame's pixel size (backing pixels on macOS, so it can exceed ``bbox``).+ size: tuple[int, int]+ #: 8-bit RGB; ``None`` when ``master`` holds a full-depth frame to derive it from.+ rgb: bytes | None = field(repr=False)+ master: MasterImage = field(repr=False)+++@dataclass(frozen=True, slots=True)+class FrozenScreens:+ """Every monitor as it looked at one instant, for an area chosen afterwards.++ The area picker shows the screens frozen while the user selects. Grabbing+ the area only once the selection was finished photographed the LIVE+ screen, so a video had played on for as long as the selection took.+ :meth:`ScreenContextService.freeze_screens` grabs at the shortcut press+ and :meth:`ScreenContextService.capture` cuts the area out of that frame.+ Memory only, never stored. ``visible`` is the window list of the same+ instant for the denylist check (``None`` = unverifiable or no denylist).+ """++ frames: tuple[_FrozenMonitor, ...]+ visible: tuple[Any, ...] | None = None++ def cut(+ self, bbox: tuple[int, int, int, int]+ ) -> tuple[tuple[int, int], bytes, MasterImage] | None:+ """``bbox`` out of the frozen monitor holding it; ``None`` when none does."""+ left, top, width, height = (int(v) for v in bbox)+ for frame in self.frames:+ m_left, m_top, m_width, m_height = frame.bbox+ if (+ left >= m_left+ and top >= m_top+ and left + width <= m_left + m_width+ and top + height <= m_top + m_height+ ):+ return _cut_frame(frame, (left - m_left, top - m_top, width, height))+ return None+++def _cut_frame(+ frame: _FrozenMonitor, local: tuple[int, int, int, int]+) -> tuple[tuple[int, int], bytes, MasterImage]:+ """A monitor-local rectangle out of one frozen frame, in the frame's own pixels."""+ x, y, width, height = local+ frame_w, frame_h = frame.size+ scale_x = frame_w / max(1, frame.bbox[2])+ scale_y = frame_h / max(1, frame.bbox[3])+ x0 = min(frame_w - 1, max(0, round(x * scale_x)))+ y0 = min(frame_h - 1, max(0, round(y * scale_y)))+ x1 = min(frame_w, max(x0 + 1, round((x + width) * scale_x)))+ y1 = min(frame_h, max(y0 + 1, round((y + height) * scale_y)))+ master = frame.master+ if master.pixels is not None:+ master = replace(master, pixels=master.pixels[y0:y1, x0:x1].copy())+ if frame.rgb is None:+ from jarvis.platform.hdr_image import scrgb_to_srgb8 # noqa: PLC0415++ rgb8 = scrgb_to_srgb8(master.pixels, master.sdr_white_nits)+ return (x1 - x0, y1 - y0), rgb8.tobytes(), master+ from PIL import Image # noqa: PLC0415++ image = Image.frombytes("RGB", frame.size, frame.rgb).crop((x0, y0, x1, y1))+ return image.size, image.tobytes(), master+++#: ``_monitor_privacy_error`` reads the visible windows itself.+_LIVE: Any = object()++class ScreenContextService:"""One-shot screen context for the on-screen bar and the voice session."""@@ -359,6 +433,67 @@ class ScreenContextService:# implement the original ``str | None`` port contract.return "capture_permission", str(issue)+ async def freeze_screens(self) -> FrozenScreens | None:+ """Grab every monitor once, now, for an area the user selects afterwards.++ Each monitor is read the way :meth:`capture` reads a ``master``+ capture (full depth on an HDR monitor). ``None`` when no monitor could+ be grabbed; :meth:`capture` then grabs the area live, as before.+ """+ if self._closed:+ return None+ started = time.perf_counter()+ monitors = await asyncio.to_thread(self.displays.monitors)+ physical = [+ m for m in (monitors[1:] if len(monitors) > 1 else monitors) if isinstance(m, dict)+ ]+ # One duplication per monitor, each on its own thread: the press must+ # not wait for the monitors one after another.+ grabbed = await asyncio.gather(+ *(self._freeze_monitor(monitor, monitors) for monitor in physical)+ )+ frames = tuple(frame for frame in grabbed if frame is not None)+ if not frames:+ return None+ visible = None+ getter = getattr(self.window_probe, "visible_windows", None)+ if self._settings.denylist and callable(getter):+ found = await asyncio.to_thread(getter)+ visible = tuple(found) if found is not None else None+ log.info(+ "screen_context: froze %d of %d monitor(s) in %.0f ms",+ len(frames),+ len(physical),+ (time.perf_counter() - started) * 1000,+ )+ return FrozenScreens(frames=frames, visible=visible)++ async def _freeze_monitor(self, monitor: dict, monitors: list[dict]) -> _FrozenMonitor | None:+ from jarvis.screen_context.targeting import _monitor_name # noqa: PLC0415++ bbox = (+ int(monitor.get("left", 0)),+ int(monitor.get("top", 0)),+ int(monitor.get("width", 0)),+ int(monitor.get("height", 0)),+ )+ if bbox[2] <= 0 or bbox[3] <= 0:+ return None+ target = CaptureTarget(+ kind=TargetKind.MONITOR,+ bbox=bbox,+ reason=TargetReason.USER_REGION,+ monitor_name=_monitor_name(monitor, monitors),+ )+ try:+ # A full-depth frame is kept as it is: only the finished area is+ # turned into 8-bit, so the press does not wait for a 4K conversion.+ size, rgb, master = await self._grab_with_master(target, derive_rgb=False)+ except Exception: # noqa: BLE001 - this monitor's area is then grabbed live+ log.info("screen_context: could not freeze monitor %s", bbox, exc_info=True)+ return None+ return _FrozenMonitor(bbox=bbox, size=size, rgb=rgb, master=master)+async def capture(self,*,@@ -366,6 +501,7 @@ class ScreenContextService:trace_id: UUID | None = None,region: tuple[int, int, int, int] | None = None,master: bool = False,+ frozen: FrozenScreens | None = None,) -> CaptureOutcome:"""Take exactly one capture. Assumes intent is already established.@@ -375,6 +511,8 @@ class ScreenContextService:``master`` also returns :attr:`ScreenContext.master`, the user'sfull-fidelity copy (HDR at full depth on an HDR monitor), redacted thesame way. It rides on the outcome only, never behind the handle.+ ``frozen`` (from :meth:`freeze_screens`) makes a ``region`` capture cut+ the area out of that earlier frame instead of grabbing the screen now."""verdict = verdict or IntentVerdict(intent=VisualIntent.SCREEN)if self._closed:@@ -494,6 +632,10 @@ class ScreenContextService:# as intersecting (fail closed), and an unavailable enumeration blocks# monitor scope rather than silently weakening the privacy rule.monitor_privacy_error = await asyncio.to_thread(self._monitor_privacy_error, target)+ if not monitor_privacy_error and frozen is not None and target.kind is TargetKind.REGION:+ # The frozen pixels show the windows of the freeze instant: a+ # denylisted window that has closed since is still in them.+ monitor_privacy_error = self._monitor_privacy_error(target, visible=frozen.visible)if monitor_privacy_error:return CaptureOutcome(status="refused",@@ -560,7 +702,16 @@ class ScreenContextService:)try:master_image = None- if master:+ cut = None+ if frozen is not None and target.kind is TargetKind.REGION:+ cut = await asyncio.to_thread(frozen.cut, target.bbox)+ if cut is None:+ log.info("screen_context: area outside the frozen screens; grabbing live")+ if cut is not None:+ size, rgb, master_image = cut+ if not master:+ master_image = None+ elif master:size, rgb, master_image = await self._grab_with_master(target)else:size, rgb = await self._grab(target)@@ -698,8 +849,8 @@ class ScreenContextService:return size, rgbasync def _grab_with_master(- self, target: CaptureTarget- ) -> tuple[tuple[int, int], bytes, MasterImage]:+ self, target: CaptureTarget, *, derive_rgb: bool = True+ ) -> tuple[tuple[int, int], bytes | None, MasterImage]:"""One shutter for both copies: the model's 8-bit frame and the user's master.On an HDR or wide-gamut monitor the rectangle is read once at full@@ -707,7 +858,8 @@ class ScreenContextService:so both show the same instant and redaction boxes line up exactly.The privacy path (a denylisted window in the rectangle) and everyother monitor keep the normal grab; the master is then that frame- plus the monitor's ICC profile.+ plus the monitor's ICC profile. ``derive_rgb=False`` leaves the 8-bit+ frame of a full-depth read as ``None`` for the caller to derive later."""from jarvis.platform.display_color import display_color # noqa: PLC0415from jarvis.platform.hdr_grab import grab_extended # noqa: PLC0415@@ -719,11 +871,14 @@ class ScreenContextService:if rect_grab:frame = await asyncio.to_thread(grab_extended, target.bbox)if frame is not None:- rgb8 = await asyncio.to_thread(- scrgb_to_srgb8, frame.pixels, frame.sdr_white_nits- )- height, width = rgb8.shape[:2]- return (width, height), rgb8.tobytes(), MasterImage(+ height, width = frame.pixels.shape[:2]+ rgb = None+ if derive_rgb:+ rgb8 = await asyncio.to_thread(+ scrgb_to_srgb8, frame.pixels, frame.sdr_white_nits+ )+ rgb = rgb8.tobytes()+ return (width, height), rgb, MasterImage(pixels=frame.pixels,hdr=True,sdr_white_nits=frame.sdr_white_nits,@@ -763,14 +918,19 @@ class ScreenContextService:# ---- assembly --------------------------------------------------------- def _monitor_privacy_error(self, target: CaptureTarget) -> str | None:- """Return a refusal when monitor-wide denylist safety is unverifiable."""+ def _monitor_privacy_error(self, target: CaptureTarget, *, visible: Any = _LIVE) -> str | None:+ """Return a refusal when monitor-wide denylist safety is unverifiable.++ ``visible`` checks an earlier window list (a frozen frame's) instead+ of the windows on screen now.+ """if target.kind is TargetKind.WINDOW or not self._settings.denylist:return Noneregion = target.kind is TargetKind.REGIONwhat = "the selected area" if region else "the monitor"- visible_getter = getattr(self.window_probe, "visible_windows", None)- visible = visible_getter() if callable(visible_getter) else None+ if visible is _LIVE:+ visible_getter = getattr(self.window_probe, "visible_windows", None)+ visible = visible_getter() if callable(visible_getter) else Noneif visible is None:return (f"I did not capture {what} because its visible windows "@@ -1387,6 +1547,7 @@ def settings_from_config(cfg: Any) -> ScreenContextSettings:__all__ = ["CaptureOutcome","CaptureStatus",+ "FrozenScreens","ScreenContextService","ScreenContextSettings","settings_from_config",diff --git a/tests/unit/appshot/test_appshot_markup.py b/tests/unit/appshot/test_appshot_markup.pyindex 48f61a45b6..a98a9b1e40 100644--- a/tests/unit/appshot/test_appshot_markup.py+++ b/tests/unit/appshot/test_appshot_markup.py@@ -376,7 +376,12 @@ class Capture:self.displays = Displays()self.markup_at_shutter: list = []- async def capture(self, *, verdict=None, trace_id=None, region=None, master=False):+ async def freeze_screens(self):+ return None++ async def capture(+ self, *, verdict=None, trace_id=None, region=None, master=False, frozen=None+ ):from jarvis.appshot.effect import shutter_markupself.markup_at_shutter.append(shutter_markup.get())diff --git a/tests/unit/appshot/test_appshot_region.py b/tests/unit/appshot/test_appshot_region.pyindex 36cfa23fe1..f588a29825 100644--- a/tests/unit/appshot/test_appshot_region.py+++ b/tests/unit/appshot/test_appshot_region.py@@ -144,9 +144,20 @@ class RegionCaptureService:def __init__(self) -> None:self.regions: list = []self.displays = Displays()-- async def capture(self, *, verdict=None, trace_id=None, region=None, master=False):+ self.steps: list[str] = []+ self.frozen = object()+ self.frozen_used: list = []++ async def freeze_screens(self):+ self.steps.append("freeze")+ return self.frozen++ async def capture(+ self, *, verdict=None, trace_id=None, region=None, master=False, frozen=None+ ):+ self.steps.append("capture")self.regions.append((verdict, region))+ self.frozen_used.append(frozen)context = ScreenContext(image=b"jpeg",mime="image/jpeg",@@ -174,6 +185,7 @@ def flow(monkeypatch):picks: list = []async def pick_region(**_kwargs):+ service.steps.append("pick")return picks.pop(0) if picks else Nonemonkeypatch.setattr(turn, "get_service", lambda bus=None: service)@@ -203,6 +215,24 @@ async def test_an_area_appshot_captures_exactly_the_selected_rectangle(flow) ->assert result.shot.delivered_to == "message"+async def test_an_area_appshot_is_the_screen_at_the_press_not_after_selecting(flow) -> None:+ # A video plays on while the user selects: the screens freeze BEFORE the+ # picker opens and the area is cut from that frame.+ service, picks = flow+ picks.append(+ region.Selection(+ screen={"x": 0.0, "y": 0.0, "w": 1920.0, "h": 1080.0, "dpr": 1.0},+ rect=(0.25, 0.5, 0.5, 0.25),+ )+ )++ result = await appshot_service.take_appshot(trigger="hotkey", scope="region")++ assert result.ok+ assert service.steps == ["freeze", "pick", "capture"]+ assert service.frozen_used == [service.frozen]++async def test_esc_on_the_picker_takes_nothing(flow) -> None:service, _picks = flowdiff --git a/tests/unit/screen_context/test_service.py b/tests/unit/screen_context/test_service.pyindex f0e3d228ce..35769eb0a6 100644--- a/tests/unit/screen_context/test_service.py+++ b/tests/unit/screen_context/test_service.py@@ -1161,3 +1161,162 @@ async def test_focus_settling_after_the_picker_does_not_void_an_area() -> None:assert outcome.status == "captured"assert len(capturer.grabs) == 1+++# --------------------------------------------------------------------------+# Frozen screens: the area is the screen at the press, not after selecting+# --------------------------------------------------------------------------++_SMALL_MONITORS = [+ {"left": 0, "top": 0, "width": 400, "height": 100},+ {"left": 0, "top": 0, "width": 200, "height": 100},+ {"left": 200, "top": 0, "width": 200, "height": 100},+]+++class _PlayingVideo:+ """A screen whose colour changes between grabs, like a playing video."""++ name = "fake-video"++ def __init__(self) -> None:+ self.colour = (255, 0, 0)+ self.grabs: list[tuple] = []++ def grab(self, bbox, *, window_handle=None):+ self.grabs.append((tuple(bbox), window_handle))+ width, height = int(bbox[2]), int(bbox[3])+ return ((width, height), bytes(self.colour) * (width * height))+++def _sdr_monitors(monkeypatch: pytest.MonkeyPatch) -> None:+ """No HDR duplication and no ICC lookup on the test host's real monitors."""+ from types import SimpleNamespace++ import jarvis.platform.display_color as display_color+ import jarvis.platform.hdr_grab as hdr_grab++ monkeypatch.setattr(hdr_grab, "grab_extended", lambda _bbox: None)+ monkeypatch.setattr(+ display_color, "display_color", lambda *_a, **_k: SimpleNamespace(icc_profile=None)+ )+++def _centre_colour(jpeg: bytes) -> tuple[int, int, int]:+ from PIL import Image++ with Image.open(io.BytesIO(jpeg)) as image:+ rgb = image.convert("RGB")+ return rgb.getpixel((rgb.width // 2, rgb.height // 2))++[email protected]("_sdr_monitors")+async def test_a_frozen_area_is_cut_from_the_press_not_grabbed_after_selecting() -> None:+ screen = _PlayingVideo()+ service = make_service(capturer=screen, displays=FakeDisplays(_SMALL_MONITORS))++ frozen = await service.freeze_screens()+ screen.colour = (0, 0, 255) # the video played on while the user selected+ outcome = await service.capture(region=(210, 10, 120, 60), master=True, frozen=frozen)++ assert outcome.status == "captured"+ # One grab per monitor at the press; none after the selection.+ assert sorted(screen.grabs) == [((0, 0, 200, 100), None), ((200, 0, 200, 100), None)]+ red, green, blue = _centre_colour(outcome.context.image)+ assert red > 200 and blue < 60+ assert outcome.context.target.bbox == (210, 10, 120, 60)++[email protected]("_sdr_monitors")+async def test_an_area_outside_the_frozen_screens_is_grabbed_live() -> None:+ screen = _PlayingVideo()+ service = make_service(capturer=screen, displays=FakeDisplays(_SMALL_MONITORS))+ frozen = _service_module.FrozenScreens(+ frames=(+ _service_module._FrozenMonitor(+ bbox=(0, 0, 200, 100),+ size=(200, 100),+ rgb=bytes((0, 255, 0)) * (200 * 100),+ master=_service_module.MasterImage(pixels=None),+ ),+ )+ )++ outcome = await service.capture(region=(210, 10, 120, 60), frozen=frozen)++ assert outcome.status == "captured"+ assert screen.grabs == [((210, 10, 120, 60), None)]+++def test_a_cut_follows_backing_pixels_and_slices_the_master() -> None:+ import numpy as np++ # A macOS-style frame: 2x the monitor's points.+ pixels = np.zeros((200, 400, 4), dtype=np.float16)+ pixels[20:40, 60:100] = 1.0+ frame = _service_module._FrozenMonitor(+ bbox=(100, 0, 200, 100),+ size=(400, 200),+ rgb=bytes(400 * 200 * 3),+ master=_service_module.MasterImage(pixels=pixels, hdr=True),+ )+ frozen = _service_module.FrozenScreens(frames=(frame,))++ size, rgb, master = frozen.cut((130, 10, 20, 10))++ assert size == (40, 20)+ assert len(rgb) == 40 * 20 * 3+ assert master.pixels.shape == (20, 40, 4)+ assert float(master.pixels.min()) == 1.0+ assert frozen.cut((290, 0, 20, 10)) is None # runs off the frozen monitor++[email protected]("_sdr_monitors")+async def test_a_denylisted_window_in_the_frozen_frame_blocks_the_area() -> None:+ class VaultClosesAfterThePress(FakeWindowProbe):+ def __init__(self) -> None:+ super().__init__()+ self.reads = 0++ def visible_windows(self):+ self.reads += 1+ if self.reads == 1:+ return (+ WindowFacts(app_name="vault", title="1Password", frame_rect=(0, 0, 150, 90)),+ )+ return (WindowFacts(app_name="editor", title="notes", frame_rect=(0, 0, 400, 100)),)++ screen = _PlayingVideo()+ service = make_service(+ settings=ScreenContextSettings(denylist=("1password",)),+ window_probe=VaultClosesAfterThePress(),+ capturer=screen,+ displays=FakeDisplays(_SMALL_MONITORS),+ )++ frozen = await service.freeze_screens()+ outcome = await service.capture(region=(10, 10, 100, 60), master=True, frozen=frozen)++ assert outcome.status == "refused"+ assert outcome.reason_kind == "policy"+ assert outcome.context is None+++def test_a_full_depth_frame_turns_only_the_cut_area_into_8_bit() -> None:+ import numpy as np++ pixels = np.zeros((100, 200, 4), dtype=np.float16)+ pixels[10:30, 20:60, :3] = 1.0 # SDR white at 80 nits+ frame = _service_module._FrozenMonitor(+ bbox=(0, 0, 200, 100),+ size=(200, 100),+ rgb=None,+ master=_service_module.MasterImage(pixels=pixels, hdr=True, sdr_white_nits=80.0),+ )++ size, rgb, master = _service_module.FrozenScreens(frames=(frame,)).cut((20, 10, 40, 20))++ assert size == (40, 20)+ assert rgb == b"\xff" * (40 * 20 * 3)+ assert master.hdr and master.pixels.shape == (20, 40, 4)