Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(appshot): take an area appshot at the shortcut press, not after selecting

rubenluetke10-beepbb164fb02ff5
diff --git a/docs/appshots.md b/docs/appshots.md
index fb068c70fc..357cded857 100644
--- a/docs/appshots.md
+++ b/docs/appshots.md
@@ -50,6 +50,10 @@ selection, so nothing stays resident. The picker provides:
- every screen is frozen under a light dim the moment the picker opens, so
nothing moves under the selection (where a frozen frame cannot be grabbed,
a dim layer over the live desktop is used instead);
+- the appshot is that same instant: every screen is grabbed at the shortcut
+ press, before the picker opens, and the finished area is cut out of that
+ frame (at full depth on an HDR monitor), so a video that plays on while
+ the area is selected does not change the picture;
- the pointer is a thin crosshair with a small ring, and two small numbers
beside it show the position — while dragging, the width and height in real
pixels. No lens, no boxed labels, no banner;
diff --git a/jarvis/appshot/service.py b/jarvis/appshot/service.py
index d1dad2cca3..bbbe9422db 100644
--- a/jarvis/appshot/service.py
+++ b/jarvis/appshot/service.py
@@ -119,6 +119,10 @@ async def take_appshot(
issue = await check() if check is not None else None
if issue is not None:
return AppshotResult(status="refused", reason_code=issue[0], message=issue[1])
+ # The picture is the screen at the press, as the picker shows it
+ # frozen — not whatever a video has played on to by the time the
+ # user finishes selecting.
+ frozen = await service.freeze_screens()
picked = await _pick_area(service, _language(config))
if isinstance(picked, AppshotResult):
return picked
@@ -134,6 +138,7 @@ async def take_appshot(
trace_id=trace_id or uuid.uuid4(),
region=bbox,
master=True,
+ frozen=frozen,
)
finally:
shutter_markup.reset(token)
diff --git a/jarvis/screen_context/service.py b/jarvis/screen_context/service.py
index 3de3ee99b2..172b898a56 100644
--- a/jarvis/screen_context/service.py
+++ b/jarvis/screen_context/service.py
@@ -32,7 +32,7 @@ import math
import secrets
import threading
import time
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
from typing import Any, Literal
from uuid import UUID, uuid4
@@ -162,6 +162,80 @@ class _Handle:
expires_at_ns: int
+@dataclass(frozen=True, slots=True)
+class _FrozenMonitor:
+ #: The monitor in capture coordinates.
+ bbox: tuple[int, int, int, int]
+ #: The frame's pixel size (backing pixels on macOS, so it can exceed ``bbox``).
+ size: tuple[int, int]
+ #: 8-bit RGB; ``None`` when ``master`` holds a full-depth frame to derive it from.
+ rgb: bytes | None = field(repr=False)
+ master: MasterImage = field(repr=False)
+
+
+@dataclass(frozen=True, slots=True)
+class FrozenScreens:
+ """Every monitor as it looked at one instant, for an area chosen afterwards.
+
+ The area picker shows the screens frozen while the user selects. Grabbing
+ the area only once the selection was finished photographed the LIVE
+ screen, so a video had played on for as long as the selection took.
+ :meth:`ScreenContextService.freeze_screens` grabs at the shortcut press
+ and :meth:`ScreenContextService.capture` cuts the area out of that frame.
+ Memory only, never stored. ``visible`` is the window list of the same
+ instant for the denylist check (``None`` = unverifiable or no denylist).
+ """
+
+ frames: tuple[_FrozenMonitor, ...]
+ visible: tuple[Any, ...] | None = None
+
+ def cut(
+ self, bbox: tuple[int, int, int, int]
+ ) -> tuple[tuple[int, int], bytes, MasterImage] | None:
+ """``bbox`` out of the frozen monitor holding it; ``None`` when none does."""
+ left, top, width, height = (int(v) for v in bbox)
+ for frame in self.frames:
+ m_left, m_top, m_width, m_height = frame.bbox
+ if (
+ left >= m_left
+ and top >= m_top
+ and left + width <= m_left + m_width
+ and top + height <= m_top + m_height
+ ):
+ return _cut_frame(frame, (left - m_left, top - m_top, width, height))
+ return None
+
+
+def _cut_frame(
+ frame: _FrozenMonitor, local: tuple[int, int, int, int]
+) -> tuple[tuple[int, int], bytes, MasterImage]:
+ """A monitor-local rectangle out of one frozen frame, in the frame's own pixels."""
+ x, y, width, height = local
+ frame_w, frame_h = frame.size
+ scale_x = frame_w / max(1, frame.bbox[2])
+ scale_y = frame_h / max(1, frame.bbox[3])
+ x0 = min(frame_w - 1, max(0, round(x * scale_x)))
+ y0 = min(frame_h - 1, max(0, round(y * scale_y)))
+ x1 = min(frame_w, max(x0 + 1, round((x + width) * scale_x)))
+ y1 = min(frame_h, max(y0 + 1, round((y + height) * scale_y)))
+ master = frame.master
+ if master.pixels is not None:
+ master = replace(master, pixels=master.pixels[y0:y1, x0:x1].copy())
+ if frame.rgb is None:
+ from jarvis.platform.hdr_image import scrgb_to_srgb8 # noqa: PLC0415
+
+ rgb8 = scrgb_to_srgb8(master.pixels, master.sdr_white_nits)
+ return (x1 - x0, y1 - y0), rgb8.tobytes(), master
+ from PIL import Image # noqa: PLC0415
+
+ image = Image.frombytes("RGB", frame.size, frame.rgb).crop((x0, y0, x1, y1))
+ return image.size, image.tobytes(), master
+
+
+#: ``_monitor_privacy_error`` reads the visible windows itself.
+_LIVE: Any = object()
+
+
class ScreenContextService:
"""One-shot screen context for the on-screen bar and the voice session."""
@@ -359,6 +433,67 @@ class ScreenContextService:
# implement the original ``str | None`` port contract.
return "capture_permission", str(issue)
+ async def freeze_screens(self) -> FrozenScreens | None:
+ """Grab every monitor once, now, for an area the user selects afterwards.
+
+ Each monitor is read the way :meth:`capture` reads a ``master``
+ capture (full depth on an HDR monitor). ``None`` when no monitor could
+ be grabbed; :meth:`capture` then grabs the area live, as before.
+ """
+ if self._closed:
+ return None
+ started = time.perf_counter()
+ monitors = await asyncio.to_thread(self.displays.monitors)
+ physical = [
+ m for m in (monitors[1:] if len(monitors) > 1 else monitors) if isinstance(m, dict)
+ ]
+ # One duplication per monitor, each on its own thread: the press must
+ # not wait for the monitors one after another.
+ grabbed = await asyncio.gather(
+ *(self._freeze_monitor(monitor, monitors) for monitor in physical)
+ )
+ frames = tuple(frame for frame in grabbed if frame is not None)
+ if not frames:
+ return None
+ visible = None
+ getter = getattr(self.window_probe, "visible_windows", None)
+ if self._settings.denylist and callable(getter):
+ found = await asyncio.to_thread(getter)
+ visible = tuple(found) if found is not None else None
+ log.info(
+ "screen_context: froze %d of %d monitor(s) in %.0f ms",
+ len(frames),
+ len(physical),
+ (time.perf_counter() - started) * 1000,
+ )
+ return FrozenScreens(frames=frames, visible=visible)
+
+ async def _freeze_monitor(self, monitor: dict, monitors: list[dict]) -> _FrozenMonitor | None:
+ from jarvis.screen_context.targeting import _monitor_name # noqa: PLC0415
+
+ bbox = (
+ int(monitor.get("left", 0)),
+ int(monitor.get("top", 0)),
+ int(monitor.get("width", 0)),
+ int(monitor.get("height", 0)),
+ )
+ if bbox[2] <= 0 or bbox[3] <= 0:
+ return None
+ target = CaptureTarget(
+ kind=TargetKind.MONITOR,
+ bbox=bbox,
+ reason=TargetReason.USER_REGION,
+ monitor_name=_monitor_name(monitor, monitors),
+ )
+ try:
+ # A full-depth frame is kept as it is: only the finished area is
+ # turned into 8-bit, so the press does not wait for a 4K conversion.
+ size, rgb, master = await self._grab_with_master(target, derive_rgb=False)
+ except Exception: # noqa: BLE001 - this monitor's area is then grabbed live
+ log.info("screen_context: could not freeze monitor %s", bbox, exc_info=True)
+ return None
+ return _FrozenMonitor(bbox=bbox, size=size, rgb=rgb, master=master)
+
async def capture(
self,
*,
@@ -366,6 +501,7 @@ class ScreenContextService:
trace_id: UUID | None = None,
region: tuple[int, int, int, int] | None = None,
master: bool = False,
+ frozen: FrozenScreens | None = None,
) -> CaptureOutcome:
"""Take exactly one capture. Assumes intent is already established.
@@ -375,6 +511,8 @@ class ScreenContextService:
``master`` also returns :attr:`ScreenContext.master`, the user's
full-fidelity copy (HDR at full depth on an HDR monitor), redacted the
same way. It rides on the outcome only, never behind the handle.
+ ``frozen`` (from :meth:`freeze_screens`) makes a ``region`` capture cut
+ the area out of that earlier frame instead of grabbing the screen now.
"""
verdict = verdict or IntentVerdict(intent=VisualIntent.SCREEN)
if self._closed:
@@ -494,6 +632,10 @@ class ScreenContextService:
# as intersecting (fail closed), and an unavailable enumeration blocks
# monitor scope rather than silently weakening the privacy rule.
monitor_privacy_error = await asyncio.to_thread(self._monitor_privacy_error, target)
+ if not monitor_privacy_error and frozen is not None and target.kind is TargetKind.REGION:
+ # The frozen pixels show the windows of the freeze instant: a
+ # denylisted window that has closed since is still in them.
+ monitor_privacy_error = self._monitor_privacy_error(target, visible=frozen.visible)
if monitor_privacy_error:
return CaptureOutcome(
status="refused",
@@ -560,7 +702,16 @@ class ScreenContextService:
)
try:
master_image = None
- if master:
+ cut = None
+ if frozen is not None and target.kind is TargetKind.REGION:
+ cut = await asyncio.to_thread(frozen.cut, target.bbox)
+ if cut is None:
+ log.info("screen_context: area outside the frozen screens; grabbing live")
+ if cut is not None:
+ size, rgb, master_image = cut
+ if not master:
+ master_image = None
+ elif master:
size, rgb, master_image = await self._grab_with_master(target)
else:
size, rgb = await self._grab(target)
@@ -698,8 +849,8 @@ class ScreenContextService:
return size, rgb
async def _grab_with_master(
- self, target: CaptureTarget
- ) -> tuple[tuple[int, int], bytes, MasterImage]:
+ self, target: CaptureTarget, *, derive_rgb: bool = True
+ ) -> tuple[tuple[int, int], bytes | None, MasterImage]:
"""One shutter for both copies: the model's 8-bit frame and the user's master.
On an HDR or wide-gamut monitor the rectangle is read once at full
@@ -707,7 +858,8 @@ class ScreenContextService:
so both show the same instant and redaction boxes line up exactly.
The privacy path (a denylisted window in the rectangle) and every
other monitor keep the normal grab; the master is then that frame
- plus the monitor's ICC profile.
+ plus the monitor's ICC profile. ``derive_rgb=False`` leaves the 8-bit
+ frame of a full-depth read as ``None`` for the caller to derive later.
"""
from jarvis.platform.display_color import display_color # noqa: PLC0415
from jarvis.platform.hdr_grab import grab_extended # noqa: PLC0415
@@ -719,11 +871,14 @@ class ScreenContextService:
if rect_grab:
frame = await asyncio.to_thread(grab_extended, target.bbox)
if frame is not None:
- rgb8 = await asyncio.to_thread(
- scrgb_to_srgb8, frame.pixels, frame.sdr_white_nits
- )
- height, width = rgb8.shape[:2]
- return (width, height), rgb8.tobytes(), MasterImage(
+ height, width = frame.pixels.shape[:2]
+ rgb = None
+ if derive_rgb:
+ rgb8 = await asyncio.to_thread(
+ scrgb_to_srgb8, frame.pixels, frame.sdr_white_nits
+ )
+ rgb = rgb8.tobytes()
+ return (width, height), rgb, MasterImage(
pixels=frame.pixels,
hdr=True,
sdr_white_nits=frame.sdr_white_nits,
@@ -763,14 +918,19 @@ class ScreenContextService:
# ---- assembly --------------------------------------------------------
- def _monitor_privacy_error(self, target: CaptureTarget) -> str | None:
- """Return a refusal when monitor-wide denylist safety is unverifiable."""
+ def _monitor_privacy_error(self, target: CaptureTarget, *, visible: Any = _LIVE) -> str | None:
+ """Return a refusal when monitor-wide denylist safety is unverifiable.
+
+ ``visible`` checks an earlier window list (a frozen frame's) instead
+ of the windows on screen now.
+ """
if target.kind is TargetKind.WINDOW or not self._settings.denylist:
return None
region = target.kind is TargetKind.REGION
what = "the selected area" if region else "the monitor"
- visible_getter = getattr(self.window_probe, "visible_windows", None)
- visible = visible_getter() if callable(visible_getter) else None
+ if visible is _LIVE:
+ visible_getter = getattr(self.window_probe, "visible_windows", None)
+ visible = visible_getter() if callable(visible_getter) else None
if visible is None:
return (
f"I did not capture {what} because its visible windows "
@@ -1387,6 +1547,7 @@ def settings_from_config(cfg: Any) -> ScreenContextSettings:
__all__ = [
"CaptureOutcome",
"CaptureStatus",
+ "FrozenScreens",
"ScreenContextService",
"ScreenContextSettings",
"settings_from_config",
diff --git a/tests/unit/appshot/test_appshot_markup.py b/tests/unit/appshot/test_appshot_markup.py
index 48f61a45b6..a98a9b1e40 100644
--- a/tests/unit/appshot/test_appshot_markup.py
+++ b/tests/unit/appshot/test_appshot_markup.py
@@ -376,7 +376,12 @@ class Capture:
self.displays = Displays()
self.markup_at_shutter: list = []
- async def capture(self, *, verdict=None, trace_id=None, region=None, master=False):
+ async def freeze_screens(self):
+ return None
+
+ async def capture(
+ self, *, verdict=None, trace_id=None, region=None, master=False, frozen=None
+ ):
from jarvis.appshot.effect import shutter_markup
self.markup_at_shutter.append(shutter_markup.get())
diff --git a/tests/unit/appshot/test_appshot_region.py b/tests/unit/appshot/test_appshot_region.py
index 36cfa23fe1..f588a29825 100644
--- a/tests/unit/appshot/test_appshot_region.py
+++ b/tests/unit/appshot/test_appshot_region.py
@@ -144,9 +144,20 @@ class RegionCaptureService:
def __init__(self) -> None:
self.regions: list = []
self.displays = Displays()
-
- async def capture(self, *, verdict=None, trace_id=None, region=None, master=False):
+ self.steps: list[str] = []
+ self.frozen = object()
+ self.frozen_used: list = []
+
+ async def freeze_screens(self):
+ self.steps.append("freeze")
+ return self.frozen
+
+ async def capture(
+ self, *, verdict=None, trace_id=None, region=None, master=False, frozen=None
+ ):
+ self.steps.append("capture")
self.regions.append((verdict, region))
+ self.frozen_used.append(frozen)
context = ScreenContext(
image=b"jpeg",
mime="image/jpeg",
@@ -174,6 +185,7 @@ def flow(monkeypatch):
picks: list = []
async def pick_region(**_kwargs):
+ service.steps.append("pick")
return picks.pop(0) if picks else None
monkeypatch.setattr(turn, "get_service", lambda bus=None: service)
@@ -203,6 +215,24 @@ async def test_an_area_appshot_captures_exactly_the_selected_rectangle(flow) ->
assert result.shot.delivered_to == "message"
+async def test_an_area_appshot_is_the_screen_at_the_press_not_after_selecting(flow) -> None:
+ # A video plays on while the user selects: the screens freeze BEFORE the
+ # picker opens and the area is cut from that frame.
+ service, picks = flow
+ picks.append(
+ region.Selection(
+ screen={"x": 0.0, "y": 0.0, "w": 1920.0, "h": 1080.0, "dpr": 1.0},
+ rect=(0.25, 0.5, 0.5, 0.25),
+ )
+ )
+
+ result = await appshot_service.take_appshot(trigger="hotkey", scope="region")
+
+ assert result.ok
+ assert service.steps == ["freeze", "pick", "capture"]
+ assert service.frozen_used == [service.frozen]
+
+
async def test_esc_on_the_picker_takes_nothing(flow) -> None:
service, _picks = flow
diff --git a/tests/unit/screen_context/test_service.py b/tests/unit/screen_context/test_service.py
index f0e3d228ce..35769eb0a6 100644
--- a/tests/unit/screen_context/test_service.py
+++ b/tests/unit/screen_context/test_service.py
@@ -1161,3 +1161,162 @@ async def test_focus_settling_after_the_picker_does_not_void_an_area() -> None:
assert outcome.status == "captured"
assert len(capturer.grabs) == 1
+
+
+# --------------------------------------------------------------------------
+# Frozen screens: the area is the screen at the press, not after selecting
+# --------------------------------------------------------------------------
+
+_SMALL_MONITORS = [
+ {"left": 0, "top": 0, "width": 400, "height": 100},
+ {"left": 0, "top": 0, "width": 200, "height": 100},
+ {"left": 200, "top": 0, "width": 200, "height": 100},
+]
+
+
+class _PlayingVideo:
+ """A screen whose colour changes between grabs, like a playing video."""
+
+ name = "fake-video"
+
+ def __init__(self) -> None:
+ self.colour = (255, 0, 0)
+ self.grabs: list[tuple] = []
+
+ def grab(self, bbox, *, window_handle=None):
+ self.grabs.append((tuple(bbox), window_handle))
+ width, height = int(bbox[2]), int(bbox[3])
+ return ((width, height), bytes(self.colour) * (width * height))
+
+
+def _sdr_monitors(monkeypatch: pytest.MonkeyPatch) -> None:
+ """No HDR duplication and no ICC lookup on the test host's real monitors."""
+ from types import SimpleNamespace
+
+ import jarvis.platform.display_color as display_color
+ import jarvis.platform.hdr_grab as hdr_grab
+
+ monkeypatch.setattr(hdr_grab, "grab_extended", lambda _bbox: None)
+ monkeypatch.setattr(
+ display_color, "display_color", lambda *_a, **_k: SimpleNamespace(icc_profile=None)
+ )
+
+
+def _centre_colour(jpeg: bytes) -> tuple[int, int, int]:
+ from PIL import Image
+
+ with Image.open(io.BytesIO(jpeg)) as image:
+ rgb = image.convert("RGB")
+ return rgb.getpixel((rgb.width // 2, rgb.height // 2))
+
+
[email protected]("_sdr_monitors")
+async def test_a_frozen_area_is_cut_from_the_press_not_grabbed_after_selecting() -> None:
+ screen = _PlayingVideo()
+ service = make_service(capturer=screen, displays=FakeDisplays(_SMALL_MONITORS))
+
+ frozen = await service.freeze_screens()
+ screen.colour = (0, 0, 255) # the video played on while the user selected
+ outcome = await service.capture(region=(210, 10, 120, 60), master=True, frozen=frozen)
+
+ assert outcome.status == "captured"
+ # One grab per monitor at the press; none after the selection.
+ assert sorted(screen.grabs) == [((0, 0, 200, 100), None), ((200, 0, 200, 100), None)]
+ red, green, blue = _centre_colour(outcome.context.image)
+ assert red > 200 and blue < 60
+ assert outcome.context.target.bbox == (210, 10, 120, 60)
+
+
[email protected]("_sdr_monitors")
+async def test_an_area_outside_the_frozen_screens_is_grabbed_live() -> None:
+ screen = _PlayingVideo()
+ service = make_service(capturer=screen, displays=FakeDisplays(_SMALL_MONITORS))
+ frozen = _service_module.FrozenScreens(
+ frames=(
+ _service_module._FrozenMonitor(
+ bbox=(0, 0, 200, 100),
+ size=(200, 100),
+ rgb=bytes((0, 255, 0)) * (200 * 100),
+ master=_service_module.MasterImage(pixels=None),
+ ),
+ )
+ )
+
+ outcome = await service.capture(region=(210, 10, 120, 60), frozen=frozen)
+
+ assert outcome.status == "captured"
+ assert screen.grabs == [((210, 10, 120, 60), None)]
+
+
+def test_a_cut_follows_backing_pixels_and_slices_the_master() -> None:
+ import numpy as np
+
+ # A macOS-style frame: 2x the monitor's points.
+ pixels = np.zeros((200, 400, 4), dtype=np.float16)
+ pixels[20:40, 60:100] = 1.0
+ frame = _service_module._FrozenMonitor(
+ bbox=(100, 0, 200, 100),
+ size=(400, 200),
+ rgb=bytes(400 * 200 * 3),
+ master=_service_module.MasterImage(pixels=pixels, hdr=True),
+ )
+ frozen = _service_module.FrozenScreens(frames=(frame,))
+
+ size, rgb, master = frozen.cut((130, 10, 20, 10))
+
+ assert size == (40, 20)
+ assert len(rgb) == 40 * 20 * 3
+ assert master.pixels.shape == (20, 40, 4)
+ assert float(master.pixels.min()) == 1.0
+ assert frozen.cut((290, 0, 20, 10)) is None # runs off the frozen monitor
+
+
[email protected]("_sdr_monitors")
+async def test_a_denylisted_window_in_the_frozen_frame_blocks_the_area() -> None:
+ class VaultClosesAfterThePress(FakeWindowProbe):
+ def __init__(self) -> None:
+ super().__init__()
+ self.reads = 0
+
+ def visible_windows(self):
+ self.reads += 1
+ if self.reads == 1:
+ return (
+ WindowFacts(app_name="vault", title="1Password", frame_rect=(0, 0, 150, 90)),
+ )
+ return (WindowFacts(app_name="editor", title="notes", frame_rect=(0, 0, 400, 100)),)
+
+ screen = _PlayingVideo()
+ service = make_service(
+ settings=ScreenContextSettings(denylist=("1password",)),
+ window_probe=VaultClosesAfterThePress(),
+ capturer=screen,
+ displays=FakeDisplays(_SMALL_MONITORS),
+ )
+
+ frozen = await service.freeze_screens()
+ outcome = await service.capture(region=(10, 10, 100, 60), master=True, frozen=frozen)
+
+ assert outcome.status == "refused"
+ assert outcome.reason_kind == "policy"
+ assert outcome.context is None
+
+
+def test_a_full_depth_frame_turns_only_the_cut_area_into_8_bit() -> None:
+ import numpy as np
+
+ pixels = np.zeros((100, 200, 4), dtype=np.float16)
+ pixels[10:30, 20:60, :3] = 1.0 # SDR white at 80 nits
+ frame = _service_module._FrozenMonitor(
+ bbox=(0, 0, 200, 100),
+ size=(200, 100),
+ rgb=None,
+ master=_service_module.MasterImage(pixels=pixels, hdr=True, sdr_white_nits=80.0),
+ )
+
+ size, rgb, master = _service_module.FrozenScreens(frames=(frame,)).cut((20, 10, 40, 20))
+
+ assert size == (40, 20)
+ assert rgb == b"\xff" * (40 * 20 * 3)
+ assert master.hdr and master.pixels.shape == (20, 40, 4)