Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

12.0 KB
Raw Blame
"""Stable-frame capture tests — injectable grabber, no real display needed."""

from __future__ import annotations

import pytest

pytest.importorskip("PIL", reason="pillow required for capture tests")

from dodaios.cu.capture import (
    DEFAULT_STABILITY_INTERVAL_S,
    Frame,
    capture_stable_frame,
    frames_differ,
    grab_region,
    grab_visual_probe,
)
from dodaios.cu.geometry import MonitorInfo


def _solid(size: tuple[int, int], rgb: tuple[int, int, int]) -> tuple[tuple[int, int], bytes]:
    w, h = size
    return ((w, h), bytes(rgb) * (w * h))


def _monitor(**kw) -> MonitorInfo:
    defaults = dict(left=0, top=0, width=192, height=108)
    defaults.update(kw)
    return MonitorInfo(**defaults)


# ---------------------------------------------------------------------------
# frames_differ
# ---------------------------------------------------------------------------


def test_identical_frames_do_not_differ():
    a = _solid((192, 108), (30, 30, 30))
    assert not frames_differ(a, a)


def test_tiny_change_stays_below_threshold():
    # One "blinking cursor": a 2x12 white block on a dark frame.
    a = _solid((192, 108), (30, 30, 30))
    pixels = bytearray(a[1])
    for row in range(12):
        for col in range(2):
            idx = ((20 + row) * 192 + (50 + col)) * 3
            pixels[idx : idx + 3] = b"\xff\xff\xff"
    b = ((192, 108), bytes(pixels))
    assert not frames_differ(a, b)


def test_large_change_differs():
    a = _solid((192, 108), (30, 30, 30))
    b = _solid((192, 108), (200, 200, 200))
    assert frames_differ(a, b)


def test_resolution_change_always_differs():
    a = _solid((192, 108), (30, 30, 30))
    b = _solid((96, 54), (30, 30, 30))
    assert frames_differ(a, b)


# ---------------------------------------------------------------------------
# capture_stable_frame
# ---------------------------------------------------------------------------


def test_stable_screen_returns_after_one_regrab():
    frame_a = _solid((192, 108), (10, 20, 30))
    calls = {"n": 0}

    def grab(bbox):
        calls["n"] += 1
        return frame_a

    frame = capture_stable_frame(
        _monitor(),
        grab=grab,
        sleep=lambda s: None,
    )
    assert isinstance(frame, Frame)
    assert frame.stable
    assert calls["n"] == 2  # initial + one confirming re-grab
    assert frame.image_width == 192 and frame.image_height == 108
    assert frame.mapper.screen_rect == (0, 0, 192, 108)
    assert frame.jpeg[:2] == b"\xff\xd8"  # JPEG magic


def test_stable_screen_uses_frame_paced_default_interval():
    frame_a = _solid((192, 108), (10, 20, 30))
    sleeps: list[float] = []

    capture_stable_frame(
        _monitor(),
        grab=lambda _bbox: frame_a,
        sleep=sleeps.append,
    )

    assert sleeps == [DEFAULT_STABILITY_INTERVAL_S]
    assert DEFAULT_STABILITY_INTERVAL_S <= 0.02


def test_stable_screen_reuses_one_call_scoped_mss_session(monkeypatch):
    import mss

    sessions = []

    class FakeShot:
        size = (192, 108)
        raw = bytes((30, 20, 10, 0)) * (192 * 108)

    class FakeMSS:
        def __init__(self):
            self.grabs = 0
            self.exits = 0
            sessions.append(self)

        def __enter__(self):
            return self

        def __exit__(self, *_exc):
            self.exits += 1

        def grab(self, _bbox):
            self.grabs += 1
            return FakeShot()

    monkeypatch.setattr(mss, "mss", FakeMSS)

    capture_stable_frame(_monitor(), sleep=lambda _seconds: None)
    capture_stable_frame(_monitor(), sleep=lambda _seconds: None)

    assert [(s.grabs, s.exits) for s in sessions] == [(2, 1), (2, 1)]


def test_call_scoped_mss_session_closes_when_regrab_fails(monkeypatch):
    import mss

    state = {"exit": 0}

    class FakeShot:
        size = (192, 108)
        raw = bytes((30, 20, 10, 0)) * (192 * 108)

    class FakeMSS:
        def __enter__(self):
            return self

        def __exit__(self, *_exc):
            state["exit"] += 1

        def grab(self, _bbox):
            if state.get("grab", 0):
                raise OSError("second grab failed")
            state["grab"] = 1
            return FakeShot()

    monkeypatch.setattr(mss, "mss", FakeMSS)

    with pytest.raises(OSError, match="second grab failed"):
        capture_stable_frame(_monitor(), sleep=lambda _seconds: None)

    assert state["exit"] == 1


def test_injected_grabber_does_not_open_mss(monkeypatch):
    import mss

    monkeypatch.setattr(
        mss,
        "mss",
        lambda: pytest.fail("injected grabber must not construct MSS"),
    )

    capture_stable_frame(
        _monitor(),
        grab=lambda _bbox: _solid((192, 108), (10, 20, 30)),
        sleep=lambda _seconds: None,
    )


def test_visual_probe_confirmation_captures_only_local_crop(monkeypatch):
    import mss

    grabs: list[dict[str, int]] = []

    class FakeShot:
        def __init__(self, bbox):
            self.size = (bbox["width"], bbox["height"])
            self.raw = bytes((30, 20, 10, 0)) * (bbox["width"] * bbox["height"])
            self.rgb = bytes((10, 20, 30)) * (bbox["width"] * bbox["height"])

    class FakeMSS:
        def __enter__(self):
            return self

        def __exit__(self, *_exc):
            return None

        def grab(self, bbox):
            grabs.append(dict(bbox))
            return FakeShot(bbox)

    monkeypatch.setattr(mss, "mss", FakeMSS)
    bbox = {"left": 0, "top": 0, "width": 192, "height": 108}

    probe = grab_visual_probe(
        bbox,
        point=(96, 54),
        radius=10,
        local_only=True,
    )

    assert probe is not None
    assert probe.global_thumb is None
    assert probe.local is not None and probe.local[0] == (20, 20)
    assert grabs == [{"left": 86, "top": 44, "width": 20, "height": 20}]


def test_visual_probe_baseline_keeps_global_and_local_evidence(monkeypatch):
    import mss

    grabs: list[dict[str, int]] = []

    class FakeShot:
        def __init__(self, bbox):
            self.size = (bbox["width"], bbox["height"])
            self.raw = bytes((30, 20, 10, 0)) * (bbox["width"] * bbox["height"])
            self.rgb = bytes((10, 20, 30)) * (bbox["width"] * bbox["height"])

    class FakeMSS:
        def __enter__(self):
            return self

        def __exit__(self, *_exc):
            return None

        def grab(self, bbox):
            grabs.append(dict(bbox))
            return FakeShot(bbox)

    monkeypatch.setattr(mss, "mss", FakeMSS)
    bbox = {"left": 0, "top": 0, "width": 192, "height": 108}

    probe = grab_visual_probe(bbox, point=(96, 54), radius=10)

    assert probe is not None
    assert probe.global_thumb is not None
    assert len(probe.global_thumb) == 96 * 54
    assert probe.local is not None and probe.local[0] == (20, 20)
    assert grabs == [
        bbox,
        {"left": 86, "top": 44, "width": 20, "height": 20},
    ]


def test_visual_probe_local_ignores_bgrx_padding(monkeypatch):
    import mss

    from dodaios.cu.verify import regions_equal

    padding = iter((0, 255))

    class FakeShot:
        size = (2, 2)

        def __init__(self, x_byte):
            self.raw = bytes((30, 20, 10, x_byte)) * 4
            self.rgb = bytes((10, 20, 30)) * 4

    class FakeMSS:
        def __enter__(self):
            return self

        def __exit__(self, *_exc):
            return None

        def grab(self, _bbox):
            return FakeShot(next(padding))

    monkeypatch.setattr(mss, "mss", FakeMSS)
    bbox = {"left": 0, "top": 0, "width": 2, "height": 2}

    first = grab_visual_probe(bbox, point=(1, 1), local_only=True)
    second = grab_visual_probe(bbox, point=(1, 1), local_only=True)

    assert first is not None and second is not None
    assert regions_equal(first.local, second.local) is True


# The macOS Screen Recording behaviour of capture (ask once on a gesture, never from
# a helper, honest refusal, wallpaper check) lives in test_capture_permissions.py.


def test_capture_guard_rechecks_window_identity_around_each_grab():
    checks = iter((True, False))
    calls = {"grab": 0}

    def grab(_bbox):
        calls["grab"] += 1
        return _solid((192, 108), (10, 20, 30))

    with pytest.raises(RuntimeError, match="during screen capture"):
        capture_stable_frame(
            _monitor(),
            grab=grab,
            sleep=lambda _s: None,
            capture_guard=lambda: next(checks),
        )

    assert calls["grab"] == 1


def test_animating_screen_times_out_unstable():
    shade = {"v": 0}

    def grab(bbox):
        shade["v"] = (shade["v"] + 60) % 250
        return _solid((192, 108), (shade["v"],) * 3)

    frame = capture_stable_frame(
        _monitor(),
        grab=grab,
        sleep=lambda s: None,
        stability_timeout_s=0.05,
    )
    assert not frame.stable


def test_settles_after_a_few_changing_frames():
    frames = [
        _solid((192, 108), (0, 0, 0)),
        _solid((192, 108), (120, 120, 120)),
        _solid((192, 108), (240, 240, 240)),
        _solid((192, 108), (240, 240, 240)),  # settled
    ]
    seq = list(frames)

    def grab(bbox):
        return seq.pop(0) if len(seq) > 1 else frames[-1]

    frame = capture_stable_frame(_monitor(), grab=grab, sleep=lambda s: None)
    assert frame.stable


def test_downscale_builds_matching_mapper():
    big = _solid((1920, 1080), (50, 60, 70))
    frame = capture_stable_frame(
        _monitor(width=1920, height=1080),
        grab=lambda bbox: big,
        sleep=lambda s: None,
        max_dimension=960,
    )
    assert frame.image_width == 960 and frame.image_height == 540
    # Model pixel center of the image -> monitor center.
    x, y = frame.mapper.image_to_screen(480, 270)
    assert abs(x - 960) <= 2 and abs(y - 540) <= 2


def test_retina_style_grab_larger_than_monitor_rect():
    # macOS: bbox in points (1440x900), grab returns 2x pixels (2880x1800).
    raw = _solid((2880, 1800), (5, 5, 5))
    frame = capture_stable_frame(
        _monitor(width=1440, height=900),
        grab=lambda bbox: raw,
        sleep=lambda s: None,
        max_dimension=1366,
    )
    # Image was downscaled from the 2880px grab; mapper still maps into the
    # 1440x900 POINT rect the input backend consumes.
    assert frame.image_width == 1366
    sx, sy = frame.mapper.image_to_screen(frame.image_width - 1, frame.image_height - 1)
    assert sx < 1440 and sy < 900


def test_blob_written_when_dir_given(tmp_path):
    frame = capture_stable_frame(
        _monitor(),
        grab=lambda bbox: _solid((192, 108), (1, 2, 3)),
        sleep=lambda s: None,
        blob_dir=tmp_path,
    )
    assert frame.blob_path is not None
    assert (tmp_path / f"{frame.sha256}.jpg").exists()


def test_thumb_identity_ignores_caret_noise_but_sees_real_change():
    from dodaios.cu.capture import screen_thumb, thumbs_similar

    base = _solid((192, 108), (30, 30, 30))
    # A caret-sized change: a small bright block on the dark frame.
    pixels = bytearray(base[1])
    for row in range(12):
        for col in range(2):
            idx = ((20 + row) * 192 + (50 + col)) * 3
            pixels[idx : idx + 3] = b"\xff\xff\xff"
    caret = ((192, 108), bytes(pixels))
    changed = _solid((192, 108), (200, 200, 200))
    assert thumbs_similar(screen_thumb(base), screen_thumb(base))
    assert thumbs_similar(screen_thumb(base), screen_thumb(caret))
    assert not thumbs_similar(screen_thumb(base), screen_thumb(changed))
    # Opaque string keys (tests / foreign callers) compare by equality.
    assert thumbs_similar("sha1", "sha1")
    assert not thumbs_similar("sha1", "sha2")


def test_frame_carries_exact_and_perceptual_identity():
    frame = capture_stable_frame(
        _monitor(),
        grab=lambda bbox: _solid((192, 108), (1, 2, 3)),
        sleep=lambda s: None,
    )
    assert len(frame.sha256) == 64
    assert len(frame.thumb) == 96 * 54


def test_grab_region_swallows_failures():
    def broken(bbox):
        raise OSError("BitBlt failed")

    assert grab_region({"left": 0, "top": 0, "width": 10, "height": 10}, grab=broken) is None
    ok = grab_region(
        {"left": 0, "top": 0, "width": 4, "height": 4},
        grab=lambda bbox: _solid((4, 4), (9, 9, 9)),
    )
    assert ok is not None and ok[0] == (4, 4)