Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
"""Native per-window capture seam for the window-centric Computer-Use path.
:func:`grab_window` returns ``((width, height), rgb_bytes)`` for one window's
CURRENT content, or ``None`` when no native per-window capture is available
on this host — callers then fall back to grabbing the window's frame rect
from the virtual desktop (mss), which is pixel-identical for the raised
foreground window Computer-Use acts on. Whatever produced the pixels, the
position translation stays with the ONE central
:class:`dodaios.cu.geometry.CoordinateMapper` built from the window's frame
rect in input units.
Per platform:
* **Windows** — Pillow's HWND capture path captures only the requested
window, so an overlapping window cannot leak into the result. A framing or
DPI-size mismatch fails closed to ``None``.
* **macOS** — ScreenCaptureKit by CGWindowID (``SCScreenshotManager``,
macOS 14+; pyobjc ``[desktop-macos]`` extra): captures the window on
whatever display it sits, at native backing resolution, independent of
monitor layout. Needs the Screen-Recording permission. Anything missing
degrades to ``None`` (rect-grab fallback via CoreGraphics/mss).
* **Linux/X11** — ``None``: the window is identified and its geometry
resolved per window id (xdotool, see
:func:`dodaios.platform.window_state.window_frame_rect`); the pixels come
from the root rect grab in the same root-pixel space. Wayland offers no
addressable global window capture (the portal screencast flow is
user-interactive) and is refused upstream with the X11/XWayland message.
Import-cleanliness (HN-7): no platform-only package at module scope; pyobjc
frameworks are imported lazily inside the darwin path. Every failure returns
``None`` — this seam must never raise into the capture loop.
"""
from __future__ import annotations
import logging
import os
import threading
from typing import Any
from dodaios.platform import detect_platform
log = logging.getLogger(__name__)
#: Completion-handler wait ceiling for one ScreenCaptureKit screenshot. SCK
#: answers in tens of milliseconds; a hung/denied call must not stall the
#: perceive loop, whose own observe timeout is 12 s.
_SCK_TIMEOUT_S = 3.0
#: Why the last :func:`grab_window` on THIS thread returned ``None``, for a
#: caller that must tell "macOS may be asking" from "no native capture here".
_PENDING_CONFIRMATION = "pending_confirmation"
_TIMEOUT = "timeout"
_failure = threading.local()
def last_grab_failure() -> str:
"""Why the last ``grab_window`` on this thread returned ``None``.
``"pending_confirmation"``: the wait timed out while the live Screen Recording
read says granted or a request is in flight, so macOS is probably showing a
confirmation (the first capture on macOS 15+ may show its own "bypass the
system picker" alert; unverified). It is PENDING, never DENIED. ``"timeout"``:
it timed out and nothing explains it. ``""``: no reason recorded (no native
path on this host, window gone, framework missing).
"""
return str(getattr(_failure, "reason", ""))
def clear_grab_failure() -> None:
"""Forget the recorded reason before a grab (a stale one must not leak in)."""
_failure.reason = ""
def _timeout_means_pending() -> bool:
"""Whether a capture timeout is the confirmation dialog rather than a denial.
True when the live state (preflight plus the window-title oracle) reads
granted, or when the permission service has a Screen Recording request in
flight (an open episode in the ``os_dialog`` phase). Silent: it never asks.
"""
try:
from dodaios.platform import screen_access # noqa: PLC0415
gate = screen_access.permission_gate()
if screen_access.state_allows_capture(screen_access.screen_recording_state(gate)):
return True
outstanding = getattr(gate, "outstanding", None)
if callable(outstanding):
return any(
"screen_recording" in episode.permissions and episode.phase == "os_dialog"
for episode in outstanding()
)
except Exception: # noqa: BLE001 - classifying a timeout must never break the seam
log.debug("Could not classify a screen capture timeout.", exc_info=True)
return False
def _note_timeout(what: str) -> None:
"""Record a SCK wait timeout as PENDING when the state explains it."""
if _timeout_means_pending():
_failure.reason = _PENDING_CONFIRMATION
log.info(
"%s timed out: macOS may be asking you to confirm screen capture. "
"Treating it as pending, not as denied.",
what,
)
else:
_failure.reason = _TIMEOUT
log.debug("%s timed out (no permission state explains it)", what)
def grab_window(
handle: int, bbox: dict[str, int],
) -> tuple[tuple[int, int], bytes] | None:
"""Capture one window natively by its platform window id.
``bbox`` is the window's frame rect in input units (advisory — the
native backends capture the window's own current bounds). Returns
``None`` whenever the host has no native per-window path; never raises.
A caller that needs to know WHY a ``None`` came back asks
:func:`last_grab_failure` on the same thread.
"""
_failure.reason = ""
try:
plat = detect_platform()
if plat == "win32":
return _grab_window_windows(int(handle), bbox)
if plat == "darwin":
return _grab_window_macos(int(handle))
return None
except Exception: # noqa: BLE001 — capture seam must never raise
log.debug("grab_window failed (non-fatal)", exc_info=True)
return None
def _grab_window_windows(
hwnd: int, bbox: dict[str, int],
) -> tuple[tuple[int, int], bytes] | None:
"""Capture exactly one HWND without desktop-compositing overlap."""
if os.name != "nt" or not hwnd:
return None
from dodaios.core.win32_dpi import per_monitor_dpi_context # noqa: PLC0415
try:
from PIL import Image, ImageGrab # noqa: PLC0415
except ImportError:
log.debug("Pillow ImageGrab unavailable")
return None
expected = (int(bbox.get("width", 0)), int(bbox.get("height", 0)))
if expected[0] <= 0 or expected[1] <= 0:
return None
with per_monitor_dpi_context():
image = ImageGrab.grab(window=hwnd)
if tuple(image.size) != expected:
from dodaios.platform.window_state import ( # noqa: PLC0415
_window_client_rect_windows,
_window_rect_windows,
)
outer = _window_rect_windows(hwnd)
client = _window_client_rect_windows(hwnd)
crop_box = None
if outer is not None and tuple(image.size) == outer[2:]:
left = int(bbox.get("left", 0)) - outer[0]
top = int(bbox.get("top", 0)) - outer[1]
crop_box = (left, top, left + expected[0], top + expected[1])
if (
crop_box[0] < 0
or crop_box[1] < 0
or crop_box[2] > image.size[0]
or crop_box[3] > image.size[1]
):
crop_box = None
if crop_box is not None:
cropped = image.crop(crop_box)
image.close()
image = cropped
elif client is not None and tuple(image.size) == client[2:]:
client_left = client[0] - int(bbox.get("left", 0))
client_top = client[1] - int(bbox.get("top", 0))
if (
client_left < 0
or client_top < 0
or client_left + image.size[0] > expected[0]
or client_top + image.size[1] > expected[1]
):
image.close()
return None
canvas = Image.new("RGB", expected, color=(0, 0, 0))
client_image = image.convert("RGB")
try:
canvas.paste(client_image, (client_left, client_top))
finally:
if client_image is not image:
client_image.close()
image.close()
image = canvas
else:
log.debug(
"HWND capture size %s does not match frame bounds %s",
image.size,
expected,
)
image.close()
return None
try:
rgb = image.convert("RGB")
try:
return (expected, rgb.tobytes())
finally:
if rgb is not image:
rgb.close()
finally:
image.close()
def _grab_window_macos(window_id: int) -> tuple[tuple[int, int], bytes] | None:
"""One ScreenCaptureKit screenshot of the window with ``window_id``.
Blocking wrapper over the async SCK API (pyobjc bridges the completion
handlers): resolve the ``SCWindow`` from ``SCShareableContent``, build a
desktop-independent window filter, size the configuration to the
window's point frame times its backing scale, and decode the returned
``CGImage`` to raw RGB via an NSBitmapImageRep PNG round-trip (PIL does
the decode — no manual bytes-per-row/BGRA handling to get subtly wrong).
Requirements probed at runtime, each miss returning ``None``:
pyobjc ScreenCaptureKit framework (``[desktop-macos]`` extra), macOS 14+
(``SCScreenshotManager``), Screen-Recording permission, window still on
screen.
"""
import io # noqa: PLC0415
import threading # noqa: PLC0415
try:
import ScreenCaptureKit as sck # noqa: PLC0415
from AppKit import NSBitmapImageRep # noqa: PLC0415
from PIL import Image # noqa: PLC0415
except Exception: # noqa: BLE001 — frameworks absent (base install)
log.debug("ScreenCaptureKit/AppKit unavailable", exc_info=True)
return None
manager = getattr(sck, "SCScreenshotManager", None)
if manager is None: # pre-14 macOS: no screenshot API on this framework
log.debug("SCScreenshotManager unavailable (macOS < 14)")
return None
# -- shareable content (async -> event) -------------------------------
content_box: dict[str, Any] = {}
content_ready = threading.Event()
def _content_handler(content, error): # pragma: no cover - live macOS only
content_box["content"] = content
content_box["error"] = error
content_ready.set()
sck.SCShareableContent.getShareableContentWithCompletionHandler_(
_content_handler,
)
if not content_ready.wait(_SCK_TIMEOUT_S):
_note_timeout("SCShareableContent")
return None
content = content_box.get("content")
if content is None or content_box.get("error") is not None:
log.debug("SCShareableContent failed: %s", content_box.get("error"))
return None
target = None
for window in content.windows() or []:
if int(window.windowID()) == window_id:
target = window
break
if target is None:
log.debug("SCK window id %d not on screen", window_id)
return None
# -- screenshot (async -> event) ---------------------------------------
sc_filter = sck.SCContentFilter.alloc().initWithDesktopIndependentWindow_(
target,
)
config = sck.SCStreamConfiguration.alloc().init()
frame = target.frame() # points
try:
scale = float(sc_filter.pointPixelScale())
except Exception: # noqa: BLE001 — attribute is macOS 14+; default 2x-safe
scale = 2.0
config.setWidth_(max(1, round(frame.size.width * scale)))
config.setHeight_(max(1, round(frame.size.height * scale)))
config.setShowsCursor_(False)
# A framed single-window image can include drop-shadow pixels outside the
# SCWindow frame. The coordinate mapper is anchored to that frame, so
# accepting shadow padding would shift every inferred target. macOS 14+
# exposes this selector; if the runtime bridge cannot provide it, fall
# back to the frame-rect capture path instead of using ambiguous pixels.
ignore_shadows = getattr(config, "setIgnoreShadowsSingleWindow_", None)
if not callable(ignore_shadows):
log.debug("SCK cannot disable single-window shadows; using rect fallback")
return None
try:
ignore_shadows(True)
except Exception: # noqa: BLE001 - unsafe framing must fail closed
log.debug("SCK single-window shadow suppression failed", exc_info=True)
return None
image_box: dict[str, Any] = {}
image_ready = threading.Event()
def _image_handler(image, error): # pragma: no cover - live macOS only
image_box["image"] = image
image_box["error"] = error
image_ready.set()
manager.captureImageWithFilter_configuration_completionHandler_(
sc_filter, config, _image_handler,
)
if not image_ready.wait(_SCK_TIMEOUT_S):
_note_timeout("SCScreenshotManager")
return None
cg_image = image_box.get("image")
if cg_image is None or image_box.get("error") is not None:
log.debug("SCK screenshot failed: %s", image_box.get("error"))
return None
# -- CGImage -> raw RGB -------------------------------------------------
rep = NSBitmapImageRep.alloc().initWithCGImage_(cg_image)
if rep is None:
return None
png = rep.representationUsingType_properties_(4, None) # NSPNGFileType
if png is None:
return None
img = Image.open(io.BytesIO(bytes(png))).convert("RGB")
return ((img.width, img.height), img.tobytes())
__all__ = ["clear_grab_failure", "grab_window", "last_grab_failure"]