Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
feat(appshot): copy an edited appshot through the OS clipboard
rubenluetke10-beepbc452846a97f
diff --git a/jarvis/platform/clipboard_image.py b/jarvis/platform/clipboard_image.pynew file mode 100644index 0000000000..ae4b79c90d--- /dev/null+++ b/jarvis/platform/clipboard_image.py@@ -0,0 +1,206 @@+"""Put a PNG image on the local desktop clipboard (Windows, macOS, Linux).++The embedded WebView cannot be trusted with ``navigator.clipboard.write`` for+images: WebView2 may never settle the promise, and WKWebView rejects it once a+click handler has awaited anything. So the desktop backend copies natively:++- **Windows** — ``CF_DIB`` (every app pastes it) plus the registered ``PNG``+ format (keeps transparency for apps that prefer it), through ctypes.+- **macOS** — ``osascript`` reading the PNG from a temporary file as+ ``«class PNGf»``; no extra package.+- **Linux** — ``wl-copy`` on Wayland, else ``xclip``; without either the copy+ reports that it cannot happen.++``write_png`` never raises and logs why a copy did not happen. A headless host+returns ``False`` without touching any OS integration.+"""++from __future__ import annotations++import ctypes+import io+import logging+import os+import shutil+import subprocess+import tempfile+import time+from pathlib import Path++from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS+from jarvis.platform import detect_platform+from jarvis.platform.capabilities import detect_capabilities++log = logging.getLogger(__name__)++_COMMAND_TIMEOUT_S = 5.0+_WINDOWS_CLIPBOARD_RETRIES = 10+_WINDOWS_CLIPBOARD_RETRY_S = 0.02+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"+#: Win32 clipboard format for a device-independent bitmap.+_CF_DIB = 8+_GMEM_MOVEABLE = 0x0002+++def is_png(data: bytes) -> bool:+ return data.startswith(_PNG_SIGNATURE)+++def write_png(png: bytes) -> bool:+ """Replace the clipboard with this PNG. ``True`` only when the OS took it."""+ if not is_png(png):+ log.warning("clipboard-image: refusing a payload that is not a PNG")+ return False+ if not detect_capabilities().display_present:+ log.info("clipboard-image: no display present; native copy is unavailable")+ return False+ platform = detect_platform()+ if platform == "win32":+ return _write_windows(png)+ if platform == "darwin":+ return _write_macos(png)+ return _write_linux(png)+++def png_to_dib(png: bytes) -> bytes:+ """The PNG as a ``CF_DIB`` payload: a BMP file without its 14-byte header."""+ from PIL import Image # noqa: PLC0415 - Pillow only when an image is copied++ with Image.open(io.BytesIO(png)) as image:+ rgb = image.convert("RGB")+ out = io.BytesIO()+ rgb.save(out, format="BMP")+ return out.getvalue()[14:]+++def _run(command: list[str], data: bytes | None = None) -> bool:+ try:+ completed = subprocess.run( # noqa: S603 - fixed, non-shell OS command+ command,+ input=data,+ capture_output=True,+ timeout=_COMMAND_TIMEOUT_S,+ check=False,+ close_fds=True,+ creationflags=NO_WINDOW_CREATIONFLAGS,+ )+ except (OSError, subprocess.SubprocessError) as exc:+ log.warning("clipboard-image: copy command unavailable (%s)", exc)+ return False+ if completed.returncode != 0:+ detail = completed.stderr.decode("utf-8", errors="replace").strip()+ log.warning(+ "clipboard-image: copy command failed with exit %d%s",+ completed.returncode,+ f" ({detail[:300]})" if detail else "",+ )+ return False+ return True+++def _write_linux(png: bytes) -> bool:+ wl_copy = shutil.which("wl-copy")+ if wl_copy and os.environ.get("WAYLAND_DISPLAY"):+ return _run([wl_copy, "--type", "image/png"], png)+ xclip = shutil.which("xclip")+ if xclip:+ return _run([xclip, "-selection", "clipboard", "-t", "image/png", "-i"], png)+ if wl_copy:+ return _run([wl_copy, "--type", "image/png"], png)+ log.info("clipboard-image: neither wl-copy nor xclip is installed")+ return False+++def _write_macos(png: bytes) -> bool:+ handle, name = tempfile.mkstemp(prefix="jarvis-appshot-", suffix=".png")+ path = Path(name)+ try:+ with os.fdopen(handle, "wb") as file:+ file.write(png)+ # The path travels as an argument to AppleScript's `item 1 of argv`,+ # never spliced into the script source.+ script = (+ "on run argv\n"+ "set the clipboard to (read (POSIX file (item 1 of argv)) as «class PNGf»)\n"+ "end run"+ )+ return _run(["/usr/bin/osascript", "-e", script, str(path)])+ except OSError as exc:+ log.warning("clipboard-image: could not stage the PNG for osascript (%s)", exc)+ return False+ finally:+ path.unlink(missing_ok=True)+++def _write_windows(png: bytes) -> bool:+ try:+ dib = png_to_dib(png)+ except Exception as exc: # noqa: BLE001 - a picture Pillow cannot read is not copyable+ log.warning("clipboard-image: could not convert the PNG to a bitmap (%s)", exc)+ return False+ try:+ from ctypes import wintypes # noqa: PLC0415++ user32 = ctypes.WinDLL("user32", use_last_error=True)+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)+ user32.OpenClipboard.argtypes = [wintypes.HWND]+ user32.OpenClipboard.restype = wintypes.BOOL+ user32.EmptyClipboard.restype = wintypes.BOOL+ user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE]+ user32.SetClipboardData.restype = wintypes.HANDLE+ user32.CloseClipboard.restype = wintypes.BOOL+ user32.RegisterClipboardFormatW.argtypes = [wintypes.LPCWSTR]+ user32.RegisterClipboardFormatW.restype = wintypes.UINT+ kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t]+ kernel32.GlobalAlloc.restype = wintypes.HGLOBAL+ kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL]+ kernel32.GlobalLock.restype = ctypes.c_void_p+ kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL]+ kernel32.GlobalFree.argtypes = [wintypes.HGLOBAL]+ except (AttributeError, OSError) as exc:+ log.warning("clipboard-image: Win32 API unavailable (%s)", exc)+ return False++ def put(fmt: int, data: bytes) -> bool:+ handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(data))+ if not handle:+ return False+ pointer = kernel32.GlobalLock(handle)+ if not pointer:+ kernel32.GlobalFree(handle)+ return False+ ctypes.memmove(pointer, data, len(data))+ kernel32.GlobalUnlock(handle)+ if not user32.SetClipboardData(fmt, handle):+ # Ownership only passes to the clipboard on success.+ kernel32.GlobalFree(handle)+ return False+ return True++ opened = False+ for _attempt in range(_WINDOWS_CLIPBOARD_RETRIES):+ if user32.OpenClipboard(None):+ opened = True+ break+ time.sleep(_WINDOWS_CLIPBOARD_RETRY_S)+ if not opened:+ log.warning("clipboard-image: another application holds the clipboard open")+ return False+ try:+ if not user32.EmptyClipboard():+ log.warning("clipboard-image: EmptyClipboard failed")+ return False+ if not put(_CF_DIB, dib):+ log.warning("clipboard-image: SetClipboardData(CF_DIB) failed")+ return False+ png_format = user32.RegisterClipboardFormatW("PNG")+ if png_format and not put(png_format, png):+ # The bitmap is already there and pastes everywhere; the PNG+ # format only adds transparency for apps that read it.+ log.info("clipboard-image: the extra PNG format was not accepted")+ return True+ finally:+ user32.CloseClipboard()+++__all__ = ["is_png", "png_to_dib", "write_png"]diff --git a/jarvis/ui/web/appshot_routes.py b/jarvis/ui/web/appshot_routes.pyindex ff45cefd95..a87c6e8d46 100644--- a/jarvis/ui/web/appshot_routes.py+++ b/jarvis/ui/web/appshot_routes.py@@ -8,6 +8,7 @@ Endpoints (mounted by the WebServer in ``_build_app()``):GET /api/appshot/latest → metadata of the last appshot.GET /api/appshot/latest/image → its picture (never cached).PUT /api/appshot/latest/image → replace it with the editor's version.+ POST /api/appshot/clipboard → copy the editor's PNG natively (desktop only).GET /api/appshot/pending → the appshot waiting for the next message.POST /api/appshot/pending/claim → hand that one to the chat composer.DELETE /api/appshot → forget every held appshot now.@@ -245,6 +246,29 @@ def _png_size(data: bytes) -> tuple[int, int] | None:return None[email protected]("/clipboard", openapi_extra={"x-jarvis-dangerous": True})+async def copy_to_clipboard(request: Request) -> dict[str, Any]:+ """Put the editor's PNG on the desktop clipboard natively.++ The embedded WebView cannot be trusted to copy images itself (see+ :mod:`jarvis.platform.clipboard_image`). Desktop only: on a browser or+ headless server the clipboard would be the server's, so this 404s there+ and the page keeps the browser's own clipboard.+ """+ if not bool(getattr(request.app.state, "native_file_actions", False)):+ raise HTTPException(status_code=404, detail="native-clipboard-disabled")+ body = await request.body()+ if not body or len(body) > _MAX_EDIT_BYTES:+ raise HTTPException(status_code=413, detail="The picture is empty or too large.")+ from jarvis.platform.clipboard_image import is_png, write_png # noqa: PLC0415++ if not is_png(body):+ raise HTTPException(status_code=400, detail="The picture is not a PNG.")+ if not await asyncio.to_thread(write_png, body):+ raise HTTPException(status_code=503, detail="native-clipboard-unavailable")+ return {"copied": True}++@router.get("/pending")async def pending() -> dict[str, Any]:from jarvis.appshot.store import get_store # noqa: PLC0415diff --git a/tests/unit/appshot/test_appshot_editor.py b/tests/unit/appshot/test_appshot_editor.pyindex 37017de5ca..587bb717bb 100644--- a/tests/unit/appshot/test_appshot_editor.py+++ b/tests/unit/appshot/test_appshot_editor.py@@ -90,3 +90,109 @@ def test_the_route_refuses_what_is_not_a_png(client, body: bytes, status: int) -def test_the_route_refuses_an_expired_appshot(client) -> None:assert client.put("/api/appshot/latest/image?id=gone", content=_png()).status_code == 404+++# -- copy: the editor's PNG goes to the OS clipboard natively ----------------+++def _copy_client(monkeypatch: pytest.MonkeyPatch, *, native: bool, accepted: bool = True):+ from jarvis.platform import clipboard_image+ from jarvis.ui.web.appshot_routes import router++ copied: list[bytes] = []++ def fake_write(png: bytes) -> bool:+ copied.append(png)+ return accepted++ monkeypatch.setattr(clipboard_image, "write_png", fake_write)+ app = FastAPI()+ app.include_router(router)+ app.state.native_file_actions = native+ return TestClient(app), copied+++def test_copy_hands_the_png_to_the_native_clipboard(monkeypatch: pytest.MonkeyPatch) -> None:+ client, copied = _copy_client(monkeypatch, native=True)+ png = _png()++ response = client.post("/api/appshot/clipboard", content=png)++ assert response.status_code == 200, response.text+ assert copied == [png]+++def test_copy_is_desktop_only(monkeypatch: pytest.MonkeyPatch) -> None:+ """On a browser/headless server the clipboard would be the server's."""+ client, copied = _copy_client(monkeypatch, native=False)++ assert client.post("/api/appshot/clipboard", content=_png()).status_code == 404+ assert copied == []+++def test_copy_refuses_what_is_not_a_png(monkeypatch: pytest.MonkeyPatch) -> None:+ client, copied = _copy_client(monkeypatch, native=True)++ assert client.post("/api/appshot/clipboard", content=b"GIF89a....").status_code == 400+ assert client.post("/api/appshot/clipboard", content=b"").status_code == 413+ assert copied == []+++def test_copy_says_when_the_os_refused(monkeypatch: pytest.MonkeyPatch) -> None:+ client, _copied = _copy_client(monkeypatch, native=True, accepted=False)++ response = client.post("/api/appshot/clipboard", content=_png())++ assert response.status_code == 503+ assert response.json()["detail"] == "native-clipboard-unavailable"+++def test_clipboard_image_needs_a_display(monkeypatch: pytest.MonkeyPatch) -> None:+ from types import SimpleNamespace++ from jarvis.platform import clipboard_image++ monkeypatch.setattr(+ clipboard_image, "detect_capabilities", lambda: SimpleNamespace(display_present=False)+ )+ assert clipboard_image.write_png(_png()) is False+ assert clipboard_image.write_png(b"not a png") is False+++def test_clipboard_image_dib_is_a_bitmap_without_the_file_header() -> None:+ from jarvis.platform.clipboard_image import png_to_dib++ dib = png_to_dib(_png(40, 30))+ # BITMAPINFOHEADER: its own size first, then width and height.+ assert int.from_bytes(dib[0:4], "little") == 40+ assert int.from_bytes(dib[4:8], "little") == 40+ assert int.from_bytes(dib[8:12], "little", signed=True) == 30+++ ("wayland", "tools", "expected"),+ [+ ("wayland-0", {"wl-copy", "xclip"}, "wl-copy"),+ ("", {"wl-copy", "xclip"}, "xclip"),+ ("", {"wl-copy"}, "wl-copy"),+ ("", set(), None),+ ],+)+def test_clipboard_image_picks_the_linux_tool(+ monkeypatch: pytest.MonkeyPatch, wayland: str, tools: set[str], expected: str | None+) -> None:+ from jarvis.platform import clipboard_image++ ran: list[list[str]] = []+ monkeypatch.setenv("WAYLAND_DISPLAY", wayland)+ monkeypatch.setattr(+ clipboard_image.shutil, "which", lambda name: f"/usr/bin/{name}" if name in tools else None+ )+ monkeypatch.setattr(+ clipboard_image, "_run", lambda command, data=None: ran.append(command) or True+ )++ ok = clipboard_image._write_linux(_png()) # noqa: SLF001++ assert ok is (expected is not None)+ assert [command[0] for command in ran] == ([f"/usr/bin/{expected}"] if expected else [])