Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

feat(appshot): copy an edited appshot through the OS clipboard

rubenluetke10-beepbc452846a97f
diff --git a/jarvis/platform/clipboard_image.py b/jarvis/platform/clipboard_image.py
new file mode 100644
index 0000000000..ae4b79c90d
--- /dev/null
+++ b/jarvis/platform/clipboard_image.py
@@ -0,0 +1,206 @@
+"""Put a PNG image on the local desktop clipboard (Windows, macOS, Linux).
+
+The embedded WebView cannot be trusted with ``navigator.clipboard.write`` for
+images: WebView2 may never settle the promise, and WKWebView rejects it once a
+click handler has awaited anything. So the desktop backend copies natively:
+
+- **Windows** — ``CF_DIB`` (every app pastes it) plus the registered ``PNG``
+ format (keeps transparency for apps that prefer it), through ctypes.
+- **macOS** — ``osascript`` reading the PNG from a temporary file as
+ ``«class PNGf»``; no extra package.
+- **Linux** — ``wl-copy`` on Wayland, else ``xclip``; without either the copy
+ reports that it cannot happen.
+
+``write_png`` never raises and logs why a copy did not happen. A headless host
+returns ``False`` without touching any OS integration.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import io
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+
+from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+from jarvis.platform import detect_platform
+from jarvis.platform.capabilities import detect_capabilities
+
+log = logging.getLogger(__name__)
+
+_COMMAND_TIMEOUT_S = 5.0
+_WINDOWS_CLIPBOARD_RETRIES = 10
+_WINDOWS_CLIPBOARD_RETRY_S = 0.02
+_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
+#: Win32 clipboard format for a device-independent bitmap.
+_CF_DIB = 8
+_GMEM_MOVEABLE = 0x0002
+
+
+def is_png(data: bytes) -> bool:
+ return data.startswith(_PNG_SIGNATURE)
+
+
+def write_png(png: bytes) -> bool:
+ """Replace the clipboard with this PNG. ``True`` only when the OS took it."""
+ if not is_png(png):
+ log.warning("clipboard-image: refusing a payload that is not a PNG")
+ return False
+ if not detect_capabilities().display_present:
+ log.info("clipboard-image: no display present; native copy is unavailable")
+ return False
+ platform = detect_platform()
+ if platform == "win32":
+ return _write_windows(png)
+ if platform == "darwin":
+ return _write_macos(png)
+ return _write_linux(png)
+
+
+def png_to_dib(png: bytes) -> bytes:
+ """The PNG as a ``CF_DIB`` payload: a BMP file without its 14-byte header."""
+ from PIL import Image # noqa: PLC0415 - Pillow only when an image is copied
+
+ with Image.open(io.BytesIO(png)) as image:
+ rgb = image.convert("RGB")
+ out = io.BytesIO()
+ rgb.save(out, format="BMP")
+ return out.getvalue()[14:]
+
+
+def _run(command: list[str], data: bytes | None = None) -> bool:
+ try:
+ completed = subprocess.run( # noqa: S603 - fixed, non-shell OS command
+ command,
+ input=data,
+ capture_output=True,
+ timeout=_COMMAND_TIMEOUT_S,
+ check=False,
+ close_fds=True,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+ except (OSError, subprocess.SubprocessError) as exc:
+ log.warning("clipboard-image: copy command unavailable (%s)", exc)
+ return False
+ if completed.returncode != 0:
+ detail = completed.stderr.decode("utf-8", errors="replace").strip()
+ log.warning(
+ "clipboard-image: copy command failed with exit %d%s",
+ completed.returncode,
+ f" ({detail[:300]})" if detail else "",
+ )
+ return False
+ return True
+
+
+def _write_linux(png: bytes) -> bool:
+ wl_copy = shutil.which("wl-copy")
+ if wl_copy and os.environ.get("WAYLAND_DISPLAY"):
+ return _run([wl_copy, "--type", "image/png"], png)
+ xclip = shutil.which("xclip")
+ if xclip:
+ return _run([xclip, "-selection", "clipboard", "-t", "image/png", "-i"], png)
+ if wl_copy:
+ return _run([wl_copy, "--type", "image/png"], png)
+ log.info("clipboard-image: neither wl-copy nor xclip is installed")
+ return False
+
+
+def _write_macos(png: bytes) -> bool:
+ handle, name = tempfile.mkstemp(prefix="jarvis-appshot-", suffix=".png")
+ path = Path(name)
+ try:
+ with os.fdopen(handle, "wb") as file:
+ file.write(png)
+ # The path travels as an argument to AppleScript's `item 1 of argv`,
+ # never spliced into the script source.
+ script = (
+ "on run argv\n"
+ "set the clipboard to (read (POSIX file (item 1 of argv)) as «class PNGf»)\n"
+ "end run"
+ )
+ return _run(["/usr/bin/osascript", "-e", script, str(path)])
+ except OSError as exc:
+ log.warning("clipboard-image: could not stage the PNG for osascript (%s)", exc)
+ return False
+ finally:
+ path.unlink(missing_ok=True)
+
+
+def _write_windows(png: bytes) -> bool:
+ try:
+ dib = png_to_dib(png)
+ except Exception as exc: # noqa: BLE001 - a picture Pillow cannot read is not copyable
+ log.warning("clipboard-image: could not convert the PNG to a bitmap (%s)", exc)
+ return False
+ try:
+ from ctypes import wintypes # noqa: PLC0415
+
+ user32 = ctypes.WinDLL("user32", use_last_error=True)
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ user32.OpenClipboard.argtypes = [wintypes.HWND]
+ user32.OpenClipboard.restype = wintypes.BOOL
+ user32.EmptyClipboard.restype = wintypes.BOOL
+ user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE]
+ user32.SetClipboardData.restype = wintypes.HANDLE
+ user32.CloseClipboard.restype = wintypes.BOOL
+ user32.RegisterClipboardFormatW.argtypes = [wintypes.LPCWSTR]
+ user32.RegisterClipboardFormatW.restype = wintypes.UINT
+ kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t]
+ kernel32.GlobalAlloc.restype = wintypes.HGLOBAL
+ kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL]
+ kernel32.GlobalLock.restype = ctypes.c_void_p
+ kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL]
+ kernel32.GlobalFree.argtypes = [wintypes.HGLOBAL]
+ except (AttributeError, OSError) as exc:
+ log.warning("clipboard-image: Win32 API unavailable (%s)", exc)
+ return False
+
+ def put(fmt: int, data: bytes) -> bool:
+ handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(data))
+ if not handle:
+ return False
+ pointer = kernel32.GlobalLock(handle)
+ if not pointer:
+ kernel32.GlobalFree(handle)
+ return False
+ ctypes.memmove(pointer, data, len(data))
+ kernel32.GlobalUnlock(handle)
+ if not user32.SetClipboardData(fmt, handle):
+ # Ownership only passes to the clipboard on success.
+ kernel32.GlobalFree(handle)
+ return False
+ return True
+
+ opened = False
+ for _attempt in range(_WINDOWS_CLIPBOARD_RETRIES):
+ if user32.OpenClipboard(None):
+ opened = True
+ break
+ time.sleep(_WINDOWS_CLIPBOARD_RETRY_S)
+ if not opened:
+ log.warning("clipboard-image: another application holds the clipboard open")
+ return False
+ try:
+ if not user32.EmptyClipboard():
+ log.warning("clipboard-image: EmptyClipboard failed")
+ return False
+ if not put(_CF_DIB, dib):
+ log.warning("clipboard-image: SetClipboardData(CF_DIB) failed")
+ return False
+ png_format = user32.RegisterClipboardFormatW("PNG")
+ if png_format and not put(png_format, png):
+ # The bitmap is already there and pastes everywhere; the PNG
+ # format only adds transparency for apps that read it.
+ log.info("clipboard-image: the extra PNG format was not accepted")
+ return True
+ finally:
+ user32.CloseClipboard()
+
+
+__all__ = ["is_png", "png_to_dib", "write_png"]
diff --git a/jarvis/ui/web/appshot_routes.py b/jarvis/ui/web/appshot_routes.py
index ff45cefd95..a87c6e8d46 100644
--- a/jarvis/ui/web/appshot_routes.py
+++ b/jarvis/ui/web/appshot_routes.py
@@ -8,6 +8,7 @@ Endpoints (mounted by the WebServer in ``_build_app()``):
GET /api/appshot/latest → metadata of the last appshot.
GET /api/appshot/latest/image → its picture (never cached).
PUT /api/appshot/latest/image → replace it with the editor's version.
+ POST /api/appshot/clipboard → copy the editor's PNG natively (desktop only).
GET /api/appshot/pending → the appshot waiting for the next message.
POST /api/appshot/pending/claim → hand that one to the chat composer.
DELETE /api/appshot → forget every held appshot now.
@@ -245,6 +246,29 @@ def _png_size(data: bytes) -> tuple[int, int] | None:
return None
[email protected]("/clipboard", openapi_extra={"x-jarvis-dangerous": True})
+async def copy_to_clipboard(request: Request) -> dict[str, Any]:
+ """Put the editor's PNG on the desktop clipboard natively.
+
+ The embedded WebView cannot be trusted to copy images itself (see
+ :mod:`jarvis.platform.clipboard_image`). Desktop only: on a browser or
+ headless server the clipboard would be the server's, so this 404s there
+ and the page keeps the browser's own clipboard.
+ """
+ if not bool(getattr(request.app.state, "native_file_actions", False)):
+ raise HTTPException(status_code=404, detail="native-clipboard-disabled")
+ body = await request.body()
+ if not body or len(body) > _MAX_EDIT_BYTES:
+ raise HTTPException(status_code=413, detail="The picture is empty or too large.")
+ from jarvis.platform.clipboard_image import is_png, write_png # noqa: PLC0415
+
+ if not is_png(body):
+ raise HTTPException(status_code=400, detail="The picture is not a PNG.")
+ if not await asyncio.to_thread(write_png, body):
+ raise HTTPException(status_code=503, detail="native-clipboard-unavailable")
+ return {"copied": True}
+
+
@router.get("/pending")
async def pending() -> dict[str, Any]:
from jarvis.appshot.store import get_store # noqa: PLC0415
diff --git a/tests/unit/appshot/test_appshot_editor.py b/tests/unit/appshot/test_appshot_editor.py
index 37017de5ca..587bb717bb 100644
--- a/tests/unit/appshot/test_appshot_editor.py
+++ b/tests/unit/appshot/test_appshot_editor.py
@@ -90,3 +90,109 @@ def test_the_route_refuses_what_is_not_a_png(client, body: bytes, status: int) -
def test_the_route_refuses_an_expired_appshot(client) -> None:
assert client.put("/api/appshot/latest/image?id=gone", content=_png()).status_code == 404
+
+
+# -- copy: the editor's PNG goes to the OS clipboard natively ----------------
+
+
+def _copy_client(monkeypatch: pytest.MonkeyPatch, *, native: bool, accepted: bool = True):
+ from jarvis.platform import clipboard_image
+ from jarvis.ui.web.appshot_routes import router
+
+ copied: list[bytes] = []
+
+ def fake_write(png: bytes) -> bool:
+ copied.append(png)
+ return accepted
+
+ monkeypatch.setattr(clipboard_image, "write_png", fake_write)
+ app = FastAPI()
+ app.include_router(router)
+ app.state.native_file_actions = native
+ return TestClient(app), copied
+
+
+def test_copy_hands_the_png_to_the_native_clipboard(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, copied = _copy_client(monkeypatch, native=True)
+ png = _png()
+
+ response = client.post("/api/appshot/clipboard", content=png)
+
+ assert response.status_code == 200, response.text
+ assert copied == [png]
+
+
+def test_copy_is_desktop_only(monkeypatch: pytest.MonkeyPatch) -> None:
+ """On a browser/headless server the clipboard would be the server's."""
+ client, copied = _copy_client(monkeypatch, native=False)
+
+ assert client.post("/api/appshot/clipboard", content=_png()).status_code == 404
+ assert copied == []
+
+
+def test_copy_refuses_what_is_not_a_png(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, copied = _copy_client(monkeypatch, native=True)
+
+ assert client.post("/api/appshot/clipboard", content=b"GIF89a....").status_code == 400
+ assert client.post("/api/appshot/clipboard", content=b"").status_code == 413
+ assert copied == []
+
+
+def test_copy_says_when_the_os_refused(monkeypatch: pytest.MonkeyPatch) -> None:
+ client, _copied = _copy_client(monkeypatch, native=True, accepted=False)
+
+ response = client.post("/api/appshot/clipboard", content=_png())
+
+ assert response.status_code == 503
+ assert response.json()["detail"] == "native-clipboard-unavailable"
+
+
+def test_clipboard_image_needs_a_display(monkeypatch: pytest.MonkeyPatch) -> None:
+ from types import SimpleNamespace
+
+ from jarvis.platform import clipboard_image
+
+ monkeypatch.setattr(
+ clipboard_image, "detect_capabilities", lambda: SimpleNamespace(display_present=False)
+ )
+ assert clipboard_image.write_png(_png()) is False
+ assert clipboard_image.write_png(b"not a png") is False
+
+
+def test_clipboard_image_dib_is_a_bitmap_without_the_file_header() -> None:
+ from jarvis.platform.clipboard_image import png_to_dib
+
+ dib = png_to_dib(_png(40, 30))
+ # BITMAPINFOHEADER: its own size first, then width and height.
+ assert int.from_bytes(dib[0:4], "little") == 40
+ assert int.from_bytes(dib[4:8], "little") == 40
+ assert int.from_bytes(dib[8:12], "little", signed=True) == 30
+
+
+ ("wayland", "tools", "expected"),
+ [
+ ("wayland-0", {"wl-copy", "xclip"}, "wl-copy"),
+ ("", {"wl-copy", "xclip"}, "xclip"),
+ ("", {"wl-copy"}, "wl-copy"),
+ ("", set(), None),
+ ],
+)
+def test_clipboard_image_picks_the_linux_tool(
+ monkeypatch: pytest.MonkeyPatch, wayland: str, tools: set[str], expected: str | None
+) -> None:
+ from jarvis.platform import clipboard_image
+
+ ran: list[list[str]] = []
+ monkeypatch.setenv("WAYLAND_DISPLAY", wayland)
+ monkeypatch.setattr(
+ clipboard_image.shutil, "which", lambda name: f"/usr/bin/{name}" if name in tools else None
+ )
+ monkeypatch.setattr(
+ clipboard_image, "_run", lambda command, data=None: ran.append(command) or True
+ )
+
+ ok = clipboard_image._write_linux(_png()) # noqa: SLF001
+
+ assert ok is (expected is not None)
+ assert [command[0] for command in ran] == ([f"/usr/bin/{expected}"] if expected else [])