A desktop coding agent with free access to frontier models.
TypeScript
fix: windows focus, scaling, lock and popups
ogsamrat93bac007aee1
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex be45daf8..b6e8ef54 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -282,6 +282,26 @@ describe("portable host: observe and act", () => {});});+describe("portable host on Windows: the agent's ancestors come from the worker", () => {+ test("an IDE the agent runs in is denied as an ancestor when the app passed none", async () => {+ const native = fakeNative();+ native.windows.push({ id: 13, pid: 700, identity: "700:1", executable: "C:\\Program Files\\Microsoft VS Code\\Code.exe", appId: "c:\\program files\\microsoft vs code\\code.exe", appName: "Code", title: "main.ts - Code", bounds: { x: 0, y: 0, width: 800, height: 600 }, dpi: 96, scale: 1, monitor: "0" });+ const call0 = native.call.bind(native);+ let asked = 0;+ native.call = async (method: string, params: Record<string, unknown> = {}) => {+ if (method === "ancestors") { asked += 1; return [{ pid: 690, name: "cmd", exe: "C:\\Windows\\System32\\cmd.exe" }, { pid: 700, name: "Code", exe: "C:\\Program Files\\Microsoft VS Code\\Code.exe" }]; }+ return call0(method, params);+ };+ const host = await createPortableHost({ native, platform: "win32", settings: () => ({ enabled: true, real_screen: true, mode: "auto" }), context: () => ({ chat: "ses_w" }), approve: { open: async () => ({ decision: "once" }), act: async () => ({ allow: true }) } });+ const c = host.connect();+ const found = await call(c, "computer_discover", {});+ expect((found.state.apps as Array<{ name: string; available?: boolean }>).find((a) => a.name === "Code")).toMatchObject({ available: false });+ expect((await call(c, "computer_open_session", { app_id: "c:\\program files\\microsoft vs code\\code.exe", mode: "control", purpose: "edit" })).state).toMatchObject({ ok: false, code: "protected_app" });+ expect(asked).toBe(1);+ await host.close();+ });+});+describe("native worker process: stuck input is released by the parent", () => {test("a worker that dies holding a button gets a --release run", async () => {const dir = mkdtempSync(path.join(os.tmpdir(), "cu-worker-"));diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index 57758792..fc344f32 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -8,6 +8,6 @@ a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjsc12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts-02be6c62de13f3c8a55a0167c751742086c16bfadb201e9ce8e7f1b955d2376d native-worker.mjs-0421dd5f0e6812df5693aadcd99f2cdf871a0c78ce93113c98d53b526d55e1b0 portable-host.mjs+a9d1df37870802ffdca408a29735d1ce2a629e93435bc46ff36a0c5f8108869a native-worker.mjs+dd4dc647994d6f52724d06856deff846cd545dc7c477b20348c14273073aa205 portable-host.mjs3adf92d8ab9fe7a1a8b595c090f6ba4f99ed10d5146c318cfbdf7b674aec9657 portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/native-worker.mjs b/apps/server/src/computer-use/shared/native-worker.mjsindex 05a32ea3..4bfe132b 100644--- a/apps/server/src/computer-use/shared/native-worker.mjs+++ b/apps/server/src/computer-use/shared/native-worker.mjs@@ -88,12 +88,19 @@ function windows() {const clientToScreen = f(u, "int __stdcall ClientToScreen(uintptr_t hwnd, _Inout_ CU2_POINT *point)");const guiInfo = f(u, "int __stdcall GetGUIThreadInfo(uint32_t thread, _Inout_ CU2_GUITHREADINFO *info)");const focusWindow = f(u, "int __stdcall SetForegroundWindow(uintptr_t hwnd)");+ const bringToTop = f(u, "int __stdcall BringWindowToTop(uintptr_t hwnd)");+ const attachInput = f(u, "int __stdcall AttachThreadInput(uint32_t attach, uint32_t to, int on)");+ const currentThread = f(k, "uint32_t __stdcall GetCurrentThreadId()");const send = f(u, "uint32_t __stdcall SendInput(uint32_t count, CU2_INPUT *inputs, int size)");const metrics = f(u, "int __stdcall GetSystemMetrics(int index)");const openProcess = f(k, "uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)");const processName = f(k, "int __stdcall QueryFullProcessImageNameW(uintptr_t process, uint32_t flags, _Out_ uint16_t *name, _Inout_ uint32_t *size)");const times = f(k, "int __stdcall GetProcessTimes(uintptr_t process, _Out_ CU2_FILETIME *created, _Out_ CU2_FILETIME *exited, _Out_ CU2_FILETIME *kernel, _Out_ CU2_FILETIME *user)");const closeHandle = f(k, "int __stdcall CloseHandle(uintptr_t handle)");+ const processEntry = koffi.struct("CU2_PROCESSENTRY32W", { dwSize: "uint32_t", cntUsage: "uint32_t", th32ProcessID: "uint32_t", th32DefaultHeapID: "uintptr_t", th32ModuleID: "uint32_t", cntThreads: "uint32_t", th32ParentProcessID: "uint32_t", pcPriClassBase: "int32_t", dwFlags: "uint32_t", szExeFile: koffi.array("uint16_t", 260) });+ const snapshot = f(k, "uintptr_t __stdcall CreateToolhelp32Snapshot(uint32_t flags, uint32_t pid)");+ const firstProcess = f(k, "int __stdcall Process32FirstW(uintptr_t snapshot, _Inout_ CU2_PROCESSENTRY32W *entry)");+ const nextProcess = f(k, "int __stdcall Process32NextW(uintptr_t snapshot, _Inout_ CU2_PROCESSENTRY32W *entry)");const openDesktop = f(u, "uintptr_t __stdcall OpenInputDesktop(uint32_t flags, int inherit, uint32_t access)");const closeDesktop = f(u, "int __stdcall CloseDesktop(uintptr_t desktop)");const userInfo = f(u, "int __stdcall GetUserObjectInformationW(uintptr_t object, int index, _Out_ uint16_t *value, uint32_t size, _Out_ uint32_t *needed)");@@ -105,6 +112,11 @@ function windows() {const select = f(gdi, "uintptr_t __stdcall SelectObject(uintptr_t hdc, uintptr_t object)");const deleteObject = f(gdi, "int __stdcall DeleteObject(uintptr_t object)");const deleteDC = f(gdi, "int __stdcall DeleteDC(uintptr_t hdc)");+ let monitorDpi = null;+ try { monitorDpi = f(koffi.load("shcore.dll"), "int32_t __stdcall GetDpiForMonitor(uintptr_t monitor, int type, _Out_ uint32_t *x, _Out_ uint32_t *y)"); } catch { /* older Windows */ }+ const bitBlt = f(gdi, "int __stdcall BitBlt(uintptr_t dest, int x, int y, int width, int height, uintptr_t source, int sx, int sy, uint32_t rop)");+ const getWindow = f(u, "uintptr_t __stdcall GetWindow(uintptr_t hwnd, uint32_t cmd)");+ const dwmValue = f(dwm, "int32_t __stdcall DwmGetWindowAttribute(uintptr_t hwnd, uint32_t attribute, _Out_ uint32_t *value, uint32_t size)");const getBits = f(gdi, "int __stdcall GetDIBits(uintptr_t hdc, uintptr_t bitmap, uint32_t start, uint32_t lines, _Out_ uint8_t *bits, CU2_BITMAPINFOHEADER *info, uint32_t usage)");// Per-monitor DPI awareness v2 for this process, then this thread.try { f(u, "int __stdcall SetProcessDpiAwarenessContext(intptr_t context)")(-4); } catch { /* older Windows */ }@@ -163,7 +175,22 @@ function windows() {const value = {};while (peek(value, 0, 0, 0, 1)) { /* pump hook messages */ }};+ let wts = null;+ try {+ const lib = koffi.load("wtsapi32.dll");+ wts = { query: f(lib, "int __stdcall WTSQuerySessionInformationW(uintptr_t server, uint32_t session, int info, _Out_ void **buffer, _Out_ uint32_t *bytes)"), free: f(lib, "void __stdcall WTSFreeMemory(void *memory)") };+ } catch { /* no WTS: the input desktop check only */ }+ // The lock screen (LockApp) runs on the Default desktop since Windows 10: ask the session itself.+ const locked = () => {+ if (!wts) return false;+ const buffer = [null], bytes = [0];+ if (!wts.query(0, 0xffffffff, 25, buffer, bytes) || !buffer[0]) return false;+ // WTSINFOEXW: Level, then (8-byte aligned) SessionId, SessionState, SessionFlags (0 = WTS_SESSIONSTATE_LOCK).+ try { const [level, , , , flags] = koffi.decode(buffer[0], "int32_t", 5); return level === 1 && flags === 0; }+ finally { wts.free(buffer[0]); }+ };const available = () => {+ if (locked()) return false;const handle = openDesktop(0, 0, 1);if (!handle) return false;try { const data = Buffer.alloc(512), size = [0]; return Boolean(userInfo(handle, 2, data, data.length, size)) && data.toString("utf16le").split("\0")[0].toLowerCase() === "default"; }@@ -190,11 +217,50 @@ function windows() {try { dpi = dpiOfWindow(id) || 96; } catch { /* older Windows */ }return { id: Number(id), pid: pid[0], thread, identity, executable, appId: executable.toLowerCase(), appName: path.win32.basename(executable, ".exe"), title: text(titleOf, id), bounds, dpi, scale: dpi / 96, monitor: String(Number(monitorOf(id, 2))) };}+ // The ancestor processes of `pid`, nearest first ([{ pid, name, exe }]): the terminal or IDE the agent runs in.+ function ancestors(pid) {+ const table = new Map();+ const snap = snapshot(2, 0);+ if (!snap || Number(snap) >= 2 ** 52) return []; // INVALID_HANDLE_VALUE+ try {+ const entry = { dwSize: koffi.sizeof(processEntry) };+ for (let ok = firstProcess(snap, entry); ok; ok = nextProcess(snap, entry)) {+ table.set(entry.th32ProcessID, { ppid: entry.th32ParentProcessID, name: Buffer.from(Uint16Array.from(entry.szExeFile).buffer).toString("utf16le").split("\0")[0] });+ entry.dwSize = koffi.sizeof(processEntry);+ }+ } finally { closeHandle(snap); }+ const info = (id) => {+ const handle = openProcess(0x1000, 0, id);+ if (!handle) return { exe: "", started: 0 };+ try {+ const name = Buffer.alloc(65536), size = [32768], created = {};+ const exe = processName(handle, 0, name, size) ? name.toString("utf16le", 0, size[0] * 2) : "";+ const started = times(handle, created, {}, {}, {}) ? created.high * 2 ** 32 + created.low : 0;+ return { exe, started };+ } finally { closeHandle(handle); }+ };+ const out = [], seen = new Set([pid]);+ let child = { ...table.get(pid), ...info(pid) };+ for (let i = 0; i < 64 && child.ppid > 4 && table.has(child.ppid) && !seen.has(child.ppid); i += 1) {+ seen.add(child.ppid);+ const parent = { pid: child.ppid, ...table.get(child.ppid), ...info(child.ppid) };+ // A parent that started after its child is a reused pid: the real parent has exited.+ if (parent.started && child.started && parent.started > child.started) break;+ // The desktop shell and system services start everything: not an app the agent runs in.+ if (/^(explorer|svchost|services|wininit|winlogon|smss|sihost)\.exe$/i.test(parent.name)) break;+ out.push({ pid: parent.pid, name: parent.name.replace(/\.exe$/i, ""), exe: parent.exe });+ child = parent;+ }+ return out;+ }function screens() {const out = [];const cb = koffi.register((monitor) => {const info = { cbSize: koffi.sizeof("CU2_MONITORINFO"), rcMonitor: {}, rcWork: {}, dwFlags: 0 };- if (monitorInfo(monitor, info)) out.push({ id: String(Number(monitor)), x: info.rcMonitor.left, y: info.rcMonitor.top, width: info.rcMonitor.right - info.rcMonitor.left, height: info.rcMonitor.bottom - info.rcMonitor.top, scale: 1, primary: (info.dwFlags & 1) === 1 });+ // Effective DPI of the monitor (the person's scaling, e.g. 144 = 150%); bounds are native pixels.+ const dx = [96], dy = [96];+ const scale = monitorDpi && monitorDpi(monitor, 0, dx, dy) === 0 && dx[0] ? dx[0] / 96 : 1;+ if (monitorInfo(monitor, info)) out.push({ id: String(Number(monitor)), x: info.rcMonitor.left, y: info.rcMonitor.top, width: info.rcMonitor.right - info.rcMonitor.left, height: info.rcMonitor.bottom - info.rcMonitor.top, scale, primary: (info.dwFlags & 1) === 1 });return 1;}, koffi.pointer(monitorProto));try { enumMonitors(0, null, cb, 0); } finally { koffi.unregister(cb); }@@ -209,7 +275,29 @@ function windows() {if (Number(ancestor(foreground(), 2)) !== expected.id) fail("focus_changed", "The approved window must stay in front. Choose Continue when ready.");return current;}- const hit = (expected, p) => Number(ancestor(fromPoint({ x: p.x, y: p.y }), 2)) === expected.id;+ // The approved window's own popups: windows it owns (bubbles, dropdowns, dialogs' popups) and the+ // process's menus (#32768). Another top-level window of the same process (a second browser window) is not.+ const ownPopup = (h, expected) => {+ if (!h || Number(h) === expected.id) return Number(h) === expected.id;+ const pid = [0]; pidOf(h, pid);+ if (pid[0] !== expected.pid) return false;+ if (text(classOf, h, 64) === "#32768") return true;+ for (let owner = getWindow(h, 4), i = 0; owner && i < 8; owner = getWindow(owner, 4), i += 1) if (Number(owner) === expected.id) return true;+ return false;+ };+ const hit = (expected, p) => ownPopup(Number(ancestor(fromPoint({ x: p.x, y: p.y }), 2)), expected);+ const cloaked = (h) => { const v = [0]; try { return dwmValue(h, 14, v, 4) === 0 && v[0] !== 0; } catch { return false; } };+ // The approved window's own popups drawn over it, lowest first (they go into its image on top of it).+ function popupsOver(current) {+ const b = current.bounds, out = [];+ for (let h = getWindow(current.id, 3), i = 0; h && i < 512; h = getWindow(h, 3), i += 1) {+ if (!visible(h) || iconic(h) || cloaked(h) || !ownPopup(Number(h), current)) continue;+ const r = {};+ if (!windowRect(h, r) || r.left >= b.x + b.width || r.top >= b.y + b.height || r.right <= b.x || r.bottom <= b.y) continue;+ out.push({ id: Number(h), r });+ }+ return out.reverse();+ }const moveTo = (x, y) => {const left = metrics(76), top = metrics(77), width = metrics(78), height = metrics(79);if (width <= 1 || height <= 1) fail("input_unavailable", "The desktop bounds could not be read.");@@ -250,8 +338,19 @@ function windows() {if (fw <= 0 || fh <= 0 || fw * fh > 33_177_600) fail("capture_unavailable", "This window is too large to capture. Resize it and observe again.");const screen = getDC(0), memory = createDC(screen), bitmap = createBitmap(screen, fw, fh), previous = select(memory, bitmap);try {- // PW_RENDERFULLCONTENT: the window's own pixels, never another window covering it.+ // PW_RENDERFULLCONTENT: the window's own pixels, never another app covering it ...if (!printWindow(current.id, memory, 2)) fail("capture_unavailable", "The window image is unavailable. Keep it visible and observe again.");+ // ... with its own popups (menus, dropdowns, bubbles) drawn on top: what a click there would hit.+ for (const popup of popupsOver(current)) {+ const pw = popup.r.right - popup.r.left, ph = popup.r.bottom - popup.r.top;+ if (pw <= 0 || ph <= 0 || pw * ph > 33_177_600) continue;+ const pdc = createDC(screen), pbm = createBitmap(screen, pw, ph), pprev = select(pdc, pbm);+ // Only the popup's visible frame (DWM bounds): its shadow margin would print black.+ const v = {};+ if (boundsOf(popup.id, 9, v, koffi.sizeof(rect)) !== 0 || v.right <= v.left) Object.assign(v, popup.r);+ try { if (printWindow(popup.id, pdc, 2)) bitBlt(memory, v.left - wr.left, v.top - wr.top, v.right - v.left, v.bottom - v.top, pdc, v.left - popup.r.left, v.top - popup.r.top, 0x00cc0020); }+ finally { select(pdc, pprev); deleteObject(pbm); deleteDC(pdc); }+ }const bits = Buffer.alloc(fw * fh * 4);const header = { biSize: 40, biWidth: fw, biHeight: -fh, biPlanes: 1, biBitCount: 32, biCompression: 0, biSizeImage: 0, biXPelsPerMeter: 0, biYPelsPerMeter: 0, biClrUsed: 0, biClrImportant: 0 };if (getBits(memory, bitmap, 0, fh, bits, header, 0) !== fh) fail("capture_unavailable", "The window image could not be read.");@@ -318,10 +417,23 @@ function windows() {} finally { release(); }}return {- poll, window, screens, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,+ poll, window, screens, ancestors, available, release, startMonitoring, stopMonitoring, check, hit, capture, act,cursor() { const p = {}; return cursorPos(p) ? { x: p.x, y: p.y } : null; },list() { const result = []; const cb = koffi.register((id) => { try { const item = window(id); if (item.title) result.push(item); } catch { /* skip */ } return 1; }, koffi.pointer(enumProto)); try { enumWindows(cb, 0); } finally { koffi.unregister(cb); } return result; },- focus(expected) { if (window(expected.id).identity !== expected.identity || !focusWindow(expected.id)) fail("focus_changed", "Bring the approved app to the front, then choose Continue."); poll(); },+ focus(expected) {+ if (window(expected.id).identity !== expected.identity) fail("focus_changed", "Bring the approved app to the front, then choose Continue.");+ const front = () => Number(ancestor(foreground(), 2)) === expected.id;+ // Windows lets the process that sent the last input set the foreground window: a zero move of ours first.+ if (!front()) { try { injectBatch([mouseItem(1, 0, 0, 0)]); } catch { /* the lock fallback below */ } focusWindow(expected.id); }+ if (!front()) {+ // Windows' foreground lock refuses a background process: join the foreground thread's input for the switch.+ const other = pidOf(foreground(), [0]), me = currentThread();+ const joined = other && other !== me ? attachInput(me, other, 1) : 0;+ try { bringToTop(expected.id); focusWindow(expected.id); } finally { if (joined) attachInput(me, other, 0); }+ }+ if (!front()) fail("focus_changed", "Bring the approved app to the front, then choose Continue.");+ poll();+ },releaseHeld({ keys = [], buttons = [] }) { for (const key of keys) downKeys.add(key); for (const b of buttons) downButtons.add(b); release(); },close() { release(); stopMonitoring(); },};@@ -755,6 +867,7 @@ process.on("message", async (message) => {if (method === "list") result = backend.list();else if (method === "window") result = backend.window(params.id);else if (method === "screens") result = backend.screens();+ else if (method === "ancestors") result = backend.ancestors ? backend.ancestors(Number(params.pid)) : [];else if (method === "cursor") result = backend.cursor();else if (method === "focus") { backend.focus(params.window); result = { generation }; }else if (method === "state") result = { generation, available: backend.available() };diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex 5b819931..24e17e75 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -191,6 +191,15 @@ export async function createPortableHost({ native, spool = null, settings, conte}, 1000);ticker.unref?.();+ // Windows: the agent's ancestor processes (its terminal or IDE) read natively when the app passed none.+ let nativeAncestors = null;+ async function ancestorsOf(ctx) {+ if (Array.isArray(ctx.ancestors) && ctx.ancestors.length) return ctx.ancestors;+ if (platform !== "win32") return ctx.ancestors ?? [];+ if (!nativeAncestors) nativeAncestors = await native.call("ancestors", { pid: process.pid }).catch(() => null);+ return nativeAncestors ?? [];+ }+async function screenOf(w) {const screens = await native.call("screens").catch(() => []);const cx = w.bounds.x + w.bounds.width / 2, cy = w.bounds.y + w.bounds.height / 2;@@ -385,9 +394,10 @@ export async function createPortableHost({ native, spool = null, settings, conteconst ctx = (await context()) ?? {};const config = settings() ?? {};const windows = await native.call("list");+ const ancestors = await ancestorsOf(ctx);const apps = new Map();for (const w of windows) {- const denied = deniedApp(appIdentity(w), { ancestors: ctx.ancestors, self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });+ const denied = deniedApp(appIdentity(w), { ancestors, self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });const key = `${w.appId}\0${w.pid}`;if (!apps.has(key)) apps.set(key, { app_id: w.appId, name: w.appName, pid: w.pid, ...(denied ? { available: false, reason: denied.reason } : {}), windows: [] });if (!denied) apps.get(key).windows.push({ id: String(w.id), title: w.title });@@ -404,7 +414,7 @@ export async function createPortableHost({ native, spool = null, settings, conteif (lease) return mcpResult(failure("session_busy", "Another computer use session is open. Close it first."));const windows = (await native.call("list")).filter((w) => w.appId === args.app_id && (args.pid === undefined || w.pid === args.pid));if (!windows.length) return mcpResult(failure("app_unavailable", "That app has no visible window. Open it yourself, then call computer_discover again."));- const denied = deniedApp(appIdentity(windows[0]), { ancestors: ctx.ancestors, self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });+ const denied = deniedApp(appIdentity(windows[0]), { ancestors: await ancestorsOf(ctx), self_pids: [...(ctx.self_pids ?? []), process.pid, native.pid].filter(Boolean), tty_pids: ctx.tty_pids, deny_apps: config.deny_apps });if (denied) return mcpResult(failure(denied.code, denied.reason));const preferred = args.window_id !== undefined ? windows.find((w) => String(w.id) === args.window_id) : windows[0];if (!preferred) return mcpResult(failure("window_unavailable", "That window is not open. Call computer_discover again."));