A desktop coding agent with free access to frontier models.
TypeScript
fix: ask once per action and keep frames whole
ogsamrat85770e2a4d4e
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.tsindex 3771d290..0de5cd89 100644--- a/apps/server/src/computer-use/media-image.test.ts+++ b/apps/server/src/computer-use/media-image.test.ts@@ -170,6 +170,34 @@ describe("media references", () => {expect(((out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[2] as Record<string, unknown>).omitted).toBe("budget");});+ test("a frame released after one chat's upload comes back whole for the next chat (thumb and full)", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const big = frame(1600, 1000, 11);+ const model = encodePng(fitLongSide(big, 800));+ const put = () => spool.put({ bytes: model, mime: "image/png", full: { bytes: encodePng(big, { level: 1 }), width: 1600, height: 1000 }, thumb: encodeJpeg(fitLongSide(big, 320)) });+ const meta = put();+ spool.ref(meta.sha256, "ses_a");+ spool.ref(meta.sha256, "ses_b");+ expect(spool.release(meta.sha256, "ses_a")).toBe(false);+ expect(spool.has(meta.sha256, meta.ext)).toBe(true);+ expect(spool.release(meta.sha256, "ses_b")).toBe(true);+ expect(spool.has(meta.sha256, meta.ext)).toBe(false);+ // The host takes the identical frame again in a new chat; the reader meets it too.+ put();+ const out = rewriteMessageMedia({ parts: [toolPart("x", [dataUrl("image/png", model)])] }, { spool, index: createMediaIndex(path.join(dir, "c.json")), limits: {}, chat: "ses_c" });+ const ref = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;+ expect(String(ref.thumb)).toStartWith("data:image/jpeg;base64,");+ expect(ref.full).toMatchObject({ width: 1600, height: 1000 });+ expect(out.files.map((f) => f.kind)).toEqual(["model", "full"]);+ // Only the reader meets it (meta kept, files gone): the model copy and a thumb come back from the message.+ spool.ref(meta.sha256, "ses_c");+ for (const sub of ["", "thumb"]) rmSync(path.join(dir, "spool", sub, sub ? `${meta.sha256}.jpg` : `${meta.sha256}.png`), { force: true });+ const back = rewriteMessageMedia({ parts: [toolPart("y", [dataUrl("image/png", model)])] }, { spool, index: createMediaIndex(path.join(dir, "d.json")), limits: {}, chat: "ses_d" });+ expect(String(((back.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>).thumb)).toStartWith("data:image/jpeg;base64,");+ expect(spool.has(meta.sha256, "png")).toBe(true);+ });+test("data URL parsing", () => {expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.tsindex 463c9b3f..ec25e7d0 100644--- a/apps/server/src/computer-use/portable-host.test.ts+++ b/apps/server/src/computer-use/portable-host.test.ts@@ -205,6 +205,27 @@ describe("portable host: observe and act", () => {await host.close();});+ test("a slow approval: the window changing meanwhile is not stale, and the same act is not asked twice", async () => {+ const r = rig("careful");+ const { host, c, session, obs } = await opened(r);+ // The person takes a while; the caret blinks or a spinner turns meanwhile.+ const slow = r.native as ReturnType<typeof fakeNative>;+ const original = r.acts.push.bind(r.acts);+ r.acts.push = (...items: ActRequest[]) => { slow.setShade(90); return original(...items); };+ const first = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the greeting", action: { type: "type", text: "hi" } });+ expect(first.state).toMatchObject({ ok: true, approval: { asked: true } });+ expect(r.acts.length).toBe(1);+ // The model retries the same act on a fresh observation: no second question within 30 s.+ const again = await call(c, "computer_act", { session_id: session, observation_id: (first.state.observation as { id: string }).id, request_id: "b", intent: "Type the greeting", action: { type: "type", text: "hi" } });+ expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });+ expect(r.acts.length).toBe(1);+ // Another act asks.+ const other = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type more", action: { type: "type", text: "there" } });+ expect(other.state).toMatchObject({ ok: true, approval: { asked: true } });+ expect(r.acts.length).toBe(2);+ await host.close();+ });+test("person input pauses; only Continue resumes; first act after takeover asks in Normal", async () => {const r = rig("normal");const { host, c, session, obs } = await opened(r);diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index d567ca11..59d50ffa 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -6,8 +6,8 @@ c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjsf11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts-06f9604aa2d36d8cda25098e26088832e8b20cf9b0d756c5d745517c935684c9 media.mjs+80e88fcc18b489720ef31dd273c15387606df3e9a2ffe4508762c5f51125b33c media.mjsc12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts02be6c62de13f3c8a55a0167c751742086c16bfadb201e9ce8e7f1b955d2376d native-worker.mjs-bceba0caf4293a06096f7d69f820d63b0b5a7bb36c12a72c8b575f3cefa591fb portable-host.mjs+42455d2fb02246b44a27b644f7f1f9bfdea9a11588442cf43f5ea57746a2403c portable-host.mjs3adf92d8ab9fe7a1a8b595c090f6ba4f99ed10d5146c318cfbdf7b674aec9657 portable-host.d.mtsdiff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjsindex 3f652c81..73b440cc 100644--- a/apps/server/src/computer-use/shared/media.mjs+++ b/apps/server/src/computer-use/shared/media.mjs@@ -109,7 +109,19 @@ export function createMediaSpool(dir) {const data = Buffer.from(bytes);const sha = sha256(data);const existing = spool.meta(sha);- if (existing) return existing;+ if (existing) {+ // Known frame: write back whatever is missing (released after an upload, swept) and keep what the meta knew.+ const merged = { ...existing };+ if (!existsSync(path.join(dir, `${sha}.${existing.ext}`))) writeAtomic(path.join(dir, `${sha}.${existing.ext}`), data);+ if (full?.bytes && full.width * full.height > existing.width * existing.height && !existsSync(path.join(dir, "full", `${sha}.png`))) {+ const fullBytes = Buffer.from(full.bytes);+ writeAtomic(path.join(dir, "full", `${sha}.png`), fullBytes);+ merged.full = { sha256: sha256(fullBytes), width: full.width, height: full.height, bytes: fullBytes.length, ext: "png" };+ }+ if (thumb?.length && !existsSync(path.join(dir, "thumb", `${sha}.jpg`))) { writeAtomic(path.join(dir, "thumb", `${sha}.jpg`), Buffer.from(thumb)); merged.thumb = thumb.length; }+ if (JSON.stringify(merged) !== JSON.stringify(existing)) writeAtomic(metaFile(sha), JSON.stringify(merged));+ return merged;+ }const info = imageInfo(data);const type = info?.mime ?? mime;const ext = extensionOf(type);@@ -240,9 +252,10 @@ export function rewriteMessageMedia(message, ctx) {if (!meta) {const thumb = thumbnailOf(parsed.bytes, parsed.mime);meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });- } else if (!ctx.spool.has(sha, meta.ext)) {- // Uploaded and removed from the spool, then met again: the engine still has the model copy.- meta = { ...ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime }), ...(meta.full ? { full: meta.full } : {}) };+ } else if (!ctx.spool.has(sha, meta.ext) || !ctx.spool.thumb(sha)) {+ // Released or swept, then met again (another chat): the engine still has the model copy; the thumb is made again.+ const thumb = thumbnailOf(parsed.bytes, parsed.mime);+ meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });}let entry = ctx.index.entry(sha);const first = !entry;diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjsindex b3be2786..2bf1ee2d 100644--- a/apps/server/src/computer-use/shared/portable-host.mjs+++ b/apps/server/src/computer-use/shared/portable-host.mjs@@ -296,33 +296,44 @@ export async function createPortableHost({ native, spool = null, settings, conteconst current = await native.call("window", { id: s.window.id });const common = { action, frame_before: { sha256: observed.sha }, window: windowRef(current), screen: s.screen, dpi: current.dpi, monitor_id: current.monitor, scale: current.bounds.width / observed.model.width };if (current.identity !== observed.window.identity || JSON.stringify(current.bounds) !== JSON.stringify(observed.window.bounds)) return finish({ ...common, ok: false, code: "window_changed", message: "The window moved or changed. Observe again.", next: "computer_observe" });- // Per-action approval by permission mode (host channel, never the engine's).- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });- let approval = { asked: false };- if (decision.ask) {- const pointList = action.type === "drag" ? action.path : action.x !== undefined ? [{ x: action.x, y: action.y }] : [];- const answer = await askAct(s, { action, intent: args.intent, reasons: decision.reasons, points: pointList, frame: observed });- approval = { asked: true, reasons: decision.reasons.map((r) => r.code), by: "person" };- if (!answer.allow) return finish({ ...common, approval, ok: false, code: "approval_denied", message: "The person did not allow this action. Ask them or choose another way." });- if (sessions.get(s.id) !== s || s.phase !== "working") return finish({ ...common, approval, ok: false, code: "paused_by_person", message: "The session was paused or ended." });- if (decision.reasons.some((r) => r.code === "document_changed")) s.approved = { ...s.approved, title: current.title };- }- s.afterTakeover = false;// Native coordinates; every point must hit the approved window.const mapped = action.type === "drag" ? action.path.map((p) => mapPoint(p, observed.model, current.bounds)) : action.x !== undefined ? [mapPoint(action, observed.model, current.bounds)] : [];- if (mapped.length) {- const hits = await native.call("hit", { window: current, points: mapped.map((m) => m.screen) });- if (!hits.inside.every(Boolean)) return finish({ ...common, approval, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });- }- // Tolerant stale check: the window must still look like the observation (caret masked).+ const hitAll = async (w) => !mapped.length || (await native.call("hit", { window: w, points: mapped.map((m) => m.screen) })).inside.every(Boolean);+ if (!(await hitAll(current))) return finish({ ...common, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });+ // Tolerant stale check (caret masked), before any question: the person is asked about what the model saw.if (action.type !== "wait") {const fresh = await native.call("capture", { window: current }, 15_000);// The observation's grid was taken after masking: mask the fresh frame the same way.paintBoxes(fresh, fresh.secure ?? []);const grid = luminanceGrid(fitLongSide(fresh, l.model_long_side));const compare = compareGrids(observed.grid, grid, { mask: observed.caret ? [observed.caret] : [] });- if (compare.stale) return finish({ ...common, approval, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });+ if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });}+ // Per-action approval by permission mode (host channel, never the engine's).+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });+ let approval = { asked: false };+ if (decision.ask) {+ // The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.+ const key = JSON.stringify([action, args.intent]);+ const remembered = s.allowed.get(key);+ if (remembered && remembered > Date.now()) {+ approval = { asked: false, reasons: decision.reasons.map((r) => r.code), by: "person" };+ } else {+ const pointList = action.type === "drag" ? action.path : action.x !== undefined ? [{ x: action.x, y: action.y }] : [];+ const answer = await askAct(s, { action, intent: args.intent, reasons: decision.reasons, points: pointList, frame: observed });+ approval = { asked: true, reasons: decision.reasons.map((r) => r.code), by: "person" };+ if (!answer.allow) return finish({ ...common, approval, ok: false, code: "approval_denied", message: "The person did not allow this action. Ask them or choose another way." });+ if (sessions.get(s.id) !== s || s.phase !== "working") return finish({ ...common, approval, ok: false, code: "paused_by_person", message: "The session was paused or ended." });+ s.allowed.set(key, Date.now() + 30_000);+ if (s.allowed.size > 64) s.allowed.delete(s.allowed.keys().next().value);+ }+ if (decision.reasons.some((r) => r.code === "document_changed")) s.approved = { ...s.approved, title: current.title };+ // While the person answered, the window may have moved or been covered: geometry and hits again (no image compare).+ const after = await native.call("window", { id: s.window.id });+ if (after.identity !== current.identity || JSON.stringify(after.bounds) !== JSON.stringify(current.bounds)) return finish({ ...common, approval, ok: false, code: "window_changed", message: "The window moved or changed. Observe again.", next: "computer_observe" });+ if (!(await hitAll(after))) return finish({ ...common, approval, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });+ }+ s.afterTakeover = false;const cursorBefore = await native.call("cursor").catch(() => null);const point = mapped.length === 1 ? { model: mapped[0].model, screen: mapped[0].screen } : undefined;const path = action.type === "drag" ? mapped.map((m) => ({ model: m.model, screen: m.screen })) : undefined;@@ -397,7 +408,7 @@ export async function createPortableHost({ native, spool = null, settings, conteconst s = {id: randomUUID(), connection, window: preferred, windows, mode: args.mode, purpose: args.purpose, phase: "approval", status: "",approval: null, chat: ctx.chat ?? null, actions: 0, requests: new Set(), observation: null, resumeObservation: null, log: [],- lastOperation: Date.now(), lastAct: 0, afterTakeover: false, generation: 0, expiresAt: Date.now() + l.approval_timeout_seconds * 1000, screen: await screenOf(preferred),+ lastOperation: Date.now(), lastAct: 0, afterTakeover: false, generation: 0, allowed: new Map(), expiresAt: Date.now() + l.approval_timeout_seconds * 1000, screen: await screenOf(preferred),};lease = s.id; sessions.set(s.id, s);update();