Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: ask once per action and keep frames whole

ogsamrat85770e2a4d4e
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.ts
index 3771d290..0de5cd89 100644
--- a/apps/server/src/computer-use/media-image.test.ts
+++ b/apps/server/src/computer-use/media-image.test.ts
@@ -170,6 +170,34 @@ describe("media references", () => {
expect(((out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[2] as Record<string, unknown>).omitted).toBe("budget");
});
+ test("a frame released after one chat's upload comes back whole for the next chat (thumb and full)", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const big = frame(1600, 1000, 11);
+ const model = encodePng(fitLongSide(big, 800));
+ const put = () => spool.put({ bytes: model, mime: "image/png", full: { bytes: encodePng(big, { level: 1 }), width: 1600, height: 1000 }, thumb: encodeJpeg(fitLongSide(big, 320)) });
+ const meta = put();
+ spool.ref(meta.sha256, "ses_a");
+ spool.ref(meta.sha256, "ses_b");
+ expect(spool.release(meta.sha256, "ses_a")).toBe(false);
+ expect(spool.has(meta.sha256, meta.ext)).toBe(true);
+ expect(spool.release(meta.sha256, "ses_b")).toBe(true);
+ expect(spool.has(meta.sha256, meta.ext)).toBe(false);
+ // The host takes the identical frame again in a new chat; the reader meets it too.
+ put();
+ const out = rewriteMessageMedia({ parts: [toolPart("x", [dataUrl("image/png", model)])] }, { spool, index: createMediaIndex(path.join(dir, "c.json")), limits: {}, chat: "ses_c" });
+ const ref = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;
+ expect(String(ref.thumb)).toStartWith("data:image/jpeg;base64,");
+ expect(ref.full).toMatchObject({ width: 1600, height: 1000 });
+ expect(out.files.map((f) => f.kind)).toEqual(["model", "full"]);
+ // Only the reader meets it (meta kept, files gone): the model copy and a thumb come back from the message.
+ spool.ref(meta.sha256, "ses_c");
+ for (const sub of ["", "thumb"]) rmSync(path.join(dir, "spool", sub, sub ? `${meta.sha256}.jpg` : `${meta.sha256}.png`), { force: true });
+ const back = rewriteMessageMedia({ parts: [toolPart("y", [dataUrl("image/png", model)])] }, { spool, index: createMediaIndex(path.join(dir, "d.json")), limits: {}, chat: "ses_d" });
+ expect(String(((back.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>).thumb)).toStartWith("data:image/jpeg;base64,");
+ expect(spool.has(meta.sha256, "png")).toBe(true);
+ });
+
test("data URL parsing", () => {
expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });
expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();
diff --git a/apps/server/src/computer-use/portable-host.test.ts b/apps/server/src/computer-use/portable-host.test.ts
index 463c9b3f..ec25e7d0 100644
--- a/apps/server/src/computer-use/portable-host.test.ts
+++ b/apps/server/src/computer-use/portable-host.test.ts
@@ -205,6 +205,27 @@ describe("portable host: observe and act", () => {
await host.close();
});
+ test("a slow approval: the window changing meanwhile is not stale, and the same act is not asked twice", async () => {
+ const r = rig("careful");
+ const { host, c, session, obs } = await opened(r);
+ // The person takes a while; the caret blinks or a spinner turns meanwhile.
+ const slow = r.native as ReturnType<typeof fakeNative>;
+ const original = r.acts.push.bind(r.acts);
+ r.acts.push = (...items: ActRequest[]) => { slow.setShade(90); return original(...items); };
+ const first = await call(c, "computer_act", { session_id: session, observation_id: obs.state.observation?.id, request_id: "a", intent: "Type the greeting", action: { type: "type", text: "hi" } });
+ expect(first.state).toMatchObject({ ok: true, approval: { asked: true } });
+ expect(r.acts.length).toBe(1);
+ // The model retries the same act on a fresh observation: no second question within 30 s.
+ const again = await call(c, "computer_act", { session_id: session, observation_id: (first.state.observation as { id: string }).id, request_id: "b", intent: "Type the greeting", action: { type: "type", text: "hi" } });
+ expect(again.state).toMatchObject({ ok: true, approval: { asked: false } });
+ expect(r.acts.length).toBe(1);
+ // Another act asks.
+ const other = await call(c, "computer_act", { session_id: session, observation_id: (again.state.observation as { id: string }).id, request_id: "c", intent: "Type more", action: { type: "type", text: "there" } });
+ expect(other.state).toMatchObject({ ok: true, approval: { asked: true } });
+ expect(r.acts.length).toBe(2);
+ await host.close();
+ });
+
test("person input pauses; only Continue resumes; first act after takeover asks in Normal", async () => {
const r = rig("normal");
const { host, c, session, obs } = await opened(r);
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index d567ca11..59d50ffa 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -6,8 +6,8 @@ c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs
103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts
17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjs
f11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts
-06f9604aa2d36d8cda25098e26088832e8b20cf9b0d756c5d745517c935684c9 media.mjs
+80e88fcc18b489720ef31dd273c15387606df3e9a2ffe4508762c5f51125b33c media.mjs
c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts
02be6c62de13f3c8a55a0167c751742086c16bfadb201e9ce8e7f1b955d2376d native-worker.mjs
-bceba0caf4293a06096f7d69f820d63b0b5a7bb36c12a72c8b575f3cefa591fb portable-host.mjs
+42455d2fb02246b44a27b644f7f1f9bfdea9a11588442cf43f5ea57746a2403c portable-host.mjs
3adf92d8ab9fe7a1a8b595c090f6ba4f99ed10d5146c318cfbdf7b674aec9657 portable-host.d.mts
diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjs
index 3f652c81..73b440cc 100644
--- a/apps/server/src/computer-use/shared/media.mjs
+++ b/apps/server/src/computer-use/shared/media.mjs
@@ -109,7 +109,19 @@ export function createMediaSpool(dir) {
const data = Buffer.from(bytes);
const sha = sha256(data);
const existing = spool.meta(sha);
- if (existing) return existing;
+ if (existing) {
+ // Known frame: write back whatever is missing (released after an upload, swept) and keep what the meta knew.
+ const merged = { ...existing };
+ if (!existsSync(path.join(dir, `${sha}.${existing.ext}`))) writeAtomic(path.join(dir, `${sha}.${existing.ext}`), data);
+ if (full?.bytes && full.width * full.height > existing.width * existing.height && !existsSync(path.join(dir, "full", `${sha}.png`))) {
+ const fullBytes = Buffer.from(full.bytes);
+ writeAtomic(path.join(dir, "full", `${sha}.png`), fullBytes);
+ merged.full = { sha256: sha256(fullBytes), width: full.width, height: full.height, bytes: fullBytes.length, ext: "png" };
+ }
+ if (thumb?.length && !existsSync(path.join(dir, "thumb", `${sha}.jpg`))) { writeAtomic(path.join(dir, "thumb", `${sha}.jpg`), Buffer.from(thumb)); merged.thumb = thumb.length; }
+ if (JSON.stringify(merged) !== JSON.stringify(existing)) writeAtomic(metaFile(sha), JSON.stringify(merged));
+ return merged;
+ }
const info = imageInfo(data);
const type = info?.mime ?? mime;
const ext = extensionOf(type);
@@ -240,9 +252,10 @@ export function rewriteMessageMedia(message, ctx) {
if (!meta) {
const thumb = thumbnailOf(parsed.bytes, parsed.mime);
meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });
- } else if (!ctx.spool.has(sha, meta.ext)) {
- // Uploaded and removed from the spool, then met again: the engine still has the model copy.
- meta = { ...ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime }), ...(meta.full ? { full: meta.full } : {}) };
+ } else if (!ctx.spool.has(sha, meta.ext) || !ctx.spool.thumb(sha)) {
+ // Released or swept, then met again (another chat): the engine still has the model copy; the thumb is made again.
+ const thumb = thumbnailOf(parsed.bytes, parsed.mime);
+ meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });
}
let entry = ctx.index.entry(sha);
const first = !entry;
diff --git a/apps/server/src/computer-use/shared/portable-host.mjs b/apps/server/src/computer-use/shared/portable-host.mjs
index b3be2786..2bf1ee2d 100644
--- a/apps/server/src/computer-use/shared/portable-host.mjs
+++ b/apps/server/src/computer-use/shared/portable-host.mjs
@@ -296,33 +296,44 @@ export async function createPortableHost({ native, spool = null, settings, conte
const current = await native.call("window", { id: s.window.id });
const common = { action, frame_before: { sha256: observed.sha }, window: windowRef(current), screen: s.screen, dpi: current.dpi, monitor_id: current.monitor, scale: current.bounds.width / observed.model.width };
if (current.identity !== observed.window.identity || JSON.stringify(current.bounds) !== JSON.stringify(observed.window.bounds)) return finish({ ...common, ok: false, code: "window_changed", message: "The window moved or changed. Observe again.", next: "computer_observe" });
- // Per-action approval by permission mode (host channel, never the engine's).
- const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });
- let approval = { asked: false };
- if (decision.ask) {
- const pointList = action.type === "drag" ? action.path : action.x !== undefined ? [{ x: action.x, y: action.y }] : [];
- const answer = await askAct(s, { action, intent: args.intent, reasons: decision.reasons, points: pointList, frame: observed });
- approval = { asked: true, reasons: decision.reasons.map((r) => r.code), by: "person" };
- if (!answer.allow) return finish({ ...common, approval, ok: false, code: "approval_denied", message: "The person did not allow this action. Ask them or choose another way." });
- if (sessions.get(s.id) !== s || s.phase !== "working") return finish({ ...common, approval, ok: false, code: "paused_by_person", message: "The session was paused or ended." });
- if (decision.reasons.some((r) => r.code === "document_changed")) s.approved = { ...s.approved, title: current.title };
- }
- s.afterTakeover = false;
// Native coordinates; every point must hit the approved window.
const mapped = action.type === "drag" ? action.path.map((p) => mapPoint(p, observed.model, current.bounds)) : action.x !== undefined ? [mapPoint(action, observed.model, current.bounds)] : [];
- if (mapped.length) {
- const hits = await native.call("hit", { window: current, points: mapped.map((m) => m.screen) });
- if (!hits.inside.every(Boolean)) return finish({ ...common, approval, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });
- }
- // Tolerant stale check: the window must still look like the observation (caret masked).
+ const hitAll = async (w) => !mapped.length || (await native.call("hit", { window: w, points: mapped.map((m) => m.screen) })).inside.every(Boolean);
+ if (!(await hitAll(current))) return finish({ ...common, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });
+ // Tolerant stale check (caret masked), before any question: the person is asked about what the model saw.
if (action.type !== "wait") {
const fresh = await native.call("capture", { window: current }, 15_000);
// The observation's grid was taken after masking: mask the fresh frame the same way.
paintBoxes(fresh, fresh.secure ?? []);
const grid = luminanceGrid(fitLongSide(fresh, l.model_long_side));
const compare = compareGrids(observed.grid, grid, { mask: observed.caret ? [observed.caret] : [] });
- if (compare.stale) return finish({ ...common, approval, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
+ if (compare.stale) return finish({ ...common, ok: false, code: "stale_observation", message: "The window changed since the observation. Observe again.", next: "computer_observe" });
}
+ // Per-action approval by permission mode (host channel, never the engine's).
+ const decision = actApproval({ mode: settings()?.mode, action, intent: args.intent, target: {}, window: { id: String(current.id), app_id: current.appId, app_name: current.appName, title: current.title }, approved: s.approved, after_takeover: s.afterTakeover });
+ let approval = { asked: false };
+ if (decision.ask) {
+ // The same action with the same intent, allowed a moment ago (a retry after a fresh observation), is not asked again.
+ const key = JSON.stringify([action, args.intent]);
+ const remembered = s.allowed.get(key);
+ if (remembered && remembered > Date.now()) {
+ approval = { asked: false, reasons: decision.reasons.map((r) => r.code), by: "person" };
+ } else {
+ const pointList = action.type === "drag" ? action.path : action.x !== undefined ? [{ x: action.x, y: action.y }] : [];
+ const answer = await askAct(s, { action, intent: args.intent, reasons: decision.reasons, points: pointList, frame: observed });
+ approval = { asked: true, reasons: decision.reasons.map((r) => r.code), by: "person" };
+ if (!answer.allow) return finish({ ...common, approval, ok: false, code: "approval_denied", message: "The person did not allow this action. Ask them or choose another way." });
+ if (sessions.get(s.id) !== s || s.phase !== "working") return finish({ ...common, approval, ok: false, code: "paused_by_person", message: "The session was paused or ended." });
+ s.allowed.set(key, Date.now() + 30_000);
+ if (s.allowed.size > 64) s.allowed.delete(s.allowed.keys().next().value);
+ }
+ if (decision.reasons.some((r) => r.code === "document_changed")) s.approved = { ...s.approved, title: current.title };
+ // While the person answered, the window may have moved or been covered: geometry and hits again (no image compare).
+ const after = await native.call("window", { id: s.window.id });
+ if (after.identity !== current.identity || JSON.stringify(after.bounds) !== JSON.stringify(current.bounds)) return finish({ ...common, approval, ok: false, code: "window_changed", message: "The window moved or changed. Observe again.", next: "computer_observe" });
+ if (!(await hitAll(after))) return finish({ ...common, approval, ok: false, code: "window_occluded", message: "Another window covers this point, or it is outside the approved window. Observe again." });
+ }
+ s.afterTakeover = false;
const cursorBefore = await native.call("cursor").catch(() => null);
const point = mapped.length === 1 ? { model: mapped[0].model, screen: mapped[0].screen } : undefined;
const path = action.type === "drag" ? mapped.map((m) => ({ model: m.model, screen: m.screen })) : undefined;
@@ -397,7 +408,7 @@ export async function createPortableHost({ native, spool = null, settings, conte
const s = {
id: randomUUID(), connection, window: preferred, windows, mode: args.mode, purpose: args.purpose, phase: "approval", status: "",
approval: null, chat: ctx.chat ?? null, actions: 0, requests: new Set(), observation: null, resumeObservation: null, log: [],
- lastOperation: Date.now(), lastAct: 0, afterTakeover: false, generation: 0, expiresAt: Date.now() + l.approval_timeout_seconds * 1000, screen: await screenOf(preferred),
+ lastOperation: Date.now(), lastAct: 0, afterTakeover: false, generation: 0, allowed: new Map(), expiresAt: Date.now() + l.approval_timeout_seconds * 1000, screen: await screenOf(preferred),
};
lease = s.id; sessions.set(s.id, s);
update();