Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix: tolerate small window changes before acting

ogsamrat812dc8004607
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.ts
index a3aa3182..25748265 100644
--- a/apps/server/src/computer-use/media-image.test.ts
+++ b/apps/server/src/computer-use/media-image.test.ts
@@ -65,11 +65,25 @@ describe("image codecs", () => {
paintBoxes(b, [{ x: 100, y: 50, width: 2, height: 14 }], [255, 255, 255]);
expect(compareGrids(luminanceGrid(a), luminanceGrid(b)).stale).toBe(false);
expect(compareGrids(luminanceGrid(a), luminanceGrid(b), { mask: [{ x: 96, y: 48, width: 10, height: 20 }] }).changed).toBe(0);
+ expect(compareGrids(luminanceGrid(a), luminanceGrid(b)).changed).toBeGreaterThan(0);
// A dialog opened: a large region changed.
const c = { ...a, data: Uint8Array.from(a.data) };
paintBoxes(c, [{ x: 40, y: 40, width: 200, height: 100 }]);
expect(compareGrids(luminanceGrid(a), luminanceGrid(c)).stale).toBe(true);
expect(compareGrids(luminanceGrid(a), luminanceGrid(frame(160, 100, 4))).stale).toBe(true);
+ // A GTK overlay scrollbar fading in at the right edge: a 3 px strip down the window.
+ const wide = frame(660, 220, 6);
+ const strip = { ...wide, data: Uint8Array.from(wide.data) };
+ paintBoxes(strip, [{ x: 655, y: 29, width: 3, height: 187 }], [120, 120, 120]);
+ expect(compareGrids(luminanceGrid(wide), luminanceGrid(strip)).stale).toBe(false);
+ // One typed word is a small change too; a cleared text area is not.
+ const typed = { ...wide, data: Uint8Array.from(wide.data) };
+ // Glyph strokes, not a solid block: a few 2 px lines.
+ paintBoxes(typed, [0, 6, 12, 18, 24, 30, 36].map((dx) => ({ x: 4 + dx, y: 30, width: 2, height: 12 })), [0, 0, 0]);
+ expect(compareGrids(luminanceGrid(wide), luminanceGrid(typed)).stale).toBe(false);
+ const cleared = { ...wide, data: Uint8Array.from(wide.data) };
+ paintBoxes(cleared, [{ x: 0, y: 26, width: 660, height: 194 }], [255, 255, 255]);
+ expect(compareGrids(luminanceGrid(wide), luminanceGrid(cleared)).stale).toBe(true);
});
});
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index b2131ce3..a56d8d77 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -4,8 +4,8 @@
07fe5681c685a1f4883ecfe3e6528c1d70cf68003f690079297f887990879d80 contract.d.mts
c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs
103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts
-17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjs
-f11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts
+a96246342720364eb3e5843ba70fbbf640365f94597fc4dbb6a7be7d95254ec4 image.mjs
+5eac31868aedd3e1d1e71b445f9a55d01316da7397f27f6db11472453bee4588 image.d.mts
98ffcc70c69299917c9804c32b786e58c1677edf7ea0732a3fe12b9f6781ba29 media.mjs
c12540410a558f4b48960c75b7a09e0660d6de679a6e49e2542427530fd9a3d4 media.d.mts
02be6c62de13f3c8a55a0167c751742086c16bfadb201e9ce8e7f1b955d2376d native-worker.mjs
diff --git a/apps/server/src/computer-use/shared/image.d.mts b/apps/server/src/computer-use/shared/image.d.mts
index b5fc59a1..0b754cf8 100644
--- a/apps/server/src/computer-use/shared/image.d.mts
+++ b/apps/server/src/computer-use/shared/image.d.mts
@@ -14,4 +14,4 @@ export function encodeJpeg(image: RgbaImage, options?: { quality?: number }): Bu
export function imageInfo(bytes: Uint8Array): { mime: string; width: number; height: number } | null;
export function extensionOf(mime: unknown): "png" | "jpg" | "gif" | "webp" | "bin";
export function luminanceGrid(image: RgbaImage, cell?: number): Grid;
-export function compareGrids(a: Grid | null | undefined, b: Grid | null | undefined, options?: { mask?: Rect[]; threshold?: number; maxCells?: number; maxSpan?: number }): { stale: boolean; changed: number; span?: number };
+export function compareGrids(a: Grid | null | undefined, b: Grid | null | undefined, options?: { mask?: Rect[]; threshold?: number; minMass?: number; fraction?: number }): { stale: boolean; changed: number; span?: number; mass?: number };
diff --git a/apps/server/src/computer-use/shared/image.mjs b/apps/server/src/computer-use/shared/image.mjs
index 9f05de0a..c076351c 100644
--- a/apps/server/src/computer-use/shared/image.mjs
+++ b/apps/server/src/computer-use/shared/image.mjs
@@ -369,20 +369,26 @@ export function luminanceGrid(image, cell = 16) {
/**
* Tolerant compare of two grids of the same image size: cells whose mean
* luminance moved more than `threshold` count as changed, except cells that
- * touch a `mask` rectangle (caret, cursor). Stale when more than `maxCells`
- * cells changed or the changed cells span more than `maxSpan` pixels.
+ * touch a `mask` rectangle (caret, cursor). The changed "mass" (luminance
+ * change × cell area, summed) estimates how many pixels changed and by how
+ * much: a blinking caret, a spinner or a fading overlay scrollbar stay under
+ * it, a dialog, a scrolled page or a new screen go over it. Stale when the
+ * mass passes max(`minMass`, `fraction` of the image at mid-grey).
*/
-export function compareGrids(a, b, { mask = [], threshold = 6, maxCells = 4, maxSpan = 48 } = {}) {
+export function compareGrids(a, b, { mask = [], threshold = 6, minMass = 60_000, fraction = 0.004 } = {}) {
if (!a || !b || a.cols !== b.cols || a.rows !== b.rows || a.cell !== b.cell) return { stale: true, changed: -1 };
- let changed = 0, minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+ let changed = 0, mass = 0, minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
for (let r = 0; r < a.rows; r += 1) for (let c = 0; c < a.cols; c += 1) {
const i = r * a.cols + c;
- if (Math.abs(a.values[i] - b.values[i]) <= threshold) continue;
+ const delta = Math.abs(a.values[i] - b.values[i]);
+ if (delta <= threshold) continue;
const x = c * a.cell, y = r * a.cell;
if (mask.some((m) => x < m.x + m.width && x + a.cell > m.x && y < m.y + m.height && y + a.cell > m.y)) continue;
changed += 1;
+ mass += delta * Math.min(a.cell, a.width - x) * Math.min(a.cell, a.height - y);
minX = Math.min(minX, x); minY = Math.min(minY, y); maxX = Math.max(maxX, x + a.cell); maxY = Math.max(maxY, y + a.cell);
}
const span = changed ? Math.max(maxX - minX, maxY - minY) : 0;
- return { stale: changed > maxCells || span > maxSpan, changed, span };
+ const limit = Math.max(minMass, fraction * a.width * a.height * 128);
+ return { stale: mass > limit, changed, span, mass: Math.round(mass) };
}