Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

feat: shared media refs and image codecs

Samrat Talukdar5612695af81c
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.ts
new file mode 100644
index 00000000..3771d290
--- /dev/null
+++ b/apps/server/src/computer-use/media-image.test.ts
@@ -0,0 +1,208 @@
+// Shared computer-use media references and image codecs (computer-use/shared/).
+import { afterAll, describe, expect, test } from "bun:test";
+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { compareGrids, decodePng, encodeJpeg, encodePng, fitLongSide, imageInfo, luminanceGrid, paintBoxes, resize } from "./shared/image.mjs";
+import {
+ createMediaIndex,
+ createMediaSpool,
+ MEDIA_ROOT,
+ mediaRefsOf,
+ parseImageDataUrl,
+ parseMediaUrl,
+ pruneImages,
+ rewriteMessageMedia,
+ sha256,
+ THUMBS_PER_MESSAGE_BYTES,
+} from "./shared/media.mjs";
+
+const dirs: string[] = [];
+function tmp() {
+ const dir = mkdtempSync(path.join(os.tmpdir(), "cu-media-"));
+ dirs.push(dir);
+ return dir;
+}
+afterAll(() => { for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); });
+
+function frame(width: number, height: number, seed: number) {
+ const data = new Uint8Array(width * height * 4);
+ for (let y = 0; y < height; y += 1) for (let x = 0; x < width; x += 1) {
+ const o = (y * width + x) * 4;
+ data[o] = (x * 7 + seed * 13) & 255; data[o + 1] = (y * 5 + seed * 3) & 255; data[o + 2] = (seed * 31) & 255; data[o + 3] = 255;
+ }
+ data[0] = seed & 255;
+ data[1] = (seed >> 8) & 255;
+ return { width, height, data };
+}
+
+const dataUrl = (mime: string, bytes: Uint8Array) => `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`;
+
+function toolPart(id: string, urls: string[]) {
+ return { id, type: "tool", tool: "computer-use_computer_act", callID: `call_${id}`, state: { status: "completed", input: {}, output: "{\"ok\":true}", attachments: urls.map((url, i) => ({ type: "file", mime: url.slice(5, url.indexOf(";")), url, filename: `shot-${i}` })), time: { start: 1, end: 2 } } };
+}
+
+describe("image codecs", () => {
+ test("PNG round trip, JPEG encode, sniffing, resize", () => {
+ const img = frame(333, 211, 1);
+ const png = encodePng(img);
+ expect(imageInfo(png)).toEqual({ mime: "image/png", width: 333, height: 211 });
+ const back = decodePng(png);
+ expect(back && Buffer.from(back.data).equals(Buffer.from(img.data))).toBe(true);
+ const jpeg = encodeJpeg(img, { quality: 80 });
+ expect(imageInfo(jpeg)).toEqual({ mime: "image/jpeg", width: 333, height: 211 });
+ expect(jpeg.subarray(-2)).toEqual(Buffer.from([0xff, 0xd9]));
+ expect(fitLongSide(frame(2560, 1600, 2), 1280)).toMatchObject({ width: 1280, height: 800 });
+ expect(resize(img, 10, 10).data.length).toBe(400);
+ expect(decodePng(Buffer.from("not a png"))).toBeNull();
+ });
+
+ test("painting boxes and the tolerant compare (caret masked)", () => {
+ const a = frame(320, 200, 4);
+ const b = { ...a, data: Uint8Array.from(a.data) };
+ // A blinking caret: a 2x14 bar.
+ paintBoxes(b, [{ x: 100, y: 50, width: 2, height: 14 }], [255, 255, 255]);
+ expect(compareGrids(luminanceGrid(a), luminanceGrid(b)).stale).toBe(false);
+ expect(compareGrids(luminanceGrid(a), luminanceGrid(b), { mask: [{ x: 96, y: 48, width: 10, height: 20 }] }).changed).toBe(0);
+ // A dialog opened: a large region changed.
+ const c = { ...a, data: Uint8Array.from(a.data) };
+ paintBoxes(c, [{ x: 40, y: 40, width: 200, height: 100 }]);
+ expect(compareGrids(luminanceGrid(a), luminanceGrid(c)).stale).toBe(true);
+ expect(compareGrids(luminanceGrid(a), luminanceGrid(frame(160, 100, 4))).stale).toBe(true);
+ });
+});
+
+describe("media references", () => {
+ test("data URLs become refs; first-seen thumb; files once per chat; user file parts untouched", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const index = createMediaIndex(path.join(dir, "index", "ses_1.json"));
+ const png = encodePng(frame(1280, 800, 9));
+ const url = dataUrl("image/png", png);
+ const message = { info: { id: "msg_1", role: "assistant" }, parts: [toolPart("p1", [url]), { type: "file", mime: "image/png", url, filename: "user.png" }] };
+ const before = JSON.stringify(message);
+ const out = rewriteMessageMedia(message, { spool, index, limits: {} });
+ expect(JSON.stringify(message)).toBe(before);
+ expect(out.changed).toBe(true);
+ const ref = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;
+ const sha = sha256(png);
+ expect(ref).toMatchObject({ type: "file", filename: "shot-0", mime: "image/png", url: `omnirush-media:${sha}.png`, sha256: sha, width: 1280, height: 800, bytes: png.length });
+ expect(String(ref.thumb)).toStartWith("data:image/jpeg;base64,");
+ const thumb = Buffer.from(String(ref.thumb).split(",")[1] ?? "", "base64");
+ expect(imageInfo(thumb)).toMatchObject({ width: 320, height: 200 });
+ expect(thumb.length).toBeLessThan(40_000);
+ expect((out.message.parts[1] as { url: string }).url).toBe(url);
+ expect(out.files).toEqual([{ path: `${MEDIA_ROOT}/${sha}.png`, file: spool.modelPath(sha, "png"), sha, kind: "model", bytes: png.length }]);
+ expect(readFileSync(out.files[0]?.file ?? "").equals(png)).toBe(true);
+ expect(mediaRefsOf(out.message)).toEqual([{ sha, ext: "png" }]);
+ expect(parseMediaUrl(ref.url)).toEqual({ sha, ext: "png" });
+ // Seen again (another read, or a resume with a fresh index object): no thumb, no new file.
+ const again = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "index", "ses_1.json")), limits: {} });
+ const ref2 = (again.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;
+ expect(ref2.thumb).toBeUndefined();
+ expect(again.files).toEqual([]);
+ // Already rewritten messages pass through.
+ expect(rewriteMessageMedia(out.message, { spool, index, limits: {} }).changed).toBe(false);
+ });
+
+ test("host frames: full resolution kept only when larger, and its own budget", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const fullImg = frame(2560, 1600, 3);
+ const model = encodeJpeg(fitLongSide(fullImg, 1280), { quality: 80 });
+ const fullPng = encodePng(fullImg, { level: 1 });
+ const meta = spool.put({ bytes: model, mime: "image/jpeg", full: { bytes: fullPng, width: 2560, height: 1600 }, thumb: encodeJpeg(fitLongSide(fullImg, 320)) });
+ expect(meta.full).toMatchObject({ width: 2560, height: 1600, ext: "png", bytes: fullPng.length });
+ const small = encodeJpeg(frame(400, 300, 5));
+ expect(spool.put({ bytes: small, mime: "image/jpeg", full: { bytes: encodePng(frame(400, 300, 5)), width: 400, height: 300 } }).full).toBeUndefined();
+ const message = { parts: [toolPart("a", [dataUrl("image/jpeg", model), dataUrl("image/jpeg", small)])] };
+ const out = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: { media_full_bytes: 10, media_model_bytes: 200 * 1024 * 1024 } });
+ const [a, b] = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments as Array<Record<string, unknown>>;
+ expect(a?.full).toEqual({ omitted: "budget" });
+ expect(b?.full).toBeUndefined();
+ expect(out.files.map((f) => f.kind)).toEqual(["model", "model"]);
+ const roomy = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {} });
+ expect(roomy.files.map((f) => f.kind)).toEqual(["model", "full", "model"]);
+ expect(((roomy.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>).full).toMatchObject({ width: 2560 });
+ spool.remove(meta.sha256);
+ expect(existsSync(spool.modelPath(meta.sha256, "jpg"))).toBe(false);
+ });
+
+ test("a 600-frame turn uploads whole: small message, 2 MB of thumbs, every frame archived under the caps", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const index = createMediaIndex(path.join(dir, "index.json"));
+ const parts = [];
+ let raw = 0;
+ for (let i = 0; i < 600; i += 1) {
+ const png = encodePng(frame(160, 100, i), { level: 1 });
+ // Pad each frame to ~150 KB (the size of a 1280x800 JPEG) with a private chunk.
+ const pad = Buffer.alloc(8 + 150_000 + 4);
+ pad.writeUInt32BE(150_000, 0);
+ pad.write("paDa", 4, "latin1");
+ const padded = Buffer.concat([png.subarray(0, png.length - 12), pad, png.subarray(png.length - 12)]);
+ const url = dataUrl("image/png", padded);
+ raw += url.length;
+ parts.push(toolPart(`p${i}`, [url]));
+ }
+ const message = { info: { id: "msg_big" }, parts };
+ expect(raw).toBeGreaterThan(64 * 1024 * 1024);
+ const out = rewriteMessageMedia(message, { spool, index, limits: {} });
+ const size = Buffer.byteLength(JSON.stringify(out.message));
+ expect(size).toBeLessThan(8 * 1024 * 1024);
+ const thumbs = (out.message.parts as Array<ReturnType<typeof toolPart>>).flatMap((p) => p.state.attachments).filter((a) => "thumb" in a).reduce((n, a) => n + String((a as { thumb: string }).thumb).length, 0);
+ expect(thumbs).toBeGreaterThan(0);
+ expect(thumbs).toBeLessThanOrEqual(THUMBS_PER_MESSAGE_BYTES);
+ expect(out.files.length).toBe(600);
+ expect(new Set(out.files.map((f) => f.path)).size).toBe(600);
+ expect(index.state.model_bytes).toBeLessThanOrEqual(200 * 1024 * 1024);
+ expect(JSON.stringify(out.message)).not.toContain("data:image/png");
+ }, 60_000);
+
+ test("model budget: past it, refs say omitted and no file is added", () => {
+ const dir = tmp();
+ const spool = createMediaSpool(path.join(dir, "spool"));
+ const pngs = [0, 1, 2].map((i) => encodePng(frame(64, 64, i)));
+ const limit = pngs[0]!.length + pngs[1]!.length;
+ const out = rewriteMessageMedia({ parts: [toolPart("a", pngs.map((p) => dataUrl("image/png", p)))] }, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: { media_model_bytes: limit } });
+ expect(out.files.length).toBe(2);
+ expect(((out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[2] as Record<string, unknown>).omitted).toBe("budget");
+ });
+
+ test("data URL parsing", () => {
+ expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });
+ expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();
+ expect(parseImageDataUrl("data:image/svg+xml;base64,AQID")).toBeNull();
+ expect(parseImageDataUrl("https://x/y.png")).toBeNull();
+ });
+});
+
+describe("model context pruning", () => {
+ function chat(count: number) {
+ const parts = Array.from({ length: count }, (_, i) => toolPart(`p${i}`, [dataUrl("image/png", encodePng(frame(8, 8, i)))]));
+ return [{ info: { role: "assistant" }, parts }];
+ }
+ const images = (messages: ReturnType<typeof chat>) => messages[0]!.parts.reduce((n, p) => n + p.state.attachments.length, 0);
+
+ test("keeps up to 12; past 12 drops the oldest 8 at a time; notes the dropped ones", () => {
+ const twelve = chat(12);
+ expect(pruneImages(twelve)).toBe(0);
+ expect(images(twelve)).toBe(12);
+ const thirteen = chat(13);
+ expect(pruneImages(thirteen)).toBe(8);
+ expect(images(thirteen)).toBe(5);
+ expect(thirteen[0]!.parts[0]!.state.output).toMatch(/\[screenshot [a-f0-9]{8} omitted\]$/);
+ const twenty = chat(20);
+ pruneImages(twenty);
+ expect(images(twenty)).toBe(12);
+ const twentyOne = chat(21);
+ pruneImages(twentyOne);
+ expect(images(twentyOne)).toBe(5);
+ // Stable prefix: the same oldest images are dropped from 13 to 20 images.
+ const at14 = chat(14);
+ pruneImages(at14);
+ expect(at14[0]!.parts.slice(0, 8).every((p) => p.state.attachments.length === 0)).toBe(true);
+ expect(at14[0]!.parts[8]!.state.attachments.length).toBe(1);
+ });
+});
diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256
index 6fb139cd..9e799cfb 100644
--- a/apps/server/src/computer-use/shared/PARITY.sha256
+++ b/apps/server/src/computer-use/shared/PARITY.sha256
@@ -4,3 +4,7 @@
07fe5681c685a1f4883ecfe3e6528c1d70cf68003f690079297f887990879d80 contract.d.mts
c601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs
103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts
+17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjs
+f11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts
+eea23441a7360c186044072fc0261a5b524e20dd639d05161dd968dc4bfae774 media.mjs
+25f747b53f5a1dbb5b2439b65bc728c508eecb62fbe2650904beffa8d3854b9c media.d.mts
diff --git a/apps/server/src/computer-use/shared/image.d.mts b/apps/server/src/computer-use/shared/image.d.mts
new file mode 100644
index 00000000..b5fc59a1
--- /dev/null
+++ b/apps/server/src/computer-use/shared/image.d.mts
@@ -0,0 +1,17 @@
+// Types of image.mjs (the same files in omnirush-cli src/computer-use/ and
+// omnirush-gui apps/server/src/computer-use/shared/; PARITY.sha256).
+
+export type RgbaImage = { width: number; height: number; data: Uint8Array };
+export type Rect = { x: number; y: number; width: number; height: number };
+export type Grid = { cell: number; cols: number; rows: number; width: number; height: number; values: Float32Array };
+
+export function encodePng(image: RgbaImage, options?: { level?: number }): Buffer;
+export function decodePng(bytes: Uint8Array, options?: { maxPixels?: number }): RgbaImage | null;
+export function resize(image: RgbaImage, width: number, height: number): RgbaImage;
+export function fitLongSide(image: RgbaImage, longSide: number): RgbaImage;
+export function paintBoxes(image: RgbaImage, boxes: Rect[] | null | undefined, color?: [number, number, number]): number;
+export function encodeJpeg(image: RgbaImage, options?: { quality?: number }): Buffer;
+export function imageInfo(bytes: Uint8Array): { mime: string; width: number; height: number } | null;
+export function extensionOf(mime: unknown): "png" | "jpg" | "gif" | "webp" | "bin";
+export function luminanceGrid(image: RgbaImage, cell?: number): Grid;
+export function compareGrids(a: Grid | null | undefined, b: Grid | null | undefined, options?: { mask?: Rect[]; threshold?: number; maxCells?: number; maxSpan?: number }): { stale: boolean; changed: number; span?: number };
diff --git a/apps/server/src/computer-use/shared/image.mjs b/apps/server/src/computer-use/shared/image.mjs
new file mode 100644
index 00000000..9f05de0a
--- /dev/null
+++ b/apps/server/src/computer-use/shared/image.mjs
@@ -0,0 +1,388 @@
+// Computer use, images: PNG encode/decode (node:zlib), an area-average
+// downscale, a small baseline JPEG encoder (4:2:0, standard tables; written
+// for this module, no third-party code), size sniffing and a tolerant frame
+// compare. Plain Node: works in Electron main, the server, engine plugins and
+// the CLI alike.
+//
+// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui
+// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.
+//
+// Pixels are RGBA, 8 bits per channel, rows top to bottom:
+// { width, height, data: Uint8Array(width * height * 4) }.
+
+import { crc32, deflateSync, inflateSync } from "node:zlib";
+
+// ---------------------------------------------------------------------- PNG
+
+const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
+
+function chunk(type, data) {
+ const name = Buffer.from(type, "latin1");
+ const length = Buffer.alloc(4);
+ length.writeUInt32BE(data.length);
+ const sum = Buffer.alloc(4);
+ sum.writeUInt32BE(crc32(data, crc32(name)) >>> 0);
+ return Buffer.concat([length, name, data, sum]);
+}
+
+/** RGBA pixels → PNG bytes (truecolor, alpha dropped when every pixel is opaque). */
+export function encodePng(image, { level = 6 } = {}) {
+ const { width, height, data } = image;
+ let opaque = true;
+ for (let i = 3; i < data.length; i += 4) if (data[i] !== 255) { opaque = false; break; }
+ const channels = opaque ? 3 : 4;
+ const stride = width * channels + 1;
+ const rows = Buffer.alloc(stride * height);
+ for (let y = 0; y < height; y += 1) {
+ let o = y * stride + 1;
+ let s = y * width * 4;
+ for (let x = 0; x < width; x += 1, s += 4) {
+ rows[o++] = data[s]; rows[o++] = data[s + 1]; rows[o++] = data[s + 2];
+ if (!opaque) rows[o++] = data[s + 3];
+ }
+ }
+ const header = Buffer.alloc(13);
+ header.writeUInt32BE(width, 0);
+ header.writeUInt32BE(height, 4);
+ header[8] = 8;
+ header[9] = opaque ? 2 : 6;
+ return Buffer.concat([PNG_SIGNATURE, chunk("IHDR", header), chunk("IDAT", deflateSync(rows, { level })), chunk("IEND", Buffer.alloc(0))]);
+}
+
+function paeth(a, b, c) {
+ const p = a + b - c;
+ const pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c);
+ return pa <= pb && pa <= pc ? a : pb <= pc ? b : c;
+}
+
+/**
+ * PNG bytes → RGBA pixels, or null for what this decoder does not read
+ * (16-bit, interlaced, a corrupt file, more than `maxPixels`).
+ */
+export function decodePng(bytes, { maxPixels = 40_000_000 } = {}) {
+ try {
+ const buf = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);
+ if (buf.length < 33 || !buf.subarray(0, 8).equals(PNG_SIGNATURE)) return null;
+ let offset = 8;
+ let width = 0, height = 0, depth = 0, type = 0, interlace = 0;
+ let palette = null, alpha = null;
+ const idat = [];
+ while (offset + 8 <= buf.length) {
+ const length = buf.readUInt32BE(offset);
+ const name = buf.toString("latin1", offset + 4, offset + 8);
+ const data = buf.subarray(offset + 8, offset + 8 + length);
+ offset += 12 + length;
+ if (name === "IHDR") { width = data.readUInt32BE(0); height = data.readUInt32BE(4); depth = data[8]; type = data[9]; interlace = data[12]; }
+ else if (name === "PLTE") palette = data;
+ else if (name === "tRNS") alpha = data;
+ else if (name === "IDAT") idat.push(data);
+ else if (name === "IEND") break;
+ }
+ if (!width || !height || depth !== 8 || interlace !== 0 || width * height > maxPixels) return null;
+ const channels = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 }[type];
+ if (!channels || (type === 3 && !palette)) return null;
+ const raw = inflateSync(Buffer.concat(idat));
+ const stride = width * channels;
+ if (raw.length < (stride + 1) * height) return null;
+ const out = new Uint8Array(width * height * 4);
+ let prev = new Uint8Array(stride);
+ const line = new Uint8Array(stride);
+ for (let y = 0; y < height; y += 1) {
+ const filter = raw[y * (stride + 1)];
+ const src = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));
+ for (let i = 0; i < stride; i += 1) {
+ const a = i >= channels ? line[i - channels] : 0;
+ const b = prev[i];
+ const c = i >= channels ? prev[i - channels] : 0;
+ const v = src[i];
+ line[i] = filter === 0 ? v : filter === 1 ? v + a : filter === 2 ? v + b : filter === 3 ? v + ((a + b) >> 1) : filter === 4 ? v + paeth(a, b, c) : v;
+ }
+ for (let x = 0; x < width; x += 1) {
+ const o = (y * width + x) * 4;
+ if (type === 6) { out[o] = line[x * 4]; out[o + 1] = line[x * 4 + 1]; out[o + 2] = line[x * 4 + 2]; out[o + 3] = line[x * 4 + 3]; }
+ else if (type === 2) { out[o] = line[x * 3]; out[o + 1] = line[x * 3 + 1]; out[o + 2] = line[x * 3 + 2]; out[o + 3] = 255; }
+ else if (type === 0) { out[o] = out[o + 1] = out[o + 2] = line[x]; out[o + 3] = 255; }
+ else if (type === 4) { out[o] = out[o + 1] = out[o + 2] = line[x * 2]; out[o + 3] = line[x * 2 + 1]; }
+ else { const p = line[x]; out[o] = palette[p * 3] ?? 0; out[o + 1] = palette[p * 3 + 1] ?? 0; out[o + 2] = palette[p * 3 + 2] ?? 0; out[o + 3] = alpha && p < alpha.length ? alpha[p] : 255; }
+ }
+ prev = Uint8Array.from(line);
+ }
+ return { width, height, data: out };
+ } catch {
+ return null;
+ }
+}
+
+// ----------------------------------------------------------------- resizing
+
+/** Area-average downscale to exactly width × height (never upscales usefully: use for shrinking). */
+export function resize(image, width, height) {
+ const sw = image.width, sh = image.height, src = image.data;
+ if (sw === width && sh === height) return { width, height, data: Uint8Array.from(src) };
+ const out = new Uint8Array(width * height * 4);
+ const xs = new Int32Array(width + 1), ys = new Int32Array(height + 1);
+ for (let x = 0; x <= width; x += 1) xs[x] = Math.min(sw, Math.round((x * sw) / width));
+ for (let y = 0; y <= height; y += 1) ys[y] = Math.min(sh, Math.round((y * sh) / height));
+ for (let y = 0; y < height; y += 1) {
+ const y0 = ys[y], y1 = Math.max(ys[y + 1], y0 + 1);
+ for (let x = 0; x < width; x += 1) {
+ const x0 = xs[x], x1 = Math.max(xs[x + 1], x0 + 1);
+ let r = 0, g = 0, b = 0, a = 0, n = 0;
+ for (let yy = y0; yy < y1 && yy < sh; yy += 1) {
+ let s = (yy * sw + x0) * 4;
+ for (let xx = x0; xx < x1 && xx < sw; xx += 1, s += 4) { r += src[s]; g += src[s + 1]; b += src[s + 2]; a += src[s + 3]; n += 1; }
+ }
+ const o = (y * width + x) * 4;
+ n = n || 1;
+ out[o] = (r / n + 0.5) | 0; out[o + 1] = (g / n + 0.5) | 0; out[o + 2] = (b / n + 0.5) | 0; out[o + 3] = (a / n + 0.5) | 0;
+ }
+ }
+ return { width, height, data: out };
+}
+
+/** Downscales so the long side is at most `longSide` (a copy when it already fits). */
+export function fitLongSide(image, longSide) {
+ const ratio = Math.min(1, longSide / Math.max(image.width, image.height));
+ if (ratio >= 1) return image;
+ return resize(image, Math.max(1, Math.round(image.width * ratio)), Math.max(1, Math.round(image.height * ratio)));
+}
+
+/** Paints rectangles ({x, y, width, height} in pixels of `image`) solid in place. */
+export function paintBoxes(image, boxes, color = [0, 0, 0]) {
+ let painted = 0;
+ for (const box of boxes ?? []) {
+ const x0 = Math.max(0, Math.floor(box.x)), y0 = Math.max(0, Math.floor(box.y));
+ const x1 = Math.min(image.width, Math.ceil(box.x + box.width)), y1 = Math.min(image.height, Math.ceil(box.y + box.height));
+ if (x1 <= x0 || y1 <= y0) continue;
+ painted += 1;
+ for (let y = y0; y < y1; y += 1) for (let x = x0; x < x1; x += 1) {
+ const o = (y * image.width + x) * 4;
+ image.data[o] = color[0]; image.data[o + 1] = color[1]; image.data[o + 2] = color[2]; image.data[o + 3] = 255;
+ }
+ }
+ return painted;
+}
+
+// --------------------------------------------------------------------- JPEG
+
+const ZIGZAG = [0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63];
+const LUMA_Q = [16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56, 14, 17, 22, 29, 51, 87, 80, 62, 18, 22, 37, 56, 68, 109, 103, 77, 24, 35, 55, 64, 81, 104, 113, 92, 49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99];
+const CHROMA_Q = [17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99, 47, 66, 99, 99, 99, 99, 99, 99, ...new Array(32).fill(99)];
+const DC_LUMA = { bits: [0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], vals: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] };
+const DC_CHROMA = { bits: [0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0], vals: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] };
+const AC_LUMA = {
+ bits: [0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d],
+ vals: [0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa],
+};
+const AC_CHROMA = {
+ bits: [0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77],
+ vals: [0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa],
+};
+
+function huffmanCodes(table) {
+ const codes = new Map();
+ let code = 0, k = 0;
+ for (let length = 1; length <= 16; length += 1) {
+ for (let i = 0; i < table.bits[length - 1]; i += 1) codes.set(table.vals[k++], [code++, length]);
+ code <<= 1;
+ }
+ return codes;
+}
+const CODES = { dcY: huffmanCodes(DC_LUMA), dcC: huffmanCodes(DC_CHROMA), acY: huffmanCodes(AC_LUMA), acC: huffmanCodes(AC_CHROMA) };
+
+function scaledTable(base, quality) {
+ const q = Math.min(100, Math.max(1, Math.round(quality)));
+ const s = q < 50 ? 5000 / q : 200 - q * 2;
+ return base.map((v) => Math.min(255, Math.max(1, Math.floor((v * s + 50) / 100))));
+}
+
+const COS = new Float64Array(64);
+for (let x = 0; x < 8; x += 1) for (let u = 0; u < 8; u += 1) COS[x * 8 + u] = Math.cos(((2 * x + 1) * u * Math.PI) / 16) * (u === 0 ? Math.SQRT1_2 : 1);
+
+/** In place: 8×8 samples (level-shifted) → DCT coefficients (natural order). */
+function fdct(block, tmp) {
+ for (let y = 0; y < 8; y += 1) for (let u = 0; u < 8; u += 1) {
+ let sum = 0;
+ for (let x = 0; x < 8; x += 1) sum += block[y * 8 + x] * COS[x * 8 + u];
+ tmp[y * 8 + u] = sum / 2;
+ }
+ for (let u = 0; u < 8; u += 1) for (let v = 0; v < 8; v += 1) {
+ let sum = 0;
+ for (let y = 0; y < 8; y += 1) sum += tmp[y * 8 + u] * COS[y * 8 + v];
+ block[v * 8 + u] = sum / 2;
+ }
+}
+
+class BitWriter {
+ constructor(size) { this.buf = Buffer.alloc(Math.max(1024, size)); this.pos = 0; this.acc = 0; this.n = 0; }
+ byte(v) { if (this.pos >= this.buf.length) { const next = Buffer.alloc(this.buf.length * 2); this.buf.copy(next); this.buf = next; } this.buf[this.pos++] = v; }
+ bits(value, length) {
+ for (let i = length - 1; i >= 0; i -= 1) {
+ this.acc = (this.acc << 1) | ((value >> i) & 1);
+ this.n += 1;
+ if (this.n === 8) { this.byte(this.acc); if (this.acc === 0xff) this.byte(0); this.acc = 0; this.n = 0; }
+ }
+ }
+ flush() { if (this.n > 0) this.bits((1 << (8 - this.n)) - 1, 8 - this.n); return this.buf.subarray(0, this.pos); }
+}
+
+function category(v) {
+ let a = Math.abs(v), n = 0;
+ while (a) { n += 1; a >>= 1; }
+ return n;
+}
+
+function encodeBlock(writer, block, table, dcCodes, acCodes, previous) {
+ const q = new Int32Array(64);
+ for (let k = 0; k < 64; k += 1) q[k] = Math.round(block[ZIGZAG[k]] / table[ZIGZAG[k]]);
+ const diff = q[0] - previous;
+ const dcCat = category(diff);
+ const dc = dcCodes.get(dcCat);
+ writer.bits(dc[0], dc[1]);
+ if (dcCat) writer.bits(diff < 0 ? diff + (1 << dcCat) - 1 : diff, dcCat);
+ let last = 63;
+ while (last > 0 && q[last] === 0) last -= 1;
+ let run = 0;
+ for (let k = 1; k <= last; k += 1) {
+ const v = q[k];
+ if (v === 0) { run += 1; continue; }
+ while (run > 15) { const z = acCodes.get(0xf0); writer.bits(z[0], z[1]); run -= 16; }
+ const c = category(v);
+ const code = acCodes.get((run << 4) | c);
+ writer.bits(code[0], code[1]);
+ writer.bits(v < 0 ? v + (1 << c) - 1 : v, c);
+ run = 0;
+ }
+ if (last < 63) { const eob = acCodes.get(0x00); writer.bits(eob[0], eob[1]); }
+ return q[0];
+}
+
+function segment(marker, payload) {
+ const head = Buffer.alloc(4);
+ head[0] = 0xff; head[1] = marker; head.writeUInt16BE(payload.length + 2, 2);
+ return Buffer.concat([head, payload]);
+}
+
+function dhtPayload(tableClass, id, table) {
+ return Buffer.from([(tableClass << 4) | id, ...table.bits, ...table.vals]);
+}
+
+/** RGBA pixels → baseline JPEG bytes (4:2:0, alpha ignored). */
+export function encodeJpeg(image, { quality = 80 } = {}) {
+ const { width, height, data } = image;
+ const qy = scaledTable(LUMA_Q, quality), qc = scaledTable(CHROMA_Q, quality);
+ const header = [
+ Buffer.from([0xff, 0xd8]),
+ segment(0xe0, Buffer.from([0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0])),
+ segment(0xdb, Buffer.from([0, ...ZIGZAG.map((i) => qy[i]), 1, ...ZIGZAG.map((i) => qc[i])])),
+ segment(0xc0, Buffer.from([8, height >> 8, height & 255, width >> 8, width & 255, 3, 1, 0x22, 0, 2, 0x11, 1, 3, 0x11, 1])),
+ segment(0xc4, Buffer.concat([dhtPayload(0, 0, DC_LUMA), dhtPayload(1, 0, AC_LUMA), dhtPayload(0, 1, DC_CHROMA), dhtPayload(1, 1, AC_CHROMA)])),
+ segment(0xda, Buffer.from([3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0])),
+ ];
+ const writer = new BitWriter((width * height) >> 2);
+ const Y = new Float64Array(64 * 4), Cb = new Float64Array(64), Cr = new Float64Array(64), tmp = new Float64Array(64);
+ const block = new Float64Array(64);
+ let pY = 0, pCb = 0, pCr = 0;
+ for (let my = 0; my < height; my += 16) {
+ for (let mx = 0; mx < width; mx += 16) {
+ Cb.fill(0); Cr.fill(0);
+ for (let y = 0; y < 16; y += 1) {
+ const sy = Math.min(height - 1, my + y);
+ for (let x = 0; x < 16; x += 1) {
+ const sx = Math.min(width - 1, mx + x);
+ const o = (sy * width + sx) * 4;
+ const r = data[o], g = data[o + 1], b = data[o + 2];
+ const yb = ((y >> 3) << 1) | (x >> 3);
+ Y[yb * 64 + (y & 7) * 8 + (x & 7)] = 0.299 * r + 0.587 * g + 0.114 * b - 128;
+ const c = (y >> 1) * 8 + (x >> 1);
+ Cb[c] += (-0.168736 * r - 0.331264 * g + 0.5 * b) / 4;
+ Cr[c] += (0.5 * r - 0.418688 * g - 0.081312 * b) / 4;
+ }
+ }
+ for (let b = 0; b < 4; b += 1) {
+ for (let i = 0; i < 64; i += 1) block[i] = Y[b * 64 + i];
+ fdct(block, tmp);
+ pY = encodeBlock(writer, block, qy, CODES.dcY, CODES.acY, pY);
+ }
+ block.set(Cb); fdct(block, tmp); pCb = encodeBlock(writer, block, qc, CODES.dcC, CODES.acC, pCb);
+ block.set(Cr); fdct(block, tmp); pCr = encodeBlock(writer, block, qc, CODES.dcC, CODES.acC, pCr);
+ }
+ }
+ return Buffer.concat([...header, Buffer.from(writer.flush()), Buffer.from([0xff, 0xd9])]);
+}
+
+// ------------------------------------------------------------------- sniffing
+
+/** { mime, width, height } of PNG, JPEG, GIF or WebP bytes, or null. */
+export function imageInfo(bytes) {
+ const b = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);
+ if (b.length >= 24 && b.subarray(0, 8).equals(PNG_SIGNATURE)) return { mime: "image/png", width: b.readUInt32BE(16), height: b.readUInt32BE(20) };
+ if (b.length >= 4 && b[0] === 0xff && b[1] === 0xd8) {
+ let o = 2;
+ while (o + 9 < b.length) {
+ if (b[o] !== 0xff) { o += 1; continue; }
+ const marker = b[o + 1];
+ if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { o += 2; continue; }
+ const length = b.readUInt16BE(o + 2);
+ if ((marker >= 0xc0 && marker <= 0xc3) || (marker >= 0xc5 && marker <= 0xc7) || (marker >= 0xc9 && marker <= 0xcb) || (marker >= 0xcd && marker <= 0xcf)) return { mime: "image/jpeg", width: b.readUInt16BE(o + 7), height: b.readUInt16BE(o + 5) };
+ o += 2 + length;
+ }
+ return { mime: "image/jpeg", width: 0, height: 0 };
+ }
+ if (b.length >= 10 && b.toString("latin1", 0, 3) === "GIF") return { mime: "image/gif", width: b.readUInt16LE(6), height: b.readUInt16LE(8) };
+ if (b.length >= 30 && b.toString("latin1", 0, 4) === "RIFF" && b.toString("latin1", 8, 12) === "WEBP") {
+ const kind = b.toString("latin1", 12, 16);
+ if (kind === "VP8X") return { mime: "image/webp", width: 1 + b.readUIntLE(24, 3), height: 1 + b.readUIntLE(27, 3) };
+ if (kind === "VP8L") { const v = b.readUInt32LE(21); return { mime: "image/webp", width: (v & 0x3fff) + 1, height: ((v >> 14) & 0x3fff) + 1 }; }
+ if (kind === "VP8 ") return { mime: "image/webp", width: b.readUInt16LE(26) & 0x3fff, height: b.readUInt16LE(28) & 0x3fff };
+ return { mime: "image/webp", width: 0, height: 0 };
+ }
+ return null;
+}
+
+/** File extension for an image mime type. */
+export function extensionOf(mime) {
+ return { "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg", "image/gif": "gif", "image/webp": "webp" }[String(mime ?? "").toLowerCase()] ?? "bin";
+}
+
+// ------------------------------------------------------------ frame compare
+
+/** Mean luminance per cell (cell × cell pixels): the fingerprint the stale check compares. */
+export function luminanceGrid(image, cell = 16) {
+ const cols = Math.ceil(image.width / cell), rows = Math.ceil(image.height / cell);
+ const sums = new Float64Array(cols * rows), counts = new Uint32Array(cols * rows);
+ const d = image.data;
+ for (let y = 0; y < image.height; y += 1) {
+ const row = ((y / cell) | 0) * cols;
+ for (let x = 0; x < image.width; x += 1) {
+ const o = (y * image.width + x) * 4;
+ const i = row + ((x / cell) | 0);
+ sums[i] += 0.299 * d[o] + 0.587 * d[o + 1] + 0.114 * d[o + 2];
+ counts[i] += 1;
+ }
+ }
+ const values = new Float32Array(cols * rows);
+ for (let i = 0; i < values.length; i += 1) values[i] = sums[i] / (counts[i] || 1);
+ return { cell, cols, rows, width: image.width, height: image.height, values };
+}
+
+/**
+ * Tolerant compare of two grids of the same image size: cells whose mean
+ * luminance moved more than `threshold` count as changed, except cells that
+ * touch a `mask` rectangle (caret, cursor). Stale when more than `maxCells`
+ * cells changed or the changed cells span more than `maxSpan` pixels.
+ */
+export function compareGrids(a, b, { mask = [], threshold = 6, maxCells = 4, maxSpan = 48 } = {}) {
+ if (!a || !b || a.cols !== b.cols || a.rows !== b.rows || a.cell !== b.cell) return { stale: true, changed: -1 };
+ let changed = 0, minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;
+ for (let r = 0; r < a.rows; r += 1) for (let c = 0; c < a.cols; c += 1) {
+ const i = r * a.cols + c;
+ if (Math.abs(a.values[i] - b.values[i]) <= threshold) continue;
+ const x = c * a.cell, y = r * a.cell;
+ if (mask.some((m) => x < m.x + m.width && x + a.cell > m.x && y < m.y + m.height && y + a.cell > m.y)) continue;
+ changed += 1;
+ minX = Math.min(minX, x); minY = Math.min(minY, y); maxX = Math.max(maxX, x + a.cell); maxY = Math.max(maxY, y + a.cell);
+ }
+ const span = changed ? Math.max(maxX - minX, maxY - minY) : 0;
+ return { stale: changed > maxCells || span > maxSpan, changed, span };
+}
diff --git a/apps/server/src/computer-use/shared/media.d.mts b/apps/server/src/computer-use/shared/media.d.mts
new file mode 100644
index 00000000..583ceb1c
--- /dev/null
+++ b/apps/server/src/computer-use/shared/media.d.mts
@@ -0,0 +1,70 @@
+// Types of media.mjs (the same files in omnirush-cli src/computer-use/ and
+// omnirush-gui apps/server/src/computer-use/shared/; PARITY.sha256).
+
+export const MEDIA_ROOT: "__media__";
+export const MEDIA_SCHEME: "omnirush-media:";
+export const THUMB_LONG_SIDE: number;
+export const THUMB_QUALITY: number;
+export const THUMBS_PER_MESSAGE_BYTES: number;
+export const INDEX_VERSION: 1;
+export const KEEP_IMAGES: number;
+export const DROP_IMAGES: number;
+
+export type FullMeta = { sha256: string; width: number; height: number; bytes: number; ext: "png" };
+export type MediaMeta = {
+ sha256: string;
+ ext: string;
+ mime: string;
+ width: number;
+ height: number;
+ bytes: number;
+ full?: FullMeta;
+ thumb?: number;
+ ocr?: "client" | "none";
+ masked?: number;
+};
+export type MediaRef = {
+ mime: string;
+ url: string;
+ sha256: string;
+ width: number;
+ height: number;
+ bytes: number;
+ full?: FullMeta | { omitted: "budget" };
+ omitted?: "budget";
+ thumb?: string;
+ [key: string]: unknown;
+};
+export type MediaFile = { path: string; file: string; sha: string; kind: "model" | "full"; bytes: number };
+
+export type MediaSpool = {
+ dir: string;
+ put(frame: { bytes: Uint8Array; mime: string; width?: number; height?: number; full?: { bytes: Uint8Array; width: number; height: number } | null; thumb?: Uint8Array | null; ocr?: "client" | "none"; masked?: number }): MediaMeta;
+ meta(sha: string): MediaMeta | null;
+ modelPath(sha: string, ext: string): string;
+ fullPath(sha: string): string;
+ thumb(sha: string): Buffer | null;
+ has(sha: string, ext: string): boolean;
+ remove(sha: string): void;
+};
+export type MediaIndexState = { v: 1; seen: Record<string, { m: 0 | 1; f: 0 | 1 }>; model_bytes: number; full_bytes: number; frames: number };
+export type MediaIndex = {
+ file: string;
+ state: MediaIndexState;
+ entry(sha: string): { m: 0 | 1; f: 0 | 1 } | null;
+ add(sha: string, stored: { model: boolean; full: boolean; modelBytes: number; fullBytes: number }): { m: 0 | 1; f: 0 | 1 };
+ save(): void;
+};
+
+export function sha256(bytes: Uint8Array): string;
+export function mediaUrl(sha: string, ext: string): string;
+export function parseMediaUrl(url: unknown): { sha: string; ext: string } | null;
+export function mediaPaths(sha: string, ext: string): { model: string; full: string };
+export function parseImageDataUrl(url: unknown): { mime: string; bytes: Buffer } | null;
+export function thumbnailOf(bytes: Uint8Array, mime: string): Buffer | null;
+export function createMediaSpool(dir: string): MediaSpool;
+export function createMediaIndex(file: string): MediaIndex;
+export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number }): { message: T; changed: boolean; files: MediaFile[] };
+export function mediaRefsOf(message: unknown): Array<{ sha: string; ext: string }>;
+export function fileBytes(file: string): number;
+export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number }): number;
diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjs
new file mode 100644
index 00000000..4675f03b
--- /dev/null
+++ b/apps/server/src/computer-use/shared/media.mjs
@@ -0,0 +1,307 @@
+// Media references: image data URLs in tool results (computer use
+// screenshots, playwright screenshots, any MCP image) are replaced by
+// `omnirush-media:<sha256>.<ext>` references while the engine's messages are
+// read, part by part, before any size cap, so a turn of hundreds of frames
+// uploads whole. The bytes go to the archive under `__media__/`, with their
+// own budget and lane:
+// __media__/<sha256>.<ext> the exact bytes the model saw
+// __media__/full/<sha256>.png full resolution, only when larger
+//
+// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui
+// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.
+//
+// A reference (it replaces the attachment's `url`, other keys are kept):
+// { mime, url: "omnirush-media:<sha>.<ext>", sha256, width, height, bytes,
+// full?: { sha256, width, height, bytes, ext } | { omitted: "budget" },
+// omitted?: "budget", (model copy not archived)
+// thumb?: "data:image/jpeg;base64,..." } (≤ 320 px; first time per chat)
+
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import path from "node:path";
+
+import { DEFAULT_LIMITS } from "./contract.mjs";
+import { decodePng, encodeJpeg, extensionOf, fitLongSide, imageInfo } from "./image.mjs";
+
+export const MEDIA_ROOT = "__media__";
+export const MEDIA_SCHEME = "omnirush-media:";
+export const THUMB_LONG_SIDE = 320;
+export const THUMB_QUALITY = 70;
+export const THUMBS_PER_MESSAGE_BYTES = 2 * 1024 * 1024;
+export const INDEX_VERSION = 1;
+
+const SHA = /^[a-f0-9]{64}$/;
+const EXT = /^(png|jpg|gif|webp)$/;
+
+function isRecord(value) {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+export function sha256(bytes) {
+ return createHash("sha256").update(bytes).digest("hex");
+}
+
+/** "omnirush-media:<sha>.<ext>" */
+export function mediaUrl(sha, ext) {
+ return `${MEDIA_SCHEME}${sha}.${ext}`;
+}
+
+/** { sha, ext } of a media reference url, or null. */
+export function parseMediaUrl(url) {
+ if (typeof url !== "string" || !url.startsWith(MEDIA_SCHEME)) return null;
+ const [sha, ext] = url.slice(MEDIA_SCHEME.length).split(".");
+ return SHA.test(sha ?? "") && EXT.test(ext ?? "") ? { sha, ext } : null;
+}
+
+/** Archive paths of a reference's files. */
+export function mediaPaths(sha, ext) {
+ return { model: `${MEDIA_ROOT}/${sha}.${ext}`, full: `${MEDIA_ROOT}/full/${sha}.png` };
+}
+
+/** { mime, bytes } of a base64 image data URL, or null (not an image, not base64, malformed). */
+export function parseImageDataUrl(url) {
+ if (typeof url !== "string" || !url.startsWith("data:image/")) return null;
+ const comma = url.indexOf(",");
+ if (comma < 0 || comma > 200) return null;
+ const head = url.slice(5, comma);
+ if (!/;base64$/i.test(head)) return null;
+ const mime = head.slice(0, head.indexOf(";")).toLowerCase();
+ if (!["image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp"].includes(mime)) return null;
+ const bytes = Buffer.from(url.slice(comma + 1), "base64");
+ return bytes.length ? { mime: mime === "image/jpg" ? "image/jpeg" : mime, bytes } : null;
+}
+
+/** A ≤ 320 px JPEG thumbnail of PNG bytes (the decoder this module has), else null. */
+export function thumbnailOf(bytes, mime) {
+ if (mime !== "image/png") return null;
+ const image = decodePng(bytes, { maxPixels: 40_000_000 });
+ if (!image) return null;
+ return encodeJpeg(fitLongSide(image, THUMB_LONG_SIDE), { quality: THUMB_QUALITY });
+}
+
+function writeAtomic(file, data) {
+ mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+ const temp = `${file}.${randomUUID()}.tmp`;
+ writeFileSync(temp, data, { mode: 0o600 });
+ renameSync(temp, file);
+}
+
+function readJson(file) {
+ try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; }
+}
+
+/**
+ * The local media spool: hosts put each frame here when they take it (model
+ * copy, full-resolution PNG, thumbnail, size), the reader finds them by sha.
+ * Frames the reader meets without a host (playwright) are put by the reader.
+ * <dir>/<sha>.<ext>, <dir>/full/<sha>.png, <dir>/thumb/<sha>.jpg, <dir>/meta/<sha>.json
+ */
+export function createMediaSpool(dir) {
+ const metaFile = (sha) => path.join(dir, "meta", `${sha}.json`);
+ const spool = {
+ dir,
+ /**
+ * Stores a frame. `bytes`/`mime`: the model copy; `full`: { bytes (PNG),
+ * width, height } when the host has a larger original; `thumb`: JPEG bytes.
+ * Returns its meta { sha256, ext, mime, width, height, bytes, full?, thumb? }.
+ */
+ put({ bytes, mime, width, height, full, thumb, ocr, masked }) {
+ const data = Buffer.from(bytes);
+ const sha = sha256(data);
+ const existing = spool.meta(sha);
+ if (existing) return existing;
+ const info = imageInfo(data);
+ const type = info?.mime ?? mime;
+ const ext = extensionOf(type);
+ const meta = { sha256: sha, ext, mime: type, width: width ?? info?.width ?? 0, height: height ?? info?.height ?? 0, bytes: data.length };
+ writeAtomic(path.join(dir, `${sha}.${ext}`), data);
+ if (full?.bytes && (full.width * full.height > meta.width * meta.height)) {
+ const fullBytes = Buffer.from(full.bytes);
+ const fullSha = sha256(fullBytes);
+ writeAtomic(path.join(dir, "full", `${sha}.png`), fullBytes);
+ meta.full = { sha256: fullSha, width: full.width, height: full.height, bytes: fullBytes.length, ext: "png" };
+ }
+ const thumbBytes = thumb ? Buffer.from(thumb) : null;
+ if (thumbBytes?.length) { writeAtomic(path.join(dir, "thumb", `${sha}.jpg`), thumbBytes); meta.thumb = thumbBytes.length; }
+ if (ocr) meta.ocr = ocr;
+ if (Number.isInteger(masked)) meta.masked = masked;
+ writeAtomic(metaFile(sha), JSON.stringify(meta));
+ return meta;
+ },
+ meta(sha) {
+ if (!SHA.test(String(sha))) return null;
+ const value = readJson(metaFile(sha));
+ return isRecord(value) && value.sha256 === sha ? value : null;
+ },
+ modelPath(sha, ext) { return path.join(dir, `${sha}.${ext}`); },
+ fullPath(sha) { return path.join(dir, "full", `${sha}.png`); },
+ thumb(sha) { try { return readFileSync(path.join(dir, "thumb", `${sha}.jpg`)); } catch { return null; } },
+ has(sha, ext) { return existsSync(path.join(dir, `${sha}.${ext}`)); },
+ /** Deletes a frame's local files (after its upload). */
+ remove(sha) {
+ if (!SHA.test(String(sha))) return;
+ const meta = spool.meta(sha);
+ for (const file of [meta ? path.join(dir, `${sha}.${meta.ext}`) : null, path.join(dir, "full", `${sha}.png`), path.join(dir, "thumb", `${sha}.jpg`), metaFile(sha)]) if (file) rmSync(file, { force: true });
+ },
+ };
+ return spool;
+}
+
+/**
+ * The per-chat media index, persisted (it survives resumes): which frames
+ * the chat has seen (a thumbnail is inlined only the first time), which went
+ * to the archive, and the bytes used of each budget.
+ */
+export function createMediaIndex(file) {
+ const loaded = readJson(file);
+ const state = isRecord(loaded) && loaded.v === INDEX_VERSION
+ ? { v: INDEX_VERSION, seen: isRecord(loaded.seen) ? loaded.seen : {}, model_bytes: Number(loaded.model_bytes) || 0, full_bytes: Number(loaded.full_bytes) || 0, frames: Number(loaded.frames) || 0 }
+ : { v: INDEX_VERSION, seen: {}, model_bytes: 0, full_bytes: 0, frames: 0 };
+ let dirty = false;
+ return {
+ file,
+ state,
+ entry(sha) { return state.seen[sha] ?? null; },
+ /** Records a frame once; `stored`: { model, full } booleans. */
+ add(sha, { model, full, modelBytes, fullBytes }) {
+ if (state.seen[sha]) return state.seen[sha];
+ const entry = { m: model ? 1 : 0, f: full ? 1 : 0 };
+ state.seen[sha] = entry;
+ state.frames += 1;
+ if (model) state.model_bytes += modelBytes;
+ if (full) state.full_bytes += fullBytes;
+ dirty = true;
+ return entry;
+ },
+ save() {
+ if (!dirty || !file) return;
+ writeAtomic(file, JSON.stringify(state));
+ dirty = false;
+ },
+ };
+}
+
+function budgets(limits) {
+ const l = isRecord(limits) ? limits : {};
+ return {
+ model: Number.isFinite(l.media_model_bytes) ? l.media_model_bytes : DEFAULT_LIMITS.media_model_bytes,
+ full: Number.isFinite(l.media_full_bytes) ? l.media_full_bytes : DEFAULT_LIMITS.media_full_bytes,
+ };
+}
+
+/**
+ * Replaces image data URLs in the tool parts of one engine message (in place
+ * on a structured copy; the input is not changed) with media references.
+ * `ctx`: { spool, index, limits, thumbBudget }. Returns
+ * { message, changed, files: [{ path, file, sha, kind: "model"|"full" }] }:
+ * the archive files this message adds (each frame once per chat).
+ * User file parts are left alone (the archive keeps those as attachments).
+ */
+export function rewriteMessageMedia(message, ctx) {
+ if (!isRecord(message) || !Array.isArray(message.parts)) return { message, changed: false, files: [] };
+ let changed = false;
+ let thumbsLeft = Number.isFinite(ctx.thumbBudget) ? ctx.thumbBudget : THUMBS_PER_MESSAGE_BYTES;
+ const files = [];
+ const budget = budgets(ctx.limits);
+ const parts = message.parts.map((part) => {
+ const attachments = isRecord(part) && part.type === "tool" && isRecord(part.state) && Array.isArray(part.state.attachments) ? part.state.attachments : null;
+ if (!attachments || !attachments.some((a) => isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/"))) return part;
+ const next = attachments.map((attachment) => {
+ const parsed = isRecord(attachment) ? parseImageDataUrl(attachment.url) : null;
+ if (!parsed) return attachment;
+ const sha = sha256(parsed.bytes);
+ let meta = ctx.spool.meta(sha);
+ if (!meta) {
+ const thumb = thumbnailOf(parsed.bytes, parsed.mime);
+ meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });
+ } else if (!ctx.spool.has(sha, meta.ext)) {
+ // Uploaded and removed from the spool, then met again: the engine still has the model copy.
+ meta = { ...ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime }), ...(meta.full ? { full: meta.full } : {}) };
+ }
+ let entry = ctx.index.entry(sha);
+ const first = !entry;
+ if (first) {
+ const model = ctx.index.state.model_bytes + meta.bytes <= budget.model;
+ const full = Boolean(meta.full) && model && ctx.index.state.full_bytes + meta.full.bytes <= budget.full && existsSync(ctx.spool.fullPath(sha));
+ entry = ctx.index.add(sha, { model, full, modelBytes: meta.bytes, fullBytes: meta.full?.bytes ?? 0 });
+ const paths = mediaPaths(sha, meta.ext);
+ if (model) files.push({ path: paths.model, file: ctx.spool.modelPath(sha, meta.ext), sha, kind: "model", bytes: meta.bytes });
+ if (full) files.push({ path: paths.full, file: ctx.spool.fullPath(sha), sha, kind: "full", bytes: meta.full.bytes });
+ }
+ const { url: _url, ...rest } = attachment;
+ const ref = { ...rest, mime: meta.mime, url: mediaUrl(sha, meta.ext), sha256: sha, width: meta.width, height: meta.height, bytes: meta.bytes };
+ if (meta.full) ref.full = entry.f ? { ...meta.full } : { omitted: "budget" };
+ if (!entry.m) ref.omitted = "budget";
+ if (first) {
+ const thumb = ctx.spool.thumb(sha);
+ if (thumb && thumb.length * 4 / 3 + 32 <= thumbsLeft) {
+ ref.thumb = `data:image/jpeg;base64,${thumb.toString("base64")}`;
+ thumbsLeft -= ref.thumb.length;
+ }
+ }
+ changed = true;
+ return ref;
+ });
+ return { ...part, state: { ...part.state, attachments: next } };
+ });
+ if (changed) ctx.index.save();
+ return { message: changed ? { ...message, parts } : message, changed, files };
+}
+
+/** Every media reference in a message's tool parts ([{ sha, ext }]). */
+export function mediaRefsOf(message) {
+ const out = [];
+ for (const part of isRecord(message) && Array.isArray(message.parts) ? message.parts : []) {
+ const attachments = isRecord(part) && isRecord(part.state) && Array.isArray(part.state.attachments) ? part.state.attachments : [];
+ for (const a of attachments) { const ref = isRecord(a) ? parseMediaUrl(a.url) : null; if (ref) out.push(ref); }
+ }
+ return out;
+}
+
+/** Bytes of a spool file, or 0. */
+export function fileBytes(file) {
+ try { return statSync(file).size; } catch { return 0; }
+}
+
+// ------------------------------------------------------- model context pruning
+
+export const KEEP_IMAGES = 12;
+export const DROP_IMAGES = 8;
+
+/**
+ * Prunes screenshots from the MODEL context (the engine plugin hook
+ * `experimental.chat.messages.transform`; the stored chat is untouched):
+ * keeps up to KEEP_IMAGES tool-result images; when the count passes it, the
+ * oldest DROP_IMAGES at a time are replaced by a text note, so the cached
+ * prefix stays stable for several steps. Mutates `messages` in place and
+ * returns the number of images dropped. `messages`: engine WithParts list.
+ */
+export function pruneImages(messages, { keep = KEEP_IMAGES, drop = DROP_IMAGES } = {}) {
+ const found = [];
+ for (const message of Array.isArray(messages) ? messages : []) {
+ for (const part of isRecord(message) && Array.isArray(message.parts) ? message.parts : []) {
+ if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !Array.isArray(part.state.attachments)) continue;
+ part.state.attachments.forEach((a, i) => { if (isRecord(a) && typeof a.mime === "string" && a.mime.startsWith("image/")) found.push({ part, index: i, attachment: a }); });
+ }
+ }
+ if (found.length <= keep) return 0;
+ // Drop in blocks of `drop`: the number dropped only changes every `drop` new images.
+ const excess = found.length - keep;
+ const count = Math.min(found.length, Math.ceil(excess / drop) * drop);
+ const byPart = new Map();
+ for (const item of found.slice(0, count)) {
+ const list = byPart.get(item.part) ?? [];
+ list.push(item);
+ byPart.set(item.part, list);
+ }
+ for (const [part, items] of byPart) {
+ const drops = new Set(items.map((i) => i.index));
+ const notes = items.map(({ attachment }) => {
+ const sha = typeof attachment.sha256 === "string" ? attachment.sha256 : parseImageDataUrl(attachment.url) ? sha256(parseImageDataUrl(attachment.url).bytes) : "";
+ return `[screenshot ${sha.slice(0, 8)} omitted]`;
+ });
+ part.state.attachments = part.state.attachments.filter((_, i) => !drops.has(i));
+ if (typeof part.state.output === "string") part.state.output = `${part.state.output}\n${notes.join("\n")}`;
+ }
+ return count;
+}