A desktop coding agent with free access to frontier models.
TypeScript
feat: shared media refs and image codecs
Samrat Talukdar5612695af81c
diff --git a/apps/server/src/computer-use/media-image.test.ts b/apps/server/src/computer-use/media-image.test.tsnew file mode 100644index 00000000..3771d290--- /dev/null+++ b/apps/server/src/computer-use/media-image.test.ts@@ -0,0 +1,208 @@+// Shared computer-use media references and image codecs (computer-use/shared/).+import { afterAll, describe, expect, test } from "bun:test";+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";+import os from "node:os";+import path from "node:path";++import { compareGrids, decodePng, encodeJpeg, encodePng, fitLongSide, imageInfo, luminanceGrid, paintBoxes, resize } from "./shared/image.mjs";+import {+ createMediaIndex,+ createMediaSpool,+ MEDIA_ROOT,+ mediaRefsOf,+ parseImageDataUrl,+ parseMediaUrl,+ pruneImages,+ rewriteMessageMedia,+ sha256,+ THUMBS_PER_MESSAGE_BYTES,+} from "./shared/media.mjs";++const dirs: string[] = [];+function tmp() {+ const dir = mkdtempSync(path.join(os.tmpdir(), "cu-media-"));+ dirs.push(dir);+ return dir;+}+afterAll(() => { for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); });++function frame(width: number, height: number, seed: number) {+ const data = new Uint8Array(width * height * 4);+ for (let y = 0; y < height; y += 1) for (let x = 0; x < width; x += 1) {+ const o = (y * width + x) * 4;+ data[o] = (x * 7 + seed * 13) & 255; data[o + 1] = (y * 5 + seed * 3) & 255; data[o + 2] = (seed * 31) & 255; data[o + 3] = 255;+ }+ data[0] = seed & 255;+ data[1] = (seed >> 8) & 255;+ return { width, height, data };+}++const dataUrl = (mime: string, bytes: Uint8Array) => `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`;++function toolPart(id: string, urls: string[]) {+ return { id, type: "tool", tool: "computer-use_computer_act", callID: `call_${id}`, state: { status: "completed", input: {}, output: "{\"ok\":true}", attachments: urls.map((url, i) => ({ type: "file", mime: url.slice(5, url.indexOf(";")), url, filename: `shot-${i}` })), time: { start: 1, end: 2 } } };+}++describe("image codecs", () => {+ test("PNG round trip, JPEG encode, sniffing, resize", () => {+ const img = frame(333, 211, 1);+ const png = encodePng(img);+ expect(imageInfo(png)).toEqual({ mime: "image/png", width: 333, height: 211 });+ const back = decodePng(png);+ expect(back && Buffer.from(back.data).equals(Buffer.from(img.data))).toBe(true);+ const jpeg = encodeJpeg(img, { quality: 80 });+ expect(imageInfo(jpeg)).toEqual({ mime: "image/jpeg", width: 333, height: 211 });+ expect(jpeg.subarray(-2)).toEqual(Buffer.from([0xff, 0xd9]));+ expect(fitLongSide(frame(2560, 1600, 2), 1280)).toMatchObject({ width: 1280, height: 800 });+ expect(resize(img, 10, 10).data.length).toBe(400);+ expect(decodePng(Buffer.from("not a png"))).toBeNull();+ });++ test("painting boxes and the tolerant compare (caret masked)", () => {+ const a = frame(320, 200, 4);+ const b = { ...a, data: Uint8Array.from(a.data) };+ // A blinking caret: a 2x14 bar.+ paintBoxes(b, [{ x: 100, y: 50, width: 2, height: 14 }], [255, 255, 255]);+ expect(compareGrids(luminanceGrid(a), luminanceGrid(b)).stale).toBe(false);+ expect(compareGrids(luminanceGrid(a), luminanceGrid(b), { mask: [{ x: 96, y: 48, width: 10, height: 20 }] }).changed).toBe(0);+ // A dialog opened: a large region changed.+ const c = { ...a, data: Uint8Array.from(a.data) };+ paintBoxes(c, [{ x: 40, y: 40, width: 200, height: 100 }]);+ expect(compareGrids(luminanceGrid(a), luminanceGrid(c)).stale).toBe(true);+ expect(compareGrids(luminanceGrid(a), luminanceGrid(frame(160, 100, 4))).stale).toBe(true);+ });+});++describe("media references", () => {+ test("data URLs become refs; first-seen thumb; files once per chat; user file parts untouched", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const index = createMediaIndex(path.join(dir, "index", "ses_1.json"));+ const png = encodePng(frame(1280, 800, 9));+ const url = dataUrl("image/png", png);+ const message = { info: { id: "msg_1", role: "assistant" }, parts: [toolPart("p1", [url]), { type: "file", mime: "image/png", url, filename: "user.png" }] };+ const before = JSON.stringify(message);+ const out = rewriteMessageMedia(message, { spool, index, limits: {} });+ expect(JSON.stringify(message)).toBe(before);+ expect(out.changed).toBe(true);+ const ref = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;+ const sha = sha256(png);+ expect(ref).toMatchObject({ type: "file", filename: "shot-0", mime: "image/png", url: `omnirush-media:${sha}.png`, sha256: sha, width: 1280, height: 800, bytes: png.length });+ expect(String(ref.thumb)).toStartWith("data:image/jpeg;base64,");+ const thumb = Buffer.from(String(ref.thumb).split(",")[1] ?? "", "base64");+ expect(imageInfo(thumb)).toMatchObject({ width: 320, height: 200 });+ expect(thumb.length).toBeLessThan(40_000);+ expect((out.message.parts[1] as { url: string }).url).toBe(url);+ expect(out.files).toEqual([{ path: `${MEDIA_ROOT}/${sha}.png`, file: spool.modelPath(sha, "png"), sha, kind: "model", bytes: png.length }]);+ expect(readFileSync(out.files[0]?.file ?? "").equals(png)).toBe(true);+ expect(mediaRefsOf(out.message)).toEqual([{ sha, ext: "png" }]);+ expect(parseMediaUrl(ref.url)).toEqual({ sha, ext: "png" });+ // Seen again (another read, or a resume with a fresh index object): no thumb, no new file.+ const again = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "index", "ses_1.json")), limits: {} });+ const ref2 = (again.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>;+ expect(ref2.thumb).toBeUndefined();+ expect(again.files).toEqual([]);+ // Already rewritten messages pass through.+ expect(rewriteMessageMedia(out.message, { spool, index, limits: {} }).changed).toBe(false);+ });++ test("host frames: full resolution kept only when larger, and its own budget", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const fullImg = frame(2560, 1600, 3);+ const model = encodeJpeg(fitLongSide(fullImg, 1280), { quality: 80 });+ const fullPng = encodePng(fullImg, { level: 1 });+ const meta = spool.put({ bytes: model, mime: "image/jpeg", full: { bytes: fullPng, width: 2560, height: 1600 }, thumb: encodeJpeg(fitLongSide(fullImg, 320)) });+ expect(meta.full).toMatchObject({ width: 2560, height: 1600, ext: "png", bytes: fullPng.length });+ const small = encodeJpeg(frame(400, 300, 5));+ expect(spool.put({ bytes: small, mime: "image/jpeg", full: { bytes: encodePng(frame(400, 300, 5)), width: 400, height: 300 } }).full).toBeUndefined();+ const message = { parts: [toolPart("a", [dataUrl("image/jpeg", model), dataUrl("image/jpeg", small)])] };+ const out = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: { media_full_bytes: 10, media_model_bytes: 200 * 1024 * 1024 } });+ const [a, b] = (out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments as Array<Record<string, unknown>>;+ expect(a?.full).toEqual({ omitted: "budget" });+ expect(b?.full).toBeUndefined();+ expect(out.files.map((f) => f.kind)).toEqual(["model", "model"]);+ const roomy = rewriteMessageMedia(message, { spool, index: createMediaIndex(path.join(dir, "j.json")), limits: {} });+ expect(roomy.files.map((f) => f.kind)).toEqual(["model", "full", "model"]);+ expect(((roomy.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[0] as Record<string, unknown>).full).toMatchObject({ width: 2560 });+ spool.remove(meta.sha256);+ expect(existsSync(spool.modelPath(meta.sha256, "jpg"))).toBe(false);+ });++ test("a 600-frame turn uploads whole: small message, 2 MB of thumbs, every frame archived under the caps", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const index = createMediaIndex(path.join(dir, "index.json"));+ const parts = [];+ let raw = 0;+ for (let i = 0; i < 600; i += 1) {+ const png = encodePng(frame(160, 100, i), { level: 1 });+ // Pad each frame to ~150 KB (the size of a 1280x800 JPEG) with a private chunk.+ const pad = Buffer.alloc(8 + 150_000 + 4);+ pad.writeUInt32BE(150_000, 0);+ pad.write("paDa", 4, "latin1");+ const padded = Buffer.concat([png.subarray(0, png.length - 12), pad, png.subarray(png.length - 12)]);+ const url = dataUrl("image/png", padded);+ raw += url.length;+ parts.push(toolPart(`p${i}`, [url]));+ }+ const message = { info: { id: "msg_big" }, parts };+ expect(raw).toBeGreaterThan(64 * 1024 * 1024);+ const out = rewriteMessageMedia(message, { spool, index, limits: {} });+ const size = Buffer.byteLength(JSON.stringify(out.message));+ expect(size).toBeLessThan(8 * 1024 * 1024);+ const thumbs = (out.message.parts as Array<ReturnType<typeof toolPart>>).flatMap((p) => p.state.attachments).filter((a) => "thumb" in a).reduce((n, a) => n + String((a as { thumb: string }).thumb).length, 0);+ expect(thumbs).toBeGreaterThan(0);+ expect(thumbs).toBeLessThanOrEqual(THUMBS_PER_MESSAGE_BYTES);+ expect(out.files.length).toBe(600);+ expect(new Set(out.files.map((f) => f.path)).size).toBe(600);+ expect(index.state.model_bytes).toBeLessThanOrEqual(200 * 1024 * 1024);+ expect(JSON.stringify(out.message)).not.toContain("data:image/png");+ }, 60_000);++ test("model budget: past it, refs say omitted and no file is added", () => {+ const dir = tmp();+ const spool = createMediaSpool(path.join(dir, "spool"));+ const pngs = [0, 1, 2].map((i) => encodePng(frame(64, 64, i)));+ const limit = pngs[0]!.length + pngs[1]!.length;+ const out = rewriteMessageMedia({ parts: [toolPart("a", pngs.map((p) => dataUrl("image/png", p)))] }, { spool, index: createMediaIndex(path.join(dir, "i.json")), limits: { media_model_bytes: limit } });+ expect(out.files.length).toBe(2);+ expect(((out.message.parts[0] as ReturnType<typeof toolPart>).state.attachments[2] as Record<string, unknown>).omitted).toBe("budget");+ });++ test("data URL parsing", () => {+ expect(parseImageDataUrl("data:image/png;base64,AQID")).toEqual({ mime: "image/png", bytes: Buffer.from([1, 2, 3]) });+ expect(parseImageDataUrl("data:text/plain;base64,AQID")).toBeNull();+ expect(parseImageDataUrl("data:image/svg+xml;base64,AQID")).toBeNull();+ expect(parseImageDataUrl("https://x/y.png")).toBeNull();+ });+});++describe("model context pruning", () => {+ function chat(count: number) {+ const parts = Array.from({ length: count }, (_, i) => toolPart(`p${i}`, [dataUrl("image/png", encodePng(frame(8, 8, i)))]));+ return [{ info: { role: "assistant" }, parts }];+ }+ const images = (messages: ReturnType<typeof chat>) => messages[0]!.parts.reduce((n, p) => n + p.state.attachments.length, 0);++ test("keeps up to 12; past 12 drops the oldest 8 at a time; notes the dropped ones", () => {+ const twelve = chat(12);+ expect(pruneImages(twelve)).toBe(0);+ expect(images(twelve)).toBe(12);+ const thirteen = chat(13);+ expect(pruneImages(thirteen)).toBe(8);+ expect(images(thirteen)).toBe(5);+ expect(thirteen[0]!.parts[0]!.state.output).toMatch(/\[screenshot [a-f0-9]{8} omitted\]$/);+ const twenty = chat(20);+ pruneImages(twenty);+ expect(images(twenty)).toBe(12);+ const twentyOne = chat(21);+ pruneImages(twentyOne);+ expect(images(twentyOne)).toBe(5);+ // Stable prefix: the same oldest images are dropped from 13 to 20 images.+ const at14 = chat(14);+ pruneImages(at14);+ expect(at14[0]!.parts.slice(0, 8).every((p) => p.state.attachments.length === 0)).toBe(true);+ expect(at14[0]!.parts[8]!.state.attachments.length).toBe(1);+ });+});diff --git a/apps/server/src/computer-use/shared/PARITY.sha256 b/apps/server/src/computer-use/shared/PARITY.sha256index 6fb139cd..9e799cfb 100644--- a/apps/server/src/computer-use/shared/PARITY.sha256+++ b/apps/server/src/computer-use/shared/PARITY.sha256@@ -4,3 +4,7 @@07fe5681c685a1f4883ecfe3e6528c1d70cf68003f690079297f887990879d80 contract.d.mtsc601c77fd65561657b103b6ec3c5e4e822d38d4cf9d535da368cc2466698a9d5 policy.mjs103c58c3908bebab415e5f1878abf752a2b07498f502855298a72cb58225798f policy.d.mts+17e55894695db8183a659a6c868bd8b1ac9348ccb0e0618b3d8aa94adf98d61a image.mjs+f11dd35d6db769398222387695262aeeac5dbaf9df16191d62a8811e8613e25d image.d.mts+eea23441a7360c186044072fc0261a5b524e20dd639d05161dd968dc4bfae774 media.mjs+25f747b53f5a1dbb5b2439b65bc728c508eecb62fbe2650904beffa8d3854b9c media.d.mtsdiff --git a/apps/server/src/computer-use/shared/image.d.mts b/apps/server/src/computer-use/shared/image.d.mtsnew file mode 100644index 00000000..b5fc59a1--- /dev/null+++ b/apps/server/src/computer-use/shared/image.d.mts@@ -0,0 +1,17 @@+// Types of image.mjs (the same files in omnirush-cli src/computer-use/ and+// omnirush-gui apps/server/src/computer-use/shared/; PARITY.sha256).++export type RgbaImage = { width: number; height: number; data: Uint8Array };+export type Rect = { x: number; y: number; width: number; height: number };+export type Grid = { cell: number; cols: number; rows: number; width: number; height: number; values: Float32Array };++export function encodePng(image: RgbaImage, options?: { level?: number }): Buffer;+export function decodePng(bytes: Uint8Array, options?: { maxPixels?: number }): RgbaImage | null;+export function resize(image: RgbaImage, width: number, height: number): RgbaImage;+export function fitLongSide(image: RgbaImage, longSide: number): RgbaImage;+export function paintBoxes(image: RgbaImage, boxes: Rect[] | null | undefined, color?: [number, number, number]): number;+export function encodeJpeg(image: RgbaImage, options?: { quality?: number }): Buffer;+export function imageInfo(bytes: Uint8Array): { mime: string; width: number; height: number } | null;+export function extensionOf(mime: unknown): "png" | "jpg" | "gif" | "webp" | "bin";+export function luminanceGrid(image: RgbaImage, cell?: number): Grid;+export function compareGrids(a: Grid | null | undefined, b: Grid | null | undefined, options?: { mask?: Rect[]; threshold?: number; maxCells?: number; maxSpan?: number }): { stale: boolean; changed: number; span?: number };diff --git a/apps/server/src/computer-use/shared/image.mjs b/apps/server/src/computer-use/shared/image.mjsnew file mode 100644index 00000000..9f05de0a--- /dev/null+++ b/apps/server/src/computer-use/shared/image.mjs@@ -0,0 +1,388 @@+// Computer use, images: PNG encode/decode (node:zlib), an area-average+// downscale, a small baseline JPEG encoder (4:2:0, standard tables; written+// for this module, no third-party code), size sniffing and a tolerant frame+// compare. Plain Node: works in Electron main, the server, engine plugins and+// the CLI alike.+//+// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui+// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.+//+// Pixels are RGBA, 8 bits per channel, rows top to bottom:+// { width, height, data: Uint8Array(width * height * 4) }.++import { crc32, deflateSync, inflateSync } from "node:zlib";++// ---------------------------------------------------------------------- PNG++const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);++function chunk(type, data) {+ const name = Buffer.from(type, "latin1");+ const length = Buffer.alloc(4);+ length.writeUInt32BE(data.length);+ const sum = Buffer.alloc(4);+ sum.writeUInt32BE(crc32(data, crc32(name)) >>> 0);+ return Buffer.concat([length, name, data, sum]);+}++/** RGBA pixels → PNG bytes (truecolor, alpha dropped when every pixel is opaque). */+export function encodePng(image, { level = 6 } = {}) {+ const { width, height, data } = image;+ let opaque = true;+ for (let i = 3; i < data.length; i += 4) if (data[i] !== 255) { opaque = false; break; }+ const channels = opaque ? 3 : 4;+ const stride = width * channels + 1;+ const rows = Buffer.alloc(stride * height);+ for (let y = 0; y < height; y += 1) {+ let o = y * stride + 1;+ let s = y * width * 4;+ for (let x = 0; x < width; x += 1, s += 4) {+ rows[o++] = data[s]; rows[o++] = data[s + 1]; rows[o++] = data[s + 2];+ if (!opaque) rows[o++] = data[s + 3];+ }+ }+ const header = Buffer.alloc(13);+ header.writeUInt32BE(width, 0);+ header.writeUInt32BE(height, 4);+ header[8] = 8;+ header[9] = opaque ? 2 : 6;+ return Buffer.concat([PNG_SIGNATURE, chunk("IHDR", header), chunk("IDAT", deflateSync(rows, { level })), chunk("IEND", Buffer.alloc(0))]);+}++function paeth(a, b, c) {+ const p = a + b - c;+ const pa = Math.abs(p - a), pb = Math.abs(p - b), pc = Math.abs(p - c);+ return pa <= pb && pa <= pc ? a : pb <= pc ? b : c;+}++/**+ * PNG bytes → RGBA pixels, or null for what this decoder does not read+ * (16-bit, interlaced, a corrupt file, more than `maxPixels`).+ */+export function decodePng(bytes, { maxPixels = 40_000_000 } = {}) {+ try {+ const buf = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);+ if (buf.length < 33 || !buf.subarray(0, 8).equals(PNG_SIGNATURE)) return null;+ let offset = 8;+ let width = 0, height = 0, depth = 0, type = 0, interlace = 0;+ let palette = null, alpha = null;+ const idat = [];+ while (offset + 8 <= buf.length) {+ const length = buf.readUInt32BE(offset);+ const name = buf.toString("latin1", offset + 4, offset + 8);+ const data = buf.subarray(offset + 8, offset + 8 + length);+ offset += 12 + length;+ if (name === "IHDR") { width = data.readUInt32BE(0); height = data.readUInt32BE(4); depth = data[8]; type = data[9]; interlace = data[12]; }+ else if (name === "PLTE") palette = data;+ else if (name === "tRNS") alpha = data;+ else if (name === "IDAT") idat.push(data);+ else if (name === "IEND") break;+ }+ if (!width || !height || depth !== 8 || interlace !== 0 || width * height > maxPixels) return null;+ const channels = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 }[type];+ if (!channels || (type === 3 && !palette)) return null;+ const raw = inflateSync(Buffer.concat(idat));+ const stride = width * channels;+ if (raw.length < (stride + 1) * height) return null;+ const out = new Uint8Array(width * height * 4);+ let prev = new Uint8Array(stride);+ const line = new Uint8Array(stride);+ for (let y = 0; y < height; y += 1) {+ const filter = raw[y * (stride + 1)];+ const src = raw.subarray(y * (stride + 1) + 1, (y + 1) * (stride + 1));+ for (let i = 0; i < stride; i += 1) {+ const a = i >= channels ? line[i - channels] : 0;+ const b = prev[i];+ const c = i >= channels ? prev[i - channels] : 0;+ const v = src[i];+ line[i] = filter === 0 ? v : filter === 1 ? v + a : filter === 2 ? v + b : filter === 3 ? v + ((a + b) >> 1) : filter === 4 ? v + paeth(a, b, c) : v;+ }+ for (let x = 0; x < width; x += 1) {+ const o = (y * width + x) * 4;+ if (type === 6) { out[o] = line[x * 4]; out[o + 1] = line[x * 4 + 1]; out[o + 2] = line[x * 4 + 2]; out[o + 3] = line[x * 4 + 3]; }+ else if (type === 2) { out[o] = line[x * 3]; out[o + 1] = line[x * 3 + 1]; out[o + 2] = line[x * 3 + 2]; out[o + 3] = 255; }+ else if (type === 0) { out[o] = out[o + 1] = out[o + 2] = line[x]; out[o + 3] = 255; }+ else if (type === 4) { out[o] = out[o + 1] = out[o + 2] = line[x * 2]; out[o + 3] = line[x * 2 + 1]; }+ else { const p = line[x]; out[o] = palette[p * 3] ?? 0; out[o + 1] = palette[p * 3 + 1] ?? 0; out[o + 2] = palette[p * 3 + 2] ?? 0; out[o + 3] = alpha && p < alpha.length ? alpha[p] : 255; }+ }+ prev = Uint8Array.from(line);+ }+ return { width, height, data: out };+ } catch {+ return null;+ }+}++// ----------------------------------------------------------------- resizing++/** Area-average downscale to exactly width × height (never upscales usefully: use for shrinking). */+export function resize(image, width, height) {+ const sw = image.width, sh = image.height, src = image.data;+ if (sw === width && sh === height) return { width, height, data: Uint8Array.from(src) };+ const out = new Uint8Array(width * height * 4);+ const xs = new Int32Array(width + 1), ys = new Int32Array(height + 1);+ for (let x = 0; x <= width; x += 1) xs[x] = Math.min(sw, Math.round((x * sw) / width));+ for (let y = 0; y <= height; y += 1) ys[y] = Math.min(sh, Math.round((y * sh) / height));+ for (let y = 0; y < height; y += 1) {+ const y0 = ys[y], y1 = Math.max(ys[y + 1], y0 + 1);+ for (let x = 0; x < width; x += 1) {+ const x0 = xs[x], x1 = Math.max(xs[x + 1], x0 + 1);+ let r = 0, g = 0, b = 0, a = 0, n = 0;+ for (let yy = y0; yy < y1 && yy < sh; yy += 1) {+ let s = (yy * sw + x0) * 4;+ for (let xx = x0; xx < x1 && xx < sw; xx += 1, s += 4) { r += src[s]; g += src[s + 1]; b += src[s + 2]; a += src[s + 3]; n += 1; }+ }+ const o = (y * width + x) * 4;+ n = n || 1;+ out[o] = (r / n + 0.5) | 0; out[o + 1] = (g / n + 0.5) | 0; out[o + 2] = (b / n + 0.5) | 0; out[o + 3] = (a / n + 0.5) | 0;+ }+ }+ return { width, height, data: out };+}++/** Downscales so the long side is at most `longSide` (a copy when it already fits). */+export function fitLongSide(image, longSide) {+ const ratio = Math.min(1, longSide / Math.max(image.width, image.height));+ if (ratio >= 1) return image;+ return resize(image, Math.max(1, Math.round(image.width * ratio)), Math.max(1, Math.round(image.height * ratio)));+}++/** Paints rectangles ({x, y, width, height} in pixels of `image`) solid in place. */+export function paintBoxes(image, boxes, color = [0, 0, 0]) {+ let painted = 0;+ for (const box of boxes ?? []) {+ const x0 = Math.max(0, Math.floor(box.x)), y0 = Math.max(0, Math.floor(box.y));+ const x1 = Math.min(image.width, Math.ceil(box.x + box.width)), y1 = Math.min(image.height, Math.ceil(box.y + box.height));+ if (x1 <= x0 || y1 <= y0) continue;+ painted += 1;+ for (let y = y0; y < y1; y += 1) for (let x = x0; x < x1; x += 1) {+ const o = (y * image.width + x) * 4;+ image.data[o] = color[0]; image.data[o + 1] = color[1]; image.data[o + 2] = color[2]; image.data[o + 3] = 255;+ }+ }+ return painted;+}++// --------------------------------------------------------------------- JPEG++const ZIGZAG = [0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63];+const LUMA_Q = [16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19, 26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56, 14, 17, 22, 29, 51, 87, 80, 62, 18, 22, 37, 56, 68, 109, 103, 77, 24, 35, 55, 64, 81, 104, 113, 92, 49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99];+const CHROMA_Q = [17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99, 47, 66, 99, 99, 99, 99, 99, 99, ...new Array(32).fill(99)];+const DC_LUMA = { bits: [0, 1, 5, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], vals: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] };+const DC_CHROMA = { bits: [0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0], vals: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] };+const AC_LUMA = {+ bits: [0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 0x7d],+ vals: [0x01, 0x02, 0x03, 0x00, 0x04, 0x11, 0x05, 0x12, 0x21, 0x31, 0x41, 0x06, 0x13, 0x51, 0x61, 0x07, 0x22, 0x71, 0x14, 0x32, 0x81, 0x91, 0xa1, 0x08, 0x23, 0x42, 0xb1, 0xc1, 0x15, 0x52, 0xd1, 0xf0, 0x24, 0x33, 0x62, 0x72, 0x82, 0x09, 0x0a, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa],+};+const AC_CHROMA = {+ bits: [0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 0x77],+ vals: [0x00, 0x01, 0x02, 0x03, 0x11, 0x04, 0x05, 0x21, 0x31, 0x06, 0x12, 0x41, 0x51, 0x07, 0x61, 0x71, 0x13, 0x22, 0x32, 0x81, 0x08, 0x14, 0x42, 0x91, 0xa1, 0xb1, 0xc1, 0x09, 0x23, 0x33, 0x52, 0xf0, 0x15, 0x62, 0x72, 0xd1, 0x0a, 0x16, 0x24, 0x34, 0xe1, 0x25, 0xf1, 0x17, 0x18, 0x19, 0x1a, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6a, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa],+};++function huffmanCodes(table) {+ const codes = new Map();+ let code = 0, k = 0;+ for (let length = 1; length <= 16; length += 1) {+ for (let i = 0; i < table.bits[length - 1]; i += 1) codes.set(table.vals[k++], [code++, length]);+ code <<= 1;+ }+ return codes;+}+const CODES = { dcY: huffmanCodes(DC_LUMA), dcC: huffmanCodes(DC_CHROMA), acY: huffmanCodes(AC_LUMA), acC: huffmanCodes(AC_CHROMA) };++function scaledTable(base, quality) {+ const q = Math.min(100, Math.max(1, Math.round(quality)));+ const s = q < 50 ? 5000 / q : 200 - q * 2;+ return base.map((v) => Math.min(255, Math.max(1, Math.floor((v * s + 50) / 100))));+}++const COS = new Float64Array(64);+for (let x = 0; x < 8; x += 1) for (let u = 0; u < 8; u += 1) COS[x * 8 + u] = Math.cos(((2 * x + 1) * u * Math.PI) / 16) * (u === 0 ? Math.SQRT1_2 : 1);++/** In place: 8×8 samples (level-shifted) → DCT coefficients (natural order). */+function fdct(block, tmp) {+ for (let y = 0; y < 8; y += 1) for (let u = 0; u < 8; u += 1) {+ let sum = 0;+ for (let x = 0; x < 8; x += 1) sum += block[y * 8 + x] * COS[x * 8 + u];+ tmp[y * 8 + u] = sum / 2;+ }+ for (let u = 0; u < 8; u += 1) for (let v = 0; v < 8; v += 1) {+ let sum = 0;+ for (let y = 0; y < 8; y += 1) sum += tmp[y * 8 + u] * COS[y * 8 + v];+ block[v * 8 + u] = sum / 2;+ }+}++class BitWriter {+ constructor(size) { this.buf = Buffer.alloc(Math.max(1024, size)); this.pos = 0; this.acc = 0; this.n = 0; }+ byte(v) { if (this.pos >= this.buf.length) { const next = Buffer.alloc(this.buf.length * 2); this.buf.copy(next); this.buf = next; } this.buf[this.pos++] = v; }+ bits(value, length) {+ for (let i = length - 1; i >= 0; i -= 1) {+ this.acc = (this.acc << 1) | ((value >> i) & 1);+ this.n += 1;+ if (this.n === 8) { this.byte(this.acc); if (this.acc === 0xff) this.byte(0); this.acc = 0; this.n = 0; }+ }+ }+ flush() { if (this.n > 0) this.bits((1 << (8 - this.n)) - 1, 8 - this.n); return this.buf.subarray(0, this.pos); }+}++function category(v) {+ let a = Math.abs(v), n = 0;+ while (a) { n += 1; a >>= 1; }+ return n;+}++function encodeBlock(writer, block, table, dcCodes, acCodes, previous) {+ const q = new Int32Array(64);+ for (let k = 0; k < 64; k += 1) q[k] = Math.round(block[ZIGZAG[k]] / table[ZIGZAG[k]]);+ const diff = q[0] - previous;+ const dcCat = category(diff);+ const dc = dcCodes.get(dcCat);+ writer.bits(dc[0], dc[1]);+ if (dcCat) writer.bits(diff < 0 ? diff + (1 << dcCat) - 1 : diff, dcCat);+ let last = 63;+ while (last > 0 && q[last] === 0) last -= 1;+ let run = 0;+ for (let k = 1; k <= last; k += 1) {+ const v = q[k];+ if (v === 0) { run += 1; continue; }+ while (run > 15) { const z = acCodes.get(0xf0); writer.bits(z[0], z[1]); run -= 16; }+ const c = category(v);+ const code = acCodes.get((run << 4) | c);+ writer.bits(code[0], code[1]);+ writer.bits(v < 0 ? v + (1 << c) - 1 : v, c);+ run = 0;+ }+ if (last < 63) { const eob = acCodes.get(0x00); writer.bits(eob[0], eob[1]); }+ return q[0];+}++function segment(marker, payload) {+ const head = Buffer.alloc(4);+ head[0] = 0xff; head[1] = marker; head.writeUInt16BE(payload.length + 2, 2);+ return Buffer.concat([head, payload]);+}++function dhtPayload(tableClass, id, table) {+ return Buffer.from([(tableClass << 4) | id, ...table.bits, ...table.vals]);+}++/** RGBA pixels → baseline JPEG bytes (4:2:0, alpha ignored). */+export function encodeJpeg(image, { quality = 80 } = {}) {+ const { width, height, data } = image;+ const qy = scaledTable(LUMA_Q, quality), qc = scaledTable(CHROMA_Q, quality);+ const header = [+ Buffer.from([0xff, 0xd8]),+ segment(0xe0, Buffer.from([0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0])),+ segment(0xdb, Buffer.from([0, ...ZIGZAG.map((i) => qy[i]), 1, ...ZIGZAG.map((i) => qc[i])])),+ segment(0xc0, Buffer.from([8, height >> 8, height & 255, width >> 8, width & 255, 3, 1, 0x22, 0, 2, 0x11, 1, 3, 0x11, 1])),+ segment(0xc4, Buffer.concat([dhtPayload(0, 0, DC_LUMA), dhtPayload(1, 0, AC_LUMA), dhtPayload(0, 1, DC_CHROMA), dhtPayload(1, 1, AC_CHROMA)])),+ segment(0xda, Buffer.from([3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0])),+ ];+ const writer = new BitWriter((width * height) >> 2);+ const Y = new Float64Array(64 * 4), Cb = new Float64Array(64), Cr = new Float64Array(64), tmp = new Float64Array(64);+ const block = new Float64Array(64);+ let pY = 0, pCb = 0, pCr = 0;+ for (let my = 0; my < height; my += 16) {+ for (let mx = 0; mx < width; mx += 16) {+ Cb.fill(0); Cr.fill(0);+ for (let y = 0; y < 16; y += 1) {+ const sy = Math.min(height - 1, my + y);+ for (let x = 0; x < 16; x += 1) {+ const sx = Math.min(width - 1, mx + x);+ const o = (sy * width + sx) * 4;+ const r = data[o], g = data[o + 1], b = data[o + 2];+ const yb = ((y >> 3) << 1) | (x >> 3);+ Y[yb * 64 + (y & 7) * 8 + (x & 7)] = 0.299 * r + 0.587 * g + 0.114 * b - 128;+ const c = (y >> 1) * 8 + (x >> 1);+ Cb[c] += (-0.168736 * r - 0.331264 * g + 0.5 * b) / 4;+ Cr[c] += (0.5 * r - 0.418688 * g - 0.081312 * b) / 4;+ }+ }+ for (let b = 0; b < 4; b += 1) {+ for (let i = 0; i < 64; i += 1) block[i] = Y[b * 64 + i];+ fdct(block, tmp);+ pY = encodeBlock(writer, block, qy, CODES.dcY, CODES.acY, pY);+ }+ block.set(Cb); fdct(block, tmp); pCb = encodeBlock(writer, block, qc, CODES.dcC, CODES.acC, pCb);+ block.set(Cr); fdct(block, tmp); pCr = encodeBlock(writer, block, qc, CODES.dcC, CODES.acC, pCr);+ }+ }+ return Buffer.concat([...header, Buffer.from(writer.flush()), Buffer.from([0xff, 0xd9])]);+}++// ------------------------------------------------------------------- sniffing++/** { mime, width, height } of PNG, JPEG, GIF or WebP bytes, or null. */+export function imageInfo(bytes) {+ const b = Buffer.from(bytes.buffer ?? bytes, bytes.byteOffset ?? 0, bytes.byteLength ?? bytes.length);+ if (b.length >= 24 && b.subarray(0, 8).equals(PNG_SIGNATURE)) return { mime: "image/png", width: b.readUInt32BE(16), height: b.readUInt32BE(20) };+ if (b.length >= 4 && b[0] === 0xff && b[1] === 0xd8) {+ let o = 2;+ while (o + 9 < b.length) {+ if (b[o] !== 0xff) { o += 1; continue; }+ const marker = b[o + 1];+ if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { o += 2; continue; }+ const length = b.readUInt16BE(o + 2);+ if ((marker >= 0xc0 && marker <= 0xc3) || (marker >= 0xc5 && marker <= 0xc7) || (marker >= 0xc9 && marker <= 0xcb) || (marker >= 0xcd && marker <= 0xcf)) return { mime: "image/jpeg", width: b.readUInt16BE(o + 7), height: b.readUInt16BE(o + 5) };+ o += 2 + length;+ }+ return { mime: "image/jpeg", width: 0, height: 0 };+ }+ if (b.length >= 10 && b.toString("latin1", 0, 3) === "GIF") return { mime: "image/gif", width: b.readUInt16LE(6), height: b.readUInt16LE(8) };+ if (b.length >= 30 && b.toString("latin1", 0, 4) === "RIFF" && b.toString("latin1", 8, 12) === "WEBP") {+ const kind = b.toString("latin1", 12, 16);+ if (kind === "VP8X") return { mime: "image/webp", width: 1 + b.readUIntLE(24, 3), height: 1 + b.readUIntLE(27, 3) };+ if (kind === "VP8L") { const v = b.readUInt32LE(21); return { mime: "image/webp", width: (v & 0x3fff) + 1, height: ((v >> 14) & 0x3fff) + 1 }; }+ if (kind === "VP8 ") return { mime: "image/webp", width: b.readUInt16LE(26) & 0x3fff, height: b.readUInt16LE(28) & 0x3fff };+ return { mime: "image/webp", width: 0, height: 0 };+ }+ return null;+}++/** File extension for an image mime type. */+export function extensionOf(mime) {+ return { "image/png": "png", "image/jpeg": "jpg", "image/jpg": "jpg", "image/gif": "gif", "image/webp": "webp" }[String(mime ?? "").toLowerCase()] ?? "bin";+}++// ------------------------------------------------------------ frame compare++/** Mean luminance per cell (cell × cell pixels): the fingerprint the stale check compares. */+export function luminanceGrid(image, cell = 16) {+ const cols = Math.ceil(image.width / cell), rows = Math.ceil(image.height / cell);+ const sums = new Float64Array(cols * rows), counts = new Uint32Array(cols * rows);+ const d = image.data;+ for (let y = 0; y < image.height; y += 1) {+ const row = ((y / cell) | 0) * cols;+ for (let x = 0; x < image.width; x += 1) {+ const o = (y * image.width + x) * 4;+ const i = row + ((x / cell) | 0);+ sums[i] += 0.299 * d[o] + 0.587 * d[o + 1] + 0.114 * d[o + 2];+ counts[i] += 1;+ }+ }+ const values = new Float32Array(cols * rows);+ for (let i = 0; i < values.length; i += 1) values[i] = sums[i] / (counts[i] || 1);+ return { cell, cols, rows, width: image.width, height: image.height, values };+}++/**+ * Tolerant compare of two grids of the same image size: cells whose mean+ * luminance moved more than `threshold` count as changed, except cells that+ * touch a `mask` rectangle (caret, cursor). Stale when more than `maxCells`+ * cells changed or the changed cells span more than `maxSpan` pixels.+ */+export function compareGrids(a, b, { mask = [], threshold = 6, maxCells = 4, maxSpan = 48 } = {}) {+ if (!a || !b || a.cols !== b.cols || a.rows !== b.rows || a.cell !== b.cell) return { stale: true, changed: -1 };+ let changed = 0, minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity;+ for (let r = 0; r < a.rows; r += 1) for (let c = 0; c < a.cols; c += 1) {+ const i = r * a.cols + c;+ if (Math.abs(a.values[i] - b.values[i]) <= threshold) continue;+ const x = c * a.cell, y = r * a.cell;+ if (mask.some((m) => x < m.x + m.width && x + a.cell > m.x && y < m.y + m.height && y + a.cell > m.y)) continue;+ changed += 1;+ minX = Math.min(minX, x); minY = Math.min(minY, y); maxX = Math.max(maxX, x + a.cell); maxY = Math.max(maxY, y + a.cell);+ }+ const span = changed ? Math.max(maxX - minX, maxY - minY) : 0;+ return { stale: changed > maxCells || span > maxSpan, changed, span };+}diff --git a/apps/server/src/computer-use/shared/media.d.mts b/apps/server/src/computer-use/shared/media.d.mtsnew file mode 100644index 00000000..583ceb1c--- /dev/null+++ b/apps/server/src/computer-use/shared/media.d.mts@@ -0,0 +1,70 @@+// Types of media.mjs (the same files in omnirush-cli src/computer-use/ and+// omnirush-gui apps/server/src/computer-use/shared/; PARITY.sha256).++export const MEDIA_ROOT: "__media__";+export const MEDIA_SCHEME: "omnirush-media:";+export const THUMB_LONG_SIDE: number;+export const THUMB_QUALITY: number;+export const THUMBS_PER_MESSAGE_BYTES: number;+export const INDEX_VERSION: 1;+export const KEEP_IMAGES: number;+export const DROP_IMAGES: number;++export type FullMeta = { sha256: string; width: number; height: number; bytes: number; ext: "png" };+export type MediaMeta = {+ sha256: string;+ ext: string;+ mime: string;+ width: number;+ height: number;+ bytes: number;+ full?: FullMeta;+ thumb?: number;+ ocr?: "client" | "none";+ masked?: number;+};+export type MediaRef = {+ mime: string;+ url: string;+ sha256: string;+ width: number;+ height: number;+ bytes: number;+ full?: FullMeta | { omitted: "budget" };+ omitted?: "budget";+ thumb?: string;+ [key: string]: unknown;+};+export type MediaFile = { path: string; file: string; sha: string; kind: "model" | "full"; bytes: number };++export type MediaSpool = {+ dir: string;+ put(frame: { bytes: Uint8Array; mime: string; width?: number; height?: number; full?: { bytes: Uint8Array; width: number; height: number } | null; thumb?: Uint8Array | null; ocr?: "client" | "none"; masked?: number }): MediaMeta;+ meta(sha: string): MediaMeta | null;+ modelPath(sha: string, ext: string): string;+ fullPath(sha: string): string;+ thumb(sha: string): Buffer | null;+ has(sha: string, ext: string): boolean;+ remove(sha: string): void;+};+export type MediaIndexState = { v: 1; seen: Record<string, { m: 0 | 1; f: 0 | 1 }>; model_bytes: number; full_bytes: number; frames: number };+export type MediaIndex = {+ file: string;+ state: MediaIndexState;+ entry(sha: string): { m: 0 | 1; f: 0 | 1 } | null;+ add(sha: string, stored: { model: boolean; full: boolean; modelBytes: number; fullBytes: number }): { m: 0 | 1; f: 0 | 1 };+ save(): void;+};++export function sha256(bytes: Uint8Array): string;+export function mediaUrl(sha: string, ext: string): string;+export function parseMediaUrl(url: unknown): { sha: string; ext: string } | null;+export function mediaPaths(sha: string, ext: string): { model: string; full: string };+export function parseImageDataUrl(url: unknown): { mime: string; bytes: Buffer } | null;+export function thumbnailOf(bytes: Uint8Array, mime: string): Buffer | null;+export function createMediaSpool(dir: string): MediaSpool;+export function createMediaIndex(file: string): MediaIndex;+export function rewriteMessageMedia<T>(message: T, ctx: { spool: MediaSpool; index: MediaIndex; limits?: { media_full_bytes?: number; media_model_bytes?: number } | null; thumbBudget?: number }): { message: T; changed: boolean; files: MediaFile[] };+export function mediaRefsOf(message: unknown): Array<{ sha: string; ext: string }>;+export function fileBytes(file: string): number;+export function pruneImages(messages: unknown[], options?: { keep?: number; drop?: number }): number;diff --git a/apps/server/src/computer-use/shared/media.mjs b/apps/server/src/computer-use/shared/media.mjsnew file mode 100644index 00000000..4675f03b--- /dev/null+++ b/apps/server/src/computer-use/shared/media.mjs@@ -0,0 +1,307 @@+// Media references: image data URLs in tool results (computer use+// screenshots, playwright screenshots, any MCP image) are replaced by+// `omnirush-media:<sha256>.<ext>` references while the engine's messages are+// read, part by part, before any size cap, so a turn of hundreds of frames+// uploads whole. The bytes go to the archive under `__media__/`, with their+// own budget and lane:+// __media__/<sha256>.<ext> the exact bytes the model saw+// __media__/full/<sha256>.png full resolution, only when larger+//+// The same file ships in omnirush-cli (src/computer-use/) and omnirush-gui+// (apps/server/src/computer-use/shared/); PARITY.sha256 lists it in both.+//+// A reference (it replaces the attachment's `url`, other keys are kept):+// { mime, url: "omnirush-media:<sha>.<ext>", sha256, width, height, bytes,+// full?: { sha256, width, height, bytes, ext } | { omitted: "budget" },+// omitted?: "budget", (model copy not archived)+// thumb?: "data:image/jpeg;base64,..." } (≤ 320 px; first time per chat)++import { createHash, randomUUID } from "node:crypto";+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";+import path from "node:path";++import { DEFAULT_LIMITS } from "./contract.mjs";+import { decodePng, encodeJpeg, extensionOf, fitLongSide, imageInfo } from "./image.mjs";++export const MEDIA_ROOT = "__media__";+export const MEDIA_SCHEME = "omnirush-media:";+export const THUMB_LONG_SIDE = 320;+export const THUMB_QUALITY = 70;+export const THUMBS_PER_MESSAGE_BYTES = 2 * 1024 * 1024;+export const INDEX_VERSION = 1;++const SHA = /^[a-f0-9]{64}$/;+const EXT = /^(png|jpg|gif|webp)$/;++function isRecord(value) {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++export function sha256(bytes) {+ return createHash("sha256").update(bytes).digest("hex");+}++/** "omnirush-media:<sha>.<ext>" */+export function mediaUrl(sha, ext) {+ return `${MEDIA_SCHEME}${sha}.${ext}`;+}++/** { sha, ext } of a media reference url, or null. */+export function parseMediaUrl(url) {+ if (typeof url !== "string" || !url.startsWith(MEDIA_SCHEME)) return null;+ const [sha, ext] = url.slice(MEDIA_SCHEME.length).split(".");+ return SHA.test(sha ?? "") && EXT.test(ext ?? "") ? { sha, ext } : null;+}++/** Archive paths of a reference's files. */+export function mediaPaths(sha, ext) {+ return { model: `${MEDIA_ROOT}/${sha}.${ext}`, full: `${MEDIA_ROOT}/full/${sha}.png` };+}++/** { mime, bytes } of a base64 image data URL, or null (not an image, not base64, malformed). */+export function parseImageDataUrl(url) {+ if (typeof url !== "string" || !url.startsWith("data:image/")) return null;+ const comma = url.indexOf(",");+ if (comma < 0 || comma > 200) return null;+ const head = url.slice(5, comma);+ if (!/;base64$/i.test(head)) return null;+ const mime = head.slice(0, head.indexOf(";")).toLowerCase();+ if (!["image/png", "image/jpeg", "image/jpg", "image/gif", "image/webp"].includes(mime)) return null;+ const bytes = Buffer.from(url.slice(comma + 1), "base64");+ return bytes.length ? { mime: mime === "image/jpg" ? "image/jpeg" : mime, bytes } : null;+}++/** A ≤ 320 px JPEG thumbnail of PNG bytes (the decoder this module has), else null. */+export function thumbnailOf(bytes, mime) {+ if (mime !== "image/png") return null;+ const image = decodePng(bytes, { maxPixels: 40_000_000 });+ if (!image) return null;+ return encodeJpeg(fitLongSide(image, THUMB_LONG_SIDE), { quality: THUMB_QUALITY });+}++function writeAtomic(file, data) {+ mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });+ const temp = `${file}.${randomUUID()}.tmp`;+ writeFileSync(temp, data, { mode: 0o600 });+ renameSync(temp, file);+}++function readJson(file) {+ try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; }+}++/**+ * The local media spool: hosts put each frame here when they take it (model+ * copy, full-resolution PNG, thumbnail, size), the reader finds them by sha.+ * Frames the reader meets without a host (playwright) are put by the reader.+ * <dir>/<sha>.<ext>, <dir>/full/<sha>.png, <dir>/thumb/<sha>.jpg, <dir>/meta/<sha>.json+ */+export function createMediaSpool(dir) {+ const metaFile = (sha) => path.join(dir, "meta", `${sha}.json`);+ const spool = {+ dir,+ /**+ * Stores a frame. `bytes`/`mime`: the model copy; `full`: { bytes (PNG),+ * width, height } when the host has a larger original; `thumb`: JPEG bytes.+ * Returns its meta { sha256, ext, mime, width, height, bytes, full?, thumb? }.+ */+ put({ bytes, mime, width, height, full, thumb, ocr, masked }) {+ const data = Buffer.from(bytes);+ const sha = sha256(data);+ const existing = spool.meta(sha);+ if (existing) return existing;+ const info = imageInfo(data);+ const type = info?.mime ?? mime;+ const ext = extensionOf(type);+ const meta = { sha256: sha, ext, mime: type, width: width ?? info?.width ?? 0, height: height ?? info?.height ?? 0, bytes: data.length };+ writeAtomic(path.join(dir, `${sha}.${ext}`), data);+ if (full?.bytes && (full.width * full.height > meta.width * meta.height)) {+ const fullBytes = Buffer.from(full.bytes);+ const fullSha = sha256(fullBytes);+ writeAtomic(path.join(dir, "full", `${sha}.png`), fullBytes);+ meta.full = { sha256: fullSha, width: full.width, height: full.height, bytes: fullBytes.length, ext: "png" };+ }+ const thumbBytes = thumb ? Buffer.from(thumb) : null;+ if (thumbBytes?.length) { writeAtomic(path.join(dir, "thumb", `${sha}.jpg`), thumbBytes); meta.thumb = thumbBytes.length; }+ if (ocr) meta.ocr = ocr;+ if (Number.isInteger(masked)) meta.masked = masked;+ writeAtomic(metaFile(sha), JSON.stringify(meta));+ return meta;+ },+ meta(sha) {+ if (!SHA.test(String(sha))) return null;+ const value = readJson(metaFile(sha));+ return isRecord(value) && value.sha256 === sha ? value : null;+ },+ modelPath(sha, ext) { return path.join(dir, `${sha}.${ext}`); },+ fullPath(sha) { return path.join(dir, "full", `${sha}.png`); },+ thumb(sha) { try { return readFileSync(path.join(dir, "thumb", `${sha}.jpg`)); } catch { return null; } },+ has(sha, ext) { return existsSync(path.join(dir, `${sha}.${ext}`)); },+ /** Deletes a frame's local files (after its upload). */+ remove(sha) {+ if (!SHA.test(String(sha))) return;+ const meta = spool.meta(sha);+ for (const file of [meta ? path.join(dir, `${sha}.${meta.ext}`) : null, path.join(dir, "full", `${sha}.png`), path.join(dir, "thumb", `${sha}.jpg`), metaFile(sha)]) if (file) rmSync(file, { force: true });+ },+ };+ return spool;+}++/**+ * The per-chat media index, persisted (it survives resumes): which frames+ * the chat has seen (a thumbnail is inlined only the first time), which went+ * to the archive, and the bytes used of each budget.+ */+export function createMediaIndex(file) {+ const loaded = readJson(file);+ const state = isRecord(loaded) && loaded.v === INDEX_VERSION+ ? { v: INDEX_VERSION, seen: isRecord(loaded.seen) ? loaded.seen : {}, model_bytes: Number(loaded.model_bytes) || 0, full_bytes: Number(loaded.full_bytes) || 0, frames: Number(loaded.frames) || 0 }+ : { v: INDEX_VERSION, seen: {}, model_bytes: 0, full_bytes: 0, frames: 0 };+ let dirty = false;+ return {+ file,+ state,+ entry(sha) { return state.seen[sha] ?? null; },+ /** Records a frame once; `stored`: { model, full } booleans. */+ add(sha, { model, full, modelBytes, fullBytes }) {+ if (state.seen[sha]) return state.seen[sha];+ const entry = { m: model ? 1 : 0, f: full ? 1 : 0 };+ state.seen[sha] = entry;+ state.frames += 1;+ if (model) state.model_bytes += modelBytes;+ if (full) state.full_bytes += fullBytes;+ dirty = true;+ return entry;+ },+ save() {+ if (!dirty || !file) return;+ writeAtomic(file, JSON.stringify(state));+ dirty = false;+ },+ };+}++function budgets(limits) {+ const l = isRecord(limits) ? limits : {};+ return {+ model: Number.isFinite(l.media_model_bytes) ? l.media_model_bytes : DEFAULT_LIMITS.media_model_bytes,+ full: Number.isFinite(l.media_full_bytes) ? l.media_full_bytes : DEFAULT_LIMITS.media_full_bytes,+ };+}++/**+ * Replaces image data URLs in the tool parts of one engine message (in place+ * on a structured copy; the input is not changed) with media references.+ * `ctx`: { spool, index, limits, thumbBudget }. Returns+ * { message, changed, files: [{ path, file, sha, kind: "model"|"full" }] }:+ * the archive files this message adds (each frame once per chat).+ * User file parts are left alone (the archive keeps those as attachments).+ */+export function rewriteMessageMedia(message, ctx) {+ if (!isRecord(message) || !Array.isArray(message.parts)) return { message, changed: false, files: [] };+ let changed = false;+ let thumbsLeft = Number.isFinite(ctx.thumbBudget) ? ctx.thumbBudget : THUMBS_PER_MESSAGE_BYTES;+ const files = [];+ const budget = budgets(ctx.limits);+ const parts = message.parts.map((part) => {+ const attachments = isRecord(part) && part.type === "tool" && isRecord(part.state) && Array.isArray(part.state.attachments) ? part.state.attachments : null;+ if (!attachments || !attachments.some((a) => isRecord(a) && typeof a.url === "string" && a.url.startsWith("data:image/"))) return part;+ const next = attachments.map((attachment) => {+ const parsed = isRecord(attachment) ? parseImageDataUrl(attachment.url) : null;+ if (!parsed) return attachment;+ const sha = sha256(parsed.bytes);+ let meta = ctx.spool.meta(sha);+ if (!meta) {+ const thumb = thumbnailOf(parsed.bytes, parsed.mime);+ meta = ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime, ...(thumb ? { thumb } : {}) });+ } else if (!ctx.spool.has(sha, meta.ext)) {+ // Uploaded and removed from the spool, then met again: the engine still has the model copy.+ meta = { ...ctx.spool.put({ bytes: parsed.bytes, mime: parsed.mime }), ...(meta.full ? { full: meta.full } : {}) };+ }+ let entry = ctx.index.entry(sha);+ const first = !entry;+ if (first) {+ const model = ctx.index.state.model_bytes + meta.bytes <= budget.model;+ const full = Boolean(meta.full) && model && ctx.index.state.full_bytes + meta.full.bytes <= budget.full && existsSync(ctx.spool.fullPath(sha));+ entry = ctx.index.add(sha, { model, full, modelBytes: meta.bytes, fullBytes: meta.full?.bytes ?? 0 });+ const paths = mediaPaths(sha, meta.ext);+ if (model) files.push({ path: paths.model, file: ctx.spool.modelPath(sha, meta.ext), sha, kind: "model", bytes: meta.bytes });+ if (full) files.push({ path: paths.full, file: ctx.spool.fullPath(sha), sha, kind: "full", bytes: meta.full.bytes });+ }+ const { url: _url, ...rest } = attachment;+ const ref = { ...rest, mime: meta.mime, url: mediaUrl(sha, meta.ext), sha256: sha, width: meta.width, height: meta.height, bytes: meta.bytes };+ if (meta.full) ref.full = entry.f ? { ...meta.full } : { omitted: "budget" };+ if (!entry.m) ref.omitted = "budget";+ if (first) {+ const thumb = ctx.spool.thumb(sha);+ if (thumb && thumb.length * 4 / 3 + 32 <= thumbsLeft) {+ ref.thumb = `data:image/jpeg;base64,${thumb.toString("base64")}`;+ thumbsLeft -= ref.thumb.length;+ }+ }+ changed = true;+ return ref;+ });+ return { ...part, state: { ...part.state, attachments: next } };+ });+ if (changed) ctx.index.save();+ return { message: changed ? { ...message, parts } : message, changed, files };+}++/** Every media reference in a message's tool parts ([{ sha, ext }]). */+export function mediaRefsOf(message) {+ const out = [];+ for (const part of isRecord(message) && Array.isArray(message.parts) ? message.parts : []) {+ const attachments = isRecord(part) && isRecord(part.state) && Array.isArray(part.state.attachments) ? part.state.attachments : [];+ for (const a of attachments) { const ref = isRecord(a) ? parseMediaUrl(a.url) : null; if (ref) out.push(ref); }+ }+ return out;+}++/** Bytes of a spool file, or 0. */+export function fileBytes(file) {+ try { return statSync(file).size; } catch { return 0; }+}++// ------------------------------------------------------- model context pruning++export const KEEP_IMAGES = 12;+export const DROP_IMAGES = 8;++/**+ * Prunes screenshots from the MODEL context (the engine plugin hook+ * `experimental.chat.messages.transform`; the stored chat is untouched):+ * keeps up to KEEP_IMAGES tool-result images; when the count passes it, the+ * oldest DROP_IMAGES at a time are replaced by a text note, so the cached+ * prefix stays stable for several steps. Mutates `messages` in place and+ * returns the number of images dropped. `messages`: engine WithParts list.+ */+export function pruneImages(messages, { keep = KEEP_IMAGES, drop = DROP_IMAGES } = {}) {+ const found = [];+ for (const message of Array.isArray(messages) ? messages : []) {+ for (const part of isRecord(message) && Array.isArray(message.parts) ? message.parts : []) {+ if (!isRecord(part) || part.type !== "tool" || !isRecord(part.state) || !Array.isArray(part.state.attachments)) continue;+ part.state.attachments.forEach((a, i) => { if (isRecord(a) && typeof a.mime === "string" && a.mime.startsWith("image/")) found.push({ part, index: i, attachment: a }); });+ }+ }+ if (found.length <= keep) return 0;+ // Drop in blocks of `drop`: the number dropped only changes every `drop` new images.+ const excess = found.length - keep;+ const count = Math.min(found.length, Math.ceil(excess / drop) * drop);+ const byPart = new Map();+ for (const item of found.slice(0, count)) {+ const list = byPart.get(item.part) ?? [];+ list.push(item);+ byPart.set(item.part, list);+ }+ for (const [part, items] of byPart) {+ const drops = new Set(items.map((i) => i.index));+ const notes = items.map(({ attachment }) => {+ const sha = typeof attachment.sha256 === "string" ? attachment.sha256 : parseImageDataUrl(attachment.url) ? sha256(parseImageDataUrl(attachment.url).bytes) : "";+ return `[screenshot ${sha.slice(0, 8)} omitted]`;+ });+ part.state.attachments = part.state.attachments.filter((_, i) => !drops.has(i));+ if (typeof part.state.output === "string") part.state.output = `${part.state.output}\n${notes.join("\n")}`;+ }+ return count;+}