Multiplayer agent harness for work.
Downscale oversized image attachments before model requests (#441)
Regan Bell7a84e9b4b7b2
diff --git a/src/core/attachments.ts b/src/core/attachments.tsindex 864dac552..e881dd305 100644--- a/src/core/attachments.ts+++ b/src/core/attachments.ts@@ -15,6 +15,7 @@ import { parseRef } from "../acl/resource-ref.ts";import { swallowAs } from "../util/errors.ts";import { hashId } from "../util/crypto.ts";import type { SecurityScreenVerdict } from "../security/security-posture.ts";+import { downscaleVisionImage } from "./image-downscale.ts";export const INBOX_DIR = "inbox";export const OUTBOX_DIR = "outbox";@@ -94,6 +95,7 @@ const MIME_BY_EXT: Record<string, string> = {jpg: "image/jpeg",jpeg: "image/jpeg",gif: "image/gif",+ webp: "image/webp",svg: "image/svg+xml",zip: "application/zip",};@@ -323,10 +325,11 @@ export async function materializeInbound(...(registered ? { artifactId: registered.id } : {}),});if (VISION_MIME_TYPES.has(mimetype) && bytes.length > 0 && bytes.length <= MAX_VISION_IMAGE_BYTES) {+ const imageBytes = await downscaleVisionImage(bytes, mimetype);images.push({name,mimeType: mimetype,- dataBase64: Buffer.from(bytes).toString("base64"),+ dataBase64: Buffer.from(imageBytes).toString("base64"),...(registered ? { artifactId: registered.id } : {}),});}diff --git a/src/core/image-downscale.ts b/src/core/image-downscale.tsnew file mode 100644index 000000000..7c29982d3--- /dev/null+++ b/src/core/image-downscale.ts@@ -0,0 +1,219 @@+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";++export const MAX_VISION_IMAGE_DIMENSION = 1568;++export interface ImageDimensions {+ width: number;+ height: number;+ format: "png" | "jpeg" | "gif" | "webp";+}++export interface ImageDownscaleDeps {+ spawn: (+ command: string,+ args: string[],+ options: { stdio: ["pipe", "pipe", "pipe"] },+ ) => ChildProcessWithoutNullStreams;+ warn: (message: string) => void;+}++const DEFAULT_DEPS: ImageDownscaleDeps = {+ spawn,+ warn: (message) => console.warn(message),+};++const FORMAT_BY_MIME = new Map<string, ImageDimensions["format"]>([+ ["image/png", "png"],+ ["image/jpeg", "jpeg"],+ ["image/gif", "gif"],+ ["image/webp", "webp"],+]);++const JPEG_SOF_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);++function readUInt24LE(bytes: Uint8Array, offset: number): number {+ return bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16);+}++function pngDimensions(bytes: Uint8Array): ImageDimensions | undefined {+ if (bytes.length < 24) return undefined;+ const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];+ if (!signature.every((v, i) => bytes[i] === v)) return undefined;+ if (Buffer.from(bytes.subarray(12, 16)).toString("ascii") !== "IHDR") return undefined;+ return { width: Buffer.from(bytes).readUInt32BE(16), height: Buffer.from(bytes).readUInt32BE(20), format: "png" };+}++function gifDimensions(bytes: Uint8Array): ImageDimensions | undefined {+ if (bytes.length < 10) return undefined;+ const header = Buffer.from(bytes.subarray(0, 6)).toString("ascii");+ if (header !== "GIF87a" && header !== "GIF89a") return undefined;+ return { width: Buffer.from(bytes).readUInt16LE(6), height: Buffer.from(bytes).readUInt16LE(8), format: "gif" };+}++function jpegDimensions(bytes: Uint8Array): ImageDimensions | undefined {+ if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined;+ let offset = 2;+ while (offset + 3 < bytes.length) {+ while (offset < bytes.length && bytes[offset] !== 0xff) offset++;+ while (offset < bytes.length && bytes[offset] === 0xff) offset++;+ if (offset >= bytes.length) return undefined;+ const marker = bytes[offset]!;+ offset++;+ if (marker === 0xd9 || marker === 0xda) return undefined;+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;+ if (offset + 2 > bytes.length) return undefined;+ const length = Buffer.from(bytes).readUInt16BE(offset);+ if (length < 2 || offset + length > bytes.length) return undefined;+ const start = offset + 2;+ if (JPEG_SOF_MARKERS.has(marker)) {+ if (start + 5 > bytes.length) return undefined;+ return {+ width: Buffer.from(bytes).readUInt16BE(start + 3),+ height: Buffer.from(bytes).readUInt16BE(start + 1),+ format: "jpeg",+ };+ }+ offset += length;+ }+ return undefined;+}++function webpDimensions(bytes: Uint8Array): ImageDimensions | undefined {+ if (bytes.length < 30) return undefined;+ if (Buffer.from(bytes.subarray(0, 4)).toString("ascii") !== "RIFF") return undefined;+ if (Buffer.from(bytes.subarray(8, 12)).toString("ascii") !== "WEBP") return undefined;+ let offset = 12;+ while (offset + 8 <= bytes.length) {+ const type = Buffer.from(bytes.subarray(offset, offset + 4)).toString("ascii");+ const length = Buffer.from(bytes).readUInt32LE(offset + 4);+ const start = offset + 8;+ if (start + length > bytes.length) return undefined;+ if (type === "VP8X" && length >= 10) {+ return { width: readUInt24LE(bytes, start + 4) + 1, height: readUInt24LE(bytes, start + 7) + 1, format: "webp" };+ }+ if (type === "VP8L" && length >= 5 && bytes[start] === 0x2f) {+ const b0 = bytes[start + 1]!;+ const b1 = bytes[start + 2]!;+ const b2 = bytes[start + 3]!;+ const b3 = bytes[start + 4]!;+ return {+ width: 1 + b0 + ((b1 & 0x3f) << 8),+ height: 1 + ((b1 & 0xc0) >> 6) + (b2 << 2) + ((b3 & 0x0f) << 10),+ format: "webp",+ };+ }+ if (+ type === "VP8 " &&+ length >= 10 &&+ bytes[start + 3] === 0x9d &&+ bytes[start + 4] === 0x01 &&+ bytes[start + 5] === 0x2a+ ) {+ return {+ width: Buffer.from(bytes).readUInt16LE(start + 6) & 0x3fff,+ height: Buffer.from(bytes).readUInt16LE(start + 8) & 0x3fff,+ format: "webp",+ };+ }+ offset = start + length + (length % 2);+ }+ return undefined;+}++export function sniffImageDimensions(bytes: Uint8Array): ImageDimensions | undefined {+ return pngDimensions(bytes) ?? jpegDimensions(bytes) ?? gifDimensions(bytes) ?? webpDimensions(bytes);+}++function imageMagickArgs(format: ImageDimensions["format"]): string[] {+ return ["-", "-resize", `${MAX_VISION_IMAGE_DIMENSION}x${MAX_VISION_IMAGE_DIMENSION}>`, `${format}:-`];+}++function ffmpegArgs(format: ImageDimensions["format"]): string[] {+ const codec = format === "jpeg" ? "mjpeg" : format;+ return [+ "-hide_banner",+ "-loglevel",+ "error",+ "-i",+ "pipe:0",+ "-vf",+ `scale='if(gt(iw,ih),min(${MAX_VISION_IMAGE_DIMENSION},iw),-2)':'if(gt(iw,ih),-2,min(${MAX_VISION_IMAGE_DIMENSION},ih))'`,+ "-frames:v",+ "1",+ "-f",+ "image2pipe",+ "-vcodec",+ codec,+ "pipe:1",+ ];+}++async function runConverter(+ bytes: Uint8Array,+ command: string,+ args: string[],+ deps: ImageDownscaleDeps,+): Promise<{ missing: boolean; output?: Uint8Array }> {+ return await new Promise((resolve) => {+ let settled = false;+ const finish = (result: { missing: boolean; output?: Uint8Array }) => {+ if (settled) return;+ settled = true;+ resolve(result);+ };+ let child: ChildProcessWithoutNullStreams;+ try {+ child = deps.spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });+ } catch (err) {+ finish({ missing: (err as NodeJS.ErrnoException).code === "ENOENT" });+ return;+ }+ const stdout: Buffer[] = [];+ child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));+ child.stderr.resume();+ child.stdin.on("error", () => {});+ child.on("error", (err: NodeJS.ErrnoException) => finish({ missing: err.code === "ENOENT" }));+ child.on("close", (code) => {+ if (code === 0) finish({ missing: false, output: Buffer.concat(stdout) });+ else finish({ missing: false });+ });+ child.stdin.end(Buffer.from(bytes));+ });+}++function outputIsUsable(bytes: Uint8Array): boolean {+ const dimensions = sniffImageDimensions(bytes);+ return (+ bytes.length > 0 &&+ (!dimensions || (dimensions.width <= MAX_VISION_IMAGE_DIMENSION && dimensions.height <= MAX_VISION_IMAGE_DIMENSION))+ );+}++export async function downscaleVisionImage(+ bytes: Uint8Array,+ mimeType: string,+ deps: ImageDownscaleDeps = DEFAULT_DEPS,+): Promise<Uint8Array> {+ const expectedFormat = FORMAT_BY_MIME.get(mimeType);+ if (!expectedFormat) return bytes;+ const dimensions = sniffImageDimensions(bytes);+ if (!dimensions) return bytes;+ if (dimensions.width <= MAX_VISION_IMAGE_DIMENSION && dimensions.height <= MAX_VISION_IMAGE_DIMENSION) return bytes;+ const attempts: Array<[string, string[]]> = [+ ["magick", imageMagickArgs(expectedFormat)],+ ["convert", imageMagickArgs(expectedFormat)],+ ["ffmpeg", ffmpegArgs(expectedFormat)],+ ];+ let sawConverter = false;+ for (const [command, args] of attempts) {+ const result = await runConverter(bytes, command, args, deps);+ if (!result.missing) sawConverter = true;+ if (result.output && outputIsUsable(result.output)) return result.output;+ }+ deps.warn(+ sawConverter+ ? "attachments: oversized image was not downscaled because image conversion failed"+ : "attachments: oversized image was not downscaled because no image converter was found on PATH",+ );+ return bytes;+}diff --git a/test/file-sharing.test.ts b/test/file-sharing.test.tsindex c26a34cc0..47f791656 100644--- a/test/file-sharing.test.ts+++ b/test/file-sharing.test.ts@@ -90,6 +90,7 @@ test("safeAttachmentName strips directories (no traversal) and keeps unicode", (test("mimeFromName maps known extensions, defaults to octet-stream", () => {assert.equal(mimeFromName("a.csv"), "text/csv");assert.equal(mimeFromName("a.PNG"), "image/png");+ assert.equal(mimeFromName("a.webp"), "image/webp");assert.equal(mimeFromName("noext"), "application/octet-stream");});diff --git a/test/image-downscale.test.ts b/test/image-downscale.test.tsnew file mode 100644index 000000000..f9d33808b--- /dev/null+++ b/test/image-downscale.test.ts@@ -0,0 +1,157 @@+import { EventEmitter } from "node:events";+import { PassThrough } from "node:stream";+import { test } from "node:test";+import assert from "node:assert/strict";+import type { ChildProcessWithoutNullStreams } from "node:child_process";+import {+ downscaleVisionImage,+ MAX_VISION_IMAGE_DIMENSION,+ sniffImageDimensions,+ type ImageDownscaleDeps,+} from "../src/core/image-downscale.ts";++interface SpawnCall {+ command: string;+ args: string[];+}++type FakeChild = EventEmitter & {+ stdin: PassThrough;+ stdout: PassThrough;+ stderr: PassThrough;+};++function png(width: number, height: number): Buffer {+ const bytes = Buffer.alloc(24);+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(bytes, 0);+ bytes.writeUInt32BE(13, 8);+ bytes.write("IHDR", 12, "ascii");+ bytes.writeUInt32BE(width, 16);+ bytes.writeUInt32BE(height, 20);+ return bytes;+}++function gif(width: number, height: number): Buffer {+ const bytes = Buffer.alloc(10);+ bytes.write("GIF89a", 0, "ascii");+ bytes.writeUInt16LE(width, 6);+ bytes.writeUInt16LE(height, 8);+ return bytes;+}++function jpeg(width: number, height: number): Buffer {+ const bytes = Buffer.from("ffd8ffe000040000ffc0000b080000000001011100ffd9", "hex");+ bytes.writeUInt16BE(height, 13);+ bytes.writeUInt16BE(width, 15);+ return bytes;+}++function webp(width: number, height: number): Buffer {+ const bytes = Buffer.alloc(30);+ bytes.write("RIFF", 0, "ascii");+ bytes.writeUInt32LE(22, 4);+ bytes.write("WEBP", 8, "ascii");+ bytes.write("VP8X", 12, "ascii");+ bytes.writeUInt32LE(10, 16);+ const encodedWidth = width - 1;+ const encodedHeight = height - 1;+ bytes[24] = encodedWidth & 0xff;+ bytes[25] = (encodedWidth >> 8) & 0xff;+ bytes[26] = (encodedWidth >> 16) & 0xff;+ bytes[27] = encodedHeight & 0xff;+ bytes[28] = (encodedHeight >> 8) & 0xff;+ bytes[29] = (encodedHeight >> 16) & 0xff;+ return bytes;+}++function fakeChild(): FakeChild {+ const child = new EventEmitter() as FakeChild;+ child.stdin = new PassThrough();+ child.stdout = new PassThrough();+ child.stderr = new PassThrough();+ return child;+}++function depsFromHandlers(+ handlers: Array<(child: FakeChild, call: SpawnCall) => void>,+ warnings: string[] = [],+ calls: SpawnCall[] = [],+): ImageDownscaleDeps {+ return {+ spawn(command, args) {+ const child = fakeChild();+ const call = { command, args };+ calls.push(call);+ const handler = handlers.shift();+ process.nextTick(() => handler?.(child, call));+ return child as unknown as ChildProcessWithoutNullStreams;+ },+ warn(message) {+ warnings.push(message);+ },+ };+}++test("sniffImageDimensions reads PNG, JPEG, GIF, and WebP headers", () => {+ assert.deepEqual(sniffImageDimensions(png(640, 480)), { width: 640, height: 480, format: "png" });+ assert.deepEqual(sniffImageDimensions(jpeg(800, 600)), { width: 800, height: 600, format: "jpeg" });+ assert.deepEqual(sniffImageDimensions(gif(320, 240)), { width: 320, height: 240, format: "gif" });+ assert.deepEqual(sniffImageDimensions(webp(1024, 768)), { width: 1024, height: 768, format: "webp" });+});++test("downscaleVisionImage leaves small images untouched without spawning a converter", async () => {+ const small = png(MAX_VISION_IMAGE_DIMENSION, 900);+ let spawned = false;+ const result = await downscaleVisionImage(small, "image/png", {+ spawn() {+ spawned = true;+ return fakeChild() as unknown as ChildProcessWithoutNullStreams;+ },+ warn() {+ assert.fail("small images should not warn");+ },+ });+ assert.equal(result, small);+ assert.equal(spawned, false);+});++test("downscaleVisionImage uses a converter for oversized images", async () => {+ const output = png(1200, 900);+ const calls: SpawnCall[] = [];+ const deps = depsFromHandlers(+ [+ (child) => {+ child.stdout.end(output);+ child.emit("close", 0);+ },+ ],+ [],+ calls,+ );+ const result = await downscaleVisionImage(png(2400, 1800), "image/png", deps);+ assert.deepEqual(Buffer.from(result), output);+ assert.equal(calls.length, 1);+ assert.equal(calls[0]!.command, "magick");+ assert.ok(calls[0]!.args.includes(`${MAX_VISION_IMAGE_DIMENSION}x${MAX_VISION_IMAGE_DIMENSION}>`));+ assert.ok(calls[0]!.args.includes("png:-"));+});++test("downscaleVisionImage falls back to original bytes when no converter is available", async () => {+ const original = png(3000, 2000);+ const warnings: string[] = [];+ const calls: SpawnCall[] = [];+ const missing = (child: FakeChild) => {+ const err = new Error("missing") as NodeJS.ErrnoException;+ err.code = "ENOENT";+ child.emit("error", err);+ };+ const deps = depsFromHandlers([missing, missing, missing], warnings, calls);+ const result = await downscaleVisionImage(original, "image/png", deps);+ assert.equal(result, original);+ assert.deepEqual(+ calls.map((c) => c.command),+ ["magick", "convert", "ffmpeg"],+ );+ assert.equal(warnings.length, 1);+ assert.match(warnings[0]!, /no image converter was found on PATH/);+});