Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Downscale oversized image attachments before model requests (#441)

Regan Bell7a84e9b4b7b2
diff --git a/src/core/attachments.ts b/src/core/attachments.ts
index 864dac552..e881dd305 100644
--- a/src/core/attachments.ts
+++ b/src/core/attachments.ts
@@ -15,6 +15,7 @@ import { parseRef } from "../acl/resource-ref.ts";
import { swallowAs } from "../util/errors.ts";
import { hashId } from "../util/crypto.ts";
import type { SecurityScreenVerdict } from "../security/security-posture.ts";
+import { downscaleVisionImage } from "./image-downscale.ts";
export const INBOX_DIR = "inbox";
export const OUTBOX_DIR = "outbox";
@@ -94,6 +95,7 @@ const MIME_BY_EXT: Record<string, string> = {
jpg: "image/jpeg",
jpeg: "image/jpeg",
gif: "image/gif",
+ webp: "image/webp",
svg: "image/svg+xml",
zip: "application/zip",
};
@@ -323,10 +325,11 @@ export async function materializeInbound(
...(registered ? { artifactId: registered.id } : {}),
});
if (VISION_MIME_TYPES.has(mimetype) && bytes.length > 0 && bytes.length <= MAX_VISION_IMAGE_BYTES) {
+ const imageBytes = await downscaleVisionImage(bytes, mimetype);
images.push({
name,
mimeType: mimetype,
- dataBase64: Buffer.from(bytes).toString("base64"),
+ dataBase64: Buffer.from(imageBytes).toString("base64"),
...(registered ? { artifactId: registered.id } : {}),
});
}
diff --git a/src/core/image-downscale.ts b/src/core/image-downscale.ts
new file mode 100644
index 000000000..7c29982d3
--- /dev/null
+++ b/src/core/image-downscale.ts
@@ -0,0 +1,219 @@
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+
+export const MAX_VISION_IMAGE_DIMENSION = 1568;
+
+export interface ImageDimensions {
+ width: number;
+ height: number;
+ format: "png" | "jpeg" | "gif" | "webp";
+}
+
+export interface ImageDownscaleDeps {
+ spawn: (
+ command: string,
+ args: string[],
+ options: { stdio: ["pipe", "pipe", "pipe"] },
+ ) => ChildProcessWithoutNullStreams;
+ warn: (message: string) => void;
+}
+
+const DEFAULT_DEPS: ImageDownscaleDeps = {
+ spawn,
+ warn: (message) => console.warn(message),
+};
+
+const FORMAT_BY_MIME = new Map<string, ImageDimensions["format"]>([
+ ["image/png", "png"],
+ ["image/jpeg", "jpeg"],
+ ["image/gif", "gif"],
+ ["image/webp", "webp"],
+]);
+
+const JPEG_SOF_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);
+
+function readUInt24LE(bytes: Uint8Array, offset: number): number {
+ return bytes[offset]! | (bytes[offset + 1]! << 8) | (bytes[offset + 2]! << 16);
+}
+
+function pngDimensions(bytes: Uint8Array): ImageDimensions | undefined {
+ if (bytes.length < 24) return undefined;
+ const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+ if (!signature.every((v, i) => bytes[i] === v)) return undefined;
+ if (Buffer.from(bytes.subarray(12, 16)).toString("ascii") !== "IHDR") return undefined;
+ return { width: Buffer.from(bytes).readUInt32BE(16), height: Buffer.from(bytes).readUInt32BE(20), format: "png" };
+}
+
+function gifDimensions(bytes: Uint8Array): ImageDimensions | undefined {
+ if (bytes.length < 10) return undefined;
+ const header = Buffer.from(bytes.subarray(0, 6)).toString("ascii");
+ if (header !== "GIF87a" && header !== "GIF89a") return undefined;
+ return { width: Buffer.from(bytes).readUInt16LE(6), height: Buffer.from(bytes).readUInt16LE(8), format: "gif" };
+}
+
+function jpegDimensions(bytes: Uint8Array): ImageDimensions | undefined {
+ if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return undefined;
+ let offset = 2;
+ while (offset + 3 < bytes.length) {
+ while (offset < bytes.length && bytes[offset] !== 0xff) offset++;
+ while (offset < bytes.length && bytes[offset] === 0xff) offset++;
+ if (offset >= bytes.length) return undefined;
+ const marker = bytes[offset]!;
+ offset++;
+ if (marker === 0xd9 || marker === 0xda) return undefined;
+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
+ if (offset + 2 > bytes.length) return undefined;
+ const length = Buffer.from(bytes).readUInt16BE(offset);
+ if (length < 2 || offset + length > bytes.length) return undefined;
+ const start = offset + 2;
+ if (JPEG_SOF_MARKERS.has(marker)) {
+ if (start + 5 > bytes.length) return undefined;
+ return {
+ width: Buffer.from(bytes).readUInt16BE(start + 3),
+ height: Buffer.from(bytes).readUInt16BE(start + 1),
+ format: "jpeg",
+ };
+ }
+ offset += length;
+ }
+ return undefined;
+}
+
+function webpDimensions(bytes: Uint8Array): ImageDimensions | undefined {
+ if (bytes.length < 30) return undefined;
+ if (Buffer.from(bytes.subarray(0, 4)).toString("ascii") !== "RIFF") return undefined;
+ if (Buffer.from(bytes.subarray(8, 12)).toString("ascii") !== "WEBP") return undefined;
+ let offset = 12;
+ while (offset + 8 <= bytes.length) {
+ const type = Buffer.from(bytes.subarray(offset, offset + 4)).toString("ascii");
+ const length = Buffer.from(bytes).readUInt32LE(offset + 4);
+ const start = offset + 8;
+ if (start + length > bytes.length) return undefined;
+ if (type === "VP8X" && length >= 10) {
+ return { width: readUInt24LE(bytes, start + 4) + 1, height: readUInt24LE(bytes, start + 7) + 1, format: "webp" };
+ }
+ if (type === "VP8L" && length >= 5 && bytes[start] === 0x2f) {
+ const b0 = bytes[start + 1]!;
+ const b1 = bytes[start + 2]!;
+ const b2 = bytes[start + 3]!;
+ const b3 = bytes[start + 4]!;
+ return {
+ width: 1 + b0 + ((b1 & 0x3f) << 8),
+ height: 1 + ((b1 & 0xc0) >> 6) + (b2 << 2) + ((b3 & 0x0f) << 10),
+ format: "webp",
+ };
+ }
+ if (
+ type === "VP8 " &&
+ length >= 10 &&
+ bytes[start + 3] === 0x9d &&
+ bytes[start + 4] === 0x01 &&
+ bytes[start + 5] === 0x2a
+ ) {
+ return {
+ width: Buffer.from(bytes).readUInt16LE(start + 6) & 0x3fff,
+ height: Buffer.from(bytes).readUInt16LE(start + 8) & 0x3fff,
+ format: "webp",
+ };
+ }
+ offset = start + length + (length % 2);
+ }
+ return undefined;
+}
+
+export function sniffImageDimensions(bytes: Uint8Array): ImageDimensions | undefined {
+ return pngDimensions(bytes) ?? jpegDimensions(bytes) ?? gifDimensions(bytes) ?? webpDimensions(bytes);
+}
+
+function imageMagickArgs(format: ImageDimensions["format"]): string[] {
+ return ["-", "-resize", `${MAX_VISION_IMAGE_DIMENSION}x${MAX_VISION_IMAGE_DIMENSION}>`, `${format}:-`];
+}
+
+function ffmpegArgs(format: ImageDimensions["format"]): string[] {
+ const codec = format === "jpeg" ? "mjpeg" : format;
+ return [
+ "-hide_banner",
+ "-loglevel",
+ "error",
+ "-i",
+ "pipe:0",
+ "-vf",
+ `scale='if(gt(iw,ih),min(${MAX_VISION_IMAGE_DIMENSION},iw),-2)':'if(gt(iw,ih),-2,min(${MAX_VISION_IMAGE_DIMENSION},ih))'`,
+ "-frames:v",
+ "1",
+ "-f",
+ "image2pipe",
+ "-vcodec",
+ codec,
+ "pipe:1",
+ ];
+}
+
+async function runConverter(
+ bytes: Uint8Array,
+ command: string,
+ args: string[],
+ deps: ImageDownscaleDeps,
+): Promise<{ missing: boolean; output?: Uint8Array }> {
+ return await new Promise((resolve) => {
+ let settled = false;
+ const finish = (result: { missing: boolean; output?: Uint8Array }) => {
+ if (settled) return;
+ settled = true;
+ resolve(result);
+ };
+ let child: ChildProcessWithoutNullStreams;
+ try {
+ child = deps.spawn(command, args, { stdio: ["pipe", "pipe", "pipe"] });
+ } catch (err) {
+ finish({ missing: (err as NodeJS.ErrnoException).code === "ENOENT" });
+ return;
+ }
+ const stdout: Buffer[] = [];
+ child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
+ child.stderr.resume();
+ child.stdin.on("error", () => {});
+ child.on("error", (err: NodeJS.ErrnoException) => finish({ missing: err.code === "ENOENT" }));
+ child.on("close", (code) => {
+ if (code === 0) finish({ missing: false, output: Buffer.concat(stdout) });
+ else finish({ missing: false });
+ });
+ child.stdin.end(Buffer.from(bytes));
+ });
+}
+
+function outputIsUsable(bytes: Uint8Array): boolean {
+ const dimensions = sniffImageDimensions(bytes);
+ return (
+ bytes.length > 0 &&
+ (!dimensions || (dimensions.width <= MAX_VISION_IMAGE_DIMENSION && dimensions.height <= MAX_VISION_IMAGE_DIMENSION))
+ );
+}
+
+export async function downscaleVisionImage(
+ bytes: Uint8Array,
+ mimeType: string,
+ deps: ImageDownscaleDeps = DEFAULT_DEPS,
+): Promise<Uint8Array> {
+ const expectedFormat = FORMAT_BY_MIME.get(mimeType);
+ if (!expectedFormat) return bytes;
+ const dimensions = sniffImageDimensions(bytes);
+ if (!dimensions) return bytes;
+ if (dimensions.width <= MAX_VISION_IMAGE_DIMENSION && dimensions.height <= MAX_VISION_IMAGE_DIMENSION) return bytes;
+ const attempts: Array<[string, string[]]> = [
+ ["magick", imageMagickArgs(expectedFormat)],
+ ["convert", imageMagickArgs(expectedFormat)],
+ ["ffmpeg", ffmpegArgs(expectedFormat)],
+ ];
+ let sawConverter = false;
+ for (const [command, args] of attempts) {
+ const result = await runConverter(bytes, command, args, deps);
+ if (!result.missing) sawConverter = true;
+ if (result.output && outputIsUsable(result.output)) return result.output;
+ }
+ deps.warn(
+ sawConverter
+ ? "attachments: oversized image was not downscaled because image conversion failed"
+ : "attachments: oversized image was not downscaled because no image converter was found on PATH",
+ );
+ return bytes;
+}
diff --git a/test/file-sharing.test.ts b/test/file-sharing.test.ts
index c26a34cc0..47f791656 100644
--- a/test/file-sharing.test.ts
+++ b/test/file-sharing.test.ts
@@ -90,6 +90,7 @@ test("safeAttachmentName strips directories (no traversal) and keeps unicode", (
test("mimeFromName maps known extensions, defaults to octet-stream", () => {
assert.equal(mimeFromName("a.csv"), "text/csv");
assert.equal(mimeFromName("a.PNG"), "image/png");
+ assert.equal(mimeFromName("a.webp"), "image/webp");
assert.equal(mimeFromName("noext"), "application/octet-stream");
});
diff --git a/test/image-downscale.test.ts b/test/image-downscale.test.ts
new file mode 100644
index 000000000..f9d33808b
--- /dev/null
+++ b/test/image-downscale.test.ts
@@ -0,0 +1,157 @@
+import { EventEmitter } from "node:events";
+import { PassThrough } from "node:stream";
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { ChildProcessWithoutNullStreams } from "node:child_process";
+import {
+ downscaleVisionImage,
+ MAX_VISION_IMAGE_DIMENSION,
+ sniffImageDimensions,
+ type ImageDownscaleDeps,
+} from "../src/core/image-downscale.ts";
+
+interface SpawnCall {
+ command: string;
+ args: string[];
+}
+
+type FakeChild = EventEmitter & {
+ stdin: PassThrough;
+ stdout: PassThrough;
+ stderr: PassThrough;
+};
+
+function png(width: number, height: number): Buffer {
+ const bytes = Buffer.alloc(24);
+ Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(bytes, 0);
+ bytes.writeUInt32BE(13, 8);
+ bytes.write("IHDR", 12, "ascii");
+ bytes.writeUInt32BE(width, 16);
+ bytes.writeUInt32BE(height, 20);
+ return bytes;
+}
+
+function gif(width: number, height: number): Buffer {
+ const bytes = Buffer.alloc(10);
+ bytes.write("GIF89a", 0, "ascii");
+ bytes.writeUInt16LE(width, 6);
+ bytes.writeUInt16LE(height, 8);
+ return bytes;
+}
+
+function jpeg(width: number, height: number): Buffer {
+ const bytes = Buffer.from("ffd8ffe000040000ffc0000b080000000001011100ffd9", "hex");
+ bytes.writeUInt16BE(height, 13);
+ bytes.writeUInt16BE(width, 15);
+ return bytes;
+}
+
+function webp(width: number, height: number): Buffer {
+ const bytes = Buffer.alloc(30);
+ bytes.write("RIFF", 0, "ascii");
+ bytes.writeUInt32LE(22, 4);
+ bytes.write("WEBP", 8, "ascii");
+ bytes.write("VP8X", 12, "ascii");
+ bytes.writeUInt32LE(10, 16);
+ const encodedWidth = width - 1;
+ const encodedHeight = height - 1;
+ bytes[24] = encodedWidth & 0xff;
+ bytes[25] = (encodedWidth >> 8) & 0xff;
+ bytes[26] = (encodedWidth >> 16) & 0xff;
+ bytes[27] = encodedHeight & 0xff;
+ bytes[28] = (encodedHeight >> 8) & 0xff;
+ bytes[29] = (encodedHeight >> 16) & 0xff;
+ return bytes;
+}
+
+function fakeChild(): FakeChild {
+ const child = new EventEmitter() as FakeChild;
+ child.stdin = new PassThrough();
+ child.stdout = new PassThrough();
+ child.stderr = new PassThrough();
+ return child;
+}
+
+function depsFromHandlers(
+ handlers: Array<(child: FakeChild, call: SpawnCall) => void>,
+ warnings: string[] = [],
+ calls: SpawnCall[] = [],
+): ImageDownscaleDeps {
+ return {
+ spawn(command, args) {
+ const child = fakeChild();
+ const call = { command, args };
+ calls.push(call);
+ const handler = handlers.shift();
+ process.nextTick(() => handler?.(child, call));
+ return child as unknown as ChildProcessWithoutNullStreams;
+ },
+ warn(message) {
+ warnings.push(message);
+ },
+ };
+}
+
+test("sniffImageDimensions reads PNG, JPEG, GIF, and WebP headers", () => {
+ assert.deepEqual(sniffImageDimensions(png(640, 480)), { width: 640, height: 480, format: "png" });
+ assert.deepEqual(sniffImageDimensions(jpeg(800, 600)), { width: 800, height: 600, format: "jpeg" });
+ assert.deepEqual(sniffImageDimensions(gif(320, 240)), { width: 320, height: 240, format: "gif" });
+ assert.deepEqual(sniffImageDimensions(webp(1024, 768)), { width: 1024, height: 768, format: "webp" });
+});
+
+test("downscaleVisionImage leaves small images untouched without spawning a converter", async () => {
+ const small = png(MAX_VISION_IMAGE_DIMENSION, 900);
+ let spawned = false;
+ const result = await downscaleVisionImage(small, "image/png", {
+ spawn() {
+ spawned = true;
+ return fakeChild() as unknown as ChildProcessWithoutNullStreams;
+ },
+ warn() {
+ assert.fail("small images should not warn");
+ },
+ });
+ assert.equal(result, small);
+ assert.equal(spawned, false);
+});
+
+test("downscaleVisionImage uses a converter for oversized images", async () => {
+ const output = png(1200, 900);
+ const calls: SpawnCall[] = [];
+ const deps = depsFromHandlers(
+ [
+ (child) => {
+ child.stdout.end(output);
+ child.emit("close", 0);
+ },
+ ],
+ [],
+ calls,
+ );
+ const result = await downscaleVisionImage(png(2400, 1800), "image/png", deps);
+ assert.deepEqual(Buffer.from(result), output);
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0]!.command, "magick");
+ assert.ok(calls[0]!.args.includes(`${MAX_VISION_IMAGE_DIMENSION}x${MAX_VISION_IMAGE_DIMENSION}>`));
+ assert.ok(calls[0]!.args.includes("png:-"));
+});
+
+test("downscaleVisionImage falls back to original bytes when no converter is available", async () => {
+ const original = png(3000, 2000);
+ const warnings: string[] = [];
+ const calls: SpawnCall[] = [];
+ const missing = (child: FakeChild) => {
+ const err = new Error("missing") as NodeJS.ErrnoException;
+ err.code = "ENOENT";
+ child.emit("error", err);
+ };
+ const deps = depsFromHandlers([missing, missing, missing], warnings, calls);
+ const result = await downscaleVisionImage(original, "image/png", deps);
+ assert.equal(result, original);
+ assert.deepEqual(
+ calls.map((c) => c.command),
+ ["magick", "convert", "ffmpeg"],
+ );
+ assert.equal(warnings.length, 1);
+ assert.match(warnings[0]!, /no image converter was found on PATH/);
+});