Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(web): safely attach HEIC photos as JPEG images (#8161)
Theo Brownebd9ed2b4bbda
diff --git a/apps/web/package.json b/apps/web/package.jsonindex 598feaec0..ea6b00eff 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -34,6 +34,7 @@"class-variance-authority": "^0.7.1","culori": "^4.0.2","effect": "catalog:",+ "heic-to": "^1.5.2","jose": "catalog:","jsonc-parser": "3.3.1","jszip": "3.10.1",diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsxindex 1ec58e0de..841bbb8f4 100644--- a/apps/web/src/components/chat/ChatComposer.tsx+++ b/apps/web/src/components/chat/ChatComposer.tsx@@ -73,7 +73,11 @@ import {type ComposerTaskStep,type ComposerTasksProgress,} from "./ComposerTasksBadge";-import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression";+import {+ compressImageForStash,+ isHeicImageFile,+ prepareImageForAttachment,+} from "../../lib/imageCompression";import {releaseAttachmentUpload,retryAttachmentUpload,@@ -2537,12 +2541,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)const acceptedFiles: File[] = [];let error: string | null = null;for (const file of files) {- if (!file.type.startsWith("image/")) {+ const isHeicImage = isHeicImageFile(file);+ if (!file.type.startsWith("image/") && !isHeicImage) {error = `Unsupported file type for '${file.name}'. Please attach image files only.`;continue;}- if (!isProviderSendTurnSupportedImageMimeType(file.type)) {- error = `'${file.name}' is not a supported image type. Attach GIF, JPEG, PNG, or WebP images.`;+ if (!isHeicImage && !isProviderSendTurnSupportedImageMimeType(file.type)) {+ error = `'${file.name}' is not a supported image type. Attach GIF, HEIC, HEIF, JPEG, PNG, or WebP images.`;continue;}if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {@@ -2562,7 +2567,10 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)for (const file of acceptedFiles) {// Images over the wire cap are downscaled to fit rather than// refused; files already within it pass through byte-for-byte.- const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);+ const compressed = await prepareImageForAttachment(+ file,+ PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,+ );if (!compressed.ok) {compressionError =compressed.reason === "unreadable"@@ -2615,7 +2623,9 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)const onComposerPaste = (event: React.ClipboardEvent<HTMLElement>) => {const files = Array.from(event.clipboardData.files);if (files.length === 0) return;- const imageFiles = files.filter((file) => file.type.startsWith("image/"));+ const imageFiles = files.filter(+ (file) => file.type.startsWith("image/") || isHeicImageFile(file),+ );if (imageFiles.length === 0) return;event.preventDefault();void addComposerImages(imageFiles);diff --git a/apps/web/src/lib/imageCompression.test.ts b/apps/web/src/lib/imageCompression.test.tsindex 63712ca7e..435c51ef8 100644--- a/apps/web/src/lib/imageCompression.test.ts+++ b/apps/web/src/lib/imageCompression.test.ts@@ -3,10 +3,20 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test";import {compressImageForStash,compressImageToByteLimit,+ isHeicImageFile,MAX_COMPRESSIBLE_SOURCE_BYTES,MAX_STASH_IMAGE_DATA_URL_CHARS,+ prepareImageForAttachment,} from "./imageCompression";+const mocks = vi.hoisted(() => ({+ heicTo: vi.fn(),+}));++vi.mock("heic-to/csp", () => ({+ heicTo: mocks.heicTo,+}));+/*** jsdom has no real canvas/codec, so the re-encode path is exercised with* stubbed `createImageBitmap` + `OffscreenCanvas`. The encoder stub returns a@@ -22,6 +32,45 @@ function makeFile(sizeBytes: number, type = "image/png"): File {return new File([new Uint8Array(sizeBytes).fill(7)], "shot.png", { type });}+function makeHeicFile(options?: {+ name?: string;+ type?: string;+ width?: number;+ height?: number;+ lastModified?: number;+}): File {+ const encoder = new TextEncoder();+ const makeBox = (name: string, ...contents: Uint8Array[]) => {+ const bytes = new Uint8Array(8 + contents.reduce((size, content) => size + content.length, 0));+ new DataView(bytes.buffer).setUint32(0, bytes.length);+ bytes.set(encoder.encode(name), 4);+ let offset = 8;+ for (const content of contents) {+ bytes.set(content, offset);+ offset += content.length;+ }+ return bytes;+ };++ const dimensions = new Uint8Array(12);+ const view = new DataView(dimensions.buffer);+ view.setUint32(4, options?.width ?? 4000);+ view.setUint32(8, options?.height ?? 3000);+ const properties = makeBox("iprp", makeBox("ipco", makeBox("ispe", dimensions)));++ return new File(+ [+ makeBox("ftyp", encoder.encode("heic"), new Uint8Array(4)),+ makeBox("meta", new Uint8Array(4), properties),+ ],+ options?.name ?? "photo.heic",+ {+ type: options?.type ?? "image/heic",+ ...(options?.lastModified !== undefined ? { lastModified: options.lastModified } : {}),+ },+ );+}+/*** Installs a fake bitmap + canvas whose encoded size follows `sizeForQuality`.* `supportsWebp: false` makes `convertToBlob` hand back a differently-typed@@ -63,6 +112,7 @@ function stubCanvasPipeline(}afterEach(() => {+ mocks.heicTo.mockReset();vi.unstubAllGlobals();globalThis.createImageBitmap = originalCreateImageBitmap;globalThis.OffscreenCanvas = originalOffscreenCanvas;@@ -251,3 +301,148 @@ describe("compressImageForStash", () => {expect(smallestRequested).toBeLessThan(800);});});++describe("HEIC attachment preparation", () => {+ it("recognizes HEIC and HEIF MIME types and case-insensitive file extensions", () => {+ expect(isHeicImageFile({ name: "photo.bin", type: "image/heic" })).toBe(true);+ expect(isHeicImageFile({ name: "photo.bin", type: "image/heif" })).toBe(true);+ expect(isHeicImageFile({ name: "photo.heic", type: "image/heic-sequence" })).toBe(false);+ expect(isHeicImageFile({ name: "photo.heif", type: "image/heif-sequence" })).toBe(false);+ expect(isHeicImageFile({ name: "IMG_1234.HEIC", type: "" })).toBe(true);+ expect(isHeicImageFile({ name: "photo.heif", type: "application/octet-stream" })).toBe(true);+ expect(isHeicImageFile({ name: "photo.png", type: "image/png" })).toBe(false);+ expect(isHeicImageFile({ name: "photo.heic", type: "image/png" })).toBe(false);+ expect(isHeicImageFile({ name: "photo.heif", type: "image/jpeg" })).toBe(false);+ });++ it("converts a HEIC photo with a missing MIME type into a named JPEG", async () => {+ const original = makeHeicFile({+ name: "IMG_1234.HEIC",+ type: "",+ lastModified: 123,+ });+ mocks.heicTo.mockResolvedValueOnce(+ new Blob([new Uint8Array([4, 5, 6, 7])], { type: "image/jpeg" }),+ );++ const result = await prepareImageForAttachment(original, 1024);++ expect(mocks.heicTo).toHaveBeenCalledWith({+ blob: original,+ type: "image/jpeg",+ quality: 0.92,+ });+ expect(result.ok && result.file.name).toBe("IMG_1234.jpg");+ expect(result.ok && result.file.type).toBe("image/jpeg");+ expect(result.ok && result.file.size).toBe(4);+ expect(result.ok && result.file.lastModified).toBe(123);+ expect(result.ok && result.recompressed).toBe(true);+ });++ it("keeps oversized converted photos in JPEG format while shrinking them", async () => {+ const original = makeHeicFile({+ name: "photo.heif",+ type: "image/heif",+ });+ mocks.heicTo.mockResolvedValueOnce(+ new Blob([new Uint8Array(2_000_000)], { type: "image/jpeg" }),+ );+ const { fillRect } = stubCanvasPipeline(() => 200_000);++ const result = await prepareImageForAttachment(original, 1_000_000);++ expect(result.ok && result.file.name).toBe("photo.jpg");+ expect(result.ok && result.file.type).toBe("image/jpeg");+ expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);+ expect(fillRect).toHaveBeenCalled();+ });++ it("compresses JPEG intermediates above the source safety ceiling", async () => {+ const original = makeHeicFile({+ name: "large.heic",+ type: "image/heic",+ });+ mocks.heicTo.mockResolvedValueOnce(+ new Blob([new Uint8Array(MAX_COMPRESSIBLE_SOURCE_BYTES + 1)], {+ type: "image/jpeg",+ }),+ );+ const { close } = stubCanvasPipeline(() => 200_000);++ const result = await prepareImageForAttachment(original, 1_000_000);++ expect(result.ok && result.file.name).toBe("large.jpg");+ expect(result.ok && result.file.type).toBe("image/jpeg");+ expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);+ expect(close).toHaveBeenCalled();+ });++ it.each([+ { label: "24 MP", width: 5712, height: 4284 },+ { label: "48 MP", width: 8064, height: 6048 },+ ])("accepts $label HEIC photos", async ({ width, height }) => {+ const original = makeHeicFile({ width, height });+ mocks.heicTo.mockResolvedValueOnce(new Blob(["jpeg"], { type: "image/jpeg" }));++ const result = await prepareImageForAttachment(original, 1024);++ expect(result.ok && result.file.type).toBe("image/jpeg");+ expect(mocks.heicTo).toHaveBeenCalledOnce();+ });++ it("rejects oversized HEIC dimensions before loading the decoder", async () => {+ const original = makeHeicFile({ width: 16_000, height: 4001 });++ expect(await prepareImageForAttachment(original, 1024)).toEqual({+ ok: false,+ reason: "too-large",+ });+ expect(mocks.heicTo).not.toHaveBeenCalled();+ });++ it("rejects invalid HEIC metadata before loading the decoder", async () => {+ const original = new File([new Uint8Array([1, 2, 3])], "broken.heic", {+ type: "image/heic",+ });++ expect(await prepareImageForAttachment(original, 1024)).toEqual({+ ok: false,+ reason: "unreadable",+ });+ expect(mocks.heicTo).not.toHaveBeenCalled();+ });++ it("reports unreadable when HEIC decoding fails", async () => {+ const original = makeHeicFile({+ name: "broken.heic",+ type: "image/heic",+ });+ mocks.heicTo.mockRejectedValueOnce(new Error("Invalid HEIC image"));++ expect(await prepareImageForAttachment(original, 1024)).toEqual({+ ok: false,+ reason: "unreadable",+ });+ });++ it("rejects unsafe HEIC sources before loading the decoder", async () => {+ const original = new File(["photo"], "large.heic", { type: "image/heic" });+ Object.defineProperty(original, "size", { value: MAX_COMPRESSIBLE_SOURCE_BYTES + 1 });++ expect(await prepareImageForAttachment(original, 1024)).toEqual({+ ok: false,+ reason: "too-large",+ });+ expect(mocks.heicTo).not.toHaveBeenCalled();+ });++ it("leaves supported images untouched without loading the HEIC decoder", async () => {+ const original = makeFile(1024);++ const result = await prepareImageForAttachment(original, 2048);++ expect(result.ok && result.file).toBe(original);+ expect(result.ok && result.recompressed).toBe(false);+ expect(mocks.heicTo).not.toHaveBeenCalled();+ });+});diff --git a/apps/web/src/lib/imageCompression.ts b/apps/web/src/lib/imageCompression.tsindex be45024f3..c699886c4 100644--- a/apps/web/src/lib/imageCompression.ts+++ b/apps/web/src/lib/imageCompression.ts@@ -8,7 +8,8 @@* `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit* via `compressImageToByteLimit` instead of rejecting the paste.*- * Images already within budget pass through untouched.+ * Supported images already within budget pass through untouched. HEIC/HEIF+ * photos are decoded to JPEG first because providers cannot consume them.*//**@@ -24,6 +25,8 @@ export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;* ImageBitmap can OOM the tab — beyond this we refuse rather than risk it.*/export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;+const MAX_HEIC_DECODE_PIXELS = 64_000_000;+const MAX_HEIC_METADATA_BYTES = 1024 * 1024;/*** Quality ladder tried in order until the encoded image fits the budget.* The floor stays high enough to avoid visible blocking on UI screenshots;@@ -32,6 +35,8 @@ export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;const QUALITY_STEPS = [0.92, 0.85, 0.78, 0.68] as const;/** Extra downscale passes applied when even the lowest quality overflows. */const FALLBACK_SCALE_STEPS = [0.75, 0.55] as const;+const HEIC_IMAGE_MIME_TYPE = /^image\/hei(?:c|f)$/i;+const HEIC_IMAGE_EXTENSION = /\.(?:heic|heif)$/i;export interface CompressedStashImage {dataUrl: string;@@ -55,6 +60,86 @@ export type CompressImageFileResult =| { ok: true; file: File; recompressed: boolean }| { ok: false; reason: ImageCompressionFailureReason };+/** Finder and some browsers omit the MIME type when dragging HEIC photos. */+export function isHeicImageFile(file: Pick<File, "name" | "type">): boolean {+ if (HEIC_IMAGE_MIME_TYPE.test(file.type)) {+ return true;+ }+ return (+ (file.type === "" || file.type.toLowerCase() === "application/octet-stream") &&+ HEIC_IMAGE_EXTENSION.test(file.name)+ );+}++interface HeicMetadataBox {+ payloadOffset: number;+ endOffset: number;+}++function findHeicMetadataBox(+ view: DataView,+ startOffset: number,+ endOffset: number,+ type: number,+): HeicMetadataBox | null {+ let offset = startOffset;+ while (offset + 8 <= endOffset) {+ let size = view.getUint32(offset);+ let headerSize = 8;+ if (size === 1) {+ if (offset + 16 > endOffset) return null;+ const extendedSize = view.getBigUint64(offset + 8);+ if (extendedSize > BigInt(Number.MAX_SAFE_INTEGER)) return null;+ size = Number(extendedSize);+ headerSize = 16;+ } else if (size === 0) {+ size = endOffset - offset;+ }+ if (size < headerSize || size > endOffset - offset) return null;++ const nextOffset = offset + size;+ if (view.getUint32(offset + 4) === type) {+ return { payloadOffset: offset + headerSize, endOffset: nextOffset };+ }+ offset = nextOffset;+ }+ return null;+}++/** Read HEIC image dimensions before the decoder allocates full RGBA buffers. */+async function validateHeicImageDimensions(+ file: File,+): Promise<ImageCompressionFailureReason | null> {+ const metadata = await file.slice(0, MAX_HEIC_METADATA_BYTES).arrayBuffer();+ const view = new DataView(metadata);+ const meta = findHeicMetadataBox(view, 0, view.byteLength, 0x6d657461);+ if (!meta || meta.payloadOffset + 4 > meta.endOffset) return "unreadable";+ const properties = findHeicMetadataBox(view, meta.payloadOffset + 4, meta.endOffset, 0x69707270);+ if (!properties) return "unreadable";+ const containers = findHeicMetadataBox(+ view,+ properties.payloadOffset,+ properties.endOffset,+ 0x6970636f,+ );+ if (!containers) return "unreadable";++ let offset = containers.payloadOffset;+ let foundImageDimensions = false;+ while (offset < containers.endOffset) {+ const image = findHeicMetadataBox(view, offset, containers.endOffset, 0x69737065);+ if (!image) break;+ if (image.payloadOffset + 12 > image.endOffset) return "unreadable";+ const width = view.getUint32(image.payloadOffset + 4);+ const height = view.getUint32(image.payloadOffset + 8);+ if (width === 0 || height === 0) return "unreadable";+ if (width > MAX_HEIC_DECODE_PIXELS / height) return "too-large";+ foundImageDimensions = true;+ offset = image.endOffset;+ }+ return foundImageDimensions ? null : "unreadable";+}+/** Chunked so a large image can't blow the argument limit of `fromCharCode`. */const BASE64_CHUNK_SIZE = 0x8000;@@ -167,6 +252,7 @@ async function encodeWithinBudget(bitmap: ImageBitmap,maxDimension: number,budgetChars: number,+ preferredMimeType?: "image/jpeg",): Promise<{ dataUrl: string; mimeType: string } | null> {const scale = Math.min(1, maxDimension / Math.max(bitmap.width, bitmap.height));const width = Math.max(1, Math.round(bitmap.width * scale));@@ -176,8 +262,11 @@ async function encodeWithinBudget(// Probe WebP once; JPEG (no alpha) needs a white matte, so the fill has to// happen before drawing and depends on which codec we end up using.- const probe = await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0);- const mimeType = probe ? "image/webp" : "image/jpeg";+ const mimeType =+ preferredMimeType ??+ ((await encodeCanvas(target.canvas, QUALITY_STEPS[0], "image/webp", 0))+ ? "image/webp"+ : "image/jpeg");if (mimeType === "image/jpeg") {target.context.fillStyle = "#ffffff";@@ -203,7 +292,11 @@ type ReencodeResult =* Shared re-encode loop: decodes `file`, then walks the quality ladder and* fallback downscale passes until an encoding fits `budgetChars`.*/-async function reencodeWithinBudget(file: File, budgetChars: number): Promise<ReencodeResult> {+async function reencodeWithinBudget(+ file: File,+ budgetChars: number,+ preferredMimeType?: "image/jpeg",+): Promise<ReencodeResult> {if (!canRecompress()) {return { ok: false, reason: "too-large" };}@@ -229,7 +322,7 @@ async function reencodeWithinBudget(file: File, budgetChars: number): Promise<Reconst targetDimension = Math.max(1, Math.round(baseDimension * dimensionScale));let encoded: { dataUrl: string; mimeType: string } | null;try {- encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars);+ encoded = await encodeWithinBudget(bitmap, targetDimension, budgetChars, preferredMimeType);} catch {// Canvas allocation, drawing, or the codec itself can throw — often// precisely *because* the target is too big (OOM on a large bitmap).@@ -300,23 +393,26 @@ export async function compressImageForStash(* `File` (WebP or JPEG). Files already within the limit pass through* untouched, preserving their exact bytes and format. Sources above* `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is- * the risk, so no amount of output budget makes them safe.+ * the risk, so no amount of output budget makes them safe. An internally+ * converted image can provide its original source size when the intermediate+ * format expands beyond that ceiling.*/export async function compressImageToByteLimit(file: File,maxBytes: number,+ options?: { preferredMimeType?: "image/jpeg"; sourceSizeBytes?: number },): Promise<CompressImageFileResult> {if (file.size <= maxBytes) {return { ok: true, file, recompressed: false };}- if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {+ if ((options?.sourceSizeBytes ?? file.size) > MAX_COMPRESSIBLE_SOURCE_BYTES) {return { ok: false, reason: "too-large" };}// The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes// into 4 chars; flooring keeps the budget a hair conservative instead of// admitting an encoding right at the byte cap.const budgetChars = Math.floor(maxBytes / 3) * 4;- const reencoded = await reencodeWithinBudget(file, budgetChars);+ const reencoded = await reencodeWithinBudget(file, budgetChars, options?.preferredMimeType);if (!reencoded.ok) {return reencoded;}@@ -330,3 +426,43 @@ export async function compressImageToByteLimit(recompressed: true,};}++/**+ * Converts HEIC/HEIF photos to provider-compatible JPEG before applying the+ * attachment size limit. The decoder is loaded only when such a photo arrives.+ */+export async function prepareImageForAttachment(+ file: File,+ maxBytes: number,+): Promise<CompressImageFileResult> {+ if (!isHeicImageFile(file)) {+ return compressImageToByteLimit(file, maxBytes);+ }++ if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {+ return { ok: false, reason: "too-large" };+ }++ let converted: Blob;+ try {+ const dimensionError = await validateHeicImageDimensions(file);+ if (dimensionError) {+ return { ok: false, reason: dimensionError };+ }+ const { heicTo } = await import("heic-to/csp");+ converted = await heicTo({ blob: file, type: "image/jpeg", quality: QUALITY_STEPS[0] });+ } catch {+ return { ok: false, reason: "unreadable" };+ }++ const jpeg = new File([converted], fileNameForMimeType(file.name || "image", "image/jpeg"), {+ type: "image/jpeg",+ lastModified: file.lastModified,+ });+ const result = await compressImageToByteLimit(jpeg, maxBytes, {+ preferredMimeType: "image/jpeg",+ sourceSizeBytes: file.size,+ });++ return result.ok ? { ...result, recompressed: true } : result;+}diff --git a/docs/user/composer.md b/docs/user/composer.mdindex b7ef57a56..35d634556 100644--- a/docs/user/composer.md+++ b/docs/user/composer.md@@ -7,6 +7,9 @@ multiple messages, then send again in the same thread.On servers that support direct uploads, images upload as soon as you add them. The send buttonbecomes available after every upload finishes. Failed uploads can be retried or removed.+On web and desktop, HEIC and HEIF photos are automatically converted to JPEG when you drag them into+the composer or paste them into a message.+## Commands and skillsType `/` to open the command menu. Type `$` to find and add a skill. Skill rows show their source,diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 3dadd8da2..490d9ca9d 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -590,6 +590,9 @@ importers:effect:specifier: 4.0.0-beta.103version: 4.0.0-beta.103(patch_hash=af36b7948b6f9c56623074662b51dade5699880c1a7c71245de73e13c3185fb6)+ heic-to:+ specifier: ^1.5.2+ version: 1.5.2jose:specifier: 'catalog:'version: 6.2.2@@ -7159,6 +7162,9 @@ packages:resolution: {integrity: sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==}+ resolution: {integrity: sha512-8Fns+lZHAWmz5U5IUxDeXKwIf3foBoKNPLxxFY4B0MkLjNuomEIHCoDbDE+x/llFK3NCEO1cu4+n3iUKY+Svmw==}+resolution: {integrity: sha512-TcRlZ0/TlyfJqquRFAWoyElVNnkdYRi/sEp4/Qy8/GYxjg8j2cS9D4MjuaQ+qimkmLN7AmO+44IznRf06mAr0w==}@@ -17839,6 +17845,8 @@ snapshots:+ [email protected]: {}+