Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
feat(web): pasting a huge screenshot now compresses it instead of erroring (#4967)
Theo Brownecbe8052020aa
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsxindex b89521a93..e92ecd497 100644--- a/apps/web/src/components/chat/ChatComposer.tsx+++ b/apps/web/src/components/chat/ChatComposer.tsx@@ -64,7 +64,7 @@ import {} from "../../promptStashStore";import { ComposerStashBadge } from "./ComposerStashBadge";import { ComposerStashMenu } from "./ComposerStashMenu";-import { compressImageForStash } from "../../lib/stashImageCompression";+import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression";import { isCommandPaletteOpen } from "../../commandPaletteBus";import { getTerminalFocusOwner } from "../../lib/terminalFocus";import { resolveShortcutCommand } from "../../keybindings";@@ -206,8 +206,6 @@ import { searchProviderSkills } from "../../providerSkillSearch";import { useMediaQuery } from "../../hooks/useMediaQuery";import type { ReviewCommentContext } from "../../reviewCommentContext";-const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`;-const runtimeModeConfig: Record<RuntimeMode,{ label: string; description: string; icon: LucideIcon }@@ -1006,6 +1004,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)* thread) can still be stashed while an earlier encode is running.*/const stashInFlightRef = useRef<Set<string>>(new Set());+ /**+ * Count of pasted images still being compressed, per thread. Reserved+ * against the attachment limit so concurrent pastes can't overshoot it,+ * and checked by `submitComposer` so a send can't race an image into the+ * next draft.+ */+ const pendingImageCompressionsRef = useRef<Map<ThreadId, number>>(new Map());// ------------------------------------------------------------------// Derived: composer send state@@ -1821,12 +1826,26 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)event?.preventDefault();return;}+ // A send while a pasted image is still compressing would strand that+ // image: the turn snapshot wouldn't include it, and it would surface+ // in the *next* draft instead. Only oversized images hit this — small+ // files clear the pending counter within a microtask.+ if (activeThreadId && (pendingImageCompressionsRef.current.get(activeThreadId) ?? 0) > 0) {+ event?.preventDefault();+ toastManager.add({+ type: "info",+ title: "Still compressing a pasted image.",+ description: "Send again once its thumbnail appears.",+ });+ return;+ }onSend(event);if (shouldBlurMobileComposerOnSubmit()) {blurMobileComposerAfterSend();}},[+ activeThreadId,blurMobileComposerAfterSend,isSendDisabled,noProviderAvailable,@@ -2273,7 +2292,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)// ------------------------------------------------------------------// Callbacks: images// ------------------------------------------------------------------- const addComposerImages = (files: File[]) => {+ const addComposerImages = async (files: File[]) => {if (!activeThreadId || files.length === 0) return;if (pendingUserInputs.length > 0) {toastManager.add({@@ -2282,40 +2301,81 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)});return;}- const nextImages: ComposerImageAttachment[] = [];- let nextImageCount = composerImagesRef.current.length;+ // Captured before the awaits below: the user may switch threads while a+ // large image is being compressed, and the attachments and errors belong+ // to the thread the paste happened in.+ const threadId = activeThreadId;++ // Validation happens synchronously so concurrent pastes see each other:+ // accepted files reserve their attachment slots (via the pending counter)+ // before the first await, keeping the total under the limit.+ const pendingCount = pendingImageCompressionsRef.current.get(threadId) ?? 0;+ let reservedCount = composerImagesRef.current.length + pendingCount;+ const acceptedFiles: File[] = [];let error: string | null = null;for (const file of files) {if (!file.type.startsWith("image/")) {error = `Unsupported file type for '${file.name}'. Please attach image files only.`;continue;}- if (file.size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {- error = `'${file.name}' exceeds the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.`;- continue;- }- if (nextImageCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {+ if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} images per message.`;break;}- const previewUrl = URL.createObjectURL(file);- nextImages.push({- type: "image",- id: randomUUID(),- name: file.name || "image",- mimeType: file.type,- sizeBytes: file.size,- previewUrl,- file,- });- nextImageCount += 1;+ acceptedFiles.push(file);+ reservedCount += 1;}- if (nextImages.length === 1 && nextImages[0]) {- addComposerImage(nextImages[0]);- } else if (nextImages.length > 1) {- addComposerImagesToDraft(nextImages);+ setThreadError(threadId, error);+ if (acceptedFiles.length === 0) return;++ pendingImageCompressionsRef.current.set(threadId, pendingCount + acceptedFiles.length);+ try {+ const nextImages: ComposerImageAttachment[] = [];+ let compressionError: string | null = null;+ for (const file of acceptedFiles) {+ // Images over the wire cap are downscaled to fit rather than+ // refused; files already within it pass through byte-for-byte.+ const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);+ if (!compressed.ok) {+ compressionError =+ compressed.reason === "unreadable"+ ? `'${file.name}' could not be read as an image.`+ : `'${file.name}' is too large to attach, even after compression.`;+ continue;+ }+ const attachmentFile = compressed.file;+ const previewUrl = URL.createObjectURL(attachmentFile);+ nextImages.push({+ type: "image",+ id: randomUUID(),+ name: attachmentFile.name || "image",+ mimeType: attachmentFile.type,+ sizeBytes: attachmentFile.size,+ previewUrl,+ file: attachmentFile,+ });+ }+ if (nextImages.length === 1 && nextImages[0]) {+ addComposerImage(nextImages[0]);+ } else if (nextImages.length > 1) {+ addComposerImagesToDraft(nextImages);+ }+ // Only failures are reported here. Success must not pass `null`: by+ // now other work (a failed send, an overlapping paste) may have set a+ // thread error this call knows nothing about, and clearing it would+ // swallow that message.+ if (compressionError !== null) {+ setThreadError(threadId, compressionError);+ }+ } finally {+ const remaining =+ (pendingImageCompressionsRef.current.get(threadId) ?? 0) - acceptedFiles.length;+ if (remaining > 0) {+ pendingImageCompressionsRef.current.set(threadId, remaining);+ } else {+ pendingImageCompressionsRef.current.delete(threadId);+ }}- setThreadError(activeThreadId, error);};const removeComposerImage = (imageId: string) => {@@ -2331,7 +2391,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)const imageFiles = files.filter((file) => file.type.startsWith("image/"));if (imageFiles.length === 0) return;event.preventDefault();- addComposerImages(imageFiles);+ void addComposerImages(imageFiles);};const onComposerDragEnter = (event: React.DragEvent<HTMLDivElement>) => {@@ -2365,7 +2425,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)dragDepthRef.current = 0;setIsDragOverComposer(false);const files = Array.from(event.dataTransfer.files);- addComposerImages(files);+ void addComposerImages(files);focusComposer();};diff --git a/apps/web/src/lib/stashImageCompression.test.ts b/apps/web/src/lib/imageCompression.test.tssimilarity index 76%rename from apps/web/src/lib/stashImageCompression.test.tsrename to apps/web/src/lib/imageCompression.test.tsindex aaffa284c..63712ca7e 100644--- a/apps/web/src/lib/stashImageCompression.test.ts+++ b/apps/web/src/lib/imageCompression.test.ts@@ -1,6 +1,11 @@import { afterEach, describe, expect, it, vi } from "vite-plus/test";-import { compressImageForStash, MAX_STASH_IMAGE_DATA_URL_CHARS } from "./stashImageCompression";+import {+ compressImageForStash,+ compressImageToByteLimit,+ MAX_COMPRESSIBLE_SOURCE_BYTES,+ MAX_STASH_IMAGE_DATA_URL_CHARS,+} from "./imageCompression";/*** jsdom has no real canvas/codec, so the re-encode path is exercised with@@ -159,6 +164,56 @@ describe("compressImageForStash", () => {});});+ it("compressImageToByteLimit passes small files through byte-for-byte", async () => {+ const bitmapSpy = vi.fn();+ vi.stubGlobal("createImageBitmap", bitmapSpy);++ const original = makeFile(1024);+ const result = await compressImageToByteLimit(original, 10 * 1024 * 1024);++ expect(result.ok).toBe(true);+ expect(result.ok && result.recompressed).toBe(false);+ // Pass-through must be the same File object, not a copy.+ expect(result.ok && result.file).toBe(original);+ expect(bitmapSpy).not.toHaveBeenCalled();+ });++ it("compressImageToByteLimit re-encodes an oversized file under the byte cap", async () => {+ stubCanvasPipeline(() => 200_000);++ const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000);++ expect(result.ok).toBe(true);+ expect(result.ok && result.recompressed).toBe(true);+ expect(result.ok && result.file.type).toBe("image/webp");+ // The re-encoded name must match the new container format.+ expect(result.ok && result.file.name).toBe("shot.webp");+ expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);+ });++ it("compressImageToByteLimit refuses sources above the decode-safety ceiling", async () => {+ const bitmapSpy = vi.fn();+ vi.stubGlobal("createImageBitmap", bitmapSpy);++ const result = await compressImageToByteLimit(+ makeFile(MAX_COMPRESSIBLE_SOURCE_BYTES + 1),+ 10 * 1024 * 1024,+ );++ expect(result).toEqual({ ok: false, reason: "too-large" });+ // The whole point of the ceiling is to never decode such a file.+ expect(bitmapSpy).not.toHaveBeenCalled();+ });++ it("compressImageToByteLimit reports too-large when no encoding fits", async () => {+ const { close } = stubCanvasPipeline(() => 3_000_000);++ const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000);++ expect(result).toEqual({ ok: false, reason: "too-large" });+ expect(close).toHaveBeenCalled();+ });+it("shrinks below the source size when the image is already under MAX_DIMENSION", async () => {// A small-but-heavy source (e.g. a dense PNG): only a real downscale can// get it under budget, since quality alone is stubbed to never suffice.diff --git a/apps/web/src/lib/stashImageCompression.ts b/apps/web/src/lib/imageCompression.tssimilarity index 63%rename from apps/web/src/lib/stashImageCompression.tsrename to apps/web/src/lib/imageCompression.tsindex 7f133fb29..1c57fdad1 100644--- a/apps/web/src/lib/stashImageCompression.ts+++ b/apps/web/src/lib/imageCompression.ts@@ -1,13 +1,14 @@/**- * Re-encoding for stashed image attachments.+ * Downscale + re-encode for image attachments that are too big for where+ * they're headed. Two consumers share the same pipeline:*- * The composer accepts images up to `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES`- * (10MB), but the stash persists them as base64 in localStorage, where the- * whole origin shares a ~5MB quota. Rather than refuse large screenshots,- * downscale + re-encode them to JPEG so a stashed prompt keeps its images.+ * - The prompt stash persists images as base64 in localStorage (~5MB origin+ * quota), so `compressImageForStash` targets a per-image character budget.+ * - The composer accepts pasted/dropped images larger than the provider's+ * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit+ * via `compressImageToByteLimit` instead of rejecting the paste.*- * Only the *stashed copy* is compressed; the live composer attachment is- * untouched, so sending without stashing still uploads the original file.+ * Images already within budget pass through untouched.*//**@@ -17,6 +18,12 @@const MAX_DIMENSION = 2048;/** Base64 budget for a single stashed image (~975KB of binary). */export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;+/**+ * Ceiling on the *source* file handed to the re-encoder. File size is a+ * proxy for pixel count, and decoding hundreds of megapixels into an+ * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it.+ */+export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;/*** Quality ladder tried in order until the encoded image fits the budget.* The floor stays high enough to avoid visible blocking on UI screenshots;@@ -35,14 +42,18 @@ export interface CompressedStashImage {}/**- * Why an image could not be stashed. Callers report these differently:+ * Why an image could not be compressed. Callers report these differently:* "too large" is a budget outcome, "unreadable" is a decode failure.*/-export type StashImageFailureReason = "too-large" | "unreadable";+export type ImageCompressionFailureReason = "too-large" | "unreadable";export type CompressStashImageResult =| { ok: true; image: CompressedStashImage }- | { ok: false; reason: StashImageFailureReason };+ | { ok: false; reason: ImageCompressionFailureReason };++export type CompressImageFileResult =+ | { ok: true; file: File; recompressed: boolean }+ | { ok: false; reason: ImageCompressionFailureReason };/** Chunked so a large image can't blow the argument limit of `fromCharCode`. */const BASE64_CHUNK_SIZE = 0x8000;@@ -73,6 +84,28 @@ function dataUrlByteLength(dataUrl: string): number {return Math.max(0, Math.floor((payload.length * 3) / 4) - padding);}+/** Base64 payload of a data URL decoded back into a `File`. */+function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {+ const payload = dataUrl.slice(dataUrl.indexOf(",") + 1);+ const binary = atob(payload);+ const bytes = new Uint8Array(binary.length);+ for (let index = 0; index < binary.length; index += 1) {+ bytes[index] = binary.charCodeAt(index);+ }+ return new File([bytes], name, { type: mimeType });+}++/**+ * Re-encoding changes the container, so a name like `shot.png` would lie+ * about its contents. Swap the extension to match the encoded mime type.+ */+function fileNameForMimeType(name: string, mimeType: string): string {+ const extension = mimeType === "image/webp" ? ".webp" : ".jpg";+ const dotIndex = name.lastIndexOf(".");+ const base = dotIndex > 0 ? name.slice(0, dotIndex) : name;+ return `${base}${extension}`;+}+function canRecompress(): boolean {return (typeof createImageBitmap === "function" &&@@ -124,10 +157,10 @@ async function encodeToDataUrl(}/**- * Draws `bitmap` scaled to fit `maxDimension` and encodes it as JPEG,- * stepping quality down until the data URL fits `budgetChars`. Returns the- * smallest encoding produced, even if it still exceeds the budget, so the- * caller can decide whether to keep or drop it.+ * Draws `bitmap` scaled to fit `maxDimension` and encodes it, stepping+ * quality down until the data URL fits `budgetChars`. Returns the smallest+ * encoding produced, even if it still exceeds the budget, so the caller can+ * decide whether to keep or drop it.*/async function encodeWithinBudget(bitmap: ImageBitmap,@@ -165,35 +198,15 @@ async function encodeWithinBudget(return smallest;}+type ReencodeResult =+ | { ok: true; dataUrl: string; mimeType: string }+ | { ok: false; reason: ImageCompressionFailureReason };+/**- * Produces the payload to persist for a stashed image.- *- * Small images are stored verbatim (preserving PNG transparency and exact- * pixels). Anything over budget is downscaled and JPEG-encoded; if it still- * doesn't fit after the fallback passes, returns `null` so the caller can- * record it as dropped.+ * Shared re-encode loop: decodes `file`, then walks the quality ladder and+ * fallback downscale passes until an encoding fits `budgetChars`.*/-export async function compressImageForStash(- file: File,- budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,-): Promise<CompressStashImageResult> {- let originalDataUrl: string;- try {- originalDataUrl = await blobToDataUrl(file);- } catch {- return { ok: false, reason: "unreadable" };- }- if (originalDataUrl.length <= budgetChars) {- return {- ok: true,- image: {- dataUrl: originalDataUrl,- mimeType: file.type,- sizeBytes: file.size,- recompressed: false,- },- };- }+async function reencodeWithinBudget(file: File, budgetChars: number): Promise<ReencodeResult> {if (!canRecompress()) {return { ok: false, reason: "too-large" };}@@ -225,22 +238,14 @@ export async function compressImageForStash(// precisely *because* the target is too big (OOM on a large bitmap).// Keep trying the smaller fallback scales rather than giving up: a// reduced pass may well succeed. The exception must never escape,- // though, since the caller finalizes the entry after this returns and- // a throw would strand it as permanently "still saving".+ // though, since callers finalize state after this returns and a+ // throw would strand it (e.g. a stash entry stuck "still saving").encodeFailed = true;continue;}encodeFailed = false;if (encoded && encoded.dataUrl.length <= budgetChars) {- return {- ok: true,- image: {- dataUrl: encoded.dataUrl,- mimeType: encoded.mimeType,- sizeBytes: dataUrlByteLength(encoded.dataUrl),- recompressed: true,- },- };+ return { ok: true, dataUrl: encoded.dataUrl, mimeType: encoded.mimeType };}}return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" };@@ -248,3 +253,83 @@ export async function compressImageForStash(bitmap.close();}}++/**+ * Produces the payload to persist for a stashed image.+ *+ * Small images are stored verbatim (preserving PNG transparency and exact+ * pixels). Anything over budget is downscaled and re-encoded; if it still+ * doesn't fit after the fallback passes, reports a failure so the caller+ * can record it as dropped.+ */+export async function compressImageForStash(+ file: File,+ budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,+): Promise<CompressStashImageResult> {+ let originalDataUrl: string;+ try {+ originalDataUrl = await blobToDataUrl(file);+ } catch {+ return { ok: false, reason: "unreadable" };+ }+ if (originalDataUrl.length <= budgetChars) {+ return {+ ok: true,+ image: {+ dataUrl: originalDataUrl,+ mimeType: file.type,+ sizeBytes: file.size,+ recompressed: false,+ },+ };+ }+ const reencoded = await reencodeWithinBudget(file, budgetChars);+ if (!reencoded.ok) {+ return reencoded;+ }+ return {+ ok: true,+ image: {+ dataUrl: reencoded.dataUrl,+ mimeType: reencoded.mimeType,+ sizeBytes: dataUrlByteLength(reencoded.dataUrl),+ recompressed: true,+ },+ };+}++/**+ * Shrinks `file` until its binary size fits `maxBytes`, returning a new+ * `File` (WebP or JPEG). Files already within the limit pass through+ * untouched, preserving their exact bytes and format. Sources above+ * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is+ * the risk, so no amount of output budget makes them safe.+ */+export async function compressImageToByteLimit(+ file: File,+ maxBytes: number,+): Promise<CompressImageFileResult> {+ if (file.size <= maxBytes) {+ return { ok: true, file, recompressed: false };+ }+ if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {+ return { ok: false, reason: "too-large" };+ }+ // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes+ // into 4 chars; flooring keeps the budget a hair conservative instead of+ // admitting an encoding right at the byte cap.+ const budgetChars = Math.floor(maxBytes / 3) * 4;+ const reencoded = await reencodeWithinBudget(file, budgetChars);+ if (!reencoded.ok) {+ return reencoded;+ }+ return {+ ok: true,+ file: dataUrlToFile(+ reencoded.dataUrl,+ fileNameForMimeType(file.name || "image", reencoded.mimeType),+ reencoded.mimeType,+ ),+ recompressed: true,+ };+}