Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

feat(web): pasting a huge screenshot now compresses it instead of erroring (#4967)

Theo Brownecbe8052020aa
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx
index b89521a93..e92ecd497 100644
--- a/apps/web/src/components/chat/ChatComposer.tsx
+++ b/apps/web/src/components/chat/ChatComposer.tsx
@@ -64,7 +64,7 @@ import {
} from "../../promptStashStore";
import { ComposerStashBadge } from "./ComposerStashBadge";
import { ComposerStashMenu } from "./ComposerStashMenu";
-import { compressImageForStash } from "../../lib/stashImageCompression";
+import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression";
import { isCommandPaletteOpen } from "../../commandPaletteBus";
import { getTerminalFocusOwner } from "../../lib/terminalFocus";
import { resolveShortcutCommand } from "../../keybindings";
@@ -206,8 +206,6 @@ import { searchProviderSkills } from "../../providerSkillSearch";
import { useMediaQuery } from "../../hooks/useMediaQuery";
import type { ReviewCommentContext } from "../../reviewCommentContext";
-const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`;
-
const runtimeModeConfig: Record<
RuntimeMode,
{ label: string; description: string; icon: LucideIcon }
@@ -1006,6 +1004,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
* thread) can still be stashed while an earlier encode is running.
*/
const stashInFlightRef = useRef<Set<string>>(new Set());
+ /**
+ * Count of pasted images still being compressed, per thread. Reserved
+ * against the attachment limit so concurrent pastes can't overshoot it,
+ * and checked by `submitComposer` so a send can't race an image into the
+ * next draft.
+ */
+ const pendingImageCompressionsRef = useRef<Map<ThreadId, number>>(new Map());
// ------------------------------------------------------------------
// Derived: composer send state
@@ -1821,12 +1826,26 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
event?.preventDefault();
return;
}
+ // A send while a pasted image is still compressing would strand that
+ // image: the turn snapshot wouldn't include it, and it would surface
+ // in the *next* draft instead. Only oversized images hit this — small
+ // files clear the pending counter within a microtask.
+ if (activeThreadId && (pendingImageCompressionsRef.current.get(activeThreadId) ?? 0) > 0) {
+ event?.preventDefault();
+ toastManager.add({
+ type: "info",
+ title: "Still compressing a pasted image.",
+ description: "Send again once its thumbnail appears.",
+ });
+ return;
+ }
onSend(event);
if (shouldBlurMobileComposerOnSubmit()) {
blurMobileComposerAfterSend();
}
},
[
+ activeThreadId,
blurMobileComposerAfterSend,
isSendDisabled,
noProviderAvailable,
@@ -2273,7 +2292,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
// ------------------------------------------------------------------
// Callbacks: images
// ------------------------------------------------------------------
- const addComposerImages = (files: File[]) => {
+ const addComposerImages = async (files: File[]) => {
if (!activeThreadId || files.length === 0) return;
if (pendingUserInputs.length > 0) {
toastManager.add({
@@ -2282,40 +2301,81 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
});
return;
}
- const nextImages: ComposerImageAttachment[] = [];
- let nextImageCount = composerImagesRef.current.length;
+ // Captured before the awaits below: the user may switch threads while a
+ // large image is being compressed, and the attachments and errors belong
+ // to the thread the paste happened in.
+ const threadId = activeThreadId;
+
+ // Validation happens synchronously so concurrent pastes see each other:
+ // accepted files reserve their attachment slots (via the pending counter)
+ // before the first await, keeping the total under the limit.
+ const pendingCount = pendingImageCompressionsRef.current.get(threadId) ?? 0;
+ let reservedCount = composerImagesRef.current.length + pendingCount;
+ const acceptedFiles: File[] = [];
let error: string | null = null;
for (const file of files) {
if (!file.type.startsWith("image/")) {
error = `Unsupported file type for '${file.name}'. Please attach image files only.`;
continue;
}
- if (file.size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) {
- error = `'${file.name}' exceeds the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.`;
- continue;
- }
- if (nextImageCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {
+ if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) {
error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} images per message.`;
break;
}
- const previewUrl = URL.createObjectURL(file);
- nextImages.push({
- type: "image",
- id: randomUUID(),
- name: file.name || "image",
- mimeType: file.type,
- sizeBytes: file.size,
- previewUrl,
- file,
- });
- nextImageCount += 1;
+ acceptedFiles.push(file);
+ reservedCount += 1;
}
- if (nextImages.length === 1 && nextImages[0]) {
- addComposerImage(nextImages[0]);
- } else if (nextImages.length > 1) {
- addComposerImagesToDraft(nextImages);
+ setThreadError(threadId, error);
+ if (acceptedFiles.length === 0) return;
+
+ pendingImageCompressionsRef.current.set(threadId, pendingCount + acceptedFiles.length);
+ try {
+ const nextImages: ComposerImageAttachment[] = [];
+ let compressionError: string | null = null;
+ for (const file of acceptedFiles) {
+ // Images over the wire cap are downscaled to fit rather than
+ // refused; files already within it pass through byte-for-byte.
+ const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);
+ if (!compressed.ok) {
+ compressionError =
+ compressed.reason === "unreadable"
+ ? `'${file.name}' could not be read as an image.`
+ : `'${file.name}' is too large to attach, even after compression.`;
+ continue;
+ }
+ const attachmentFile = compressed.file;
+ const previewUrl = URL.createObjectURL(attachmentFile);
+ nextImages.push({
+ type: "image",
+ id: randomUUID(),
+ name: attachmentFile.name || "image",
+ mimeType: attachmentFile.type,
+ sizeBytes: attachmentFile.size,
+ previewUrl,
+ file: attachmentFile,
+ });
+ }
+ if (nextImages.length === 1 && nextImages[0]) {
+ addComposerImage(nextImages[0]);
+ } else if (nextImages.length > 1) {
+ addComposerImagesToDraft(nextImages);
+ }
+ // Only failures are reported here. Success must not pass `null`: by
+ // now other work (a failed send, an overlapping paste) may have set a
+ // thread error this call knows nothing about, and clearing it would
+ // swallow that message.
+ if (compressionError !== null) {
+ setThreadError(threadId, compressionError);
+ }
+ } finally {
+ const remaining =
+ (pendingImageCompressionsRef.current.get(threadId) ?? 0) - acceptedFiles.length;
+ if (remaining > 0) {
+ pendingImageCompressionsRef.current.set(threadId, remaining);
+ } else {
+ pendingImageCompressionsRef.current.delete(threadId);
+ }
}
- setThreadError(activeThreadId, error);
};
const removeComposerImage = (imageId: string) => {
@@ -2331,7 +2391,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
const imageFiles = files.filter((file) => file.type.startsWith("image/"));
if (imageFiles.length === 0) return;
event.preventDefault();
- addComposerImages(imageFiles);
+ void addComposerImages(imageFiles);
};
const onComposerDragEnter = (event: React.DragEvent<HTMLDivElement>) => {
@@ -2365,7 +2425,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
dragDepthRef.current = 0;
setIsDragOverComposer(false);
const files = Array.from(event.dataTransfer.files);
- addComposerImages(files);
+ void addComposerImages(files);
focusComposer();
};
diff --git a/apps/web/src/lib/stashImageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts
similarity index 76%
rename from apps/web/src/lib/stashImageCompression.test.ts
rename to apps/web/src/lib/imageCompression.test.ts
index aaffa284c..63712ca7e 100644
--- a/apps/web/src/lib/stashImageCompression.test.ts
+++ b/apps/web/src/lib/imageCompression.test.ts
@@ -1,6 +1,11 @@
import { afterEach, describe, expect, it, vi } from "vite-plus/test";
-import { compressImageForStash, MAX_STASH_IMAGE_DATA_URL_CHARS } from "./stashImageCompression";
+import {
+ compressImageForStash,
+ compressImageToByteLimit,
+ MAX_COMPRESSIBLE_SOURCE_BYTES,
+ MAX_STASH_IMAGE_DATA_URL_CHARS,
+} from "./imageCompression";
/**
* jsdom has no real canvas/codec, so the re-encode path is exercised with
@@ -159,6 +164,56 @@ describe("compressImageForStash", () => {
});
});
+ it("compressImageToByteLimit passes small files through byte-for-byte", async () => {
+ const bitmapSpy = vi.fn();
+ vi.stubGlobal("createImageBitmap", bitmapSpy);
+
+ const original = makeFile(1024);
+ const result = await compressImageToByteLimit(original, 10 * 1024 * 1024);
+
+ expect(result.ok).toBe(true);
+ expect(result.ok && result.recompressed).toBe(false);
+ // Pass-through must be the same File object, not a copy.
+ expect(result.ok && result.file).toBe(original);
+ expect(bitmapSpy).not.toHaveBeenCalled();
+ });
+
+ it("compressImageToByteLimit re-encodes an oversized file under the byte cap", async () => {
+ stubCanvasPipeline(() => 200_000);
+
+ const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000);
+
+ expect(result.ok).toBe(true);
+ expect(result.ok && result.recompressed).toBe(true);
+ expect(result.ok && result.file.type).toBe("image/webp");
+ // The re-encoded name must match the new container format.
+ expect(result.ok && result.file.name).toBe("shot.webp");
+ expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000);
+ });
+
+ it("compressImageToByteLimit refuses sources above the decode-safety ceiling", async () => {
+ const bitmapSpy = vi.fn();
+ vi.stubGlobal("createImageBitmap", bitmapSpy);
+
+ const result = await compressImageToByteLimit(
+ makeFile(MAX_COMPRESSIBLE_SOURCE_BYTES + 1),
+ 10 * 1024 * 1024,
+ );
+
+ expect(result).toEqual({ ok: false, reason: "too-large" });
+ // The whole point of the ceiling is to never decode such a file.
+ expect(bitmapSpy).not.toHaveBeenCalled();
+ });
+
+ it("compressImageToByteLimit reports too-large when no encoding fits", async () => {
+ const { close } = stubCanvasPipeline(() => 3_000_000);
+
+ const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000);
+
+ expect(result).toEqual({ ok: false, reason: "too-large" });
+ expect(close).toHaveBeenCalled();
+ });
+
it("shrinks below the source size when the image is already under MAX_DIMENSION", async () => {
// A small-but-heavy source (e.g. a dense PNG): only a real downscale can
// get it under budget, since quality alone is stubbed to never suffice.
diff --git a/apps/web/src/lib/stashImageCompression.ts b/apps/web/src/lib/imageCompression.ts
similarity index 63%
rename from apps/web/src/lib/stashImageCompression.ts
rename to apps/web/src/lib/imageCompression.ts
index 7f133fb29..1c57fdad1 100644
--- a/apps/web/src/lib/stashImageCompression.ts
+++ b/apps/web/src/lib/imageCompression.ts
@@ -1,13 +1,14 @@
/**
- * Re-encoding for stashed image attachments.
+ * Downscale + re-encode for image attachments that are too big for where
+ * they're headed. Two consumers share the same pipeline:
*
- * The composer accepts images up to `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES`
- * (10MB), but the stash persists them as base64 in localStorage, where the
- * whole origin shares a ~5MB quota. Rather than refuse large screenshots,
- * downscale + re-encode them to JPEG so a stashed prompt keeps its images.
+ * - The prompt stash persists images as base64 in localStorage (~5MB origin
+ * quota), so `compressImageForStash` targets a per-image character budget.
+ * - The composer accepts pasted/dropped images larger than the provider's
+ * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit
+ * via `compressImageToByteLimit` instead of rejecting the paste.
*
- * Only the *stashed copy* is compressed; the live composer attachment is
- * untouched, so sending without stashing still uploads the original file.
+ * Images already within budget pass through untouched.
*/
/**
@@ -17,6 +18,12 @@
const MAX_DIMENSION = 2048;
/** Base64 budget for a single stashed image (~975KB of binary). */
export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000;
+/**
+ * Ceiling on the *source* file handed to the re-encoder. File size is a
+ * proxy for pixel count, and decoding hundreds of megapixels into an
+ * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it.
+ */
+export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024;
/**
* Quality ladder tried in order until the encoded image fits the budget.
* The floor stays high enough to avoid visible blocking on UI screenshots;
@@ -35,14 +42,18 @@ export interface CompressedStashImage {
}
/**
- * Why an image could not be stashed. Callers report these differently:
+ * Why an image could not be compressed. Callers report these differently:
* "too large" is a budget outcome, "unreadable" is a decode failure.
*/
-export type StashImageFailureReason = "too-large" | "unreadable";
+export type ImageCompressionFailureReason = "too-large" | "unreadable";
export type CompressStashImageResult =
| { ok: true; image: CompressedStashImage }
- | { ok: false; reason: StashImageFailureReason };
+ | { ok: false; reason: ImageCompressionFailureReason };
+
+export type CompressImageFileResult =
+ | { ok: true; file: File; recompressed: boolean }
+ | { ok: false; reason: ImageCompressionFailureReason };
/** Chunked so a large image can't blow the argument limit of `fromCharCode`. */
const BASE64_CHUNK_SIZE = 0x8000;
@@ -73,6 +84,28 @@ function dataUrlByteLength(dataUrl: string): number {
return Math.max(0, Math.floor((payload.length * 3) / 4) - padding);
}
+/** Base64 payload of a data URL decoded back into a `File`. */
+function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File {
+ const payload = dataUrl.slice(dataUrl.indexOf(",") + 1);
+ const binary = atob(payload);
+ const bytes = new Uint8Array(binary.length);
+ for (let index = 0; index < binary.length; index += 1) {
+ bytes[index] = binary.charCodeAt(index);
+ }
+ return new File([bytes], name, { type: mimeType });
+}
+
+/**
+ * Re-encoding changes the container, so a name like `shot.png` would lie
+ * about its contents. Swap the extension to match the encoded mime type.
+ */
+function fileNameForMimeType(name: string, mimeType: string): string {
+ const extension = mimeType === "image/webp" ? ".webp" : ".jpg";
+ const dotIndex = name.lastIndexOf(".");
+ const base = dotIndex > 0 ? name.slice(0, dotIndex) : name;
+ return `${base}${extension}`;
+}
+
function canRecompress(): boolean {
return (
typeof createImageBitmap === "function" &&
@@ -124,10 +157,10 @@ async function encodeToDataUrl(
}
/**
- * Draws `bitmap` scaled to fit `maxDimension` and encodes it as JPEG,
- * stepping quality down until the data URL fits `budgetChars`. Returns the
- * smallest encoding produced, even if it still exceeds the budget, so the
- * caller can decide whether to keep or drop it.
+ * Draws `bitmap` scaled to fit `maxDimension` and encodes it, stepping
+ * quality down until the data URL fits `budgetChars`. Returns the smallest
+ * encoding produced, even if it still exceeds the budget, so the caller can
+ * decide whether to keep or drop it.
*/
async function encodeWithinBudget(
bitmap: ImageBitmap,
@@ -165,35 +198,15 @@ async function encodeWithinBudget(
return smallest;
}
+type ReencodeResult =
+ | { ok: true; dataUrl: string; mimeType: string }
+ | { ok: false; reason: ImageCompressionFailureReason };
+
/**
- * Produces the payload to persist for a stashed image.
- *
- * Small images are stored verbatim (preserving PNG transparency and exact
- * pixels). Anything over budget is downscaled and JPEG-encoded; if it still
- * doesn't fit after the fallback passes, returns `null` so the caller can
- * record it as dropped.
+ * Shared re-encode loop: decodes `file`, then walks the quality ladder and
+ * fallback downscale passes until an encoding fits `budgetChars`.
*/
-export async function compressImageForStash(
- file: File,
- budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,
-): Promise<CompressStashImageResult> {
- let originalDataUrl: string;
- try {
- originalDataUrl = await blobToDataUrl(file);
- } catch {
- return { ok: false, reason: "unreadable" };
- }
- if (originalDataUrl.length <= budgetChars) {
- return {
- ok: true,
- image: {
- dataUrl: originalDataUrl,
- mimeType: file.type,
- sizeBytes: file.size,
- recompressed: false,
- },
- };
- }
+async function reencodeWithinBudget(file: File, budgetChars: number): Promise<ReencodeResult> {
if (!canRecompress()) {
return { ok: false, reason: "too-large" };
}
@@ -225,22 +238,14 @@ export async function compressImageForStash(
// precisely *because* the target is too big (OOM on a large bitmap).
// Keep trying the smaller fallback scales rather than giving up: a
// reduced pass may well succeed. The exception must never escape,
- // though, since the caller finalizes the entry after this returns and
- // a throw would strand it as permanently "still saving".
+ // though, since callers finalize state after this returns and a
+ // throw would strand it (e.g. a stash entry stuck "still saving").
encodeFailed = true;
continue;
}
encodeFailed = false;
if (encoded && encoded.dataUrl.length <= budgetChars) {
- return {
- ok: true,
- image: {
- dataUrl: encoded.dataUrl,
- mimeType: encoded.mimeType,
- sizeBytes: dataUrlByteLength(encoded.dataUrl),
- recompressed: true,
- },
- };
+ return { ok: true, dataUrl: encoded.dataUrl, mimeType: encoded.mimeType };
}
}
return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" };
@@ -248,3 +253,83 @@ export async function compressImageForStash(
bitmap.close();
}
}
+
+/**
+ * Produces the payload to persist for a stashed image.
+ *
+ * Small images are stored verbatim (preserving PNG transparency and exact
+ * pixels). Anything over budget is downscaled and re-encoded; if it still
+ * doesn't fit after the fallback passes, reports a failure so the caller
+ * can record it as dropped.
+ */
+export async function compressImageForStash(
+ file: File,
+ budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS,
+): Promise<CompressStashImageResult> {
+ let originalDataUrl: string;
+ try {
+ originalDataUrl = await blobToDataUrl(file);
+ } catch {
+ return { ok: false, reason: "unreadable" };
+ }
+ if (originalDataUrl.length <= budgetChars) {
+ return {
+ ok: true,
+ image: {
+ dataUrl: originalDataUrl,
+ mimeType: file.type,
+ sizeBytes: file.size,
+ recompressed: false,
+ },
+ };
+ }
+ const reencoded = await reencodeWithinBudget(file, budgetChars);
+ if (!reencoded.ok) {
+ return reencoded;
+ }
+ return {
+ ok: true,
+ image: {
+ dataUrl: reencoded.dataUrl,
+ mimeType: reencoded.mimeType,
+ sizeBytes: dataUrlByteLength(reencoded.dataUrl),
+ recompressed: true,
+ },
+ };
+}
+
+/**
+ * Shrinks `file` until its binary size fits `maxBytes`, returning a new
+ * `File` (WebP or JPEG). Files already within the limit pass through
+ * untouched, preserving their exact bytes and format. Sources above
+ * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is
+ * the risk, so no amount of output budget makes them safe.
+ */
+export async function compressImageToByteLimit(
+ file: File,
+ maxBytes: number,
+): Promise<CompressImageFileResult> {
+ if (file.size <= maxBytes) {
+ return { ok: true, file, recompressed: false };
+ }
+ if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) {
+ return { ok: false, reason: "too-large" };
+ }
+ // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes
+ // into 4 chars; flooring keeps the budget a hair conservative instead of
+ // admitting an encoding right at the byte cap.
+ const budgetChars = Math.floor(maxBytes / 3) * 4;
+ const reencoded = await reencodeWithinBudget(file, budgetChars);
+ if (!reencoded.ok) {
+ return reencoded;
+ }
+ return {
+ ok: true,
+ file: dataUrlToFile(
+ reencoded.dataUrl,
+ fileNameForMimeType(file.name || "image", reencoded.mimeType),
+ reencoded.mimeType,
+ ),
+ recompressed: true,
+ };
+}