Skip to content

armgit / fx0

Public Imported from GitHub
0

A lightweight design agent built with libfx. Describe a page and an agent writes it as one HTML file.

TypeScript

Resize attached images in the browser and check them on the server

Esteban Suarez1243f47f71bd
diff --git a/app/api/prompt/route.ts b/app/api/prompt/route.ts
index 5e5dd89..b3213fe 100644
--- a/app/api/prompt/route.ts
+++ b/app/api/prompt/route.ts
@@ -10,14 +10,11 @@ import { checkBotId } from 'botid/server'
import { trialEnabled, trialIpHash, trialStore, type TrialCredential, type TrialReservation } from '../../../lib/trial'
import { TRIAL_MODEL, usesFreePrompts } from '../../../lib/trial-policy'
import { DEFAULT_MODEL, log } from '../../../lib/session'
+import { validatePromptImages, validatePromptPreviews } from '../../../lib/prompt-images'
import { PROMPT_TEXT_LIMIT_ERROR, PROMPT_TEXT_MAX_BYTES, promptTextBytes } from '../../../lib/prompt-limits'
export const maxDuration = 800
-// libfx stores each prompt, images included, before it acknowledges it; this
-// keeps one prompt's images well inside its stored-input limits.
-const MAX_DURABLE_IMAGE_BYTES = 3 * 1024 * 1024
-
// Catalog files ship with this route (outputFileTracingIncludes in next.config.mjs).
async function catalogDesignMd(id: string, name: string): Promise<DesignMdInput> {
const text = await readFile(path.join(process.cwd(), 'lib', 'design-md', `${id}.md`), 'utf8')
@@ -43,34 +40,17 @@ export const POST = withPromptRoute(async (request, context) => {
}
const text = typeof body?.text === 'string' ? body.text.trim() : ''
- const images = Array.isArray(body?.images)
- ? body.images.filter(
- (i): i is { type: 'image'; data: string; mimeType: string } =>
- i?.type === 'image' && typeof i?.data === 'string' && typeof i?.mimeType === 'string',
- )
- : undefined
- if (!text && !images?.length) return Response.json({ error: 'empty prompt' }, { status: 400 })
+ // Checked whole before any reservation or write: a refusal leaves the composer's draft.
+ const checkedImages = validatePromptImages(body?.images)
+ if ('error' in checkedImages) return Response.json({ error: checkedImages.error }, { status: 413 })
+ const images = checkedImages.images.length ? checkedImages.images : undefined
+ if (!text && !images) return Response.json({ error: 'empty prompt' }, { status: 400 })
// Rejected whole, never cut: the composer keeps the draft and shows this error.
if (promptTextBytes(text) > PROMPT_TEXT_MAX_BYTES) return Response.json({ error: PROMPT_TEXT_LIMIT_ERROR }, { status: 413 })
- if ((images?.length ?? 0) > 8) return Response.json({ error: 'prompt cannot contain more than 8 images' }, { status: 413 })
- let imageBytes = 0
- for (const [index, image] of (images ?? []).entries()) {
- if (!image.mimeType || image.mimeType.length > 128 || image.data.length > 5 * 1024 * 1024) {
- return Response.json({ error: `invalid or oversized image ${index + 1}` }, { status: 413 })
- }
- imageBytes += image.data.length
- }
- if (imageBytes > MAX_DURABLE_IMAGE_BYTES) {
- return Response.json({ error: 'prompt images exceed the 3 MiB limit' }, { status: 413 })
- }
// Thumbnails persisted with the user's event so a reload still shows the images.
// The full images above go to the model; these are display-only and bounded.
- const previews = Array.isArray(body?.previews)
- ? body.previews.filter((preview): preview is string =>
- typeof preview === 'string' && /^data:image\/(jpeg|png|webp);base64,[A-Za-z0-9+/=]+$/.test(preview) && preview.length <= 96 * 1024)
- .slice(0, images?.length ?? 0)
- : []
+ const previews = validatePromptPreviews(body?.previews, images?.length ?? 0)
// Settings picked before the chat existed; ignored once the chat has state.
const generation = body?.generation && typeof body.generation === 'object' ? body.generation : {}
diff --git a/components/attach-image.ts b/components/attach-image.ts
new file mode 100644
index 0000000..c19d0ac
--- /dev/null
+++ b/components/attach-image.ts
@@ -0,0 +1,104 @@
+import { imageThumbnail } from './image-thumbnail'
+
+export interface NormalizedImage {
+ data: string
+ mimeType: string
+ preview: string
+ thumbnail: string | null
+}
+
+// What the composer sends for one attached image: at most 2048 px on a side, usually
+// WebP, small enough that several fit the prompt's 3 MiB. The same bytes go to the model.
+const MAX_SIDE = 2048
+const TARGET_BYTES = 700 * 1024
+const MAX_SOURCE_BYTES = 40 * 1024 * 1024
+// Refused before drawing: a tiny file can decode to an enormous bitmap.
+const MAX_SOURCE_PIXELS = 60_000_000
+const KEPT_AS_IS = ['image/png', 'image/jpeg', 'image/webp']
+const UNSUPPORTED = 'this image format is not supported here. use PNG, JPEG, GIF or WebP.'
+
+function readDataUrl(blob: Blob): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const reader = new FileReader()
+ reader.onload = () => resolve(String(reader.result))
+ reader.onerror = () => reject(new Error('unable to read the selected image.'))
+ reader.readAsDataURL(blob)
+ })
+}
+
+async function decode(file: File): Promise<CanvasImageSource & { width: number; height: number }> {
+ try {
+ return await createImageBitmap(file)
+ } catch {
+ const url = URL.createObjectURL(file)
+ try {
+ const image = new Image()
+ image.src = url
+ await image.decode()
+ return Object.assign(image, { width: image.naturalWidth, height: image.naturalHeight })
+ } catch {
+ throw new Error(UNSUPPORTED)
+ } finally {
+ URL.revokeObjectURL(url)
+ }
+ }
+}
+
+function hasTransparency(context: CanvasRenderingContext2D, width: number, height: number): boolean {
+ const pixels = context.getImageData(0, 0, width, height).data
+ for (let index = 3; index < pixels.length; index += 4 * 16) if (pixels[index] < 255) return true
+ return false
+}
+
+function encode(canvas: HTMLCanvasElement, type: string, quality: number): Promise<Blob | null> {
+ return new Promise((resolve) => canvas.toBlob(resolve, type, quality))
+}
+
+async function finish(blob: Blob): Promise<NormalizedImage> {
+ const preview = await readDataUrl(blob)
+ return { data: preview.slice(preview.indexOf(',') + 1), mimeType: blob.type, preview, thumbnail: await imageThumbnail(preview) }
+}
+
+/** One picked, pasted or dropped file as the composer sends it, or an error to show. */
+export async function normalizeImage(file: File, { webp = true } = {}): Promise<NormalizedImage> {
+ if (file.size > MAX_SOURCE_BYTES) throw new Error('the selected image is larger than 40 MB.')
+ const source = await decode(file)
+ if (source.width * source.height > MAX_SOURCE_PIXELS) throw new Error('the selected image has too many pixels.')
+ // Already small enough: sent as is, so a GIF keeps its animation and nothing is re-encoded.
+ const small = file.size <= TARGET_BYTES && Math.max(source.width, source.height) <= MAX_SIDE
+ if (small && (KEPT_AS_IS.includes(file.type) || file.type === 'image/gif')) return finish(file)
+
+ const canvas = document.createElement('canvas')
+ const context = canvas.getContext('2d')
+ if (!context) throw new Error('unable to read the selected image.')
+ let transparent: boolean | null = null
+ let smallest: Blob | null = null
+ for (const side of [MAX_SIDE, 1600, 1200]) {
+ const scale = Math.min(1, side / Math.max(source.width, source.height))
+ canvas.width = Math.max(1, Math.round(source.width * scale))
+ canvas.height = Math.max(1, Math.round(source.height * scale))
+ context.clearRect(0, 0, canvas.width, canvas.height)
+ context.drawImage(source, 0, 0, canvas.width, canvas.height)
+ for (const quality of [0.82, 0.7, 0.55]) {
+ let blob = webp ? await encode(canvas, 'image/webp', quality) : null
+ // Safari can hand back PNG for WebP; then PNG keeps transparency, JPEG the rest.
+ if (!blob || blob.type !== 'image/webp') {
+ transparent ??= hasTransparency(context, canvas.width, canvas.height)
+ if (transparent) blob = await encode(canvas, 'image/png', 1)
+ else {
+ context.globalCompositeOperation = 'destination-over'
+ context.fillStyle = '#fff'
+ context.fillRect(0, 0, canvas.width, canvas.height)
+ context.globalCompositeOperation = 'source-over'
+ blob = await encode(canvas, 'image/jpeg', quality)
+ }
+ }
+ if (!blob) continue
+ if (!smallest || blob.size < smallest.size) smallest = blob
+ if (blob.size <= TARGET_BYTES) return finish(blob)
+ if (blob.type === 'image/png') break
+ }
+ }
+ if (!smallest) throw new Error('unable to read the selected image.')
+ return finish(smallest)
+}
diff --git a/components/chat-app.tsx b/components/chat-app.tsx
index 13b01cc..abe9a18 100644
--- a/components/chat-app.tsx
+++ b/components/chat-app.tsx
@@ -1314,9 +1314,10 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultMod
}
try {
- const previews = images.length
- ? (await Promise.all(images.map((image) => imageThumbnail(image.preview)))).filter((preview): preview is string => !!preview)
- : []
+ // Made at attach time; computed here only for images restored from a guest draft.
+ // Sent only when every image has one, so none is shown on the wrong image.
+ const thumbnails = await Promise.all(images.map((image) => image.thumbnail ?? imageThumbnail(image.preview)))
+ const previews = thumbnails.every(Boolean) ? thumbnails as string[] : []
const promptSession = !creating ? targetSession : reservedId ?? 'new'
const response = await fetchUi(`/api/prompt?session=${encodeURIComponent(promptSession)}`, {
method: 'POST',
diff --git a/components/composer.tsx b/components/composer.tsx
index f78aa61..996cc4a 100644
--- a/components/composer.tsx
+++ b/components/composer.tsx
@@ -5,11 +5,12 @@ import { type GuestDraft } from './guest-draft'
import { AuthRouteError, fetchUi } from './auth-ui'
import { IconFileText } from './geist-icons'
import ImagePreview from './image-preview'
+import { normalizeImage } from './attach-image'
import { Button } from './ui/button'
import { Popover, PopoverContent, PopoverTrigger } from './ui/popover'
import { checkDesignMdUpload, DESIGN_MD_CATALOG, DESIGN_MD_MAX_BYTES } from '../lib/design-md'
import { fastClick } from '../lib/fast-click'
-import { PROMPT_TEXT_LIMIT_ERROR, PROMPT_TEXT_MAX_BYTES, promptTextBytes } from '../lib/prompt-limits'
+import { PROMPT_IMAGE_MAX_COUNT, PROMPT_IMAGES_MAX_BASE64, PROMPT_TEXT_LIMIT_ERROR, PROMPT_TEXT_MAX_BYTES, promptTextBytes } from '../lib/prompt-limits'
import { focusPrompt, registerPrompt } from '../lib/prompt-focus'
import { Textarea } from './ui/textarea'
import { Tooltip, TooltipContent, TooltipTrigger } from './ui/tooltip'
@@ -33,6 +34,8 @@ export interface AttachedImage {
data: string
mimeType: string
preview: string
+ // Made when the image is attached; missing for one restored from a guest draft.
+ thumbnail?: string | null
}
// A new chat's DESIGN.md: a catalog file by id, or an uploaded file's text.
@@ -40,8 +43,8 @@ const DOUBLE_ESCAPE_MS = 500
export type DesignMdPick = { id: string; name: string } | { name: string; text: string }
-const IMAGE_CAP = 8
-const IMAGE_DATA_CAP = 3 * 1024 * 1024
+const IMAGE_CAP = PROMPT_IMAGE_MAX_COUNT
+const IMAGE_DATA_CAP = PROMPT_IMAGES_MAX_BASE64
const PROMPT_MIN_HEIGHT = 64
const PROMPT_MAX_HEIGHT = 160
@@ -86,20 +89,6 @@ function loadModelCatalog() {
return modelCatalogPromise
}
-function readImage(file: File): Promise<AttachedImage> {
- return new Promise((resolve, reject) => {
- const reader = new FileReader()
- reader.onload = () => {
- const result = String(reader.result)
- const comma = result.indexOf(',')
- if (!result.startsWith('data:image/') || comma < 0)
- return reject(new Error('the selected file is not a supported image.'))
- resolve({ data: result.slice(comma + 1), mimeType: file.type || 'image/png', preview: result })
- }
- reader.onerror = () => reject(new Error('unable to read the selected image.'))
- reader.readAsDataURL(file)
- })
-}
// Geist ships only the outlined bolt; dropping its inner counter fills the shape.
const LIGHTNING_OUTLINE =
@@ -663,7 +652,7 @@ export default function Composer({
if (acceptedFiles.length === 0) return
setAttachmentError('')
try {
- const nextImages = await Promise.all(acceptedFiles.slice(0, IMAGE_CAP).map(readImage))
+ const nextImages = await Promise.all(acceptedFiles.slice(0, IMAGE_CAP).map((file) => normalizeImage(file)))
setImages((current) => {
const available = Math.max(0, IMAGE_CAP - current.length)
const combined = [...current, ...nextImages.slice(0, available)]
@@ -675,7 +664,7 @@ export default function Composer({
return combined
})
} catch (error) {
- setAttachmentError(`${(error as Error).message} Choose another image.`)
+ setAttachmentError(`${(error as Error).message} choose another image.`)
}
}
diff --git a/lib/image-sniff.ts b/lib/image-sniff.ts
index b1f2868..43863fc 100644
--- a/lib/image-sniff.ts
+++ b/lib/image-sniff.ts
@@ -20,3 +20,41 @@ export function sniffImageType(bytes: Uint8Array): SniffedImageType | null {
}
return null
}
+
+const u16be = (b: Uint8Array, i: number) => (b[i] << 8) | b[i + 1]
+const u16le = (b: Uint8Array, i: number) => b[i] | (b[i + 1] << 8)
+const u24le = (b: Uint8Array, i: number) => b[i] | (b[i + 1] << 8) | (b[i + 2] << 16)
+const JPEG_FRAMES = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf])
+
+/** Width and height from an image's header, or null when it is malformed or truncated. */
+export function imageDimensions(bytes: Uint8Array, type: SniffedImageType): { width: number; height: number } | null {
+ if (type === 'image/png') return bytes.length >= 24 && ascii(bytes, 12, 16) === 'IHDR'
+ ? { width: (bytes[16] << 24 | bytes[17] << 16 | bytes[18] << 8 | bytes[19]) >>> 0, height: (bytes[20] << 24 | bytes[21] << 16 | bytes[22] << 8 | bytes[23]) >>> 0 }
+ : null
+ if (type === 'image/gif') return bytes.length >= 10 ? { width: u16le(bytes, 6), height: u16le(bytes, 8) } : null
+ if (type === 'image/webp') {
+ const chunk = ascii(bytes, 12, 16)
+ if (chunk === 'VP8 ' && bytes.length >= 30) return { width: u16le(bytes, 26) & 0x3fff, height: u16le(bytes, 28) & 0x3fff }
+ if (chunk === 'VP8L' && bytes.length >= 25 && bytes[20] === 0x2f) {
+ return { width: 1 + (((bytes[22] & 0x3f) << 8) | bytes[21]), height: 1 + (((bytes[24] & 0x0f) << 10) | (bytes[23] << 2) | ((bytes[22] & 0xc0) >> 6)) }
+ }
+ if (chunk === 'VP8X' && bytes.length >= 30) return { width: 1 + u24le(bytes, 24), height: 1 + u24le(bytes, 27) }
+ return null
+ }
+ if (type === 'image/jpeg') {
+ // Walk the segments to the first frame header, baseline or progressive.
+ let index = 2
+ while (index + 9 <= bytes.length) {
+ if (bytes[index] !== 0xff) return null
+ const marker = bytes[index + 1]
+ if (marker === 0xff) { index++; continue }
+ if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd8)) { index += 2; continue }
+ if (JPEG_FRAMES.has(marker)) return { width: u16be(bytes, index + 7), height: u16be(bytes, index + 5) }
+ const length = u16be(bytes, index + 2)
+ if (length < 2) return null
+ index += 2 + length
+ }
+ return null
+ }
+ return null
+}
diff --git a/lib/prompt-images.ts b/lib/prompt-images.ts
new file mode 100644
index 0000000..0e3d090
--- /dev/null
+++ b/lib/prompt-images.ts
@@ -0,0 +1,49 @@
+// @ts-expect-error Node strip-types tests require the source extension.
+import { imageDimensions, sniffImageType } from './image-sniff.ts'
+// @ts-expect-error Node strip-types tests require the source extension.
+import { PROMPT_IMAGE_MAX_COUNT, PROMPT_IMAGE_TYPES, PROMPT_IMAGES_MAX_BASE64 } from './prompt-limits.ts'
+
+export interface PromptImage { type: 'image'; data: string; mimeType: string }
+
+// libfx takes at most 3.75 MiB per image and drops one wider or taller than 8000 px.
+const MAX_IMAGE_BYTES = 3.75 * 1024 * 1024
+const MAX_SIDE = 8000
+
+/**
+ * A prompt's images, or the reason they are refused. Each must be canonical base64 of
+ * a PNG, JPEG, GIF or WebP whose bytes match its declared type and whose header gives
+ * sane dimensions, so nothing malformed reaches the journal or the model.
+ */
+export function validatePromptImages(value: unknown): { images: PromptImage[] } | { error: string } {
+ if (value == null) return { images: [] }
+ if (!Array.isArray(value)) return { error: 'images must be a list' }
+ if (value.length > PROMPT_IMAGE_MAX_COUNT) return { error: `prompt cannot contain more than ${PROMPT_IMAGE_MAX_COUNT} images` }
+ const images: PromptImage[] = []
+ let total = 0
+ for (const [index, image] of value.entries()) {
+ const label = `image ${index + 1}`
+ if (!image || image.type !== 'image' || typeof image.data !== 'string' || typeof image.mimeType !== 'string') return { error: `${label} is invalid` }
+ if (!(PROMPT_IMAGE_TYPES as readonly string[]).includes(image.mimeType)) return { error: `${label} must be PNG, JPEG, GIF or WebP` }
+ total += image.data.length
+ if (total > PROMPT_IMAGES_MAX_BASE64) return { error: 'prompt images exceed the 3 MiB limit' }
+ if (!image.data || image.data.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(image.data)) return { error: `${label} is not valid base64` }
+ const bytes = Buffer.from(image.data, 'base64')
+ if (bytes.toString('base64') !== image.data) return { error: `${label} is not valid base64` }
+ if (bytes.length > MAX_IMAGE_BYTES) return { error: `${label} is larger than 3.75 MiB` }
+ const type = sniffImageType(bytes)
+ if (!type || type !== image.mimeType) return { error: `${label} is not the ${image.mimeType.slice(6).toUpperCase()} it claims to be` }
+ const size = imageDimensions(bytes, type)
+ if (!size || !size.width || !size.height) return { error: `${label} is damaged or truncated` }
+ if (size.width > MAX_SIDE || size.height > MAX_SIDE) return { error: `${label} is larger than ${MAX_SIDE} px on a side` }
+ images.push({ type: 'image', data: image.data, mimeType: image.mimeType })
+ }
+ return { images }
+}
+
+const PREVIEW = /^data:image\/(?:jpeg|png|webp);base64,[A-Za-z0-9+/=]+$/
+
+/** Display thumbnails, one per image in order, or none: a missing one would shift the rest. */
+export function validatePromptPreviews(value: unknown, count: number): string[] {
+ if (!Array.isArray(value) || value.length !== count) return []
+ return value.every((preview) => typeof preview === 'string' && preview.length <= 96 * 1024 && PREVIEW.test(preview)) ? value : []
+}
diff --git a/lib/prompt-limits.ts b/lib/prompt-limits.ts
index 7f6c4c9..cce8e0b 100644
--- a/lib/prompt-limits.ts
+++ b/lib/prompt-limits.ts
@@ -8,3 +8,9 @@ export const PROMPT_TEXT_LIMIT_ERROR = 'prompt text must be 256 KB or smaller.'
export function promptTextBytes(text: string): number {
return new TextEncoder().encode(text).byteLength
}
+
+// A prompt's images, checked in the composer and again by /api/prompt. Their base64
+// stays within 3 MiB so the whole prompt fits one function request body.
+export const PROMPT_IMAGE_MAX_COUNT = 8
+export const PROMPT_IMAGES_MAX_BASE64 = 3 * 1024 * 1024
+export const PROMPT_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'] as const
diff --git a/package.json b/package.json
index 4872614..aa4cfca 100644
--- a/package.json
+++ b/package.json
@@ -9,12 +9,12 @@
"build": "next build",
"typecheck": "next typegen && tsc --noEmit --incremental false",
"start": "next start",
- "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/runtime.test.mjs",
+ "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/prompt-images.test.mjs tests/runtime.test.mjs",
"test:auth": "node --experimental-strip-types --test tests/auth.test.mjs tests/auth-callback.test.mjs tests/signout.test.mjs tests/ownership.test.mjs tests/history.test.mjs tests/route-security.test.mjs tests/gateway-session.test.mjs tests/libfx-transport.test.mjs tests/libfx-agent.test.mjs tests/auth-ui.test.mjs tests/html-ui.test.mjs tests/guest-ui.test.mjs tests/guest-draft.test.mjs tests/trial.test.mjs tests/turn-funding.test.mjs",
"test:journal:integration": "node --env-file-if-exists=.env.local --experimental-strip-types --test tests/journal.integration.mjs",
"test:smoke": "node tests/local-auth-smoke.mjs",
"db:migrate": "node --env-file-if-exists=.env.local scripts/migrate-database.mjs",
- "test:artifact:browser": "node --experimental-strip-types --test tests/design-artifact.browser.mjs",
+ "test:artifact:browser": "node --experimental-strip-types --test tests/design-artifact.browser.mjs tests/attach-image.browser.mjs",
"test:budget:integration": "node --experimental-strip-types --test tests/turn-budget.integration.mjs",
"test:turns:integration": "node --experimental-strip-types --test tests/turn-work.integration.mjs",
"test:trial:integration": "node --experimental-strip-types --test tests/trial.integration.mjs",
diff --git a/tests/attach-image.browser.mjs b/tests/attach-image.browser.mjs
new file mode 100644
index 0000000..0c76615
--- /dev/null
+++ b/tests/attach-image.browser.mjs
@@ -0,0 +1,90 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { execFile } from 'node:child_process'
+import http from 'node:http'
+import { existsSync } from 'node:fs'
+import { mkdtemp, readFile, rm } from 'node:fs/promises'
+import { stripTypeScriptTypes } from 'node:module'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+
+// Opt-in: runs real Chrome headless on this machine. Set CHROME_PATH for another browser binary.
+const CHROME = process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
+
+// The composer's attachment code, types stripped and imports inlined, run on images the
+// page draws: what reaches the prompt must be at most 2048 px, usually WebP, and small.
+async function attachCode() {
+ const read = async (name) => stripTypeScriptTypes(await readFile(new URL(`../components/${name}`, import.meta.url), 'utf8'))
+ .replace(/^import .*$/gm, '').replace(/^export /gm, '')
+ return `${await read('image-thumbnail.ts')}\n${await read('attach-image.ts')}`
+}
+
+const PAGE = (code) => `<!doctype html><html><head><title>attach</title></head><body><pre id="out">{}</pre><script>${code}
+const canvasFile = async (width, height, { type = 'image/png', transparent = false, quality } = {}) => {
+ const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height
+ const context = canvas.getContext('2d')
+ if (!transparent) { const gradient = context.createLinearGradient(0, 0, width, height); gradient.addColorStop(0, '#2a6'); gradient.addColorStop(1, '#a26'); context.fillStyle = gradient; context.fillRect(0, 0, width, height) }
+ for (let i = 0; i < 400; i++) { context.fillStyle = 'hsl(' + (i * 37 % 360) + ' 70% 50%)'; context.beginPath(); context.arc((i * 97) % width, (i * 53) % height, 20 + i % 60, 0, 7); context.fill() }
+ const blob = await new Promise((resolve) => canvas.toBlob(resolve, type, quality))
+ return new File([blob], 'image', { type })
+}
+const describe = async (result) => {
+ const image = new Image(); image.src = result.preview; await image.decode()
+ return { type: result.mimeType, width: image.naturalWidth, height: image.naturalHeight, bytes: atob(result.data).length, thumbnail: !!result.thumbnail && result.thumbnail.startsWith('data:image/jpeg') }
+}
+;(async () => {
+ const out = {}
+ const run = async (name, make, options) => {
+ try { out[name] = await describe(await normalizeImage(await make(), options)) } catch (error) { out[name] = { error: error.message } }
+ }
+ await run('photo', () => canvasFile(4000, 3000))
+ await run('transparentNoWebp', () => canvasFile(3000, 2000, { transparent: true }), { webp: false })
+ await run('opaqueNoWebp', () => canvasFile(3000, 2000), { webp: false })
+ const small = await canvasFile(800, 600, { type: 'image/jpeg', quality: 0.8 })
+ const keptSmall = await normalizeImage(small)
+ out.smallJpeg = { ...(await describe(keptSmall)), unchanged: atob(keptSmall.data).length === small.size }
+ await run('gif', async () => new File([Uint8Array.from(atob('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7'), (c) => c.charCodeAt(0))], 'a.gif', { type: 'image/gif' }))
+ await run('notAnImage', async () => new File(['<svg onload=alert(1)>'], 'x.heic', { type: 'image/heic' }))
+ document.getElementById('out').textContent = JSON.stringify(out)
+ await fetch('/done')
+})()
+</script><img alt="" src="/hold"></body></html>`
+
+test('in Chrome, attached images are resized, re-encoded and kept small, and unreadable files are refused', async () => {
+ assert.ok(existsSync(CHROME), `Chrome not found at ${CHROME}; set CHROME_PATH`)
+ const dir = await mkdtemp(join(tmpdir(), 'fx0-attach-'))
+ const page = PAGE(await attachCode())
+ // The page's load waits on /hold, which answers once the script reports /done, so
+ // Chrome dumps the page only after the asynchronous encoding has finished.
+ let release
+ const done = new Promise((resolve) => { release = resolve })
+ const server = http.createServer((request, response) => {
+ if (request.url === '/done') { release(); return response.end() }
+ if (request.url === '/hold') return void Promise.race([done, new Promise((resolve) => setTimeout(resolve, 30_000))]).then(() => response.writeHead(404).end())
+ response.writeHead(200, { 'content-type': 'text/html' }).end(page)
+ })
+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
+ try {
+ const { stdout } = await promisify(execFile)(CHROME, [
+ '--headless=new', '--disable-gpu', '--no-first-run', '--no-default-browser-check',
+ `--user-data-dir=${join(dir, 'profile')}`, '--dump-dom', `http://127.0.0.1:${server.address().port}/`,
+ ], { timeout: 90_000, maxBuffer: 16 * 1024 * 1024 })
+ const out = JSON.parse(stdout.match(/<pre id="out">([^<]*)<\/pre>/)[1])
+ assert.equal(out.photo.type, 'image/webp')
+ assert.deepEqual([out.photo.width, out.photo.height], [2048, 1536])
+ assert.ok(out.photo.bytes <= 700 * 1024, `photo is ${out.photo.bytes} bytes`)
+ assert.equal(out.photo.thumbnail, true)
+ assert.equal(out.transparentNoWebp.type, 'image/png', 'without WebP, transparency keeps PNG')
+ assert.ok(Math.max(out.transparentNoWebp.width, out.transparentNoWebp.height) <= 2048)
+ assert.equal(out.opaqueNoWebp.type, 'image/jpeg', 'without WebP, an opaque image becomes JPEG')
+ assert.ok(out.opaqueNoWebp.bytes <= 700 * 1024)
+ assert.deepEqual({ type: out.smallJpeg.type, width: out.smallJpeg.width, unchanged: out.smallJpeg.unchanged }, { type: 'image/jpeg', width: 800, unchanged: true })
+ assert.equal(out.gif.type, 'image/gif', 'a small GIF is sent as is, animation intact')
+ assert.match(out.notAnImage.error, /not supported here/)
+ } finally {
+ server.closeAllConnections()
+ server.close()
+ await rm(dir, { recursive: true, force: true })
+ }
+})
diff --git a/tests/prompt-images.test.mjs b/tests/prompt-images.test.mjs
new file mode 100644
index 0000000..4bebee8
--- /dev/null
+++ b/tests/prompt-images.test.mjs
@@ -0,0 +1,72 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { imageDimensions } from '../lib/image-sniff.ts'
+import { validatePromptImages, validatePromptPreviews } from '../lib/prompt-images.ts'
+
+const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==', 'base64')
+const GIF = Buffer.from('R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7', 'base64')
+const u16 = (n) => [n >> 8, n & 255]
+const jpeg = (frame, width, height) => Buffer.from([0xff, 0xd8,
+ 0xff, 0xe0, 0, 16, 0x4a, 0x46, 0x49, 0x46, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0,
+ 0xff, frame, 0, 17, 8, ...u16(height), ...u16(width), 3, 1, 0x22, 0, 2, 0x11, 1, 3, 0x11, 1, 0xff, 0xd9])
+const riff = (chunk, body) => {
+ const payload = Buffer.concat([Buffer.from(chunk), Buffer.from([body.length, 0, 0, 0]), Buffer.from(body)])
+ return Buffer.concat([Buffer.from('RIFF'), Buffer.from([payload.length + 4, 0, 0, 0]), Buffer.from('WEBP'), payload])
+}
+const webpLossy = (width, height) => riff('VP8 ', [0, 0, 0, 0x9d, 0x01, 0x2a, width & 255, width >> 8, height & 255, height >> 8])
+const webpLossless = (width, height) => {
+ const w = width - 1, h = height - 1
+ return riff('VP8L', [0x2f, w & 255, ((w >> 8) & 0x3f) | ((h & 3) << 6), (h >> 2) & 255, (h >> 10) & 0x0f])
+}
+const webpExtended = (width, height) => {
+ const w = width - 1, h = height - 1
+ return riff('VP8X', [0, 0, 0, 0, w & 255, (w >> 8) & 255, w >> 16, h & 255, (h >> 8) & 255, h >> 16])
+}
+const image = (bytes, mimeType) => ({ type: 'image', data: Buffer.from(bytes).toString('base64'), mimeType })
+
+test('dimensions come from every supported header, including progressive JPEG and each WebP layout', () => {
+ assert.deepEqual(imageDimensions(PNG, 'image/png'), { width: 1, height: 1 })
+ assert.deepEqual(imageDimensions(GIF, 'image/gif'), { width: 1, height: 1 })
+ assert.deepEqual(imageDimensions(jpeg(0xc0, 640, 480), 'image/jpeg'), { width: 640, height: 480 })
+ assert.deepEqual(imageDimensions(jpeg(0xc2, 1920, 1080), 'image/jpeg'), { width: 1920, height: 1080 })
+ assert.deepEqual(imageDimensions(webpLossy(800, 600), 'image/webp'), { width: 800, height: 600 })
+ assert.deepEqual(imageDimensions(webpLossless(2048, 1365), 'image/webp'), { width: 2048, height: 1365 })
+ assert.deepEqual(imageDimensions(webpExtended(4000, 3000), 'image/webp'), { width: 4000, height: 3000 })
+ assert.equal(imageDimensions(jpeg(0xc0, 640, 480).subarray(0, 24), 'image/jpeg'), null, 'truncated before the frame')
+ assert.equal(imageDimensions(PNG.subarray(0, 20), 'image/png'), null)
+})
+
+test('valid images pass unchanged', () => {
+ const images = [image(PNG, 'image/png'), image(GIF, 'image/gif'), image(jpeg(0xc2, 1920, 1080), 'image/jpeg'), image(webpLossless(2048, 1365), 'image/webp')]
+ assert.deepEqual(validatePromptImages(images), { images })
+ assert.deepEqual(validatePromptImages(undefined), { images: [] })
+})
+
+test('anything malformed, mislabeled, oversized or unsupported is refused with the image it is about', () => {
+ const refuse = (images, error) => assert.deepEqual(validatePromptImages(images), { error })
+ refuse('nope', 'images must be a list')
+ refuse([image(PNG, 'image/png'), { type: 'image', data: 1, mimeType: 'image/png' }], 'image 2 is invalid')
+ refuse([image(PNG, 'image/svg+xml')], 'image 1 must be PNG, JPEG, GIF or WebP')
+ refuse([image(PNG, 'image/heic')], 'image 1 must be PNG, JPEG, GIF or WebP')
+ refuse([{ ...image(PNG, 'image/png'), data: PNG.toString('base64').replace(/=+$/, '') }], 'image 1 is not valid base64')
+ refuse([{ ...image(PNG, 'image/png'), data: ` ${PNG.toString('base64').slice(1)}` }], 'image 1 is not valid base64')
+ // Non-zero padding bits decode to the same bytes but are not the canonical encoding.
+ assert.ok(PNG.toString('base64').endsWith('g=='))
+ refuse([{ type: 'image', data: PNG.toString('base64').replace(/g==$/, 'h=='), mimeType: 'image/png' }], 'image 1 is not valid base64')
+ refuse([image(PNG, 'image/jpeg')], 'image 1 is not the JPEG it claims to be')
+ refuse([image(Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"></svg>'), 'image/png')], 'image 1 is not the PNG it claims to be')
+ refuse([image(jpeg(0xc0, 640, 480).subarray(0, 24), 'image/jpeg')], 'image 1 is damaged or truncated')
+ refuse([image(webpExtended(9000, 100), 'image/webp')], 'image 1 is larger than 8000 px on a side')
+ refuse(Array.from({ length: 9 }, () => image(PNG, 'image/png')), 'prompt cannot contain more than 8 images')
+ const big = Buffer.concat([PNG.subarray(0, 24), Buffer.alloc(1.2 * 1024 * 1024)])
+ refuse([image(big, 'image/png'), image(big, 'image/png')], 'prompt images exceed the 3 MiB limit')
+})
+
+test('thumbnails are kept only when every image has one, so none lands on the wrong image', () => {
+ const thumb = `data:image/jpeg;base64,${PNG.toString('base64')}`
+ assert.deepEqual(validatePromptPreviews([thumb, thumb], 2), [thumb, thumb])
+ assert.deepEqual(validatePromptPreviews([thumb], 2), [])
+ assert.deepEqual(validatePromptPreviews([thumb, 'javascript:alert(1)'], 2), [])
+ assert.deepEqual(validatePromptPreviews([thumb, `data:image/jpeg;base64,${'A'.repeat(96 * 1024)}`], 2), [])
+ assert.deepEqual(validatePromptPreviews('x', 0), [])
+})