A lightweight design agent built with libfx. Describe a page and an agent writes it as one HTML file.
TypeScript
Let the agent screenshot its published design
Esteban Suarezcb4eba73fa91
diff --git a/app/api/artifact-frame/route.ts b/app/api/artifact-frame/route.tsindex e30a921..3c59664 100644--- a/app/api/artifact-frame/route.ts+++ b/app/api/artifact-frame/route.ts@@ -1,5 +1,6 @@import { ARTIFACT_IMAGE_PATH, artifactImageSrc } from '../../../lib/artifact-image'import { DESIGN_ARTIFACT_EMBEDDED_CSP, designFrameCsp, designFrameHtml, prepareDesignArtifactHtml, remoteImagesEnabled } from '../../../lib/design-artifact'+import { DESIGN_CAPTURE_RELAY, designCaptureScript } from '../../../lib/design-capture'import { markRemoteImages } from '../../../lib/remote-image'import { readDesignArtifactAt } from '../../../lib/journal'import { readGeneration } from '../../../lib/history'@@ -32,7 +33,10 @@ export const GET = withSessionRoute(async (request, { id: sessionId, user }) =>? markRemoteImages(artifact, crypto.randomUUID().replaceAll('-', ''), { remote: remoteImagesEnabled() }): { html: artifact, images: [] }const images = marked.images.map(({ marker, url }) => ({ marker, src: artifactImageSrc(sessionId, url) }))- const prepared = marked.html == null ? null : prepareDesignArtifactHtml(marked.html, DESIGN_ARTIFACT_EMBEDDED_CSP, readyToken)+ // Every preview can be photographed where it is (lib/design-capture.ts), on screen or as+ // the hidden copy screenshot_design loads when no preview shows the revision.+ const prepared = marked.html == null ? null : prepareDesignArtifactHtml(marked.html, DESIGN_ARTIFACT_EMBEDDED_CSP, readyToken,+ designCaptureScript(readyToken))// Every stored design passed validation, so this only guards against one that// somehow lacks a doctype: it is never shown without its CSP.if (artifact != null && prepared == null) log(sessionId, 'artifact:unrenderable', { revision: past ? revision : live!.revision })@@ -41,7 +45,7 @@ export const GET = withSessionRoute(async (request, { id: sessionId, user }) =>artifact != null ? 'This design could not be prepared for preview.': past ? 'This version is not in the history.' : 'No design artifact has been published for this session.'}</p></body></html>`- const html = designFrameHtml(preparedArtifact, readyToken, images)+ const html = designFrameHtml(preparedArtifact, readyToken, images, DESIGN_CAPTURE_RELAY)return new Response(html, {status: artifact == null ? 404 : 200,headers: {diff --git a/app/api/capture/route.ts b/app/api/capture/route.tsnew file mode 100644index 0000000..59eb23d--- /dev/null+++ b/app/api/capture/route.ts@@ -0,0 +1,95 @@+import { CAPTURE_FAILURES, CAPTURE_TIMING_HEADER, MAX_CAPTURE_BYTES, MAX_CAPTURE_TILES, type CaptureFailure, type FrameTiming, type ShotTiming } from '../../../lib/capture-protocol'+import { imageDimensions, sniffImageType } from '../../../lib/image-sniff'+import { acknowledgeCapture, answerCapture } from '../../../lib/journal'+import { withSessionRoute } from '../../../lib/route-auth'+import { log } from '../../../lib/session'++const CAPTURE_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/+const CAPTURE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp'])+const MAX_SIDE = 16384++type TabTiming = Pick<ShotTiming, 'path' | 'receivedAt' | 'sentAt' | 'previewWaitMs' | 'roundTripMs' | 'frame'>++// The tab's part of the trace. It only labels the photo, so anything malformed is dropped.+function tabTiming(header: string | null): TabTiming | null {+ try {+ const value = JSON.parse(header ?? 'null')+ const number = (n: unknown) => (typeof n === 'number' && Number.isFinite(n) ? n : null)+ const [receivedAt, sentAt, previewWaitMs, roundTripMs] = [value?.receivedAt, value?.sentAt, value?.previewWaitMs, value?.roundTripMs].map(number)+ if (receivedAt === null || sentAt === null || previewWaitMs === null || roundTripMs === null) return null+ const frameParts = [value.frame?.fontsMs, value.frame?.libraryMs, value.frame?.encodeMs].map(number)+ const frame: FrameTiming | null = frameParts.every((n) => n !== null)+ ? { fontsMs: frameParts[0]!, libraryMs: frameParts[1]!, encodeMs: frameParts[2]! }+ : null+ return { path: value.path === 'hidden' ? 'hidden' : 'preview', receivedAt, sentAt, previewWaitMs, roundTripMs, frame }+ } catch {+ return null+ }+}++// A tab's reply for screenshot_design (lib/capture-protocol.ts): `?ack=1` as it takes the+// request, then either `?failed=<reason>` or the photo itself, one image or a full page's+// tiles, as multipart form data. Every reply counts only for the capture the tool call is+// waiting on; `ok: false` tells the tab it is no longer open.+const handle = (arrivedAt: number, arrived: number) => withSessionRoute(async (request, { id: sessionId }) => {+ const authMs = performance.now() - arrived+ const params = new URL(request.url).searchParams+ const cid = params.get('cid') ?? ''+ if (!CAPTURE_ID.test(cid)) return Response.json({ error: 'invalid capture' }, { status: 400 })+ if (params.get('ack') === '1') return Response.json({ ok: await acknowledgeCapture(sessionId, cid) })+ const failed = params.get('failed')+ if (failed !== null) {+ if (!Object.hasOwn(CAPTURE_FAILURES, failed)) return Response.json({ error: 'invalid capture' }, { status: 400 })+ const accepted = await answerCapture(sessionId, cid, { kind: 'failed', reason: failed as CaptureFailure })+ log(sessionId, 'capture:failed', { reason: failed, accepted })+ return Response.json({ ok: accepted })+ }+ // A refused photo still answers the capture, so the call stops waiting and says why.+ const refuse = async (status: number, error: string, reason: CaptureFailure) => {+ const accepted = await answerCapture(sessionId, cid, { kind: 'failed', reason })+ log(sessionId, 'capture:refused', { status, reason, accepted })+ return Response.json({ error }, { status })+ }+ // A multipart body adds a few hundred bytes per image around the images themselves. A+ // browser always sends its length; one without it is refused before it is read.+ const length = Number(request.headers.get('content-length'))+ if (!(length > 0)) return refuse(411, 'length required', 'refused')+ if (length > MAX_CAPTURE_BYTES + 4096) return refuse(413, 'capture too large', 'too-large')+ const reading = performance.now()+ let files: File[]+ try {+ files = (await request.formData()).getAll('image').filter((entry): entry is File => entry instanceof File)+ } catch {+ return refuse(400, 'invalid capture', 'refused')+ }+ if (!files.length || files.length > MAX_CAPTURE_TILES) return refuse(400, 'invalid capture', 'refused')+ const bytes = files.reduce((sum, file) => sum + file.size, 0)+ if (bytes > MAX_CAPTURE_BYTES) return refuse(413, 'capture too large', 'too-large')+ const images: { mimeType: string; data: string }[] = []+ for (const file of files) {+ const bytes = new Uint8Array(await file.arrayBuffer())+ // A header that names an image is not enough: one the model's provider cannot size would+ // fail the whole model request.+ const mimeType = sniffImageType(bytes)+ const dimensions = mimeType && CAPTURE_TYPES.has(mimeType) ? imageDimensions(bytes, mimeType) : null+ if (!mimeType || !dimensions || dimensions.width < 1 || dimensions.height < 1 || dimensions.width > MAX_SIDE || dimensions.height > MAX_SIDE) {+ return refuse(415, 'capture is not an image', 'refused')+ }+ images.push({ mimeType, data: Buffer.from(bytes).toString('base64') })+ }+ const readMs = performance.now() - reading+ // The viewport the tab photographed only labels the photo for the model.+ const size = (name: string) => { const n = Number(params.get(name)); return Number.isInteger(n) && n > 0 && n <= MAX_SIDE ? n : undefined }+ const tab = tabTiming(request.headers.get(CAPTURE_TIMING_HEADER))+ // The timing travels with the photo, so the waiting tool call can log the whole trace.+ const submitting = performance.now()+ const timing: ShotTiming | undefined = tab ? { ...tab, arrivedAt, authMs, readMs, storingAt: Date.now() } : undefined+ const accepted = await answerCapture(sessionId, cid, {+ kind: 'shot', shot: { images, width: size('w'), height: size('h'), truncated: params.get('truncated') === '1', timing },+ })+ // Not accepted: the call stopped waiting (closed by a timeout, a cancel or an earlier answer).+ log(sessionId, accepted ? 'capture:accepted' : 'capture:closed', { images: images.length, bytes, submitMs: Math.round(performance.now() - submitting) })+ return Response.json({ ok: accepted })+})++export const POST = (request: Request) => handle(Date.now(), performance.now())(request)diff --git a/components/chat-app.tsx b/components/chat-app.tsxindex 513b9a7..f29e1a6 100644--- a/components/chat-app.tsx+++ b/components/chat-app.tsx@@ -37,6 +37,7 @@ const DebugPanel = dynamic(() => import('./debug-panel'), { ssr: false })import DesignExport from './design-export'import { Skeleton } from './ui/skeleton'import PreviewPane, { type PreviewSource } from './preview-pane'+import DesignCapture, { acknowledgeCapture, type CaptureRequest } from './design-capture'import Sidebar from './sidebar'import SidebarGestures, { type SidebarGestureControls } from './sidebar-gestures'import { Button } from './ui/button'@@ -280,6 +281,7 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultModconst [entries, setEntries] = useState<ChatEntry[]>(initialState.entries)const [artifactRevision, setArtifactRevision] = useState(initialState.view.artifactRevision)const [designDraft, setDesignDraft] = useState<{ id: string; html: string } | null>(null)+ const [captureRequest, setCaptureRequest] = useState<CaptureRequest | null>(null)const [viewedRevision, setViewedRevision] = useState<{ sessionId: string; revision: number; latest: number } | null>(null)const [activeTurnId, setActiveTurnId] = useState<number | null>(initialState.activeTurnId)const [turnStartedAt, setTurnStartedAt] = useState<number | null>(initialState.activeTurnStartedAt)@@ -728,6 +730,19 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultMod}}if (item.type === 'heartbeat') return+ // screenshot_design asks every tab; only a visible one can paint the design, so only it+ // takes the request. It acknowledges at once, alongside the capture; a capture that is no+ // longer open (an event replayed after a reconnect) answers false and is dropped.+ if (item.type === 'capture') {+ if (item.cid && document.visibilityState === 'visible') {+ const request: CaptureRequest = { cid: item.cid, sessionId: sourceSessionId, revision: item.artifactRevision ?? 0, full: item.captureFull === true, receivedAt: Date.now() }+ setCaptureRequest(request)+ void acknowledgeCapture(request).then((open) => {+ if (!open) setCaptureRequest((current) => (current?.cid === request.cid ? null : current))+ })+ }+ return+ }if (item.type === 'turn_start') {setAwaiting(false)setActiveTurnId(item.id ?? null)@@ -2092,10 +2107,20 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultModdraft={designDraft?.html ?? null}intro={view.snapshot.transcript.entries.length === 0}onClose={closePreview}+ capture={captureRequest && !captureRequest.offscreen && captureRequest.sessionId === view.sessionId ? captureRequest : null}+ onCaptureDone={(cid, answered) => setCaptureRequest((current) =>+ current?.cid !== cid ? current : answered ? null : { ...current, offscreen: true })}/>))}</>)}+ {captureRequest && (captureRequest.offscreen || !previewOpen) && (+ <DesignCapture+ key={captureRequest.cid}+ request={captureRequest}+ onDone={(cid) => setCaptureRequest((current) => (current?.cid === cid ? null : current))}+ />+ )}</main><span role="status" aria-live="polite" className="sr-only">diff --git a/components/chat-state.ts b/components/chat-state.tsindex 419e59c..9f3aa47 100644--- a/components/chat-state.ts+++ b/components/chat-state.ts@@ -267,7 +267,10 @@ export function applyStreamItem(entries: ChatEntry[], item: StreamItem): ChatEnt...entry,toolStatus: item.isError ? 'error' : 'done',content: item.content,- durationMs: entry.startedAt != null && item.at != null ? item.at - entry.startedAt : undefined,+ previewWaitMs: item.previewWaitMs,+ durationMs: entry.startedAt != null && item.at != null+ ? Math.max(0, item.at - entry.startedAt - (item.previewWaitMs ?? 0))+ : undefined,}return next}diff --git a/components/design-capture.tsx b/components/design-capture.tsxnew file mode 100644index 0000000..fcb16d5--- /dev/null+++ b/components/design-capture.tsx@@ -0,0 +1,170 @@+'use client'++import { useEffect, useRef } from 'react'+import {+ CAPTURE_FAILURES,+ CAPTURE_PAINT_MS,+ CAPTURE_REPLY,+ CAPTURE_REQUEST,+ CAPTURE_TIMING_HEADER,+ CAPTURE_TAB_MS,+ CAPTURE_VIEWPORT,+ MAX_CAPTURE_BYTES,+ MAX_CAPTURE_TILES,+ type CaptureFailure,+ type CaptureReply,+ type FrameTiming,+} from '../lib/capture-protocol'+import { fetchUi } from './auth-ui'++// The app side of the screenshot protocol (lib/capture-protocol.ts). The tab acknowledges a+// request as it receives it; the preview on screen answers when it shows the asked revision+// (PreviewPane), otherwise DesignCapture loads a hidden copy and answers from it. Both send+// what the frame returned with sendCapture: the photo, or the reason there is none.++export interface CaptureRequest {+ cid: string+ sessionId: string+ // When this tab received the request, on this tab's clock: the preview wait counts from it.+ receivedAt: number+ // The revision the tool asked for; the preview on screen answers only if it shows it.+ revision: number+ // The whole page in viewport-high tiles instead of the visible viewport.+ full?: boolean+ // The preview on screen could not answer, so the hidden copy does.+ offscreen?: boolean+}++const captureUrl = (request: CaptureRequest, params: Record<string, string>) =>+ `/api/capture?${new URLSearchParams({ session: request.sessionId, cid: request.cid, ...params })}`++/** Tells the waiting call this tab took the request; false when it is no longer open. */+export async function acknowledgeCapture(request: CaptureRequest): Promise<boolean> {+ try {+ const response = await fetchUi(captureUrl(request, { ack: '1' }), { method: 'POST' })+ return response.ok && (await response.json()).ok === true+ } catch {+ return false+ }+}++/** Asks one preview wrapper for its design's photo; null if it sends no reply in time. */+export function captureFrame(target: Window, cid: string, full = false): Promise<CaptureReply | null> {+ return new Promise((resolve) => {+ const done = (reply: CaptureReply | null) => {+ clearTimeout(timer)+ window.removeEventListener('message', onMessage)+ resolve(reply)+ }+ const onMessage = (event: MessageEvent) => {+ if (event.origin !== window.location.origin || event.source !== target) return+ if (event.data?.type !== CAPTURE_REPLY || event.data.cid !== cid) return+ const blobs: unknown = event.data.blobs+ if (Array.isArray(blobs) && blobs.every((blob) => blob instanceof Blob)) done(event.data as CaptureReply)+ }+ const timer = setTimeout(() => done(null), CAPTURE_TAB_MS)+ window.addEventListener('message', onMessage)+ target.postMessage({ type: CAPTURE_REQUEST, cid, full }, window.location.origin)+ })+}++/** Where a photo came from, and when this tab asked the frame for it (this tab's clock). */+export interface CaptureSource {+ path: 'preview' | 'hidden'+ askedAt: number+}++async function postCapture(url: string, init: RequestInit): Promise<void> {+ try {+ await fetchUi(url, { method: 'POST', ...init })+ } catch {+ // The waiting call times out instead.+ }+}++/** Tells the waiting call this tab cannot send a photo, and why. */+export const reportCaptureFailure = (request: CaptureRequest, reason: CaptureFailure) =>+ postCapture(captureUrl(request, { failed: reason }), {})++/**+ * Sends a frame's reply to /api/capture: its photo with this tab's part of the trace, or+ * the failure. No reply in time, the frame's own error, or a photo past the limits (a+ * design can answer for itself) all become failures, so the call never waits for nothing.+ */+export async function sendCapture(request: CaptureRequest, reply: CaptureReply | null, source: CaptureSource): Promise<void> {+ if (!reply) return reportCaptureFailure(request, 'no-reply')+ if (reply.error || !reply.blobs.length) {+ return reportCaptureFailure(request, reply.error && Object.hasOwn(CAPTURE_FAILURES, reply.error) ? reply.error : 'capture-failed')+ }+ const bytes = reply.blobs.reduce((sum, blob) => sum + blob.size, 0)+ if (reply.blobs.length > MAX_CAPTURE_TILES || bytes > MAX_CAPTURE_BYTES) return reportCaptureFailure(request, 'too-large')+ const size = (n: unknown) => (Number.isFinite(n) ? `${Math.round(n as number)}` : '')+ const form = new FormData()+ for (const blob of reply.blobs) form.append('image', blob)+ const sentAt = Date.now()+ // The frame's timing comes from the design's own document: kept only as plain numbers.+ const { fontsMs, libraryMs, encodeMs } = reply.timing ?? {}+ const frame = [fontsMs, libraryMs, encodeMs].every(Number.isFinite) ? { fontsMs, libraryMs, encodeMs } as FrameTiming : null+ const timing = {+ path: source.path, receivedAt: request.receivedAt, sentAt,+ previewWaitMs: source.askedAt - request.receivedAt, roundTripMs: sentAt - source.askedAt, frame,+ }+ await postCapture(captureUrl(request, { w: size(reply.width), h: size(reply.height), ...(reply.truncated ? { truncated: '1' } : {}) }), {+ body: form, headers: { [CAPTURE_TIMING_HEADER]: JSON.stringify(timing) },+ })+}++// The hidden copy loads the preview's own URL, so a revision this tab already loaded comes+// from the browser's cache. It stays inside the viewport, invisible: Chrome stops rendering+// cross-origin frames outside it, and the design would never paint. A design that focuses+// an element would take the keyboard from the composer, so focus goes back where it was.+export default function DesignCapture({ request, onDone }: { request: CaptureRequest; onDone: (cid: string) => void }) {+ const frame = useRef<HTMLIFrameElement>(null)+ const latest = useRef({ request, onDone })+ latest.current = { request, onDone }+ const { cid, sessionId, revision } = request++ useEffect(() => {+ let started = false+ const onPainted = (event: MessageEvent) => {+ const target = frame.current?.contentWindow+ if (started || !target || event.source !== target || event.origin !== window.location.origin) return+ if (event.data?.type !== 'fx0-artifact-painted') return+ started = true+ const askedAt = Date.now()+ void captureFrame(target, cid, latest.current.request.full)+ .then((reply) => sendCapture(latest.current.request, reply, { path: 'hidden', askedAt }))+ .finally(() => latest.current.onDone(cid))+ }+ const timer = setTimeout(() => {+ if (started) return+ void reportCaptureFailure(latest.current.request, 'not-painted').finally(() => latest.current.onDone(cid))+ }, CAPTURE_PAINT_MS)+ const focused = document.activeElement+ const onBlur = () => setTimeout(() => {+ if (document.activeElement !== frame.current) return+ if (focused instanceof HTMLElement && focused !== document.body && focused.isConnected) focused.focus()+ else frame.current?.blur()+ })+ window.addEventListener('message', onPainted)+ window.addEventListener('blur', onBlur)+ return () => {+ clearTimeout(timer)+ window.removeEventListener('message', onPainted)+ window.removeEventListener('blur', onBlur)+ }+ }, [cid])++ return (+ <div aria-hidden className="pointer-events-none fixed inset-0 -z-10 overflow-hidden opacity-0">+ <iframe+ ref={frame}+ src={`/api/artifact-frame?session=${encodeURIComponent(sessionId)}&v=${revision}`}+ referrerPolicy="no-referrer"+ tabIndex={-1}+ title="Design screenshot"+ style={{ ...CAPTURE_VIEWPORT, border: 0 }}+ />+ </div>+ )+}diff --git a/components/preview-pane.tsx b/components/preview-pane.tsxindex ec1c6e0..a1fb539 100644--- a/components/preview-pane.tsx+++ b/components/preview-pane.tsx@@ -3,6 +3,8 @@import { useCallback, useEffect, useRef, useState } from 'react'import { designLinkUrl } from '../lib/design-link'import { EMPTY_DRAFT_DOCUMENT, patchDraftDocument } from '../lib/draft-preview'+import { CAPTURE_PAINT_MS, CAPTURE_PREVIEW_START_MS } from '../lib/capture-protocol'+import { captureFrame, reportCaptureFailure, sendCapture, type CaptureRequest } from './design-capture'import DesignLinkDialog from './design-link-dialog'import { HowItWorksPanel } from './how-it-works'@@ -18,6 +20,8 @@ export default function PreviewPane({draft = null,intro,onClose,+ capture = null,+ onCaptureDone,}: {source: PreviewSource// A past revision opened from the transcript's "View" link, never the latest; null is the latest.@@ -27,6 +31,10 @@ export default function PreviewPane({// Only an empty chat explains fx0; a chat waiting on its first design stays blank.intro: booleanonClose: () => void+ // A screenshot_design request; the preview answers from the frame on screen.+ capture?: CaptureRequest | null+ // Whether this preview answered (a photo or a failure); when not, the hidden copy should.+ onCaptureDone?: (cid: string, answered: boolean) => void}) {const session = encodeURIComponent(source.sessionId)const previewUrl = source.revision > 0@@ -109,6 +117,57 @@ export default function PreviewPane({if (fallback.current) clearTimeout(fallback.current)}}, [])+ // screenshot_design photographs the design on screen in place: no reload, and the photo is+ // what the user sees. A past revision on screen or a frame with no size (behind the chat on+ // a phone) hands the request to the hidden copy at once (onCaptureDone(cid, false)), and so+ // does a revision that has not started loading here after CAPTURE_PREVIEW_START_MS. A slot+ // loading the asked revision is waited for, up to CAPTURE_PAINT_MS: the hidden copy would+ // load the same document from scratch. Once the frame is asked, its answer is final.+ const captureDone = useRef(onCaptureDone)+ captureDone.current = onCaptureDone+ const waitingCapture = useRef<{ request: CaptureRequest; timer: ReturnType<typeof setTimeout>; loading: boolean } | null>(null)+ const settleCapture = (request: CaptureRequest): boolean => {+ if (waitingCapture.current?.request !== request) return false+ clearTimeout(waitingCapture.current.timer)+ waitingCapture.current = null+ return true+ }+ const handOffCapture = (request: CaptureRequest) => {+ if (settleCapture(request)) captureDone.current?.(request.cid, false)+ }+ const failCapture = (request: CaptureRequest) => {+ if (settleCapture(request)) void reportCaptureFailure(request, 'not-painted').finally(() => captureDone.current?.(request.cid, true))+ }+ useEffect(() => {+ if (!capture) return+ const request = capture+ waitingCapture.current = { request, timer: setTimeout(() => handOffCapture(request), CAPTURE_PREVIEW_START_MS), loading: false }+ return () => handOffCapture(request)+ }, [capture?.cid]) // eslint-disable-line react-hooks/exhaustive-deps+ useEffect(() => {+ const waiting = waitingCapture.current+ if (!waiting) return+ const { request } = waiting+ if (viewing != null) return handOffCapture(request)+ if (!previewUrl?.endsWith(`&v=${request.revision}`)) return+ const index = slots.indexOf(previewUrl)+ const frame = index < 0 ? null : frames.current[index]+ if (!frame) return+ if (frame.getBoundingClientRect().width < 1) return handOffCapture(request)+ if (front !== index || !frame.contentWindow) {+ if (!waiting.loading) {+ clearTimeout(waiting.timer)+ waiting.loading = true+ waiting.timer = setTimeout(() => failCapture(request), CAPTURE_PAINT_MS)+ }+ return+ }+ settleCapture(request)+ const askedAt = Date.now()+ void captureFrame(frame.contentWindow, request.cid, request.full)+ .then((reply) => sendCapture(request, reply, { path: 'preview', askedAt }))+ .finally(() => captureDone.current?.(request.cid, true))+ }, [capture?.cid, front, slots, previewUrl, viewing]) // eslint-disable-line react-hooks/exhaustive-depsconst caughtUp = !previewUrl || (front != null && slots[front] === previewUrl)const [heldDraft, setHeldDraft] = useState<string | null>(null)const liveDraft = draft && renderable(draft) ? draft : nulldiff --git a/components/thread.tsx b/components/thread.tsxindex 5b24a85..5bf2ed9 100644--- a/components/thread.tsx+++ b/components/thread.tsx@@ -150,6 +150,8 @@ export interface ChatEntry {content?: stringstartedAt?: numberdurationMs?: number+ // Part of a tool call spent waiting for the preview, left out of durationMs.+ previewWaitMs?: numberartifactRevision?: numberqid?: numberquestions?: { question: string; options: { label: string; description?: string }[] }[]diff --git a/components/tool-widget.tsx b/components/tool-widget.tsxindex 7885c42..2a2f696 100644--- a/components/tool-widget.tsx+++ b/components/tool-widget.tsx@@ -25,6 +25,7 @@ const TOOL_WIDGETS: Record<string, ToolWidgetConfig> = {read_design: { label: 'read design', icon: IconFileText },publish_design: { label: 'publish design', icon: IconPencilEdit },edit_design: { label: 'edit design', icon: IconPencilEdit },+ screenshot_design: { label: 'screenshot', icon: IconFileText },web_fetch: { label: 'fetch url', icon: IconGlobe },web_search: { label: 'web search', icon: IconGlobe },ask_user_question: { label: 'ask a question', icon: IconQuestion },@@ -43,6 +44,7 @@ const TOOL_ACTIVITY: Record<string, { group: string; one: string; many: string }publish_design: { group: 'edits', one: 'published a design', many: 'published designs' },edit_design: { group: 'edits', one: 'edited a design', many: 'edited designs' },read_design: { group: 'reads', one: 'read a design', many: 'read designs' },+ screenshot_design: { group: 'reads', one: 'checked the design', many: 'checked the design' },web_fetch: { group: 'web', one: 'used the web', many: 'used the web' },web_search: { group: 'web', one: 'searched the web', many: 'searched the web' },ask_user_question: { group: 'questions', one: 'asked for input', many: 'asked for input' },@@ -91,6 +93,7 @@ function ToolWidget({status,content,durationMs,+ previewWaitMs,callId,}: {name: string@@ -98,6 +101,7 @@ function ToolWidget({status: 'running' | 'done' | 'error'content?: stringdurationMs?: number+ previewWaitMs?: numbercallId?: number}) {const [open, setOpen] = useState(false)@@ -117,7 +121,7 @@ function ToolWidget({{command && <code className="min-w-0 flex-1 truncate font-mono text-xs text-foreground-tertiary">{command}</code>}<span className="ml-auto flex shrink-0 items-center gap-2 text-xs text-muted-foreground tabular-nums">{callId != null && <span title="Tool call ID">#{callId}</span>}- {duration && <span>{duration}</span>}+ {duration && <span title={previewWaitMs ? `plus ${formatDuration(previewWaitMs)} waiting for the preview to load` : undefined}>{duration}</span>}<span role="status" className={`font-medium ${state.className}`}>{state.label}</span>@@ -248,6 +252,7 @@ export function ActivityGroup({status={entry.toolStatus ?? 'done'}content={entry.content}durationMs={entry.durationMs}+ previewWaitMs={entry.previewWaitMs}callId={entry.callId}/>)diff --git a/lib/agent.ts b/lib/agent.tsindex f1e3514..cd31a44 100644--- a/lib/agent.ts+++ b/lib/agent.ts@@ -4,6 +4,7 @@ import { assertGenerationCredential } from './gateway-session'import { remoteImagesEnabled } from './design-artifact'import { prefetchRemoteImages } from './remote-image-cache'import { deleteDesignAssets, type PendingDesignAssets } from './design-assets'+import { CAPTURE_FAILURES, captureTrace } from './capture-protocol'import { chatSession, durableAgent, runSubagent, type ChatRuntime } from './libfx-agent'import { project } from './projector'import {@@ -18,13 +19,16 @@ import type { TrialCredential } from './trial'import type { SessionData } from './secure-session'import {answerActiveQuestion,+ awaitCapture,claimSessionCleanup,clearActiveQuestion,+ closeCapture,currentSeq,finishSessionCleanup,getActiveQuestion,getActiveTurn,nextQuestionId,+ openCapture,projectedCursor,publishDesignArtifact,QUESTION_WAIT_MS,@@ -33,6 +37,7 @@ import {readEvents,readSession,readSessionView,+ recordPreviewWait,releaseSessionCleanup,renewSessionCleanup,SessionCleanupError,@@ -40,6 +45,8 @@ import {submitPrompt as journalPrompt,takeAnswer,updateSession,+ type CaptureOutcome,+ type CaptureShot,type InitialGeneration,type StoredSession,} from './journal'@@ -109,6 +116,7 @@ function buildToolHost(host: SessionState, turnId: number): ToolHost {},readResult: (callId) => host.toolOutputs.get(callId),askUser: (questions, signal) => askUser(host, turnId, questions, signal),+ captureDesign: (signal, options) => captureDesign(host, turnId, signal, options),log: (event, detail) => log(host.id, event, { turn: turnId, ...detail }),spawnSubagent: async (task, signal) => {log(host.id, 'subagent:start', { task: task.slice(0, 120) })@@ -373,6 +381,55 @@ async function askUser(host: SessionState, turnId: number, questions: AskQuestiothrow new Error(signal.aborted ? 'cancelled' : 'no answer within 10 minutes')}+// screenshot_design (lib/capture-protocol.ts): the tool call asks every open tab for a+// photo of the published design and blocks until one arrives through Redis, wherever the+// turn runs.+// What the model is told when no photo arrives; a tab's own failure says why in its words.+const CAPTURE_ERRORS: Record<Exclude<CaptureOutcome['kind'], 'shot' | 'failed'>, string> = {+ unanswered: 'no fx0 tab is open and in front to take it',+ timeout: 'the fx0 tab took the request but sent nothing in time',+ cancelled: 'cancelled',+}++async function captureDesign(+ host: SessionState,+ turnId: number,+ signal: AbortSignal,+ { toolCallId, full = false }: { toolCallId?: string; full?: boolean } = {},+): Promise<CaptureShot & { revision: number }> {+ const started = performance.now()+ const cid = crypto.randomUUID()+ const [{ artifactRevision: revision }] = await Promise.all([readSessionView(host.id), openCapture(host.id, cid)])+ if (!revision) {+ await closeCapture(host.id, cid)+ throw new Error('no design has been published yet')+ }+ const emitting = performance.now()+ try {+ await emit(host, {+ type: 'capture', cid, artifactRevision: revision, turnId,+ ...(full ? { captureFull: true as const } : {}),+ })+ } catch (error) {+ await closeCapture(host.id, cid).catch(() => {})+ throw error+ }+ const emitted = performance.now()+ const emittedAt = Date.now()+ const outcome = await awaitCapture(host.id, cid, signal)+ const received = performance.now()+ const server = { totalMs: received - started, setupMs: emitting - started, emitMs: emitted - emitting, emittedAt, receivedAt: Date.now() }+ if (outcome.kind !== 'shot') {+ log(host.id, 'capture:trace', { turn: turnId, outcome: outcome.kind, ...(outcome.kind === 'failed' ? { reason: outcome.reason } : {}), totalMs: Math.round(server.totalMs) })+ throw new Error(outcome.kind === 'failed' ? CAPTURE_FAILURES[outcome.reason] : CAPTURE_ERRORS[outcome.kind])+ }+ const { shot } = outcome+ log(host.id, 'capture:trace', { turn: turnId, outcome: 'shot', ...(shot.timing ? captureTrace(server, shot.timing) : { totalMs: Math.round(server.totalMs) }) })+ const previewWaitMs = shot.timing?.previewWaitMs ?? 0+ if (previewWaitMs > 0 && toolCallId) await recordPreviewWait(host.id, toolCallId, previewWaitMs)+ return { ...shot, revision }+}+export async function answerQuestion(host: SessionState, qid: number, answers: Record<string, string>): Promise<boolean> {const accepted = await answerActiveQuestion(host.id, qid, answers)log(host.id, accepted ? 'question:answered' : 'question:answer-rejected', { qid })diff --git a/lib/capture-protocol.ts b/lib/capture-protocol.tsnew file mode 100644index 0000000..9743f29--- /dev/null+++ b/lib/capture-protocol.ts@@ -0,0 +1,122 @@+// The design screenshot protocol, shared by the server, the frame and the app.+//+// 1. A screenshot_design call opens a capture in Redis and emits a `capture` event, then+// blocks on Redis (lib/journal.ts). A tab that receives the event acknowledges it at+// once; a call nobody acknowledges within CAPTURE_ACK_MS fails fast, and an acknowledged+// one waits for a photo or a failure.+// 2. The tab answers: the preview on screen when it shows the asked revision, otherwise a+// hidden copy of the frame (components/design-capture.tsx).+// 3. The app asks the frame's wrapper, the wrapper asks the design, and the capture library+// photographs the design inside its own frame, where the CSP allows no network+// (lib/design-capture.ts). The photo travels back as Blobs, or the reason there is none.+// 4. The tab posts the photo, or the failure, to /api/capture, which hands it to the call.++/** The app asks a preview wrapper for a photo. */+export const CAPTURE_REQUEST = 'fx0-capture'+/** A preview wrapper returns the design's photo, or none. */+export const CAPTURE_REPLY = 'fx0-artifact-captured'+/** The tab's part of a capture's trace, sent with the photo as JSON. */+export const CAPTURE_TIMING_HEADER = 'x-fx0-capture-timing'++/** Where the frame's time went, on the frame's own clock. */+export interface FrameTiming {+ fontsMs: number+ libraryMs: number+ encodeMs: number+}++/** Why a tab that acknowledged a capture sends no photo. */+export const CAPTURE_FAILURES = {+ 'too-large': 'the page did not fit the screenshot size limit even at low quality',+ 'capture-failed': 'the capture library failed on this design',+ 'not-painted': 'the design did not finish painting in time',+ 'no-reply': 'the design did not answer the capture request in time',+ 'refused': 'the photo the tab sent was refused as not a valid image',+} as const+export type CaptureFailure = keyof typeof CAPTURE_FAILURES++export interface CaptureReply {+ type: typeof CAPTURE_REPLY+ cid: string+ // Set when the frame could not photograph the design; blobs is then empty.+ error?: CaptureFailure+ // The viewport, or the whole page cut into viewport-high tiles, top to bottom.+ blobs: Blob[]+ width?: number+ // The photographed height: the viewport, or the page.+ height?: number+ // A full page taller than MAX_CAPTURE_TILES viewports: the tiles stop before its end.+ truncated?: boolean+ timing?: FrameTiming+}++/** What the tab and /api/capture add to a photo for its trace. */+export interface ShotTiming {+ path: 'preview' | 'hidden'+ receivedAt: number // tab clock+ sentAt: number // tab clock+ previewWaitMs: number+ roundTripMs: number+ frame: FrameTiming | null+ arrivedAt: number // server clock+ authMs: number+ readMs: number+ storingAt: number // server clock, as the Redis write starts+}++/** Every image of one capture together, held in Redis as base64 within a single request. */+export const MAX_CAPTURE_BYTES = 600 * 1024+/** A full page is returned as at most this many viewport-high tiles: one tool result's image limit. */+export const MAX_CAPTURE_TILES = 8+/** How long the tool call waits for a tab to acknowledge the request before failing. */+export const CAPTURE_ACK_MS = 5_000+/** The capture library's own limit inside the frame. */+export const CAPTURE_LIBRARY_MS = 20_000+/** How long a tab may take to return a photo once it asks the frame. */+export const CAPTURE_TAB_MS = CAPTURE_LIBRARY_MS + 5_000+/** How long the preview may take to start loading the asked revision before the hidden copy answers. */+export const CAPTURE_PREVIEW_START_MS = 3_000+/** How long a frame loading the asked revision may take to paint it. */+export const CAPTURE_PAINT_MS = 15_000+/** How long an acknowledged call waits: a hand-off to the hidden copy after a slow start still fits. */+export const CAPTURE_WAIT_MS = CAPTURE_PREVIEW_START_MS + CAPTURE_PAINT_MS + CAPTURE_TAB_MS + 5_000+/** The hidden copy's viewport, when no preview is on screen. */+export const CAPTURE_VIEWPORT = { width: 1280, height: 800 }++/** What the tool call measured itself, on the turn's server clock. */+interface ServerTiming {+ totalMs: number+ setupMs: number+ emitMs: number+ emittedAt: number // server clock, as the request left+ receivedAt: number // server clock, as the photo arrived+}++/**+ * One capture, end to end, in whole milliseconds, as the tool call logs it (capture:trace).+ * Each part is measured on one clock, except delivery, upload and store-and-wake, which+ * compare two machines' clocks; delivery plus upload (networkMs) is exact all the same.+ */+export function captureTrace(server: ServerTiming, shot: ShotTiming) {+ const ms = (n: number) => Math.round(n)+ const deliveryMs = shot.receivedAt - server.emittedAt+ const uploadMs = shot.arrivedAt - shot.sentAt+ const frame = shot.frame+ return {+ path: shot.path,+ totalMs: ms(server.totalMs),+ setupMs: ms(server.setupMs),+ emitMs: ms(server.emitMs),+ deliveryMs: ms(deliveryMs),+ previewWaitMs: ms(shot.previewWaitMs),+ messagingMs: ms(frame ? shot.roundTripMs - frame.fontsMs - frame.libraryMs - frame.encodeMs : shot.roundTripMs),+ fontsMs: frame ? ms(frame.fontsMs) : null,+ libraryMs: frame ? ms(frame.libraryMs) : null,+ encodeMs: frame ? ms(frame.encodeMs) : null,+ uploadMs: ms(uploadMs),+ networkMs: ms(deliveryMs + uploadMs),+ authMs: ms(shot.authMs),+ readMs: ms(shot.readMs),+ storeAndWakeMs: ms(server.receivedAt - shot.storingAt),+ }+}diff --git a/lib/design-artifact.ts b/lib/design-artifact.tsindex ad1714b..2e4e1e0 100644--- a/lib/design-artifact.ts+++ b/lib/design-artifact.ts@@ -74,8 +74,9 @@ export const MAX_EMBEDDED_IMAGE_BYTES = 4 * 1024 * 1024// and relays its heartbeat, paint and link messages to the app. Each image marker in the// design becomes its image, loaded from `src` (cached by the browser across revisions),// within MAX_EMBEDDED_IMAGE_BYTES counted per occurrence; a missing image is blank. Only-// then does the design start, so it never needs the network.-export function designFrameHtml(preparedArtifact: string, readyToken: string, images: { marker: string; src: string }[] = []): string {+// then does the design start, so it never needs the network. `relay` is more wrapper+// script (lib/design-capture.ts), run before the design loads.+export function designFrameHtml(preparedArtifact: string, readyToken: string, images: { marker: string; src: string }[] = [], relay = ''): string {// The design reaches its frame as a blob: URL the wrapper makes from this string,// not as a data: URL: browsers refuse data: URLs past 2 MiB, which embedded images// reach. A blob: document is local like data:, so it inherits this wrapper's CSP and@@ -109,6 +110,8 @@ frame.setAttribute('credentialless','');let readySignals=0;let consumedSignals=0;let lastHeartbeat=Date.now();+// A relay can hold off restores while the design is busy on its behalf (lib/design-capture.ts).+let holdUntil=0;const restore=()=>{if(!artifact)return;lastHeartbeat=Date.now();frame.src=artifact;};addEventListener('message',event=>{if(event.source!==frame.contentWindow||event.data?.token!==token)return;@@ -135,9 +138,9 @@ setInterval(()=>{const now=Date.now(),paused=now-lastCheck>3000;lastCheck=now;if(document.visibilityState!=='visible'||paused){lastHeartbeat=now;return;}- if(now-lastHeartbeat>3000)restore();+ if(now-lastHeartbeat>3000&&now>holdUntil)restore();},500);-const read=blob=>new Promise(done=>{const reader=new FileReader();reader.onload=()=>done(String(reader.result));reader.onerror=()=>done(null);reader.readAsDataURL(blob)});+${relay}const read=blob=>new Promise(done=>{const reader=new FileReader();reader.onload=()=>done(String(reader.result));reader.onerror=()=>done(null);reader.readAsDataURL(blob)});const load=async src=>{try{const response=await fetch(src,{credentials:'same-origin'});if(!response.ok)return null;const blob=await response.blob();return /^image\\/(png|jpeg|gif|webp|avif)$/.test(blob.type)?await read(blob):null}catch{return null}};Promise.all(images.map(image=>load(image.src))).then(loaded=>{let html=design,total=0;@@ -182,13 +185,14 @@ export function normalizeDesignArtifactHtml(html: string): string {// The CSP and bootstrap go right after the doctype, so they are the first things// the parser puts in <head>. A design stored before normalization renders the same// way; one without a doctype is not rendered at all (null), never without its CSP.-export function prepareDesignArtifactHtml(html: string, embeddedCsp?: string, readyToken?: string): string | null {+// `scripts` follow the bootstrap, under the same CSP (lib/design-capture.ts).+export function prepareDesignArtifactHtml(html: string, embeddedCsp?: string, readyToken?: string, scripts = ''): string | null {const document = normalizeDesignArtifactHtml(html)if (!document.startsWith(DOCTYPE)) return nullconst csp = embeddedCsp? `<meta http-equiv="Content-Security-Policy" content="${escapeAttribute(embeddedCsp)}">`: ''- return DOCTYPE + csp + designArtifactBootstrap(readyToken) + document.slice(DOCTYPE.length)+ return DOCTYPE + csp + designArtifactBootstrap(readyToken) + scripts + document.slice(DOCTYPE.length)}// The start of a document, with invisible characters spelled out, so a rejecteddiff --git a/lib/design-capture.ts b/lib/design-capture.tsnew file mode 100644index 0000000..b32861d--- /dev/null+++ b/lib/design-capture.ts@@ -0,0 +1,102 @@+// @ts-expect-error Node strip-types tests require the source extension.+import { CAPTURE_LIBRARY_MS, CAPTURE_REPLY, CAPTURE_REQUEST, MAX_CAPTURE_BYTES, MAX_CAPTURE_TILES } from './capture-protocol.ts'+// @ts-expect-error Node strip-types tests require the source extension.+import { FAST_HTML_TO_IMAGE } from './vendor/fast-html-to-image.ts'++// The frame side of the screenshot protocol (lib/capture-protocol.ts). The capture library+// runs inside the design's own frame, under the CSP that gives the design no network, so+// it cannot fetch anything the design names either: the photo shows what the screen shows.+// The design's scripts share that realm and can tamper with their own photo; that only+// misleads the model about its own work, and /api/capture still checks the bytes.++/** The library and its message handler, injected into every prepared design. */+export function designCaptureScript(readyToken: string): string {+ // Encodes at once: canvas.toBlob waits for an idle moment a busy page never has. WebP is+ // about half the bytes of a JPEG at the same quality, which the upload feels most; a+ // browser that cannot write WebP (Safari) returns a PNG for it, and gets JPEG instead.+ // Nothing runs until the first capture, so a preview that is never photographed pays+ // only for parsing the library.+ const handler = `(() => {+ const token = ${JSON.stringify(readyToken)}, maxBytes = ${MAX_CAPTURE_BYTES}, maxTiles = ${MAX_CAPTURE_TILES};+ let type = null;+ const encode = (canvas, quality) => {+ if (!type) {+ const probe = document.createElement("canvas");+ probe.width = probe.height = 1;+ type = probe.toDataURL("image/webp").startsWith("data:image/webp") ? "image/webp" : "image/jpeg";+ }+ const url = canvas.toDataURL(type, quality), binary = atob(url.slice(url.indexOf(",") + 1));+ const bytes = new Uint8Array(binary.length);+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);+ return new Blob([bytes], { type });+ };+ const total = (blobs) => blobs.reduce((sum, blob) => sum + blob.size, 0);+ // A remainder under 15% of a viewport joins the last tile instead of becoming its own.+ const slack = () => Math.round(innerHeight * 0.15);+ // The page, cut into viewport-high tiles from the top.+ const tiles = (page) => {+ const height = innerHeight, count = Math.min(maxTiles, Math.max(1, Math.ceil((page.height - slack()) / height)));+ return Array.from({ length: count }, (_, i) => {+ const tile = document.createElement("canvas");+ tile.width = page.width;+ tile.height = i === count - 1 ? Math.min(page.height - i * height, height + slack()) : height;+ tile.getContext("2d").drawImage(page, 0, i * height, page.width, tile.height, 0, 0, page.width, tile.height);+ return tile;+ });+ };+ addEventListener("message", async (event) => {+ if (event.source !== parent || event.data?.type !== "capture" || typeof event.data.cid !== "string") return;+ const { cid, full } = event.data;+ const reply = (data) => parent.postMessage({ type: "captured", token, cid, ...data }, "*");+ try {+ const t0 = performance.now();+ await document.fonts.ready;+ const t1 = performance.now();+ const options = { pixelRatio: 1, backgroundColor: "#ffffff", timeoutMs: ${CAPTURE_LIBRARY_MS} };+ // A full page is the document from its top, as tall as its content (which can overflow+ // an html{height:100%} box) but no taller than the tiles can carry.+ const root = document.documentElement, pageHeight = Math.max(root.scrollHeight, document.body?.scrollHeight ?? 0);+ const region = full+ ? { x: 0, y: root.getBoundingClientRect().top, width: innerWidth, height: Math.min(pageHeight, maxTiles * innerHeight + slack()) }+ : { x: 0, y: 0, width: innerWidth, height: innerHeight };+ const page = await (await FastHtmlToImage.captureRegion(region, options)).toCanvas();+ const t2 = performance.now();+ const parts = full ? tiles(page) : [page];+ // One pass at a good quality; over the limit, one more at the quality the size suggests.+ let blobs = parts.map((part) => encode(part, 0.85));+ if (total(blobs) > maxBytes) {+ const quality = Math.min(0.75, Math.max(0.3, 0.85 * (maxBytes / total(blobs)) * 0.9));+ blobs = parts.map((part) => encode(part, quality));+ }+ const t3 = performance.now();+ if (total(blobs) > maxBytes) return reply({ error: "too-large" });+ const covered = parts.reduce((sum, part) => sum + part.height, 0);+ reply({+ blobs, width: page.width, height: page.height, truncated: full && covered < pageHeight,+ timing: { fontsMs: t1 - t0, libraryMs: t2 - t1, encodeMs: t3 - t2 },+ });+ } catch {+ reply({ error: "capture-failed" });+ }+ });+})()`+ return `<script>${FAST_HTML_TO_IMAGE}</script><script>${handler}</script>`+}++// Added to the wrapper's script (designFrameHtml), where `frame`, `token` and `holdUntil`+// are in scope. Only the app may ask for a photo, and only the design's reply is passed+// on. A large page can block its own heartbeat while it is photographed, so the wrapper+// holds off restoring it until the reply or the library's time limit.+export const DESIGN_CAPTURE_RELAY = `addEventListener('message',event=>{+ const data=event.data;+ if(event.source===parent&&event.origin===location.origin&&data?.type===${JSON.stringify(CAPTURE_REQUEST)}&&typeof data.cid==='string'){+ holdUntil=Date.now()+${CAPTURE_LIBRARY_MS};+ frame.contentWindow?.postMessage({type:'capture',cid:data.cid,full:data.full===true},'*');+ }else if(event.source===frame.contentWindow&&data?.token===token&&data.type==='captured'&&typeof data.cid==='string'){+ holdUntil=0;+ const blobs=Array.isArray(data.blobs)?data.blobs.filter(blob=>blob instanceof Blob):[];+ const error=typeof data.error==='string'?data.error:undefined;+ parent.postMessage({type:${JSON.stringify(CAPTURE_REPLY)},cid:data.cid,blobs,error,width:data.width,height:data.height,truncated:data.truncated===true,timing:data.timing},location.origin);+ }+});+`diff --git a/lib/events.ts b/lib/events.tsindex c081bf6..070652e 100644--- a/lib/events.ts+++ b/lib/events.ts@@ -19,6 +19,7 @@ export interface StreamItem {| 'heartbeat'| 'question'| 'question_resolved'+ | 'capture'| 'reasoning'| 'done'| 'error'@@ -55,6 +56,14 @@ export interface StreamItem {stopReason?: stringartifactRevision?: numberqid?: number+ /** screenshot_design's request id; an open tab answers it through /api/capture. */+ cid?: string+ /** A capture of the whole page in viewport-high tiles, not just the visible viewport. */+ captureFull?: true+ /** The model's id for a tool call, on its tool_end. */+ toolCallId?: string+ /** On a screenshot call's tool_end: time it spent waiting for the preview, not its own work. */+ previewWaitMs?: numberinstance?: string/** Count only — image bytes are never broadcast over the chat stream. */imageCount?: numberdiff --git a/lib/journal.ts b/lib/journal.tsindex 61b5a01..4005b9c 100644--- a/lib/journal.ts+++ b/lib/journal.ts@@ -4,6 +4,8 @@ import { Redis } from '@upstash/redis'// @ts-expect-error Node's strip-types integration test requires the source extension.import { assetReference, rewriteRemoteImages, validateDesignArtifactHtml } from './design-artifact.ts'// @ts-expect-error Node's strip-types integration test requires the source extension.+import { CAPTURE_ACK_MS, CAPTURE_WAIT_MS, type CaptureFailure, type ShotTiming } from './capture-protocol.ts'+// @ts-expect-error Node's strip-types integration test requires the source extension.import { type DesignMdInput } from './design-md.ts'// @ts-expect-error Node's strip-types integration test requires the source extension.import { type SessionInfo, type SessionView, type StreamItem } from './events.ts'@@ -16,16 +18,40 @@ const PREFIX = process.env.FX0_REDIS_PREFIX?.trim() || 'fx0:v1'let client: Redis | null = null+function redisConfig(): { url: string; token: string } {+ const url = process.env.KV_REST_API_URL+ const token = process.env.KV_REST_API_TOKEN+ if (!url || !token) throw new Error('Redis journal is not configured (KV_REST_API_URL/KV_REST_API_TOKEN)')+ return { url, token }+}+export function redis(): Redis {- if (!client) {- const url = process.env.KV_REST_API_URL- const token = process.env.KV_REST_API_TOKEN- if (!url || !token) throw new Error('Redis journal is not configured (KV_REST_API_URL/KV_REST_API_TOKEN)')- client = new Redis({ url, token })- }+ if (!client) client = new Redis(redisConfig())return client}+// BLPOP over Upstash's REST API, which the client library does not expose: one request+// blocks in Redis for up to `seconds` and returns as soon as a value is pushed. Aborting+// ends the request; the caller withdraws whatever it was waiting for (see awaitCapture).+// A request that outlives its block by much has stalled, and fails like any other error.+async function blockingPop(listKey: string, seconds: number, signal: AbortSignal): Promise<string | null> {+ const { url, token } = redisConfig()+ try {+ const response = await fetch(url, {+ method: 'POST',+ headers: { authorization: `Bearer ${token}` },+ body: JSON.stringify(['BLPOP', listKey, String(seconds)]),+ signal: AbortSignal.any([signal, AbortSignal.timeout((seconds + 5) * 1000)]),+ })+ if (!response.ok) throw new Error(`Redis BLPOP failed with ${response.status}`)+ const { result } = await response.json() as { result: [string, string] | null }+ return result?.[1] ?? null+ } catch (error) {+ if (signal.aborted) return null+ throw error+ }+}+// Events committed by this process, announced the moment Redis confirms them. A// stream in the same process (a prompt's own response, where its worker also runs)// delivers them without waiting for its next journal poll; polling still covers@@ -425,6 +451,9 @@ const SESSION_PARTS = ['active','active-question','answer',+ 'capture',+ 'capture-replies',+ 'preview-waits','design-artifact','design-md','event-seq',@@ -827,6 +856,152 @@ export async function clearActiveQuestion(sessionId: string, qid: number): Promireturn (await redis().eval<unknown[], number>(CLEAR_QUESTION, [key(sessionId, 'active-question')], [String(qid)])) === 1}+export interface CaptureShot {+ // The viewport, or a full page's viewport-high tiles from the top.+ images: { mimeType: string; data: string }[]+ width?: number+ height?: number+ // A full page taller than the tiles cover.+ truncated?: boolean+ // The tab's and /api/capture's part of the capture's trace (lib/capture-protocol.ts).+ timing?: ShotTiming+}++// The Redis side of the screenshot protocol (lib/capture-protocol.ts). The tool call opens a+// capture and blocks on the chat's reply list; /api/capture pushes the tab's acknowledgement,+// then its photo or failure, so the call wakes the moment each arrives, on whichever instance+// runs the turn. No polling. A key whose format changes gets a new name, so instances of two+// deploys never misread it.++/** What a tab answered for a capture it acknowledged. */+export type CaptureAnswer = { kind: 'shot'; shot: CaptureShot } | { kind: 'failed'; reason: CaptureFailure }++/** How a capture ended for the waiting call. */+export type CaptureOutcome =+ | CaptureAnswer+ | { kind: 'unanswered' } // no tab acknowledged it within the ack window+ | { kind: 'timeout' } // a tab acknowledged it, then sent nothing in time+ | { kind: 'cancelled' }++const OPEN_CAPTURE = `+if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end+redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])+redis.call('DEL', KEYS[2])+return 1+`+export async function openCapture(sessionId: string, cid: string): Promise<void> {+ const set = await redis().eval<unknown[], number>(OPEN_CAPTURE,+ [key(sessionId, 'capture'), key(sessionId, 'capture-replies'), key(sessionId, 'deleting')],+ [cid, String(CAPTURE_ACK_MS + CAPTURE_WAIT_MS)])+ if (!set) throw new SessionCleanupError()+}++// A reply counts only for the capture still open. An acknowledgement leaves it open for the+// whole wait it starts; a photo or a failure closes it, so the first answer is the only one.+const REPLY_CAPTURE = `+if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end+if ARGV[4] == '1' then redis.call('DEL', KEYS[1]) else redis.call('PEXPIRE', KEYS[1], ARGV[3]) end+redis.call('RPUSH', KEYS[2], ARGV[2])+redis.call('PEXPIRE', KEYS[2], ARGV[3])+return 1+`+async function replyCapture(sessionId: string, cid: string, reply: { kind: 'ack' } | CaptureAnswer): Promise<boolean> {+ return (await redis().eval<unknown[], number>(REPLY_CAPTURE,+ [key(sessionId, 'capture'), key(sessionId, 'capture-replies')],+ [cid, JSON.stringify({ cid, ...reply }), String(CAPTURE_ACK_MS + CAPTURE_WAIT_MS), reply.kind === 'ack' ? '0' : '1'])) === 1+}++/** A tab took the request; false when the capture is no longer open (a replayed event). */+export const acknowledgeCapture = (sessionId: string, cid: string) => replyCapture(sessionId, cid, { kind: 'ack' })+/** The tab's photo or failure; false when the capture is no longer open or already answered. */+export const answerCapture = (sessionId: string, cid: string, answer: CaptureAnswer) => replyCapture(sessionId, cid, answer)++// Upstash ends long requests on its own terms; waiting in slices keeps each one short. A+// slice that fails is retried while the wait has time left, so one bad request does not+// refuse a photo already on its way.+const CAPTURE_BLOCK_SECONDS = 10+const CAPTURE_POP_ATTEMPTS = 3++function replyFor(value: unknown, cid: string): { kind: 'ack' } | CaptureAnswer | null {+ const stored = parseJson<{ cid?: string; kind?: string; shot?: CaptureShot; reason?: CaptureFailure }>(value)+ if (stored?.cid !== cid) return null+ if (stored.kind === 'ack') return { kind: 'ack' }+ if (stored.kind === 'failed' && stored.reason) return { kind: 'failed', reason: stored.reason }+ return stored.kind === 'shot' && stored.shot ? { kind: 'shot', shot: stored.shot } : null+}++/**+ * Waits for the open capture: first for a tab to acknowledge it within `ackMs`, then for+ * its photo or failure within CAPTURE_WAIT_MS. However it ends, the capture is withdrawn,+ * so a reply sent afterwards is refused instead of left behind.+ */+export async function awaitCapture(sessionId: string, cid: string, signal: AbortSignal, ackMs = CAPTURE_ACK_MS): Promise<CaptureOutcome> {+ const replies = key(sessionId, 'capture-replies')+ let acknowledged = false+ let deadline = Date.now() + ackMs+ let failures = 0+ try {+ for (let left = ackMs; left > 0 && !signal.aborted; left = deadline - Date.now()) {+ let value: string | null+ try {+ value = await blockingPop(replies, Math.min(CAPTURE_BLOCK_SECONDS, Math.ceil(left / 1000)), signal)+ failures = 0+ } catch (error) {+ if (++failures >= CAPTURE_POP_ATTEMPTS) throw error+ continue+ }+ const reply = value == null ? null : replyFor(value, cid)+ if (reply?.kind === 'ack') {+ if (!acknowledged) deadline = Date.now() + CAPTURE_WAIT_MS+ acknowledged = true+ } else if (reply) return reply+ }+ } catch (error) {+ await closeCapture(sessionId, cid).catch(() => {})+ throw error+ }+ // An answer accepted just before the capture closed is already in the list.+ if (!await closeCapture(sessionId, cid)) {+ for (let late = await redis().lpop<unknown>(replies); late != null; late = await redis().lpop<unknown>(replies)) {+ const reply = replyFor(late, cid)+ if (reply && reply.kind !== 'ack') return reply+ }+ }+ if (signal.aborted) return { kind: 'cancelled' }+ return { kind: acknowledged ? 'timeout' : 'unanswered' }+}++// How long a screenshot call waited for the preview, by the model's id for the call. The+// call records it before it returns, so it is there when the projector writes the call's+// tool_end; the transcript leaves that wait out of the call's time.+const PREVIEW_WAITS_TTL_MS = 86_400_000++export async function recordPreviewWait(sessionId: string, toolCallId: string, ms: number): Promise<void> {+ const waits = key(sessionId, 'preview-waits')+ await redis().pipeline().hset(waits, { [toolCallId]: Math.round(ms) }).pexpire(waits, PREVIEW_WAITS_TTL_MS).exec()+}++export async function readPreviewWaits(sessionId: string, toolCallIds: string[]): Promise<Map<string, number>> {+ if (!toolCallIds.length) return new Map()+ const values = await redis().hmget<Record<string, unknown>>(key(sessionId, 'preview-waits'), ...toolCallIds)+ const waits = new Map<string, number>()+ for (const id of toolCallIds) {+ const ms = Number(values?.[id])+ if (Number.isFinite(ms) && ms > 0) waits.set(id, ms)+ }+ return waits+}++// Withdraws the capture when its tool call stops waiting; true if it was still open.+const CLOSE_CAPTURE = `+if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end+redis.call('DEL', KEYS[1])+return 1+`+export async function closeCapture(sessionId: string, cid: string): Promise<boolean> {+ return (await redis().eval<unknown[], number>(CLOSE_CAPTURE, [key(sessionId, 'capture')], [cid])) === 1+}+// The first libfx session id stored for a chat wins, so two first prompts racing// on the same chat still share one conversation. The reverse key lets the agent's// fetch find the chat from the x-session-id libfx sends with each model request.diff --git a/lib/libfx-agent.ts b/lib/libfx-agent.tsindex 0575ced..70f9bc3 100644--- a/lib/libfx-agent.ts+++ b/lib/libfx-agent.ts@@ -160,8 +160,10 @@ async function chatOfRequest(sessionId: string | null): Promise<string | null> {// Reads are safe to run twice after a crash; publishing the same document twice// is too. Edits, questions and subagents are not, so libfx tells the model a// call cut off by a crash may have run instead of running it again.-const IDEMPOTENT = new Set(['read_design', 'publish_design', 'web_fetch', 'read_tool_result', 'capability_search'])-const WRITES = new Set(['publish_design', 'edit_design'])+const IDEMPOTENT = new Set(['read_design', 'publish_design', 'screenshot_design', 'web_fetch', 'read_tool_result', 'capability_search'])+// A tool that writes never runs beside another call. A chat has one capture open at a time+// (lib/journal.ts), so two screenshots in one response must take turns too.+const WRITES = new Set(['publish_design', 'edit_design', 'screenshot_design'])function turnContext(ctx: FxToolContext): TurnContext {const context = ctx.context as Partial<TurnContext> | undefineddiff --git a/lib/projector.ts b/lib/projector.tsindex 1472c80..4122111 100644--- a/lib/projector.ts+++ b/lib/projector.ts@@ -6,6 +6,7 @@ import {projectEvents,projectedCursor,readEvents,+ readPreviewWaits,readSession,readTurnWork,type TurnSettings,@@ -78,7 +79,7 @@ export function itemsOf(line: FxStreamLine, at: number): Entry[] {case 'tool_start':return [{ type: 'tool_start', name: line.name || 'tool', command: toolLabel(line as Record<string, unknown>), turnId: id, at }]case 'tool_end':- return [{ type: 'tool_end', name: line.name || 'tool', isError: line.isError === true, content: typeof line.content === 'string' ? tail(line.content) : undefined, turnId: id, at }]+ return [{ type: 'tool_end', name: line.name || 'tool', isError: line.isError === true, content: typeof line.content === 'string' ? tail(line.content) : undefined, toolCallId: line.id, turnId: id, at }]case 'turn_end': {const failed = line.stopReason === 'error' || line.stopReason === 'unknown' || (line.stopReason === 'refused' && line.error?.message)return [failed@@ -205,6 +206,17 @@ export interface FollowOptions {idleMs: number}+// The preview wait a screenshot call recorded (lib/journal.ts), onto its tool_end.+export async function addPreviewWaits(sessionId: string, entries: Entry[]): Promise<void> {+ const ends = entries.filter((entry) => entry.type === 'tool_end' && entry.toolCallId && entry.name === 'screenshot_design')+ if (!ends.length) return+ const waits = await readPreviewWaits(sessionId, ends.map((entry) => entry.toolCallId!))+ for (const entry of ends) {+ const ms = waits.get(entry.toolCallId!)+ if (ms) entry.previewWaitMs = ms+ }+}+// Follows the chat's libfx stream from the projected cursor. Returns when the// chat stays idle for idleMs, when the signal aborts, or when another projector// moved the cursor first (that one carries on).@@ -241,6 +253,7 @@ export async function project(agent: FxAgent, host: SessionState, libfxSessionIdconst id = line.type === 'turn_start' ? messageIdOf(line) : nullif (id != null && line.type === 'turn_start') starts[id] = await readTurnStart(host, id, line.input)}+ await addPreviewWaits(host.id, entries)const items = await projectEvents(host.id, cursor, toCursor, entries, starts)if (!items) return falsecursor = toCursordiff --git a/lib/tools.ts b/lib/tools.tsindex e522622..c2cfca6 100644--- a/lib/tools.ts+++ b/lib/tools.ts@@ -1,4 +1,4 @@-import type { FxHostTool } from 'libfx'+import type { FxHostTool, FxToolResult } from 'libfx'// @ts-expect-error Node strip-types tests require the source extension.import { fetchPublicText } from './public-fetch.ts'@@ -10,6 +10,11 @@ export interface ToolHost {storeResult(callId: number, text: string): voidreadResult(callId: number): string | undefinedaskUser(questions: AskQuestion[], signal: AbortSignal): Promise<Record<string, string>>+ /** A photo of the published design from an open fx0 tab; throws when none sends one in time. */+ captureDesign(+ signal: AbortSignal,+ options?: { toolCallId?: string; full?: boolean },+ ): Promise<{ images: { mimeType: string; data: string }[]; revision: number; width?: number; height?: number; truncated?: boolean }>log(event: string, detail?: Record<string, unknown>): voidspawnSubagent(task: string, signal: AbortSignal): Promise<string>capabilitySearch(query: string): { name: string; kind: string; description: string }[]@@ -116,6 +121,37 @@ export function createTools(host: ToolHost, opts: { allowSubagent?: boolean } =},}+ const screenshotDesign: FxHostTool = {+ name: 'screenshot_design',+ description:+ 'Screenshot the published design as the preview renders it, at the size the user sees it (1280x800 when the preview is closed). By default it shows the visible viewport; set full_page to see the whole page, top to bottom, in viewport-high images (at most 8). Use it to check the result when appearance matters, such as matching a reference or a layout you cannot verify from the markup, not after every edit; use full_page when the sections below the first screen matter. Needs an fx0 tab open and in front; on failure, continue without it.',+ inputSchema: {+ type: 'object',+ properties: { full_page: { type: 'boolean', description: 'Whole page in viewport-high images instead of the visible viewport.' } },+ additionalProperties: false,+ },+ async execute(input: { full_page?: boolean }, { signal, executionId }): Promise<FxToolResult | { error: string }> {+ host.reportInvocation('screenshot_design')+ const full = input?.full_page === true+ const t0 = Date.now()+ try {+ const shot = await host.captureDesign(signal, { toolCallId: executionId, full })+ const base64Bytes = shot.images.reduce((sum, image) => sum + image.data.length, 0)+ host.log('tool:screenshot_design:done', { ms: Date.now() - t0, revision: shot.revision, images: shot.images.length, base64Bytes })+ const size = `${shot.width ?? '?'}x${shot.height ?? '?'} CSS px`+ const text = full+ ? `Full-page screenshot of design revision ${shot.revision}, ${size}, in ${shot.images.length} viewport-high image${shot.images.length === 1 ? '' : 's'}, top to bottom.` ++ (shot.truncated ? ' The page continues below the last image.' : '')+ : `Screenshot of design revision ${shot.revision}, ${size}, as the preview shows it.`+ return { type: 'libfx.tool-result', text, images: shot.images.map((image) => ({ type: 'image', ...image })) }+ } catch (error) {+ const message = error instanceof Error ? error.message : 'unknown error'+ host.log('tool:screenshot_design:error', { ms: Date.now() - t0, error: message })+ return { error: `screenshot unavailable: ${message}` }+ }+ },+ }+const webFetch: FxHostTool = {name: 'web_fetch',description:@@ -238,7 +274,7 @@ export function createTools(host: ToolHost, opts: { allowSubagent?: boolean } =},}- const tools: FxHostTool[] = [readDesign, publishDesign, editDesign, webFetch, askUser, readToolResult, capabilitySearch]+ const tools: FxHostTool[] = [readDesign, publishDesign, editDesign, screenshotDesign, webFetch, askUser, readToolResult, capabilitySearch]if (allowSubagent) {tools.push({diff --git a/package.json b/package.jsonindex f239754..433a264 100644--- a/package.json+++ b/package.json@@ -9,7 +9,7 @@"build": "next build","typecheck": "next typegen && tsc --noEmit --incremental false","start": "next start",- "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/prompt-images.test.mjs tests/design-assets.test.mjs tests/runtime.test.mjs",+ "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/prompt-images.test.mjs tests/design-assets.test.mjs tests/capture-protocol.test.mjs tests/runtime.test.mjs","test:auth": "node --experimental-strip-types --test tests/auth.test.mjs tests/auth-callback.test.mjs tests/signout.test.mjs tests/ownership.test.mjs tests/history.test.mjs tests/route-security.test.mjs tests/gateway-session.test.mjs tests/libfx-transport.test.mjs tests/libfx-agent.test.mjs tests/auth-ui.test.mjs tests/html-ui.test.mjs tests/guest-ui.test.mjs tests/guest-draft.test.mjs tests/trial.test.mjs tests/turn-funding.test.mjs","test:journal:integration": "node --env-file-if-exists=.env.local --experimental-strip-types --test tests/journal.integration.mjs","test:smoke": "node tests/local-auth-smoke.mjs",diff --git a/tests/capture-protocol.test.mjs b/tests/capture-protocol.test.mjsnew file mode 100644index 0000000..61ea09d--- /dev/null+++ b/tests/capture-protocol.test.mjs@@ -0,0 +1,29 @@+import assert from 'node:assert/strict'+import test from 'node:test'+import { captureTrace } from '../lib/capture-protocol.ts'++test('a capture trace splits the call into parts, and its network total survives a clock offset', () => {+ // The tab's clock runs 40 ms ahead of the servers'.+ const skew = 40+ const server = { totalMs: 700, setupMs: 9, emitMs: 6, emittedAt: 1_000_000, receivedAt: 1_000_685 }+ const shot = {+ path: 'preview', receivedAt: 1_000_150 + skew, sentAt: 1_000_520 + skew, previewWaitMs: 300, roundTripMs: 70,+ frame: { fontsMs: 1, libraryMs: 25, encodeMs: 30 }, arrivedAt: 1_000_670, authMs: 3, readMs: 1, storingAt: 1_000_675,+ }+ const trace = captureTrace(server, shot)+ assert.deepEqual(trace, {+ path: 'preview', totalMs: 700, setupMs: 9, emitMs: 6,+ deliveryMs: 150 + skew, previewWaitMs: 300, messagingMs: 14, fontsMs: 1, libraryMs: 25, encodeMs: 30,+ uploadMs: 150 - skew, networkMs: 300, authMs: 3, readMs: 1, storeAndWakeMs: 10,+ })+})++test("a capture trace without the frame's timing counts the whole round trip as messaging", () => {+ const trace = captureTrace(+ { totalMs: 100, setupMs: 1, emitMs: 1, emittedAt: 0, receivedAt: 100 },+ { path: 'hidden', receivedAt: 10, sentAt: 60, previewWaitMs: 20, roundTripMs: 30, frame: null, arrivedAt: 80, authMs: 2, readMs: 0, storingAt: 90 },+ )+ assert.equal(trace.messagingMs, 30)+ assert.equal(trace.fontsMs, null)+ assert.equal(trace.path, 'hidden')+})diff --git a/tests/core.test.mjs b/tests/core.test.mjsindex f3560cb..888daed 100644--- a/tests/core.test.mjs+++ b/tests/core.test.mjs@@ -69,6 +69,7 @@ function toolHost() {storeResult: () => {},readResult: () => undefined,askUser: async () => ({}),+ captureDesign: async () => ({ images: [{ mimeType: 'image/jpeg', data: 'AAAA' }], revision: 1 }),log: () => {},spawnSubagent: async () => 'done',capabilitySearch: () => [],@@ -113,6 +114,7 @@ test('design mode exposes only design and common host tools', async () => {'read_design','publish_design','edit_design',+ 'screenshot_design','web_fetch','ask_user_question','read_tool_result',@@ -128,6 +130,23 @@ test('design mode exposes only design and common host tools', async () => {)})+test('screenshot_design gives the model its images with what they show, or why there are none', async () => {+ const signal = new AbortController().signal+ const calls = []+ const host = { ...toolHost(), captureDesign: async (_signal, options) => {+ calls.push(options)+ return { images: [{ mimeType: 'image/webp', data: 'AAAA' }, { mimeType: 'image/webp', data: 'BBBB' }], width: 1280, height: 2400, truncated: true, revision: 3 }+ } }+ const screenshot = createTools(host).find((tool) => tool.name === 'screenshot_design')+ const page = await screenshot.execute({ full_page: true }, { signal, executionId: 'call_9' })+ assert.deepEqual(calls, [{ toolCallId: 'call_9', full: true }])+ assert.deepEqual(page.images, [{ type: 'image', mimeType: 'image/webp', data: 'AAAA' }, { type: 'image', mimeType: 'image/webp', data: 'BBBB' }])+ assert.equal(page.text, 'Full-page screenshot of design revision 3, 1280x2400 CSS px, in 2 viewport-high images, top to bottom. The page continues below the last image.')+ const failing = createTools({ ...toolHost(), captureDesign: async () => { throw new Error('no fx0 tab is open and in front to take it') } })+ assert.deepEqual(await failing.find((tool) => tool.name === 'screenshot_design').execute({}, { signal, executionId: 'call_10' }),+ { error: 'screenshot unavailable: no fx0 tab is open and in front to take it' })+})+test('turn events update an existing sidebar session without mutating it', () => {const current = [{ id: 'chat', title: 'Landing page', createdAt: 10, lastActivity: 10, busy: false }]const started = upsertSession(current, 'chat', { type: 'turn_start', id: 1, at: 11 })diff --git a/tests/design-artifact.browser.mjs b/tests/design-artifact.browser.mjsindex e603e93..901d408 100644--- a/tests/design-artifact.browser.mjs+++ b/tests/design-artifact.browser.mjs@@ -10,6 +10,7 @@ import { pathToFileURL } from 'node:url'import { promisify } from 'node:util'import { markRemoteImages } from '../lib/remote-image.ts'import { DESIGN_ARTIFACT_EMBEDDED_CSP, DESIGN_FRAME_CSP, designFrameCsp, designFrameHtml, MAX_EMBEDDED_IMAGE_BYTES, prepareDesignArtifactHtml, validateDesignArtifactHtml } from '../lib/design-artifact.ts'+import { DESIGN_CAPTURE_RELAY, designCaptureScript } from '../lib/design-capture.ts'// Opt-in: runs real Chrome headless on this machine. Set CHROME_PATH for another browser binary.const CHROME = process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'@@ -190,6 +191,86 @@ for (const name of ['small', 'large', 'repeat']) open(name)}})+// screenshot_design: the app mounts a preview the way DesignCapture does+// (fixed, invisible, under the page, in a phone-sized window) and asks it for photos once+// it has painted: the viewport, then the whole page. The viewport must come back as one+// 1280x800 WebP of the design, the page as viewport-high tiles from the top (even when+// html and body are only viewport-high and the content overflows them), and the remote+// url() the design names must never reach the network, capture included.+test('in Chrome, a hidden preview photographs its viewport and its whole page without any network', async () => {+ assert.ok(existsSync(CHROME), `Chrome not found at ${CHROME}; set CHROME_PATH`)+ const requests = []+ const server = http.createServer((request, response) => {+ const url = new URL(request.url, 'http://test')+ requests.push(url.pathname)+ if (url.pathname === '/frame') {+ const stored = validateDesignArtifactHtml(`<!doctype html><html><head><style>html,body{margin:0;height:100%}+section{height:800px}.a{background:#ff0066}.b{background:#00cc66}.c{height:400px;background:#3366ff}+.leak{background:url(http://127.0.0.1:${server.address().port}/leak.png)}</style></head><body>+<section class="a"><h1 class="leak" style="color:#fff;margin:0;padding:40px">fx0</h1></section><section class="b"></section><section class="c"></section></body></html>`)+ assert.equal(stored.ok, true, stored.error)+ const token = 'browser-test'+ response.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'content-security-policy': DESIGN_FRAME_CSP })+ response.end(designFrameHtml(prepareDesignArtifactHtml(stored.html, DESIGN_ARTIFACT_EMBEDDED_CSP, token, designCaptureScript(token)), token, [], DESIGN_CAPTURE_RELAY))+ } else if (url.pathname === '/hold') {+ setTimeout(() => response.writeHead(404).end(), 6000)+ } else if (url.pathname === '/') {+ response.writeHead(200, { 'content-type': 'text/html' }).end(`<!doctype html><html><head><title>app</title></head><body style="margin:0">+<main style="position:fixed;inset:0;background:#fff"><pre id="out">{}</pre></main>+<div style="position:fixed;inset:0;z-index:-10;overflow:hidden;opacity:0;pointer-events:none"><iframe id="capture" src="/frame" style="width:1280px;height:800px;border:0"></iframe></div>+<script>+const frame = document.getElementById('capture')+const out = {}+const show = () => { document.getElementById('out').textContent = JSON.stringify(out) }+const describe = async (blob) => {+ const bitmap = await createImageBitmap(blob)+ const canvas = new OffscreenCanvas(bitmap.width, bitmap.height)+ const context = canvas.getContext('2d')+ context.drawImage(bitmap, 0, 0)+ return { type: blob.type, width: bitmap.width, height: bitmap.height, color: [...context.getImageData(1200, bitmap.height - 20, 1, 1).data.slice(0, 3)] }+}+addEventListener('message', async ({ data, source }) => {+ if (source !== frame.contentWindow) return+ if (data?.type === 'fx0-artifact-painted' && !out.asked) {+ out.asked = true+ frame.contentWindow.postMessage({ type: 'fx0-capture', cid: 'viewport' }, location.origin)+ }+ if (data?.type !== 'fx0-artifact-captured') return+ out[data.cid] = { height: data.height, truncated: data.truncated, timing: data.timing, images: await Promise.all(data.blobs.map(describe)) }+ if (data.cid === 'viewport') frame.contentWindow.postMessage({ type: 'fx0-capture', cid: 'page', full: true }, location.origin)+ show()+})+</script><img alt="" src="/hold"></body></html>`)+ } else {+ response.writeHead(404).end()+ }+ })+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))+ const dir = await mkdtemp(join(tmpdir(), 'fx0-design-capture-'))+ try {+ const { stdout } = await promisify(execFile)(CHROME, [+ '--headless=new', '--disable-gpu', '--no-first-run', '--no-default-browser-check', '--window-size=390,844',+ `--user-data-dir=${join(dir, 'profile')}`, '--dump-dom', `http://127.0.0.1:${server.address().port}/`,+ ], { timeout: 60_000, maxBuffer: 16 * 1024 * 1024 })+ const out = JSON.parse(stdout.match(/<pre id="out">([^<]*)<\/pre>/)[1])+ const near = (rgb, target) => rgb.every((value, i) => Math.abs(value - target[i]) < 24)+ assert.equal(out.viewport.images.length, 1)+ assert.deepEqual(out.viewport.images.map(({ type, width, height }) => [type, width, height]), [['image/webp', 1280, 800]], 'Chrome sends WebP')+ assert.ok(near(out.viewport.images[0].color, [255, 0, 102]), `the viewport is the design's first section, got ${out.viewport.images[0].color}`)+ for (const part of ['fontsMs', 'libraryMs', 'encodeMs']) assert.ok(out.viewport.timing?.[part] >= 0, `the frame reports ${part}`)+ assert.equal(out.page.height, 2000)+ assert.equal(out.page.truncated, false)+ assert.deepEqual(out.page.images.map(({ width, height }) => [width, height]), [[1280, 800], [1280, 800], [1280, 400]])+ const colors = [[255, 0, 102], [0, 204, 102], [51, 102, 255]]+ out.page.images.forEach((image, i) => assert.ok(near(image.color, colors[i]), `tile ${i + 1} shows section ${i + 1}, got ${image.color}`))+ assert.ok(!requests.includes('/leak.png'), 'the url() the design names never reached the network')+ } finally {+ server.closeAllConnections()+ server.close()+ await rm(dir, { recursive: true, force: true })+ }+})+// The draft has no network, so a remote image must show the soft placeholder, never// the browser's broken-image icon. The draft cleaner runs in Chrome, types stripped.test('in Chrome, a draft shows a placeholder for remote images instead of a broken icon', async () => {diff --git a/tests/journal.integration.mjs b/tests/journal.integration.mjsindex 9e1a801..f7bc395 100644--- a/tests/journal.integration.mjs+++ b/tests/journal.integration.mjs@@ -368,6 +368,55 @@ test('journal Redis integration uses isolated durable session state', async (t)ok: false, reason: 'fenced', error: 'the chat was deleted before publishing',})})+ await t.test('a capture is acknowledged, answered once, and fails fast when no tab takes it', async () => {+ const sessionId = randomUUID()+ const shot = { images: [{ mimeType: 'image/webp', data: 'AAAA' }, { mimeType: 'image/webp', data: 'BBBB' }], width: 1280, height: 1600 }+ const replies = key(sessionId, 'capture-replies')++ // Acknowledged, then answered with a photo: the call wakes as the photo arrives.+ await journal.openCapture(sessionId, 'c1')+ const started = Date.now()+ const waiting = journal.awaitCapture(sessionId, 'c1', AbortSignal.timeout(15_000))+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c0'), false, 'another capture id is refused')+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c1'), true)+ await new Promise((resolve) => setTimeout(resolve, 500))+ assert.equal(await journal.answerCapture(sessionId, 'c1', { kind: 'shot', shot }), true)+ assert.deepEqual(await waiting, { kind: 'shot', shot })+ assert.ok(Date.now() - started < 2_500, `woke ${Date.now() - started} ms after waiting began, the photo came at 500 ms`)+ assert.equal(await journal.answerCapture(sessionId, 'c1', { kind: 'failed', reason: 'too-large' }), false, 'an answer is accepted once')+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c1'), false, 'a replayed request finds its capture closed')++ // A tab's failure ends the wait with its reason.+ await journal.openCapture(sessionId, 'c2')+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c2'), true)+ assert.equal(await journal.answerCapture(sessionId, 'c2', { kind: 'failed', reason: 'too-large' }), true)+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c2', AbortSignal.timeout(5_000)), { kind: 'failed', reason: 'too-large' })++ // No tab acknowledges: the call gives up after the ack window, not the whole wait.+ await journal.openCapture(sessionId, 'c3')+ const silent = Date.now()+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c3', AbortSignal.timeout(15_000), 1_000), { kind: 'unanswered' })+ assert.ok(Date.now() - silent < 3_000, `gave up after ${Date.now() - silent} ms`)+ assert.equal(await journal.answerCapture(sessionId, 'c3', { kind: 'shot', shot }), false, 'a late photo is refused')++ // Cancelled: the capture is withdrawn.+ await journal.openCapture(sessionId, 'c4')+ const cancel = new AbortController()+ setTimeout(() => cancel.abort(), 300)+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c4', cancel.signal), { kind: 'cancelled' })+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c4'), false, 'a cancelled wait withdraws its capture')+ assert.equal(await redis.exists(key(sessionId, 'capture'), replies), 0, 'nothing is left behind')+ })++ await t.test('a screenshot call records its preview wait under its call id, for its tool_end', async () => {+ const sessionId = randomUUID()+ await journal.recordPreviewWait(sessionId, 'call_a', 515.4)+ await journal.recordPreviewWait(sessionId, 'call_b', 0)+ assert.deepEqual([...await journal.readPreviewWaits(sessionId, ['call_a', 'call_b', 'call_missing'])], [['call_a', 515]])+ assert.deepEqual([...await journal.readPreviewWaits(sessionId, [])], [])+ assert.ok(await redis.pttl(key(sessionId, 'preview-waits')) > 0, 'the waits expire with the chat idle')+ })+await t.test('a chat credential is sealed, expires with its token, and sign-out drops only its own login', async () => {const session = (loginId, accessToken) => ({userId: 'user-a', loginId, loginAt: 1, email: '[email protected]', accessToken, teamId: 'team_a',diff --git a/tests/libfx-agent.test.mjs b/tests/libfx-agent.test.mjsindex 244b12d..260eb4f 100644--- a/tests/libfx-agent.test.mjs+++ b/tests/libfx-agent.test.mjs@@ -60,6 +60,12 @@ globalThis.__fx0Agent = {durableAgent({ toolHost: async () => { throw new Error('no tools here') } }, 'instructions')const agentFetch = agentOptions.fetch+test('screenshots in one response take turns: a chat has one capture open at a time', () => {+ const screenshot = agentOptions.tools.find((tool) => tool.name === 'screenshot_design')+ assert.equal(screenshot.idempotent, true)+ assert.equal(screenshot.writes, true, 'a tool that writes never runs beside another call')+})+test('a recovered sponsored chat uses its recorded trial funding instead of requiring a customer card', async (t) => {const previous = { key: process.env.AI_GATEWAY_API_KEY, enabled: process.env.FX0_FREE_PROMPTS }process.env.AI_GATEWAY_API_KEY = 'synthetic-server-key'diff --git a/tests/route-security.test.mjs b/tests/route-security.test.mjsindex b7be32c..193bff2 100644--- a/tests/route-security.test.mjs+++ b/tests/route-security.test.mjs@@ -113,6 +113,8 @@ const mocks = {return { html: s.liveRevision === 0 ? null : '<!doctype html><html><head><title>private design</title></head><body>private artifact</body></html>', revision: s.liveRevision ?? 1 };}export async function currentSeq(id) { record('seq', id); return 0; }+ export async function acknowledgeCapture(id) { record('capture-ack', id); return true; }+ export async function answerCapture(id, cid, answer) { record('capture-answer', id); globalThis.securityTest.captureAnswer = answer; return true; }`,'lib/agent.ts': `function effect(name, host) {@@ -263,7 +265,7 @@ function request(path, method = 'GET', id = own, body, origin = process.env.APP_return req}const routes = {}-for (const name of ['answer', 'artifact', 'artifact-frame', 'artifact-image', 'cancel', 'debug', 'events',+for (const name of ['answer', 'artifact', 'artifact-frame', 'capture', 'artifact-image', 'cancel', 'debug', 'events','credits', 'model', 'prompt', 'session', 'sessions', 'teams', 'warm']) {routes[name] = await import(`../app/api/${name}/route.ts`)}@@ -421,6 +423,63 @@ test('session routes return indistinguishable 404 for unknown and foreign IDs be}})+test('a tab acknowledges a capture and reports a known failure; an unknown reason is refused', async () => {+ const s = reset()+ const post = (query) => routes.capture.POST(new Request(`https://fx0.example/api/capture?session=${own}&cid=${'1'.repeat(8)}-1111-1111-1111-${'1'.repeat(12)}&${query}`, {+ method: 'POST', headers: { origin: process.env.APP_URL },+ }))+ const ack = await post('ack=1')+ assert.equal(ack.status, 200)+ assert.deepEqual(await ack.json(), { ok: true })+ assert.ok(s.trace.includes('capture-ack'))+ const failed = await post('failed=too-large')+ assert.deepEqual(await failed.json(), { ok: true })+ assert.deepEqual(s.captureAnswer, { kind: 'failed', reason: 'too-large' })+ assert.equal((await post('failed=__proto__')).status, 400)+ assert.equal((await post('failed=anything')).status, 400)+ const unsized = await post('w=1')+ assert.equal(unsized.status, 411, 'a photo without a length is refused before it is read')+})++test('a photo reaches the waiting call; one the route refuses still answers it, with the reason', async () => {+ const s = reset()+ const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', 'base64')+ const upload = async (files, query = 'w=1280&h=800', length) => {+ const form = new FormData()+ for (const file of files) form.append('image', new Blob([file], { type: 'image/png' }))+ const encoded = new Response(form)+ const body = await encoded.arrayBuffer()+ return routes.capture.POST(new Request(`https://fx0.example/api/capture?session=${own}&cid=${'2'.repeat(8)}-2222-2222-2222-${'2'.repeat(12)}&${query}`, {+ method: 'POST', body,+ headers: { origin: process.env.APP_URL, 'content-type': encoded.headers.get('content-type'), 'content-length': String(length ?? body.byteLength) },+ }))+ }+ const accepted = await upload([PNG], 'w=1280&h=2400&truncated=1')+ assert.deepEqual(await accepted.json(), { ok: true })+ assert.equal(s.captureAnswer.kind, 'shot')+ assert.deepEqual(s.captureAnswer.shot.images, [{ mimeType: 'image/png', data: PNG.toString('base64') }])+ assert.deepEqual([s.captureAnswer.shot.width, s.captureAnswer.shot.height, s.captureAnswer.shot.truncated], [1280, 2400, true])+ for (const [files, length, status, reason] of [+ [[Buffer.from('not an image at all')], undefined, 415, 'refused'],+ [Array(9).fill(PNG), undefined, 400, 'refused'],+ [[PNG], 10 * 1024 * 1024, 413, 'too-large'],+ ]) {+ s.captureAnswer = null+ assert.equal((await upload(files, undefined, length)).status, status)+ assert.deepEqual(s.captureAnswer, { kind: 'failed', reason }, `a ${status} answers the call`)+ }+})++test('a capture for an unknown or foreign chat is refused before its body or Redis', async () => {+ for (const id of [foreign, fresh]) {+ const s = reset()+ const response = await routes.capture.POST(request('capture', 'POST', id))+ assert.equal(response.status, 404)+ assert.deepEqual(s.trace, ['auth', 'owner'])+ assert.deepEqual(s.redisReads, [])+ }+})+test('the same owner can steer, enqueue or recover a chat after switching teams, with one owner check', async () => {for (const [name, method, body] of sessionRoutes) {const oldContext = reset()@@ -440,7 +499,7 @@ test('the same owner can steer, enqueue or recover a chat after switching teams,test('session routes and creation reject unverified team grants before runtime or writes', async () => {for (const teamId of [undefined, 'user_a']) {- for (const [name, method, body] of [...sessionRoutes, ['sessions', 'POST']]) {+ for (const [name, method, body] of [...sessionRoutes, ['sessions', 'POST'], ['capture', 'POST']]) {const s = reset({ session: session({ teamId }) })const response = await routes[name][method](request(name, method, own, body))assert.equal(response.status, 403, `${method} ${name}`)@@ -598,8 +657,10 @@ test('the artifact preview paints no white of its own before the design does', areset()const html = await (await routes['artifact-frame'].GET(request('artifact-frame', 'GET', own, undefined, 'null'))).text()// The app's color scheme keeps the wrapper transparent over the app's background.- assert.match(html, /:root\{color-scheme:light dark\}/)- assert.doesNotMatch(html, /background:#fff/)+ // The design (and the capture library) travel inside the wrapper's script; only its own markup counts.+ const wrapper = html.slice(0, html.indexOf('<script>'))+ assert.match(wrapper, /:root\{color-scheme:light dark\}/)+ assert.doesNotMatch(wrapper, /background:#fff/)})test('the browser keeps a preview only when its URL names the revision that is live', async () => {diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjsindex 014d87e..6c1f0ac 100644--- a/tests/runtime.test.mjs+++ b/tests/runtime.test.mjs@@ -7,11 +7,12 @@ import test from 'node:test'// own integration test; the agent's fetch has tests/libfx-agent.test.mjs.const journalNames = ['answerActiveQuestion', 'appendEvent', 'bindLibfxSession', 'chatOfLibfxSession', 'claimSessionCleanup',- 'clearActiveQuestion', 'currentSeq', 'emitLive', 'finishSessionCleanup', 'finishTurnWork', 'getActiveQuestion',- 'getActiveTurn', 'nextQuestionId', 'projectEvents', 'projectedCursor', 'publishDesignArtifact', 'readDesignArtifact',- 'readDesignAssets', 'readDesignMd', 'readEvents', 'recordDesignAssets', 'releaseDesignAssets', 'reserveDesignAssets',- 'finishDesignAssetCleanup', 'pendingDesignAssetCleanups', 'queueDesignAssetCleanup',- 'readSession', 'readSessionView', 'readTurnWork', 'releaseSessionCleanup', 'renewSessionCleanup',+ 'awaitCapture', 'clearActiveQuestion', 'closeCapture', 'currentSeq', 'emitLive', 'finishDesignAssetCleanup',+ 'finishSessionCleanup', 'finishTurnWork', 'getActiveQuestion', 'getActiveTurn', 'nextQuestionId', 'openCapture',+ 'pendingDesignAssetCleanups', 'projectEvents', 'projectedCursor', 'publishDesignArtifact',+ 'queueDesignAssetCleanup', 'readDesignArtifact', 'readDesignAssets', 'readDesignMd', 'readEvents',+ 'readPreviewWaits', 'readSession', 'readSessionView', 'readTurnWork', 'recordDesignAssets', 'recordPreviewWait',+ 'releaseDesignAssets', 'releaseSessionCleanup', 'renewSessionCleanup', 'reserveDesignAssets','setActiveQuestion', 'submitPrompt', 'takeAnswer', 'updateSession',]const dataUrl = (source) => `data:text/javascript,${encodeURIComponent(source)}`@@ -449,11 +450,12 @@ test('the agent offers HTML and common tools only; a tool runs on the chat its tconst f = fixture(t)runtime.agent()const names = f.agentOptions.tools.map((tool) => tool.name)- assert.deepEqual(names, ['read_design', 'publish_design', 'edit_design', 'web_fetch', 'ask_user_question', 'read_tool_result', 'capability_search', 'subagent', 'web_search'])+ assert.deepEqual(names, ['read_design', 'publish_design', 'edit_design', 'screenshot_design', 'web_fetch', 'ask_user_question', 'read_tool_result', 'capability_search', 'subagent', 'web_search'])const edit = f.agentOptions.tools.find((tool) => tool.name === 'edit_design')assert.equal(edit.idempotent, false)assert.equal(edit.writes, true)assert.equal(f.agentOptions.tools.find((tool) => tool.name === 'read_design').idempotent, true)+ assert.equal(f.agentOptions.tools.find((tool) => tool.name === 'screenshot_design').idempotent, true)const ctx = { signal: new AbortController().signal, executionId: 'call-1', sessionId: 'wrun_1', context: { chatId: 'tool-chat', turnId: 4 } }const result = await edit.execute({ edits: [{ old_string: 'old', new_string: 'new' }, { old_string: '<body>', new_string: '<body class="x">' }] }, ctx)@@ -622,6 +624,20 @@ test('a turn\'s history and subagents keep the settings it started with after aassert.equal(history.fast, false)})+test('the projector puts the preview wait of a screenshot call on its tool_end, and only there', async (t) => {+ const f = fixture(t)+ const entries = [+ { type: 'tool_end', name: 'screenshot_design', toolCallId: 'shot', turnId: 1, at: 1 },+ { type: 'tool_end', name: 'read_design', toolCallId: 'read', turnId: 1, at: 2 },+ { type: 'tool_end', name: 'screenshot_design', toolCallId: 'unrecorded', turnId: 1, at: 3 },+ ]+ const asked = []+ f.journal.readPreviewWaits = async (_id, ids) => { asked.push(...ids); return new Map([['shot', 515], ['read', 9]]) }+ await projector.addPreviewWaits('chat', entries)+ assert.deepEqual(asked, ['shot', 'unrecorded'], 'only screenshot calls are looked up')+ assert.deepEqual(entries.map((entry) => entry.previewWaitMs), [515, undefined, undefined])+})+test('the projector copies a turn once however many follow it, then records its history and design', async (t) => {const f = fixture(t, { libfxSessionId: 'wrun_7', preferredModel: null, preferredEffort: null, preferredFast: null })const host = boundHost('project-chat')diff --git a/tests/transcript.test.mjs b/tests/transcript.test.mjsindex 4134ad6..3de16d6 100644--- a/tests/transcript.test.mjs+++ b/tests/transcript.test.mjs@@ -84,3 +84,19 @@ test('a page may publish its checkpoint after responding without delaying the trawait job()assert.equal(published, true)})++test('a screenshot call reports its own time, not the time it waited for the preview to load', () => {+ const view = { artifactRevision: 1 }+ const state = compactTranscript([+ { type: 'turn_start', id: 1, seq: 1, at: 0 },+ // The capture event may land before the call's tool_start (the projector writes libfx's+ // lines in batches); the wait travels on the tool_end instead.+ { type: 'capture', turnId: 1, seq: 2, cid: 'c1', artifactRevision: 1, at: 1_009 },+ { type: 'tool_start', turnId: 1, seq: 3, name: 'screenshot_design', at: 1_000 },+ { type: 'tool_end', turnId: 1, seq: 4, name: 'screenshot_design', toolCallId: 'call_1', previewWaitMs: 515, content: 'Screenshot', at: 2_208 },+ ], view)+ const tool = state.entries.find((entry) => entry.type === 'tool')+ assert.equal(tool.durationMs, 693)+ assert.equal(tool.previewWaitMs, 515)+ assert.equal(state.entries.filter((entry) => entry.type === 'tool').length, 1, 'capture events add no rows')+})