Skip to content

armgit / fx0

Public Imported from GitHub
0

A lightweight design agent built with libfx. Describe a page and an agent writes it as one HTML file.

TypeScript

Let the agent screenshot its published design

Esteban Suarezcb4eba73fa91
diff --git a/app/api/artifact-frame/route.ts b/app/api/artifact-frame/route.ts
index e30a921..3c59664 100644
--- a/app/api/artifact-frame/route.ts
+++ b/app/api/artifact-frame/route.ts
@@ -1,5 +1,6 @@
import { ARTIFACT_IMAGE_PATH, artifactImageSrc } from '../../../lib/artifact-image'
import { DESIGN_ARTIFACT_EMBEDDED_CSP, designFrameCsp, designFrameHtml, prepareDesignArtifactHtml, remoteImagesEnabled } from '../../../lib/design-artifact'
+import { DESIGN_CAPTURE_RELAY, designCaptureScript } from '../../../lib/design-capture'
import { markRemoteImages } from '../../../lib/remote-image'
import { readDesignArtifactAt } from '../../../lib/journal'
import { readGeneration } from '../../../lib/history'
@@ -32,7 +33,10 @@ export const GET = withSessionRoute(async (request, { id: sessionId, user }) =>
? markRemoteImages(artifact, crypto.randomUUID().replaceAll('-', ''), { remote: remoteImagesEnabled() })
: { html: artifact, images: [] }
const images = marked.images.map(({ marker, url }) => ({ marker, src: artifactImageSrc(sessionId, url) }))
- const prepared = marked.html == null ? null : prepareDesignArtifactHtml(marked.html, DESIGN_ARTIFACT_EMBEDDED_CSP, readyToken)
+ // Every preview can be photographed where it is (lib/design-capture.ts), on screen or as
+ // the hidden copy screenshot_design loads when no preview shows the revision.
+ const prepared = marked.html == null ? null : prepareDesignArtifactHtml(marked.html, DESIGN_ARTIFACT_EMBEDDED_CSP, readyToken,
+ designCaptureScript(readyToken))
// Every stored design passed validation, so this only guards against one that
// somehow lacks a doctype: it is never shown without its CSP.
if (artifact != null && prepared == null) log(sessionId, 'artifact:unrenderable', { revision: past ? revision : live!.revision })
@@ -41,7 +45,7 @@ export const GET = withSessionRoute(async (request, { id: sessionId, user }) =>
artifact != null ? 'This design could not be prepared for preview.'
: past ? 'This version is not in the history.' : 'No design artifact has been published for this session.'
}</p></body></html>`
- const html = designFrameHtml(preparedArtifact, readyToken, images)
+ const html = designFrameHtml(preparedArtifact, readyToken, images, DESIGN_CAPTURE_RELAY)
return new Response(html, {
status: artifact == null ? 404 : 200,
headers: {
diff --git a/app/api/capture/route.ts b/app/api/capture/route.ts
new file mode 100644
index 0000000..59eb23d
--- /dev/null
+++ b/app/api/capture/route.ts
@@ -0,0 +1,95 @@
+import { CAPTURE_FAILURES, CAPTURE_TIMING_HEADER, MAX_CAPTURE_BYTES, MAX_CAPTURE_TILES, type CaptureFailure, type FrameTiming, type ShotTiming } from '../../../lib/capture-protocol'
+import { imageDimensions, sniffImageType } from '../../../lib/image-sniff'
+import { acknowledgeCapture, answerCapture } from '../../../lib/journal'
+import { withSessionRoute } from '../../../lib/route-auth'
+import { log } from '../../../lib/session'
+
+const CAPTURE_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/
+const CAPTURE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp'])
+const MAX_SIDE = 16384
+
+type TabTiming = Pick<ShotTiming, 'path' | 'receivedAt' | 'sentAt' | 'previewWaitMs' | 'roundTripMs' | 'frame'>
+
+// The tab's part of the trace. It only labels the photo, so anything malformed is dropped.
+function tabTiming(header: string | null): TabTiming | null {
+ try {
+ const value = JSON.parse(header ?? 'null')
+ const number = (n: unknown) => (typeof n === 'number' && Number.isFinite(n) ? n : null)
+ const [receivedAt, sentAt, previewWaitMs, roundTripMs] = [value?.receivedAt, value?.sentAt, value?.previewWaitMs, value?.roundTripMs].map(number)
+ if (receivedAt === null || sentAt === null || previewWaitMs === null || roundTripMs === null) return null
+ const frameParts = [value.frame?.fontsMs, value.frame?.libraryMs, value.frame?.encodeMs].map(number)
+ const frame: FrameTiming | null = frameParts.every((n) => n !== null)
+ ? { fontsMs: frameParts[0]!, libraryMs: frameParts[1]!, encodeMs: frameParts[2]! }
+ : null
+ return { path: value.path === 'hidden' ? 'hidden' : 'preview', receivedAt, sentAt, previewWaitMs, roundTripMs, frame }
+ } catch {
+ return null
+ }
+}
+
+// A tab's reply for screenshot_design (lib/capture-protocol.ts): `?ack=1` as it takes the
+// request, then either `?failed=<reason>` or the photo itself, one image or a full page's
+// tiles, as multipart form data. Every reply counts only for the capture the tool call is
+// waiting on; `ok: false` tells the tab it is no longer open.
+const handle = (arrivedAt: number, arrived: number) => withSessionRoute(async (request, { id: sessionId }) => {
+ const authMs = performance.now() - arrived
+ const params = new URL(request.url).searchParams
+ const cid = params.get('cid') ?? ''
+ if (!CAPTURE_ID.test(cid)) return Response.json({ error: 'invalid capture' }, { status: 400 })
+ if (params.get('ack') === '1') return Response.json({ ok: await acknowledgeCapture(sessionId, cid) })
+ const failed = params.get('failed')
+ if (failed !== null) {
+ if (!Object.hasOwn(CAPTURE_FAILURES, failed)) return Response.json({ error: 'invalid capture' }, { status: 400 })
+ const accepted = await answerCapture(sessionId, cid, { kind: 'failed', reason: failed as CaptureFailure })
+ log(sessionId, 'capture:failed', { reason: failed, accepted })
+ return Response.json({ ok: accepted })
+ }
+ // A refused photo still answers the capture, so the call stops waiting and says why.
+ const refuse = async (status: number, error: string, reason: CaptureFailure) => {
+ const accepted = await answerCapture(sessionId, cid, { kind: 'failed', reason })
+ log(sessionId, 'capture:refused', { status, reason, accepted })
+ return Response.json({ error }, { status })
+ }
+ // A multipart body adds a few hundred bytes per image around the images themselves. A
+ // browser always sends its length; one without it is refused before it is read.
+ const length = Number(request.headers.get('content-length'))
+ if (!(length > 0)) return refuse(411, 'length required', 'refused')
+ if (length > MAX_CAPTURE_BYTES + 4096) return refuse(413, 'capture too large', 'too-large')
+ const reading = performance.now()
+ let files: File[]
+ try {
+ files = (await request.formData()).getAll('image').filter((entry): entry is File => entry instanceof File)
+ } catch {
+ return refuse(400, 'invalid capture', 'refused')
+ }
+ if (!files.length || files.length > MAX_CAPTURE_TILES) return refuse(400, 'invalid capture', 'refused')
+ const bytes = files.reduce((sum, file) => sum + file.size, 0)
+ if (bytes > MAX_CAPTURE_BYTES) return refuse(413, 'capture too large', 'too-large')
+ const images: { mimeType: string; data: string }[] = []
+ for (const file of files) {
+ const bytes = new Uint8Array(await file.arrayBuffer())
+ // A header that names an image is not enough: one the model's provider cannot size would
+ // fail the whole model request.
+ const mimeType = sniffImageType(bytes)
+ const dimensions = mimeType && CAPTURE_TYPES.has(mimeType) ? imageDimensions(bytes, mimeType) : null
+ if (!mimeType || !dimensions || dimensions.width < 1 || dimensions.height < 1 || dimensions.width > MAX_SIDE || dimensions.height > MAX_SIDE) {
+ return refuse(415, 'capture is not an image', 'refused')
+ }
+ images.push({ mimeType, data: Buffer.from(bytes).toString('base64') })
+ }
+ const readMs = performance.now() - reading
+ // The viewport the tab photographed only labels the photo for the model.
+ const size = (name: string) => { const n = Number(params.get(name)); return Number.isInteger(n) && n > 0 && n <= MAX_SIDE ? n : undefined }
+ const tab = tabTiming(request.headers.get(CAPTURE_TIMING_HEADER))
+ // The timing travels with the photo, so the waiting tool call can log the whole trace.
+ const submitting = performance.now()
+ const timing: ShotTiming | undefined = tab ? { ...tab, arrivedAt, authMs, readMs, storingAt: Date.now() } : undefined
+ const accepted = await answerCapture(sessionId, cid, {
+ kind: 'shot', shot: { images, width: size('w'), height: size('h'), truncated: params.get('truncated') === '1', timing },
+ })
+ // Not accepted: the call stopped waiting (closed by a timeout, a cancel or an earlier answer).
+ log(sessionId, accepted ? 'capture:accepted' : 'capture:closed', { images: images.length, bytes, submitMs: Math.round(performance.now() - submitting) })
+ return Response.json({ ok: accepted })
+})
+
+export const POST = (request: Request) => handle(Date.now(), performance.now())(request)
diff --git a/components/chat-app.tsx b/components/chat-app.tsx
index 513b9a7..f29e1a6 100644
--- a/components/chat-app.tsx
+++ b/components/chat-app.tsx
@@ -37,6 +37,7 @@ const DebugPanel = dynamic(() => import('./debug-panel'), { ssr: false })
import DesignExport from './design-export'
import { Skeleton } from './ui/skeleton'
import PreviewPane, { type PreviewSource } from './preview-pane'
+import DesignCapture, { acknowledgeCapture, type CaptureRequest } from './design-capture'
import Sidebar from './sidebar'
import SidebarGestures, { type SidebarGestureControls } from './sidebar-gestures'
import { Button } from './ui/button'
@@ -280,6 +281,7 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultMod
const [entries, setEntries] = useState<ChatEntry[]>(initialState.entries)
const [artifactRevision, setArtifactRevision] = useState(initialState.view.artifactRevision)
const [designDraft, setDesignDraft] = useState<{ id: string; html: string } | null>(null)
+ const [captureRequest, setCaptureRequest] = useState<CaptureRequest | null>(null)
const [viewedRevision, setViewedRevision] = useState<{ sessionId: string; revision: number; latest: number } | null>(null)
const [activeTurnId, setActiveTurnId] = useState<number | null>(initialState.activeTurnId)
const [turnStartedAt, setTurnStartedAt] = useState<number | null>(initialState.activeTurnStartedAt)
@@ -728,6 +730,19 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultMod
}
}
if (item.type === 'heartbeat') return
+ // screenshot_design asks every tab; only a visible one can paint the design, so only it
+ // takes the request. It acknowledges at once, alongside the capture; a capture that is no
+ // longer open (an event replayed after a reconnect) answers false and is dropped.
+ if (item.type === 'capture') {
+ if (item.cid && document.visibilityState === 'visible') {
+ const request: CaptureRequest = { cid: item.cid, sessionId: sourceSessionId, revision: item.artifactRevision ?? 0, full: item.captureFull === true, receivedAt: Date.now() }
+ setCaptureRequest(request)
+ void acknowledgeCapture(request).then((open) => {
+ if (!open) setCaptureRequest((current) => (current?.cid === request.cid ? null : current))
+ })
+ }
+ return
+ }
if (item.type === 'turn_start') {
setAwaiting(false)
setActiveTurnId(item.id ?? null)
@@ -2092,10 +2107,20 @@ export default function ChatApp({ account, sidebar, boot, preference, defaultMod
draft={designDraft?.html ?? null}
intro={view.snapshot.transcript.entries.length === 0}
onClose={closePreview}
+ capture={captureRequest && !captureRequest.offscreen && captureRequest.sessionId === view.sessionId ? captureRequest : null}
+ onCaptureDone={(cid, answered) => setCaptureRequest((current) =>
+ current?.cid !== cid ? current : answered ? null : { ...current, offscreen: true })}
/>
))}
</>
)}
+ {captureRequest && (captureRequest.offscreen || !previewOpen) && (
+ <DesignCapture
+ key={captureRequest.cid}
+ request={captureRequest}
+ onDone={(cid) => setCaptureRequest((current) => (current?.cid === cid ? null : current))}
+ />
+ )}
</main>
<span role="status" aria-live="polite" className="sr-only">
diff --git a/components/chat-state.ts b/components/chat-state.ts
index 419e59c..9f3aa47 100644
--- a/components/chat-state.ts
+++ b/components/chat-state.ts
@@ -267,7 +267,10 @@ export function applyStreamItem(entries: ChatEntry[], item: StreamItem): ChatEnt
...entry,
toolStatus: item.isError ? 'error' : 'done',
content: item.content,
- durationMs: entry.startedAt != null && item.at != null ? item.at - entry.startedAt : undefined,
+ previewWaitMs: item.previewWaitMs,
+ durationMs: entry.startedAt != null && item.at != null
+ ? Math.max(0, item.at - entry.startedAt - (item.previewWaitMs ?? 0))
+ : undefined,
}
return next
}
diff --git a/components/design-capture.tsx b/components/design-capture.tsx
new file mode 100644
index 0000000..fcb16d5
--- /dev/null
+++ b/components/design-capture.tsx
@@ -0,0 +1,170 @@
+'use client'
+
+import { useEffect, useRef } from 'react'
+import {
+ CAPTURE_FAILURES,
+ CAPTURE_PAINT_MS,
+ CAPTURE_REPLY,
+ CAPTURE_REQUEST,
+ CAPTURE_TIMING_HEADER,
+ CAPTURE_TAB_MS,
+ CAPTURE_VIEWPORT,
+ MAX_CAPTURE_BYTES,
+ MAX_CAPTURE_TILES,
+ type CaptureFailure,
+ type CaptureReply,
+ type FrameTiming,
+} from '../lib/capture-protocol'
+import { fetchUi } from './auth-ui'
+
+// The app side of the screenshot protocol (lib/capture-protocol.ts). The tab acknowledges a
+// request as it receives it; the preview on screen answers when it shows the asked revision
+// (PreviewPane), otherwise DesignCapture loads a hidden copy and answers from it. Both send
+// what the frame returned with sendCapture: the photo, or the reason there is none.
+
+export interface CaptureRequest {
+ cid: string
+ sessionId: string
+ // When this tab received the request, on this tab's clock: the preview wait counts from it.
+ receivedAt: number
+ // The revision the tool asked for; the preview on screen answers only if it shows it.
+ revision: number
+ // The whole page in viewport-high tiles instead of the visible viewport.
+ full?: boolean
+ // The preview on screen could not answer, so the hidden copy does.
+ offscreen?: boolean
+}
+
+const captureUrl = (request: CaptureRequest, params: Record<string, string>) =>
+ `/api/capture?${new URLSearchParams({ session: request.sessionId, cid: request.cid, ...params })}`
+
+/** Tells the waiting call this tab took the request; false when it is no longer open. */
+export async function acknowledgeCapture(request: CaptureRequest): Promise<boolean> {
+ try {
+ const response = await fetchUi(captureUrl(request, { ack: '1' }), { method: 'POST' })
+ return response.ok && (await response.json()).ok === true
+ } catch {
+ return false
+ }
+}
+
+/** Asks one preview wrapper for its design's photo; null if it sends no reply in time. */
+export function captureFrame(target: Window, cid: string, full = false): Promise<CaptureReply | null> {
+ return new Promise((resolve) => {
+ const done = (reply: CaptureReply | null) => {
+ clearTimeout(timer)
+ window.removeEventListener('message', onMessage)
+ resolve(reply)
+ }
+ const onMessage = (event: MessageEvent) => {
+ if (event.origin !== window.location.origin || event.source !== target) return
+ if (event.data?.type !== CAPTURE_REPLY || event.data.cid !== cid) return
+ const blobs: unknown = event.data.blobs
+ if (Array.isArray(blobs) && blobs.every((blob) => blob instanceof Blob)) done(event.data as CaptureReply)
+ }
+ const timer = setTimeout(() => done(null), CAPTURE_TAB_MS)
+ window.addEventListener('message', onMessage)
+ target.postMessage({ type: CAPTURE_REQUEST, cid, full }, window.location.origin)
+ })
+}
+
+/** Where a photo came from, and when this tab asked the frame for it (this tab's clock). */
+export interface CaptureSource {
+ path: 'preview' | 'hidden'
+ askedAt: number
+}
+
+async function postCapture(url: string, init: RequestInit): Promise<void> {
+ try {
+ await fetchUi(url, { method: 'POST', ...init })
+ } catch {
+ // The waiting call times out instead.
+ }
+}
+
+/** Tells the waiting call this tab cannot send a photo, and why. */
+export const reportCaptureFailure = (request: CaptureRequest, reason: CaptureFailure) =>
+ postCapture(captureUrl(request, { failed: reason }), {})
+
+/**
+ * Sends a frame's reply to /api/capture: its photo with this tab's part of the trace, or
+ * the failure. No reply in time, the frame's own error, or a photo past the limits (a
+ * design can answer for itself) all become failures, so the call never waits for nothing.
+ */
+export async function sendCapture(request: CaptureRequest, reply: CaptureReply | null, source: CaptureSource): Promise<void> {
+ if (!reply) return reportCaptureFailure(request, 'no-reply')
+ if (reply.error || !reply.blobs.length) {
+ return reportCaptureFailure(request, reply.error && Object.hasOwn(CAPTURE_FAILURES, reply.error) ? reply.error : 'capture-failed')
+ }
+ const bytes = reply.blobs.reduce((sum, blob) => sum + blob.size, 0)
+ if (reply.blobs.length > MAX_CAPTURE_TILES || bytes > MAX_CAPTURE_BYTES) return reportCaptureFailure(request, 'too-large')
+ const size = (n: unknown) => (Number.isFinite(n) ? `${Math.round(n as number)}` : '')
+ const form = new FormData()
+ for (const blob of reply.blobs) form.append('image', blob)
+ const sentAt = Date.now()
+ // The frame's timing comes from the design's own document: kept only as plain numbers.
+ const { fontsMs, libraryMs, encodeMs } = reply.timing ?? {}
+ const frame = [fontsMs, libraryMs, encodeMs].every(Number.isFinite) ? { fontsMs, libraryMs, encodeMs } as FrameTiming : null
+ const timing = {
+ path: source.path, receivedAt: request.receivedAt, sentAt,
+ previewWaitMs: source.askedAt - request.receivedAt, roundTripMs: sentAt - source.askedAt, frame,
+ }
+ await postCapture(captureUrl(request, { w: size(reply.width), h: size(reply.height), ...(reply.truncated ? { truncated: '1' } : {}) }), {
+ body: form, headers: { [CAPTURE_TIMING_HEADER]: JSON.stringify(timing) },
+ })
+}
+
+// The hidden copy loads the preview's own URL, so a revision this tab already loaded comes
+// from the browser's cache. It stays inside the viewport, invisible: Chrome stops rendering
+// cross-origin frames outside it, and the design would never paint. A design that focuses
+// an element would take the keyboard from the composer, so focus goes back where it was.
+export default function DesignCapture({ request, onDone }: { request: CaptureRequest; onDone: (cid: string) => void }) {
+ const frame = useRef<HTMLIFrameElement>(null)
+ const latest = useRef({ request, onDone })
+ latest.current = { request, onDone }
+ const { cid, sessionId, revision } = request
+
+ useEffect(() => {
+ let started = false
+ const onPainted = (event: MessageEvent) => {
+ const target = frame.current?.contentWindow
+ if (started || !target || event.source !== target || event.origin !== window.location.origin) return
+ if (event.data?.type !== 'fx0-artifact-painted') return
+ started = true
+ const askedAt = Date.now()
+ void captureFrame(target, cid, latest.current.request.full)
+ .then((reply) => sendCapture(latest.current.request, reply, { path: 'hidden', askedAt }))
+ .finally(() => latest.current.onDone(cid))
+ }
+ const timer = setTimeout(() => {
+ if (started) return
+ void reportCaptureFailure(latest.current.request, 'not-painted').finally(() => latest.current.onDone(cid))
+ }, CAPTURE_PAINT_MS)
+ const focused = document.activeElement
+ const onBlur = () => setTimeout(() => {
+ if (document.activeElement !== frame.current) return
+ if (focused instanceof HTMLElement && focused !== document.body && focused.isConnected) focused.focus()
+ else frame.current?.blur()
+ })
+ window.addEventListener('message', onPainted)
+ window.addEventListener('blur', onBlur)
+ return () => {
+ clearTimeout(timer)
+ window.removeEventListener('message', onPainted)
+ window.removeEventListener('blur', onBlur)
+ }
+ }, [cid])
+
+ return (
+ <div aria-hidden className="pointer-events-none fixed inset-0 -z-10 overflow-hidden opacity-0">
+ <iframe
+ ref={frame}
+ src={`/api/artifact-frame?session=${encodeURIComponent(sessionId)}&v=${revision}`}
+ referrerPolicy="no-referrer"
+ tabIndex={-1}
+ title="Design screenshot"
+ style={{ ...CAPTURE_VIEWPORT, border: 0 }}
+ />
+ </div>
+ )
+}
diff --git a/components/preview-pane.tsx b/components/preview-pane.tsx
index ec1c6e0..a1fb539 100644
--- a/components/preview-pane.tsx
+++ b/components/preview-pane.tsx
@@ -3,6 +3,8 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { designLinkUrl } from '../lib/design-link'
import { EMPTY_DRAFT_DOCUMENT, patchDraftDocument } from '../lib/draft-preview'
+import { CAPTURE_PAINT_MS, CAPTURE_PREVIEW_START_MS } from '../lib/capture-protocol'
+import { captureFrame, reportCaptureFailure, sendCapture, type CaptureRequest } from './design-capture'
import DesignLinkDialog from './design-link-dialog'
import { HowItWorksPanel } from './how-it-works'
@@ -18,6 +20,8 @@ export default function PreviewPane({
draft = null,
intro,
onClose,
+ capture = null,
+ onCaptureDone,
}: {
source: PreviewSource
// A past revision opened from the transcript's "View" link, never the latest; null is the latest.
@@ -27,6 +31,10 @@ export default function PreviewPane({
// Only an empty chat explains fx0; a chat waiting on its first design stays blank.
intro: boolean
onClose: () => void
+ // A screenshot_design request; the preview answers from the frame on screen.
+ capture?: CaptureRequest | null
+ // Whether this preview answered (a photo or a failure); when not, the hidden copy should.
+ onCaptureDone?: (cid: string, answered: boolean) => void
}) {
const session = encodeURIComponent(source.sessionId)
const previewUrl = source.revision > 0
@@ -109,6 +117,57 @@ export default function PreviewPane({
if (fallback.current) clearTimeout(fallback.current)
}
}, [])
+ // screenshot_design photographs the design on screen in place: no reload, and the photo is
+ // what the user sees. A past revision on screen or a frame with no size (behind the chat on
+ // a phone) hands the request to the hidden copy at once (onCaptureDone(cid, false)), and so
+ // does a revision that has not started loading here after CAPTURE_PREVIEW_START_MS. A slot
+ // loading the asked revision is waited for, up to CAPTURE_PAINT_MS: the hidden copy would
+ // load the same document from scratch. Once the frame is asked, its answer is final.
+ const captureDone = useRef(onCaptureDone)
+ captureDone.current = onCaptureDone
+ const waitingCapture = useRef<{ request: CaptureRequest; timer: ReturnType<typeof setTimeout>; loading: boolean } | null>(null)
+ const settleCapture = (request: CaptureRequest): boolean => {
+ if (waitingCapture.current?.request !== request) return false
+ clearTimeout(waitingCapture.current.timer)
+ waitingCapture.current = null
+ return true
+ }
+ const handOffCapture = (request: CaptureRequest) => {
+ if (settleCapture(request)) captureDone.current?.(request.cid, false)
+ }
+ const failCapture = (request: CaptureRequest) => {
+ if (settleCapture(request)) void reportCaptureFailure(request, 'not-painted').finally(() => captureDone.current?.(request.cid, true))
+ }
+ useEffect(() => {
+ if (!capture) return
+ const request = capture
+ waitingCapture.current = { request, timer: setTimeout(() => handOffCapture(request), CAPTURE_PREVIEW_START_MS), loading: false }
+ return () => handOffCapture(request)
+ }, [capture?.cid]) // eslint-disable-line react-hooks/exhaustive-deps
+ useEffect(() => {
+ const waiting = waitingCapture.current
+ if (!waiting) return
+ const { request } = waiting
+ if (viewing != null) return handOffCapture(request)
+ if (!previewUrl?.endsWith(`&v=${request.revision}`)) return
+ const index = slots.indexOf(previewUrl)
+ const frame = index < 0 ? null : frames.current[index]
+ if (!frame) return
+ if (frame.getBoundingClientRect().width < 1) return handOffCapture(request)
+ if (front !== index || !frame.contentWindow) {
+ if (!waiting.loading) {
+ clearTimeout(waiting.timer)
+ waiting.loading = true
+ waiting.timer = setTimeout(() => failCapture(request), CAPTURE_PAINT_MS)
+ }
+ return
+ }
+ settleCapture(request)
+ const askedAt = Date.now()
+ void captureFrame(frame.contentWindow, request.cid, request.full)
+ .then((reply) => sendCapture(request, reply, { path: 'preview', askedAt }))
+ .finally(() => captureDone.current?.(request.cid, true))
+ }, [capture?.cid, front, slots, previewUrl, viewing]) // eslint-disable-line react-hooks/exhaustive-deps
const caughtUp = !previewUrl || (front != null && slots[front] === previewUrl)
const [heldDraft, setHeldDraft] = useState<string | null>(null)
const liveDraft = draft && renderable(draft) ? draft : null
diff --git a/components/thread.tsx b/components/thread.tsx
index 5b24a85..5bf2ed9 100644
--- a/components/thread.tsx
+++ b/components/thread.tsx
@@ -150,6 +150,8 @@ export interface ChatEntry {
content?: string
startedAt?: number
durationMs?: number
+ // Part of a tool call spent waiting for the preview, left out of durationMs.
+ previewWaitMs?: number
artifactRevision?: number
qid?: number
questions?: { question: string; options: { label: string; description?: string }[] }[]
diff --git a/components/tool-widget.tsx b/components/tool-widget.tsx
index 7885c42..2a2f696 100644
--- a/components/tool-widget.tsx
+++ b/components/tool-widget.tsx
@@ -25,6 +25,7 @@ const TOOL_WIDGETS: Record<string, ToolWidgetConfig> = {
read_design: { label: 'read design', icon: IconFileText },
publish_design: { label: 'publish design', icon: IconPencilEdit },
edit_design: { label: 'edit design', icon: IconPencilEdit },
+ screenshot_design: { label: 'screenshot', icon: IconFileText },
web_fetch: { label: 'fetch url', icon: IconGlobe },
web_search: { label: 'web search', icon: IconGlobe },
ask_user_question: { label: 'ask a question', icon: IconQuestion },
@@ -43,6 +44,7 @@ const TOOL_ACTIVITY: Record<string, { group: string; one: string; many: string }
publish_design: { group: 'edits', one: 'published a design', many: 'published designs' },
edit_design: { group: 'edits', one: 'edited a design', many: 'edited designs' },
read_design: { group: 'reads', one: 'read a design', many: 'read designs' },
+ screenshot_design: { group: 'reads', one: 'checked the design', many: 'checked the design' },
web_fetch: { group: 'web', one: 'used the web', many: 'used the web' },
web_search: { group: 'web', one: 'searched the web', many: 'searched the web' },
ask_user_question: { group: 'questions', one: 'asked for input', many: 'asked for input' },
@@ -91,6 +93,7 @@ function ToolWidget({
status,
content,
durationMs,
+ previewWaitMs,
callId,
}: {
name: string
@@ -98,6 +101,7 @@ function ToolWidget({
status: 'running' | 'done' | 'error'
content?: string
durationMs?: number
+ previewWaitMs?: number
callId?: number
}) {
const [open, setOpen] = useState(false)
@@ -117,7 +121,7 @@ function ToolWidget({
{command && <code className="min-w-0 flex-1 truncate font-mono text-xs text-foreground-tertiary">{command}</code>}
<span className="ml-auto flex shrink-0 items-center gap-2 text-xs text-muted-foreground tabular-nums">
{callId != null && <span title="Tool call ID">#{callId}</span>}
- {duration && <span>{duration}</span>}
+ {duration && <span title={previewWaitMs ? `plus ${formatDuration(previewWaitMs)} waiting for the preview to load` : undefined}>{duration}</span>}
<span role="status" className={`font-medium ${state.className}`}>
{state.label}
</span>
@@ -248,6 +252,7 @@ export function ActivityGroup({
status={entry.toolStatus ?? 'done'}
content={entry.content}
durationMs={entry.durationMs}
+ previewWaitMs={entry.previewWaitMs}
callId={entry.callId}
/>
)
diff --git a/lib/agent.ts b/lib/agent.ts
index f1e3514..cd31a44 100644
--- a/lib/agent.ts
+++ b/lib/agent.ts
@@ -4,6 +4,7 @@ import { assertGenerationCredential } from './gateway-session'
import { remoteImagesEnabled } from './design-artifact'
import { prefetchRemoteImages } from './remote-image-cache'
import { deleteDesignAssets, type PendingDesignAssets } from './design-assets'
+import { CAPTURE_FAILURES, captureTrace } from './capture-protocol'
import { chatSession, durableAgent, runSubagent, type ChatRuntime } from './libfx-agent'
import { project } from './projector'
import {
@@ -18,13 +19,16 @@ import type { TrialCredential } from './trial'
import type { SessionData } from './secure-session'
import {
answerActiveQuestion,
+ awaitCapture,
claimSessionCleanup,
clearActiveQuestion,
+ closeCapture,
currentSeq,
finishSessionCleanup,
getActiveQuestion,
getActiveTurn,
nextQuestionId,
+ openCapture,
projectedCursor,
publishDesignArtifact,
QUESTION_WAIT_MS,
@@ -33,6 +37,7 @@ import {
readEvents,
readSession,
readSessionView,
+ recordPreviewWait,
releaseSessionCleanup,
renewSessionCleanup,
SessionCleanupError,
@@ -40,6 +45,8 @@ import {
submitPrompt as journalPrompt,
takeAnswer,
updateSession,
+ type CaptureOutcome,
+ type CaptureShot,
type InitialGeneration,
type StoredSession,
} from './journal'
@@ -109,6 +116,7 @@ function buildToolHost(host: SessionState, turnId: number): ToolHost {
},
readResult: (callId) => host.toolOutputs.get(callId),
askUser: (questions, signal) => askUser(host, turnId, questions, signal),
+ captureDesign: (signal, options) => captureDesign(host, turnId, signal, options),
log: (event, detail) => log(host.id, event, { turn: turnId, ...detail }),
spawnSubagent: async (task, signal) => {
log(host.id, 'subagent:start', { task: task.slice(0, 120) })
@@ -373,6 +381,55 @@ async function askUser(host: SessionState, turnId: number, questions: AskQuestio
throw new Error(signal.aborted ? 'cancelled' : 'no answer within 10 minutes')
}
+// screenshot_design (lib/capture-protocol.ts): the tool call asks every open tab for a
+// photo of the published design and blocks until one arrives through Redis, wherever the
+// turn runs.
+// What the model is told when no photo arrives; a tab's own failure says why in its words.
+const CAPTURE_ERRORS: Record<Exclude<CaptureOutcome['kind'], 'shot' | 'failed'>, string> = {
+ unanswered: 'no fx0 tab is open and in front to take it',
+ timeout: 'the fx0 tab took the request but sent nothing in time',
+ cancelled: 'cancelled',
+}
+
+async function captureDesign(
+ host: SessionState,
+ turnId: number,
+ signal: AbortSignal,
+ { toolCallId, full = false }: { toolCallId?: string; full?: boolean } = {},
+): Promise<CaptureShot & { revision: number }> {
+ const started = performance.now()
+ const cid = crypto.randomUUID()
+ const [{ artifactRevision: revision }] = await Promise.all([readSessionView(host.id), openCapture(host.id, cid)])
+ if (!revision) {
+ await closeCapture(host.id, cid)
+ throw new Error('no design has been published yet')
+ }
+ const emitting = performance.now()
+ try {
+ await emit(host, {
+ type: 'capture', cid, artifactRevision: revision, turnId,
+ ...(full ? { captureFull: true as const } : {}),
+ })
+ } catch (error) {
+ await closeCapture(host.id, cid).catch(() => {})
+ throw error
+ }
+ const emitted = performance.now()
+ const emittedAt = Date.now()
+ const outcome = await awaitCapture(host.id, cid, signal)
+ const received = performance.now()
+ const server = { totalMs: received - started, setupMs: emitting - started, emitMs: emitted - emitting, emittedAt, receivedAt: Date.now() }
+ if (outcome.kind !== 'shot') {
+ log(host.id, 'capture:trace', { turn: turnId, outcome: outcome.kind, ...(outcome.kind === 'failed' ? { reason: outcome.reason } : {}), totalMs: Math.round(server.totalMs) })
+ throw new Error(outcome.kind === 'failed' ? CAPTURE_FAILURES[outcome.reason] : CAPTURE_ERRORS[outcome.kind])
+ }
+ const { shot } = outcome
+ log(host.id, 'capture:trace', { turn: turnId, outcome: 'shot', ...(shot.timing ? captureTrace(server, shot.timing) : { totalMs: Math.round(server.totalMs) }) })
+ const previewWaitMs = shot.timing?.previewWaitMs ?? 0
+ if (previewWaitMs > 0 && toolCallId) await recordPreviewWait(host.id, toolCallId, previewWaitMs)
+ return { ...shot, revision }
+}
+
export async function answerQuestion(host: SessionState, qid: number, answers: Record<string, string>): Promise<boolean> {
const accepted = await answerActiveQuestion(host.id, qid, answers)
log(host.id, accepted ? 'question:answered' : 'question:answer-rejected', { qid })
diff --git a/lib/capture-protocol.ts b/lib/capture-protocol.ts
new file mode 100644
index 0000000..9743f29
--- /dev/null
+++ b/lib/capture-protocol.ts
@@ -0,0 +1,122 @@
+// The design screenshot protocol, shared by the server, the frame and the app.
+//
+// 1. A screenshot_design call opens a capture in Redis and emits a `capture` event, then
+// blocks on Redis (lib/journal.ts). A tab that receives the event acknowledges it at
+// once; a call nobody acknowledges within CAPTURE_ACK_MS fails fast, and an acknowledged
+// one waits for a photo or a failure.
+// 2. The tab answers: the preview on screen when it shows the asked revision, otherwise a
+// hidden copy of the frame (components/design-capture.tsx).
+// 3. The app asks the frame's wrapper, the wrapper asks the design, and the capture library
+// photographs the design inside its own frame, where the CSP allows no network
+// (lib/design-capture.ts). The photo travels back as Blobs, or the reason there is none.
+// 4. The tab posts the photo, or the failure, to /api/capture, which hands it to the call.
+
+/** The app asks a preview wrapper for a photo. */
+export const CAPTURE_REQUEST = 'fx0-capture'
+/** A preview wrapper returns the design's photo, or none. */
+export const CAPTURE_REPLY = 'fx0-artifact-captured'
+/** The tab's part of a capture's trace, sent with the photo as JSON. */
+export const CAPTURE_TIMING_HEADER = 'x-fx0-capture-timing'
+
+/** Where the frame's time went, on the frame's own clock. */
+export interface FrameTiming {
+ fontsMs: number
+ libraryMs: number
+ encodeMs: number
+}
+
+/** Why a tab that acknowledged a capture sends no photo. */
+export const CAPTURE_FAILURES = {
+ 'too-large': 'the page did not fit the screenshot size limit even at low quality',
+ 'capture-failed': 'the capture library failed on this design',
+ 'not-painted': 'the design did not finish painting in time',
+ 'no-reply': 'the design did not answer the capture request in time',
+ 'refused': 'the photo the tab sent was refused as not a valid image',
+} as const
+export type CaptureFailure = keyof typeof CAPTURE_FAILURES
+
+export interface CaptureReply {
+ type: typeof CAPTURE_REPLY
+ cid: string
+ // Set when the frame could not photograph the design; blobs is then empty.
+ error?: CaptureFailure
+ // The viewport, or the whole page cut into viewport-high tiles, top to bottom.
+ blobs: Blob[]
+ width?: number
+ // The photographed height: the viewport, or the page.
+ height?: number
+ // A full page taller than MAX_CAPTURE_TILES viewports: the tiles stop before its end.
+ truncated?: boolean
+ timing?: FrameTiming
+}
+
+/** What the tab and /api/capture add to a photo for its trace. */
+export interface ShotTiming {
+ path: 'preview' | 'hidden'
+ receivedAt: number // tab clock
+ sentAt: number // tab clock
+ previewWaitMs: number
+ roundTripMs: number
+ frame: FrameTiming | null
+ arrivedAt: number // server clock
+ authMs: number
+ readMs: number
+ storingAt: number // server clock, as the Redis write starts
+}
+
+/** Every image of one capture together, held in Redis as base64 within a single request. */
+export const MAX_CAPTURE_BYTES = 600 * 1024
+/** A full page is returned as at most this many viewport-high tiles: one tool result's image limit. */
+export const MAX_CAPTURE_TILES = 8
+/** How long the tool call waits for a tab to acknowledge the request before failing. */
+export const CAPTURE_ACK_MS = 5_000
+/** The capture library's own limit inside the frame. */
+export const CAPTURE_LIBRARY_MS = 20_000
+/** How long a tab may take to return a photo once it asks the frame. */
+export const CAPTURE_TAB_MS = CAPTURE_LIBRARY_MS + 5_000
+/** How long the preview may take to start loading the asked revision before the hidden copy answers. */
+export const CAPTURE_PREVIEW_START_MS = 3_000
+/** How long a frame loading the asked revision may take to paint it. */
+export const CAPTURE_PAINT_MS = 15_000
+/** How long an acknowledged call waits: a hand-off to the hidden copy after a slow start still fits. */
+export const CAPTURE_WAIT_MS = CAPTURE_PREVIEW_START_MS + CAPTURE_PAINT_MS + CAPTURE_TAB_MS + 5_000
+/** The hidden copy's viewport, when no preview is on screen. */
+export const CAPTURE_VIEWPORT = { width: 1280, height: 800 }
+
+/** What the tool call measured itself, on the turn's server clock. */
+interface ServerTiming {
+ totalMs: number
+ setupMs: number
+ emitMs: number
+ emittedAt: number // server clock, as the request left
+ receivedAt: number // server clock, as the photo arrived
+}
+
+/**
+ * One capture, end to end, in whole milliseconds, as the tool call logs it (capture:trace).
+ * Each part is measured on one clock, except delivery, upload and store-and-wake, which
+ * compare two machines' clocks; delivery plus upload (networkMs) is exact all the same.
+ */
+export function captureTrace(server: ServerTiming, shot: ShotTiming) {
+ const ms = (n: number) => Math.round(n)
+ const deliveryMs = shot.receivedAt - server.emittedAt
+ const uploadMs = shot.arrivedAt - shot.sentAt
+ const frame = shot.frame
+ return {
+ path: shot.path,
+ totalMs: ms(server.totalMs),
+ setupMs: ms(server.setupMs),
+ emitMs: ms(server.emitMs),
+ deliveryMs: ms(deliveryMs),
+ previewWaitMs: ms(shot.previewWaitMs),
+ messagingMs: ms(frame ? shot.roundTripMs - frame.fontsMs - frame.libraryMs - frame.encodeMs : shot.roundTripMs),
+ fontsMs: frame ? ms(frame.fontsMs) : null,
+ libraryMs: frame ? ms(frame.libraryMs) : null,
+ encodeMs: frame ? ms(frame.encodeMs) : null,
+ uploadMs: ms(uploadMs),
+ networkMs: ms(deliveryMs + uploadMs),
+ authMs: ms(shot.authMs),
+ readMs: ms(shot.readMs),
+ storeAndWakeMs: ms(server.receivedAt - shot.storingAt),
+ }
+}
diff --git a/lib/design-artifact.ts b/lib/design-artifact.ts
index ad1714b..2e4e1e0 100644
--- a/lib/design-artifact.ts
+++ b/lib/design-artifact.ts
@@ -74,8 +74,9 @@ export const MAX_EMBEDDED_IMAGE_BYTES = 4 * 1024 * 1024
// and relays its heartbeat, paint and link messages to the app. Each image marker in the
// design becomes its image, loaded from `src` (cached by the browser across revisions),
// within MAX_EMBEDDED_IMAGE_BYTES counted per occurrence; a missing image is blank. Only
-// then does the design start, so it never needs the network.
-export function designFrameHtml(preparedArtifact: string, readyToken: string, images: { marker: string; src: string }[] = []): string {
+// then does the design start, so it never needs the network. `relay` is more wrapper
+// script (lib/design-capture.ts), run before the design loads.
+export function designFrameHtml(preparedArtifact: string, readyToken: string, images: { marker: string; src: string }[] = [], relay = ''): string {
// The design reaches its frame as a blob: URL the wrapper makes from this string,
// not as a data: URL: browsers refuse data: URLs past 2 MiB, which embedded images
// reach. A blob: document is local like data:, so it inherits this wrapper's CSP and
@@ -109,6 +110,8 @@ frame.setAttribute('credentialless','');
let readySignals=0;
let consumedSignals=0;
let lastHeartbeat=Date.now();
+// A relay can hold off restores while the design is busy on its behalf (lib/design-capture.ts).
+let holdUntil=0;
const restore=()=>{if(!artifact)return;lastHeartbeat=Date.now();frame.src=artifact;};
addEventListener('message',event=>{
if(event.source!==frame.contentWindow||event.data?.token!==token)return;
@@ -135,9 +138,9 @@ setInterval(()=>{
const now=Date.now(),paused=now-lastCheck>3000;
lastCheck=now;
if(document.visibilityState!=='visible'||paused){lastHeartbeat=now;return;}
- if(now-lastHeartbeat>3000)restore();
+ if(now-lastHeartbeat>3000&&now>holdUntil)restore();
},500);
-const read=blob=>new Promise(done=>{const reader=new FileReader();reader.onload=()=>done(String(reader.result));reader.onerror=()=>done(null);reader.readAsDataURL(blob)});
+${relay}const read=blob=>new Promise(done=>{const reader=new FileReader();reader.onload=()=>done(String(reader.result));reader.onerror=()=>done(null);reader.readAsDataURL(blob)});
const load=async src=>{try{const response=await fetch(src,{credentials:'same-origin'});if(!response.ok)return null;const blob=await response.blob();return /^image\\/(png|jpeg|gif|webp|avif)$/.test(blob.type)?await read(blob):null}catch{return null}};
Promise.all(images.map(image=>load(image.src))).then(loaded=>{
let html=design,total=0;
@@ -182,13 +185,14 @@ export function normalizeDesignArtifactHtml(html: string): string {
// The CSP and bootstrap go right after the doctype, so they are the first things
// the parser puts in <head>. A design stored before normalization renders the same
// way; one without a doctype is not rendered at all (null), never without its CSP.
-export function prepareDesignArtifactHtml(html: string, embeddedCsp?: string, readyToken?: string): string | null {
+// `scripts` follow the bootstrap, under the same CSP (lib/design-capture.ts).
+export function prepareDesignArtifactHtml(html: string, embeddedCsp?: string, readyToken?: string, scripts = ''): string | null {
const document = normalizeDesignArtifactHtml(html)
if (!document.startsWith(DOCTYPE)) return null
const csp = embeddedCsp
? `<meta http-equiv="Content-Security-Policy" content="${escapeAttribute(embeddedCsp)}">`
: ''
- return DOCTYPE + csp + designArtifactBootstrap(readyToken) + document.slice(DOCTYPE.length)
+ return DOCTYPE + csp + designArtifactBootstrap(readyToken) + scripts + document.slice(DOCTYPE.length)
}
// The start of a document, with invisible characters spelled out, so a rejected
diff --git a/lib/design-capture.ts b/lib/design-capture.ts
new file mode 100644
index 0000000..b32861d
--- /dev/null
+++ b/lib/design-capture.ts
@@ -0,0 +1,102 @@
+// @ts-expect-error Node strip-types tests require the source extension.
+import { CAPTURE_LIBRARY_MS, CAPTURE_REPLY, CAPTURE_REQUEST, MAX_CAPTURE_BYTES, MAX_CAPTURE_TILES } from './capture-protocol.ts'
+// @ts-expect-error Node strip-types tests require the source extension.
+import { FAST_HTML_TO_IMAGE } from './vendor/fast-html-to-image.ts'
+
+// The frame side of the screenshot protocol (lib/capture-protocol.ts). The capture library
+// runs inside the design's own frame, under the CSP that gives the design no network, so
+// it cannot fetch anything the design names either: the photo shows what the screen shows.
+// The design's scripts share that realm and can tamper with their own photo; that only
+// misleads the model about its own work, and /api/capture still checks the bytes.
+
+/** The library and its message handler, injected into every prepared design. */
+export function designCaptureScript(readyToken: string): string {
+ // Encodes at once: canvas.toBlob waits for an idle moment a busy page never has. WebP is
+ // about half the bytes of a JPEG at the same quality, which the upload feels most; a
+ // browser that cannot write WebP (Safari) returns a PNG for it, and gets JPEG instead.
+ // Nothing runs until the first capture, so a preview that is never photographed pays
+ // only for parsing the library.
+ const handler = `(() => {
+ const token = ${JSON.stringify(readyToken)}, maxBytes = ${MAX_CAPTURE_BYTES}, maxTiles = ${MAX_CAPTURE_TILES};
+ let type = null;
+ const encode = (canvas, quality) => {
+ if (!type) {
+ const probe = document.createElement("canvas");
+ probe.width = probe.height = 1;
+ type = probe.toDataURL("image/webp").startsWith("data:image/webp") ? "image/webp" : "image/jpeg";
+ }
+ const url = canvas.toDataURL(type, quality), binary = atob(url.slice(url.indexOf(",") + 1));
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+ return new Blob([bytes], { type });
+ };
+ const total = (blobs) => blobs.reduce((sum, blob) => sum + blob.size, 0);
+ // A remainder under 15% of a viewport joins the last tile instead of becoming its own.
+ const slack = () => Math.round(innerHeight * 0.15);
+ // The page, cut into viewport-high tiles from the top.
+ const tiles = (page) => {
+ const height = innerHeight, count = Math.min(maxTiles, Math.max(1, Math.ceil((page.height - slack()) / height)));
+ return Array.from({ length: count }, (_, i) => {
+ const tile = document.createElement("canvas");
+ tile.width = page.width;
+ tile.height = i === count - 1 ? Math.min(page.height - i * height, height + slack()) : height;
+ tile.getContext("2d").drawImage(page, 0, i * height, page.width, tile.height, 0, 0, page.width, tile.height);
+ return tile;
+ });
+ };
+ addEventListener("message", async (event) => {
+ if (event.source !== parent || event.data?.type !== "capture" || typeof event.data.cid !== "string") return;
+ const { cid, full } = event.data;
+ const reply = (data) => parent.postMessage({ type: "captured", token, cid, ...data }, "*");
+ try {
+ const t0 = performance.now();
+ await document.fonts.ready;
+ const t1 = performance.now();
+ const options = { pixelRatio: 1, backgroundColor: "#ffffff", timeoutMs: ${CAPTURE_LIBRARY_MS} };
+ // A full page is the document from its top, as tall as its content (which can overflow
+ // an html{height:100%} box) but no taller than the tiles can carry.
+ const root = document.documentElement, pageHeight = Math.max(root.scrollHeight, document.body?.scrollHeight ?? 0);
+ const region = full
+ ? { x: 0, y: root.getBoundingClientRect().top, width: innerWidth, height: Math.min(pageHeight, maxTiles * innerHeight + slack()) }
+ : { x: 0, y: 0, width: innerWidth, height: innerHeight };
+ const page = await (await FastHtmlToImage.captureRegion(region, options)).toCanvas();
+ const t2 = performance.now();
+ const parts = full ? tiles(page) : [page];
+ // One pass at a good quality; over the limit, one more at the quality the size suggests.
+ let blobs = parts.map((part) => encode(part, 0.85));
+ if (total(blobs) > maxBytes) {
+ const quality = Math.min(0.75, Math.max(0.3, 0.85 * (maxBytes / total(blobs)) * 0.9));
+ blobs = parts.map((part) => encode(part, quality));
+ }
+ const t3 = performance.now();
+ if (total(blobs) > maxBytes) return reply({ error: "too-large" });
+ const covered = parts.reduce((sum, part) => sum + part.height, 0);
+ reply({
+ blobs, width: page.width, height: page.height, truncated: full && covered < pageHeight,
+ timing: { fontsMs: t1 - t0, libraryMs: t2 - t1, encodeMs: t3 - t2 },
+ });
+ } catch {
+ reply({ error: "capture-failed" });
+ }
+ });
+})()`
+ return `<script>${FAST_HTML_TO_IMAGE}</script><script>${handler}</script>`
+}
+
+// Added to the wrapper's script (designFrameHtml), where `frame`, `token` and `holdUntil`
+// are in scope. Only the app may ask for a photo, and only the design's reply is passed
+// on. A large page can block its own heartbeat while it is photographed, so the wrapper
+// holds off restoring it until the reply or the library's time limit.
+export const DESIGN_CAPTURE_RELAY = `addEventListener('message',event=>{
+ const data=event.data;
+ if(event.source===parent&&event.origin===location.origin&&data?.type===${JSON.stringify(CAPTURE_REQUEST)}&&typeof data.cid==='string'){
+ holdUntil=Date.now()+${CAPTURE_LIBRARY_MS};
+ frame.contentWindow?.postMessage({type:'capture',cid:data.cid,full:data.full===true},'*');
+ }else if(event.source===frame.contentWindow&&data?.token===token&&data.type==='captured'&&typeof data.cid==='string'){
+ holdUntil=0;
+ const blobs=Array.isArray(data.blobs)?data.blobs.filter(blob=>blob instanceof Blob):[];
+ const error=typeof data.error==='string'?data.error:undefined;
+ parent.postMessage({type:${JSON.stringify(CAPTURE_REPLY)},cid:data.cid,blobs,error,width:data.width,height:data.height,truncated:data.truncated===true,timing:data.timing},location.origin);
+ }
+});
+`
diff --git a/lib/events.ts b/lib/events.ts
index c081bf6..070652e 100644
--- a/lib/events.ts
+++ b/lib/events.ts
@@ -19,6 +19,7 @@ export interface StreamItem {
| 'heartbeat'
| 'question'
| 'question_resolved'
+ | 'capture'
| 'reasoning'
| 'done'
| 'error'
@@ -55,6 +56,14 @@ export interface StreamItem {
stopReason?: string
artifactRevision?: number
qid?: number
+ /** screenshot_design's request id; an open tab answers it through /api/capture. */
+ cid?: string
+ /** A capture of the whole page in viewport-high tiles, not just the visible viewport. */
+ captureFull?: true
+ /** The model's id for a tool call, on its tool_end. */
+ toolCallId?: string
+ /** On a screenshot call's tool_end: time it spent waiting for the preview, not its own work. */
+ previewWaitMs?: number
instance?: string
/** Count only — image bytes are never broadcast over the chat stream. */
imageCount?: number
diff --git a/lib/journal.ts b/lib/journal.ts
index 61b5a01..4005b9c 100644
--- a/lib/journal.ts
+++ b/lib/journal.ts
@@ -4,6 +4,8 @@ import { Redis } from '@upstash/redis'
// @ts-expect-error Node's strip-types integration test requires the source extension.
import { assetReference, rewriteRemoteImages, validateDesignArtifactHtml } from './design-artifact.ts'
// @ts-expect-error Node's strip-types integration test requires the source extension.
+import { CAPTURE_ACK_MS, CAPTURE_WAIT_MS, type CaptureFailure, type ShotTiming } from './capture-protocol.ts'
+// @ts-expect-error Node's strip-types integration test requires the source extension.
import { type DesignMdInput } from './design-md.ts'
// @ts-expect-error Node's strip-types integration test requires the source extension.
import { type SessionInfo, type SessionView, type StreamItem } from './events.ts'
@@ -16,16 +18,40 @@ const PREFIX = process.env.FX0_REDIS_PREFIX?.trim() || 'fx0:v1'
let client: Redis | null = null
+function redisConfig(): { url: string; token: string } {
+ const url = process.env.KV_REST_API_URL
+ const token = process.env.KV_REST_API_TOKEN
+ if (!url || !token) throw new Error('Redis journal is not configured (KV_REST_API_URL/KV_REST_API_TOKEN)')
+ return { url, token }
+}
+
export function redis(): Redis {
- if (!client) {
- const url = process.env.KV_REST_API_URL
- const token = process.env.KV_REST_API_TOKEN
- if (!url || !token) throw new Error('Redis journal is not configured (KV_REST_API_URL/KV_REST_API_TOKEN)')
- client = new Redis({ url, token })
- }
+ if (!client) client = new Redis(redisConfig())
return client
}
+// BLPOP over Upstash's REST API, which the client library does not expose: one request
+// blocks in Redis for up to `seconds` and returns as soon as a value is pushed. Aborting
+// ends the request; the caller withdraws whatever it was waiting for (see awaitCapture).
+// A request that outlives its block by much has stalled, and fails like any other error.
+async function blockingPop(listKey: string, seconds: number, signal: AbortSignal): Promise<string | null> {
+ const { url, token } = redisConfig()
+ try {
+ const response = await fetch(url, {
+ method: 'POST',
+ headers: { authorization: `Bearer ${token}` },
+ body: JSON.stringify(['BLPOP', listKey, String(seconds)]),
+ signal: AbortSignal.any([signal, AbortSignal.timeout((seconds + 5) * 1000)]),
+ })
+ if (!response.ok) throw new Error(`Redis BLPOP failed with ${response.status}`)
+ const { result } = await response.json() as { result: [string, string] | null }
+ return result?.[1] ?? null
+ } catch (error) {
+ if (signal.aborted) return null
+ throw error
+ }
+}
+
// Events committed by this process, announced the moment Redis confirms them. A
// stream in the same process (a prompt's own response, where its worker also runs)
// delivers them without waiting for its next journal poll; polling still covers
@@ -425,6 +451,9 @@ const SESSION_PARTS = [
'active',
'active-question',
'answer',
+ 'capture',
+ 'capture-replies',
+ 'preview-waits',
'design-artifact',
'design-md',
'event-seq',
@@ -827,6 +856,152 @@ export async function clearActiveQuestion(sessionId: string, qid: number): Promi
return (await redis().eval<unknown[], number>(CLEAR_QUESTION, [key(sessionId, 'active-question')], [String(qid)])) === 1
}
+export interface CaptureShot {
+ // The viewport, or a full page's viewport-high tiles from the top.
+ images: { mimeType: string; data: string }[]
+ width?: number
+ height?: number
+ // A full page taller than the tiles cover.
+ truncated?: boolean
+ // The tab's and /api/capture's part of the capture's trace (lib/capture-protocol.ts).
+ timing?: ShotTiming
+}
+
+// The Redis side of the screenshot protocol (lib/capture-protocol.ts). The tool call opens a
+// capture and blocks on the chat's reply list; /api/capture pushes the tab's acknowledgement,
+// then its photo or failure, so the call wakes the moment each arrives, on whichever instance
+// runs the turn. No polling. A key whose format changes gets a new name, so instances of two
+// deploys never misread it.
+
+/** What a tab answered for a capture it acknowledged. */
+export type CaptureAnswer = { kind: 'shot'; shot: CaptureShot } | { kind: 'failed'; reason: CaptureFailure }
+
+/** How a capture ended for the waiting call. */
+export type CaptureOutcome =
+ | CaptureAnswer
+ | { kind: 'unanswered' } // no tab acknowledged it within the ack window
+ | { kind: 'timeout' } // a tab acknowledged it, then sent nothing in time
+ | { kind: 'cancelled' }
+
+const OPEN_CAPTURE = `
+if redis.call('EXISTS', KEYS[3]) == 1 then return 0 end
+redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
+redis.call('DEL', KEYS[2])
+return 1
+`
+export async function openCapture(sessionId: string, cid: string): Promise<void> {
+ const set = await redis().eval<unknown[], number>(OPEN_CAPTURE,
+ [key(sessionId, 'capture'), key(sessionId, 'capture-replies'), key(sessionId, 'deleting')],
+ [cid, String(CAPTURE_ACK_MS + CAPTURE_WAIT_MS)])
+ if (!set) throw new SessionCleanupError()
+}
+
+// A reply counts only for the capture still open. An acknowledgement leaves it open for the
+// whole wait it starts; a photo or a failure closes it, so the first answer is the only one.
+const REPLY_CAPTURE = `
+if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
+if ARGV[4] == '1' then redis.call('DEL', KEYS[1]) else redis.call('PEXPIRE', KEYS[1], ARGV[3]) end
+redis.call('RPUSH', KEYS[2], ARGV[2])
+redis.call('PEXPIRE', KEYS[2], ARGV[3])
+return 1
+`
+async function replyCapture(sessionId: string, cid: string, reply: { kind: 'ack' } | CaptureAnswer): Promise<boolean> {
+ return (await redis().eval<unknown[], number>(REPLY_CAPTURE,
+ [key(sessionId, 'capture'), key(sessionId, 'capture-replies')],
+ [cid, JSON.stringify({ cid, ...reply }), String(CAPTURE_ACK_MS + CAPTURE_WAIT_MS), reply.kind === 'ack' ? '0' : '1'])) === 1
+}
+
+/** A tab took the request; false when the capture is no longer open (a replayed event). */
+export const acknowledgeCapture = (sessionId: string, cid: string) => replyCapture(sessionId, cid, { kind: 'ack' })
+/** The tab's photo or failure; false when the capture is no longer open or already answered. */
+export const answerCapture = (sessionId: string, cid: string, answer: CaptureAnswer) => replyCapture(sessionId, cid, answer)
+
+// Upstash ends long requests on its own terms; waiting in slices keeps each one short. A
+// slice that fails is retried while the wait has time left, so one bad request does not
+// refuse a photo already on its way.
+const CAPTURE_BLOCK_SECONDS = 10
+const CAPTURE_POP_ATTEMPTS = 3
+
+function replyFor(value: unknown, cid: string): { kind: 'ack' } | CaptureAnswer | null {
+ const stored = parseJson<{ cid?: string; kind?: string; shot?: CaptureShot; reason?: CaptureFailure }>(value)
+ if (stored?.cid !== cid) return null
+ if (stored.kind === 'ack') return { kind: 'ack' }
+ if (stored.kind === 'failed' && stored.reason) return { kind: 'failed', reason: stored.reason }
+ return stored.kind === 'shot' && stored.shot ? { kind: 'shot', shot: stored.shot } : null
+}
+
+/**
+ * Waits for the open capture: first for a tab to acknowledge it within `ackMs`, then for
+ * its photo or failure within CAPTURE_WAIT_MS. However it ends, the capture is withdrawn,
+ * so a reply sent afterwards is refused instead of left behind.
+ */
+export async function awaitCapture(sessionId: string, cid: string, signal: AbortSignal, ackMs = CAPTURE_ACK_MS): Promise<CaptureOutcome> {
+ const replies = key(sessionId, 'capture-replies')
+ let acknowledged = false
+ let deadline = Date.now() + ackMs
+ let failures = 0
+ try {
+ for (let left = ackMs; left > 0 && !signal.aborted; left = deadline - Date.now()) {
+ let value: string | null
+ try {
+ value = await blockingPop(replies, Math.min(CAPTURE_BLOCK_SECONDS, Math.ceil(left / 1000)), signal)
+ failures = 0
+ } catch (error) {
+ if (++failures >= CAPTURE_POP_ATTEMPTS) throw error
+ continue
+ }
+ const reply = value == null ? null : replyFor(value, cid)
+ if (reply?.kind === 'ack') {
+ if (!acknowledged) deadline = Date.now() + CAPTURE_WAIT_MS
+ acknowledged = true
+ } else if (reply) return reply
+ }
+ } catch (error) {
+ await closeCapture(sessionId, cid).catch(() => {})
+ throw error
+ }
+ // An answer accepted just before the capture closed is already in the list.
+ if (!await closeCapture(sessionId, cid)) {
+ for (let late = await redis().lpop<unknown>(replies); late != null; late = await redis().lpop<unknown>(replies)) {
+ const reply = replyFor(late, cid)
+ if (reply && reply.kind !== 'ack') return reply
+ }
+ }
+ if (signal.aborted) return { kind: 'cancelled' }
+ return { kind: acknowledged ? 'timeout' : 'unanswered' }
+}
+
+// How long a screenshot call waited for the preview, by the model's id for the call. The
+// call records it before it returns, so it is there when the projector writes the call's
+// tool_end; the transcript leaves that wait out of the call's time.
+const PREVIEW_WAITS_TTL_MS = 86_400_000
+
+export async function recordPreviewWait(sessionId: string, toolCallId: string, ms: number): Promise<void> {
+ const waits = key(sessionId, 'preview-waits')
+ await redis().pipeline().hset(waits, { [toolCallId]: Math.round(ms) }).pexpire(waits, PREVIEW_WAITS_TTL_MS).exec()
+}
+
+export async function readPreviewWaits(sessionId: string, toolCallIds: string[]): Promise<Map<string, number>> {
+ if (!toolCallIds.length) return new Map()
+ const values = await redis().hmget<Record<string, unknown>>(key(sessionId, 'preview-waits'), ...toolCallIds)
+ const waits = new Map<string, number>()
+ for (const id of toolCallIds) {
+ const ms = Number(values?.[id])
+ if (Number.isFinite(ms) && ms > 0) waits.set(id, ms)
+ }
+ return waits
+}
+
+// Withdraws the capture when its tool call stops waiting; true if it was still open.
+const CLOSE_CAPTURE = `
+if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end
+redis.call('DEL', KEYS[1])
+return 1
+`
+export async function closeCapture(sessionId: string, cid: string): Promise<boolean> {
+ return (await redis().eval<unknown[], number>(CLOSE_CAPTURE, [key(sessionId, 'capture')], [cid])) === 1
+}
+
// The first libfx session id stored for a chat wins, so two first prompts racing
// on the same chat still share one conversation. The reverse key lets the agent's
// fetch find the chat from the x-session-id libfx sends with each model request.
diff --git a/lib/libfx-agent.ts b/lib/libfx-agent.ts
index 0575ced..70f9bc3 100644
--- a/lib/libfx-agent.ts
+++ b/lib/libfx-agent.ts
@@ -160,8 +160,10 @@ async function chatOfRequest(sessionId: string | null): Promise<string | null> {
// Reads are safe to run twice after a crash; publishing the same document twice
// is too. Edits, questions and subagents are not, so libfx tells the model a
// call cut off by a crash may have run instead of running it again.
-const IDEMPOTENT = new Set(['read_design', 'publish_design', 'web_fetch', 'read_tool_result', 'capability_search'])
-const WRITES = new Set(['publish_design', 'edit_design'])
+const IDEMPOTENT = new Set(['read_design', 'publish_design', 'screenshot_design', 'web_fetch', 'read_tool_result', 'capability_search'])
+// A tool that writes never runs beside another call. A chat has one capture open at a time
+// (lib/journal.ts), so two screenshots in one response must take turns too.
+const WRITES = new Set(['publish_design', 'edit_design', 'screenshot_design'])
function turnContext(ctx: FxToolContext): TurnContext {
const context = ctx.context as Partial<TurnContext> | undefined
diff --git a/lib/projector.ts b/lib/projector.ts
index 1472c80..4122111 100644
--- a/lib/projector.ts
+++ b/lib/projector.ts
@@ -6,6 +6,7 @@ import {
projectEvents,
projectedCursor,
readEvents,
+ readPreviewWaits,
readSession,
readTurnWork,
type TurnSettings,
@@ -78,7 +79,7 @@ export function itemsOf(line: FxStreamLine, at: number): Entry[] {
case 'tool_start':
return [{ type: 'tool_start', name: line.name || 'tool', command: toolLabel(line as Record<string, unknown>), turnId: id, at }]
case 'tool_end':
- return [{ type: 'tool_end', name: line.name || 'tool', isError: line.isError === true, content: typeof line.content === 'string' ? tail(line.content) : undefined, turnId: id, at }]
+ return [{ type: 'tool_end', name: line.name || 'tool', isError: line.isError === true, content: typeof line.content === 'string' ? tail(line.content) : undefined, toolCallId: line.id, turnId: id, at }]
case 'turn_end': {
const failed = line.stopReason === 'error' || line.stopReason === 'unknown' || (line.stopReason === 'refused' && line.error?.message)
return [failed
@@ -205,6 +206,17 @@ export interface FollowOptions {
idleMs: number
}
+// The preview wait a screenshot call recorded (lib/journal.ts), onto its tool_end.
+export async function addPreviewWaits(sessionId: string, entries: Entry[]): Promise<void> {
+ const ends = entries.filter((entry) => entry.type === 'tool_end' && entry.toolCallId && entry.name === 'screenshot_design')
+ if (!ends.length) return
+ const waits = await readPreviewWaits(sessionId, ends.map((entry) => entry.toolCallId!))
+ for (const entry of ends) {
+ const ms = waits.get(entry.toolCallId!)
+ if (ms) entry.previewWaitMs = ms
+ }
+}
+
// Follows the chat's libfx stream from the projected cursor. Returns when the
// chat stays idle for idleMs, when the signal aborts, or when another projector
// moved the cursor first (that one carries on).
@@ -241,6 +253,7 @@ export async function project(agent: FxAgent, host: SessionState, libfxSessionId
const id = line.type === 'turn_start' ? messageIdOf(line) : null
if (id != null && line.type === 'turn_start') starts[id] = await readTurnStart(host, id, line.input)
}
+ await addPreviewWaits(host.id, entries)
const items = await projectEvents(host.id, cursor, toCursor, entries, starts)
if (!items) return false
cursor = toCursor
diff --git a/lib/tools.ts b/lib/tools.ts
index e522622..c2cfca6 100644
--- a/lib/tools.ts
+++ b/lib/tools.ts
@@ -1,4 +1,4 @@
-import type { FxHostTool } from 'libfx'
+import type { FxHostTool, FxToolResult } from 'libfx'
// @ts-expect-error Node strip-types tests require the source extension.
import { fetchPublicText } from './public-fetch.ts'
@@ -10,6 +10,11 @@ export interface ToolHost {
storeResult(callId: number, text: string): void
readResult(callId: number): string | undefined
askUser(questions: AskQuestion[], signal: AbortSignal): Promise<Record<string, string>>
+ /** A photo of the published design from an open fx0 tab; throws when none sends one in time. */
+ captureDesign(
+ signal: AbortSignal,
+ options?: { toolCallId?: string; full?: boolean },
+ ): Promise<{ images: { mimeType: string; data: string }[]; revision: number; width?: number; height?: number; truncated?: boolean }>
log(event: string, detail?: Record<string, unknown>): void
spawnSubagent(task: string, signal: AbortSignal): Promise<string>
capabilitySearch(query: string): { name: string; kind: string; description: string }[]
@@ -116,6 +121,37 @@ export function createTools(host: ToolHost, opts: { allowSubagent?: boolean } =
},
}
+ const screenshotDesign: FxHostTool = {
+ name: 'screenshot_design',
+ description:
+ 'Screenshot the published design as the preview renders it, at the size the user sees it (1280x800 when the preview is closed). By default it shows the visible viewport; set full_page to see the whole page, top to bottom, in viewport-high images (at most 8). Use it to check the result when appearance matters, such as matching a reference or a layout you cannot verify from the markup, not after every edit; use full_page when the sections below the first screen matter. Needs an fx0 tab open and in front; on failure, continue without it.',
+ inputSchema: {
+ type: 'object',
+ properties: { full_page: { type: 'boolean', description: 'Whole page in viewport-high images instead of the visible viewport.' } },
+ additionalProperties: false,
+ },
+ async execute(input: { full_page?: boolean }, { signal, executionId }): Promise<FxToolResult | { error: string }> {
+ host.reportInvocation('screenshot_design')
+ const full = input?.full_page === true
+ const t0 = Date.now()
+ try {
+ const shot = await host.captureDesign(signal, { toolCallId: executionId, full })
+ const base64Bytes = shot.images.reduce((sum, image) => sum + image.data.length, 0)
+ host.log('tool:screenshot_design:done', { ms: Date.now() - t0, revision: shot.revision, images: shot.images.length, base64Bytes })
+ const size = `${shot.width ?? '?'}x${shot.height ?? '?'} CSS px`
+ const text = full
+ ? `Full-page screenshot of design revision ${shot.revision}, ${size}, in ${shot.images.length} viewport-high image${shot.images.length === 1 ? '' : 's'}, top to bottom.` +
+ (shot.truncated ? ' The page continues below the last image.' : '')
+ : `Screenshot of design revision ${shot.revision}, ${size}, as the preview shows it.`
+ return { type: 'libfx.tool-result', text, images: shot.images.map((image) => ({ type: 'image', ...image })) }
+ } catch (error) {
+ const message = error instanceof Error ? error.message : 'unknown error'
+ host.log('tool:screenshot_design:error', { ms: Date.now() - t0, error: message })
+ return { error: `screenshot unavailable: ${message}` }
+ }
+ },
+ }
+
const webFetch: FxHostTool = {
name: 'web_fetch',
description:
@@ -238,7 +274,7 @@ export function createTools(host: ToolHost, opts: { allowSubagent?: boolean } =
},
}
- const tools: FxHostTool[] = [readDesign, publishDesign, editDesign, webFetch, askUser, readToolResult, capabilitySearch]
+ const tools: FxHostTool[] = [readDesign, publishDesign, editDesign, screenshotDesign, webFetch, askUser, readToolResult, capabilitySearch]
if (allowSubagent) {
tools.push({
diff --git a/package.json b/package.json
index f239754..433a264 100644
--- a/package.json
+++ b/package.json
@@ -9,7 +9,7 @@
"build": "next build",
"typecheck": "next typegen && tsc --noEmit --incremental false",
"start": "next start",
- "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/prompt-images.test.mjs tests/design-assets.test.mjs tests/runtime.test.mjs",
+ "test:core": "node --experimental-strip-types --test tests/core.test.mjs tests/transcript.test.mjs tests/design-draft.test.mjs tests/design-publish.test.mjs tests/event-delivery.test.mjs tests/event-stream.test.mjs tests/public-fetch.test.mjs tests/remote-image.test.mjs tests/prompt-images.test.mjs tests/design-assets.test.mjs tests/capture-protocol.test.mjs tests/runtime.test.mjs",
"test:auth": "node --experimental-strip-types --test tests/auth.test.mjs tests/auth-callback.test.mjs tests/signout.test.mjs tests/ownership.test.mjs tests/history.test.mjs tests/route-security.test.mjs tests/gateway-session.test.mjs tests/libfx-transport.test.mjs tests/libfx-agent.test.mjs tests/auth-ui.test.mjs tests/html-ui.test.mjs tests/guest-ui.test.mjs tests/guest-draft.test.mjs tests/trial.test.mjs tests/turn-funding.test.mjs",
"test:journal:integration": "node --env-file-if-exists=.env.local --experimental-strip-types --test tests/journal.integration.mjs",
"test:smoke": "node tests/local-auth-smoke.mjs",
diff --git a/tests/capture-protocol.test.mjs b/tests/capture-protocol.test.mjs
new file mode 100644
index 0000000..61ea09d
--- /dev/null
+++ b/tests/capture-protocol.test.mjs
@@ -0,0 +1,29 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { captureTrace } from '../lib/capture-protocol.ts'
+
+test('a capture trace splits the call into parts, and its network total survives a clock offset', () => {
+ // The tab's clock runs 40 ms ahead of the servers'.
+ const skew = 40
+ const server = { totalMs: 700, setupMs: 9, emitMs: 6, emittedAt: 1_000_000, receivedAt: 1_000_685 }
+ const shot = {
+ path: 'preview', receivedAt: 1_000_150 + skew, sentAt: 1_000_520 + skew, previewWaitMs: 300, roundTripMs: 70,
+ frame: { fontsMs: 1, libraryMs: 25, encodeMs: 30 }, arrivedAt: 1_000_670, authMs: 3, readMs: 1, storingAt: 1_000_675,
+ }
+ const trace = captureTrace(server, shot)
+ assert.deepEqual(trace, {
+ path: 'preview', totalMs: 700, setupMs: 9, emitMs: 6,
+ deliveryMs: 150 + skew, previewWaitMs: 300, messagingMs: 14, fontsMs: 1, libraryMs: 25, encodeMs: 30,
+ uploadMs: 150 - skew, networkMs: 300, authMs: 3, readMs: 1, storeAndWakeMs: 10,
+ })
+})
+
+test("a capture trace without the frame's timing counts the whole round trip as messaging", () => {
+ const trace = captureTrace(
+ { totalMs: 100, setupMs: 1, emitMs: 1, emittedAt: 0, receivedAt: 100 },
+ { path: 'hidden', receivedAt: 10, sentAt: 60, previewWaitMs: 20, roundTripMs: 30, frame: null, arrivedAt: 80, authMs: 2, readMs: 0, storingAt: 90 },
+ )
+ assert.equal(trace.messagingMs, 30)
+ assert.equal(trace.fontsMs, null)
+ assert.equal(trace.path, 'hidden')
+})
diff --git a/tests/core.test.mjs b/tests/core.test.mjs
index f3560cb..888daed 100644
--- a/tests/core.test.mjs
+++ b/tests/core.test.mjs
@@ -69,6 +69,7 @@ function toolHost() {
storeResult: () => {},
readResult: () => undefined,
askUser: async () => ({}),
+ captureDesign: async () => ({ images: [{ mimeType: 'image/jpeg', data: 'AAAA' }], revision: 1 }),
log: () => {},
spawnSubagent: async () => 'done',
capabilitySearch: () => [],
@@ -113,6 +114,7 @@ test('design mode exposes only design and common host tools', async () => {
'read_design',
'publish_design',
'edit_design',
+ 'screenshot_design',
'web_fetch',
'ask_user_question',
'read_tool_result',
@@ -128,6 +130,23 @@ test('design mode exposes only design and common host tools', async () => {
)
})
+test('screenshot_design gives the model its images with what they show, or why there are none', async () => {
+ const signal = new AbortController().signal
+ const calls = []
+ const host = { ...toolHost(), captureDesign: async (_signal, options) => {
+ calls.push(options)
+ return { images: [{ mimeType: 'image/webp', data: 'AAAA' }, { mimeType: 'image/webp', data: 'BBBB' }], width: 1280, height: 2400, truncated: true, revision: 3 }
+ } }
+ const screenshot = createTools(host).find((tool) => tool.name === 'screenshot_design')
+ const page = await screenshot.execute({ full_page: true }, { signal, executionId: 'call_9' })
+ assert.deepEqual(calls, [{ toolCallId: 'call_9', full: true }])
+ assert.deepEqual(page.images, [{ type: 'image', mimeType: 'image/webp', data: 'AAAA' }, { type: 'image', mimeType: 'image/webp', data: 'BBBB' }])
+ assert.equal(page.text, 'Full-page screenshot of design revision 3, 1280x2400 CSS px, in 2 viewport-high images, top to bottom. The page continues below the last image.')
+ const failing = createTools({ ...toolHost(), captureDesign: async () => { throw new Error('no fx0 tab is open and in front to take it') } })
+ assert.deepEqual(await failing.find((tool) => tool.name === 'screenshot_design').execute({}, { signal, executionId: 'call_10' }),
+ { error: 'screenshot unavailable: no fx0 tab is open and in front to take it' })
+})
+
test('turn events update an existing sidebar session without mutating it', () => {
const current = [{ id: 'chat', title: 'Landing page', createdAt: 10, lastActivity: 10, busy: false }]
const started = upsertSession(current, 'chat', { type: 'turn_start', id: 1, at: 11 })
diff --git a/tests/design-artifact.browser.mjs b/tests/design-artifact.browser.mjs
index e603e93..901d408 100644
--- a/tests/design-artifact.browser.mjs
+++ b/tests/design-artifact.browser.mjs
@@ -10,6 +10,7 @@ import { pathToFileURL } from 'node:url'
import { promisify } from 'node:util'
import { markRemoteImages } from '../lib/remote-image.ts'
import { DESIGN_ARTIFACT_EMBEDDED_CSP, DESIGN_FRAME_CSP, designFrameCsp, designFrameHtml, MAX_EMBEDDED_IMAGE_BYTES, prepareDesignArtifactHtml, validateDesignArtifactHtml } from '../lib/design-artifact.ts'
+import { DESIGN_CAPTURE_RELAY, designCaptureScript } from '../lib/design-capture.ts'
// Opt-in: runs real Chrome headless on this machine. Set CHROME_PATH for another browser binary.
const CHROME = process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
@@ -190,6 +191,86 @@ for (const name of ['small', 'large', 'repeat']) open(name)
}
})
+// screenshot_design: the app mounts a preview the way DesignCapture does
+// (fixed, invisible, under the page, in a phone-sized window) and asks it for photos once
+// it has painted: the viewport, then the whole page. The viewport must come back as one
+// 1280x800 WebP of the design, the page as viewport-high tiles from the top (even when
+// html and body are only viewport-high and the content overflows them), and the remote
+// url() the design names must never reach the network, capture included.
+test('in Chrome, a hidden preview photographs its viewport and its whole page without any network', async () => {
+ assert.ok(existsSync(CHROME), `Chrome not found at ${CHROME}; set CHROME_PATH`)
+ const requests = []
+ const server = http.createServer((request, response) => {
+ const url = new URL(request.url, 'http://test')
+ requests.push(url.pathname)
+ if (url.pathname === '/frame') {
+ const stored = validateDesignArtifactHtml(`<!doctype html><html><head><style>html,body{margin:0;height:100%}
+section{height:800px}.a{background:#ff0066}.b{background:#00cc66}.c{height:400px;background:#3366ff}
+.leak{background:url(http://127.0.0.1:${server.address().port}/leak.png)}</style></head><body>
+<section class="a"><h1 class="leak" style="color:#fff;margin:0;padding:40px">fx0</h1></section><section class="b"></section><section class="c"></section></body></html>`)
+ assert.equal(stored.ok, true, stored.error)
+ const token = 'browser-test'
+ response.writeHead(200, { 'content-type': 'text/html; charset=utf-8', 'content-security-policy': DESIGN_FRAME_CSP })
+ response.end(designFrameHtml(prepareDesignArtifactHtml(stored.html, DESIGN_ARTIFACT_EMBEDDED_CSP, token, designCaptureScript(token)), token, [], DESIGN_CAPTURE_RELAY))
+ } else if (url.pathname === '/hold') {
+ setTimeout(() => response.writeHead(404).end(), 6000)
+ } else if (url.pathname === '/') {
+ response.writeHead(200, { 'content-type': 'text/html' }).end(`<!doctype html><html><head><title>app</title></head><body style="margin:0">
+<main style="position:fixed;inset:0;background:#fff"><pre id="out">{}</pre></main>
+<div style="position:fixed;inset:0;z-index:-10;overflow:hidden;opacity:0;pointer-events:none"><iframe id="capture" src="/frame" style="width:1280px;height:800px;border:0"></iframe></div>
+<script>
+const frame = document.getElementById('capture')
+const out = {}
+const show = () => { document.getElementById('out').textContent = JSON.stringify(out) }
+const describe = async (blob) => {
+ const bitmap = await createImageBitmap(blob)
+ const canvas = new OffscreenCanvas(bitmap.width, bitmap.height)
+ const context = canvas.getContext('2d')
+ context.drawImage(bitmap, 0, 0)
+ return { type: blob.type, width: bitmap.width, height: bitmap.height, color: [...context.getImageData(1200, bitmap.height - 20, 1, 1).data.slice(0, 3)] }
+}
+addEventListener('message', async ({ data, source }) => {
+ if (source !== frame.contentWindow) return
+ if (data?.type === 'fx0-artifact-painted' && !out.asked) {
+ out.asked = true
+ frame.contentWindow.postMessage({ type: 'fx0-capture', cid: 'viewport' }, location.origin)
+ }
+ if (data?.type !== 'fx0-artifact-captured') return
+ out[data.cid] = { height: data.height, truncated: data.truncated, timing: data.timing, images: await Promise.all(data.blobs.map(describe)) }
+ if (data.cid === 'viewport') frame.contentWindow.postMessage({ type: 'fx0-capture', cid: 'page', full: true }, location.origin)
+ show()
+})
+</script><img alt="" src="/hold"></body></html>`)
+ } else {
+ response.writeHead(404).end()
+ }
+ })
+ await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
+ const dir = await mkdtemp(join(tmpdir(), 'fx0-design-capture-'))
+ try {
+ const { stdout } = await promisify(execFile)(CHROME, [
+ '--headless=new', '--disable-gpu', '--no-first-run', '--no-default-browser-check', '--window-size=390,844',
+ `--user-data-dir=${join(dir, 'profile')}`, '--dump-dom', `http://127.0.0.1:${server.address().port}/`,
+ ], { timeout: 60_000, maxBuffer: 16 * 1024 * 1024 })
+ const out = JSON.parse(stdout.match(/<pre id="out">([^<]*)<\/pre>/)[1])
+ const near = (rgb, target) => rgb.every((value, i) => Math.abs(value - target[i]) < 24)
+ assert.equal(out.viewport.images.length, 1)
+ assert.deepEqual(out.viewport.images.map(({ type, width, height }) => [type, width, height]), [['image/webp', 1280, 800]], 'Chrome sends WebP')
+ assert.ok(near(out.viewport.images[0].color, [255, 0, 102]), `the viewport is the design's first section, got ${out.viewport.images[0].color}`)
+ for (const part of ['fontsMs', 'libraryMs', 'encodeMs']) assert.ok(out.viewport.timing?.[part] >= 0, `the frame reports ${part}`)
+ assert.equal(out.page.height, 2000)
+ assert.equal(out.page.truncated, false)
+ assert.deepEqual(out.page.images.map(({ width, height }) => [width, height]), [[1280, 800], [1280, 800], [1280, 400]])
+ const colors = [[255, 0, 102], [0, 204, 102], [51, 102, 255]]
+ out.page.images.forEach((image, i) => assert.ok(near(image.color, colors[i]), `tile ${i + 1} shows section ${i + 1}, got ${image.color}`))
+ assert.ok(!requests.includes('/leak.png'), 'the url() the design names never reached the network')
+ } finally {
+ server.closeAllConnections()
+ server.close()
+ await rm(dir, { recursive: true, force: true })
+ }
+})
+
// The draft has no network, so a remote image must show the soft placeholder, never
// the browser's broken-image icon. The draft cleaner runs in Chrome, types stripped.
test('in Chrome, a draft shows a placeholder for remote images instead of a broken icon', async () => {
diff --git a/tests/journal.integration.mjs b/tests/journal.integration.mjs
index 9e1a801..f7bc395 100644
--- a/tests/journal.integration.mjs
+++ b/tests/journal.integration.mjs
@@ -368,6 +368,55 @@ test('journal Redis integration uses isolated durable session state', async (t)
ok: false, reason: 'fenced', error: 'the chat was deleted before publishing',
})
})
+ await t.test('a capture is acknowledged, answered once, and fails fast when no tab takes it', async () => {
+ const sessionId = randomUUID()
+ const shot = { images: [{ mimeType: 'image/webp', data: 'AAAA' }, { mimeType: 'image/webp', data: 'BBBB' }], width: 1280, height: 1600 }
+ const replies = key(sessionId, 'capture-replies')
+
+ // Acknowledged, then answered with a photo: the call wakes as the photo arrives.
+ await journal.openCapture(sessionId, 'c1')
+ const started = Date.now()
+ const waiting = journal.awaitCapture(sessionId, 'c1', AbortSignal.timeout(15_000))
+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c0'), false, 'another capture id is refused')
+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c1'), true)
+ await new Promise((resolve) => setTimeout(resolve, 500))
+ assert.equal(await journal.answerCapture(sessionId, 'c1', { kind: 'shot', shot }), true)
+ assert.deepEqual(await waiting, { kind: 'shot', shot })
+ assert.ok(Date.now() - started < 2_500, `woke ${Date.now() - started} ms after waiting began, the photo came at 500 ms`)
+ assert.equal(await journal.answerCapture(sessionId, 'c1', { kind: 'failed', reason: 'too-large' }), false, 'an answer is accepted once')
+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c1'), false, 'a replayed request finds its capture closed')
+
+ // A tab's failure ends the wait with its reason.
+ await journal.openCapture(sessionId, 'c2')
+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c2'), true)
+ assert.equal(await journal.answerCapture(sessionId, 'c2', { kind: 'failed', reason: 'too-large' }), true)
+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c2', AbortSignal.timeout(5_000)), { kind: 'failed', reason: 'too-large' })
+
+ // No tab acknowledges: the call gives up after the ack window, not the whole wait.
+ await journal.openCapture(sessionId, 'c3')
+ const silent = Date.now()
+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c3', AbortSignal.timeout(15_000), 1_000), { kind: 'unanswered' })
+ assert.ok(Date.now() - silent < 3_000, `gave up after ${Date.now() - silent} ms`)
+ assert.equal(await journal.answerCapture(sessionId, 'c3', { kind: 'shot', shot }), false, 'a late photo is refused')
+
+ // Cancelled: the capture is withdrawn.
+ await journal.openCapture(sessionId, 'c4')
+ const cancel = new AbortController()
+ setTimeout(() => cancel.abort(), 300)
+ assert.deepEqual(await journal.awaitCapture(sessionId, 'c4', cancel.signal), { kind: 'cancelled' })
+ assert.equal(await journal.acknowledgeCapture(sessionId, 'c4'), false, 'a cancelled wait withdraws its capture')
+ assert.equal(await redis.exists(key(sessionId, 'capture'), replies), 0, 'nothing is left behind')
+ })
+
+ await t.test('a screenshot call records its preview wait under its call id, for its tool_end', async () => {
+ const sessionId = randomUUID()
+ await journal.recordPreviewWait(sessionId, 'call_a', 515.4)
+ await journal.recordPreviewWait(sessionId, 'call_b', 0)
+ assert.deepEqual([...await journal.readPreviewWaits(sessionId, ['call_a', 'call_b', 'call_missing'])], [['call_a', 515]])
+ assert.deepEqual([...await journal.readPreviewWaits(sessionId, [])], [])
+ assert.ok(await redis.pttl(key(sessionId, 'preview-waits')) > 0, 'the waits expire with the chat idle')
+ })
+
await t.test('a chat credential is sealed, expires with its token, and sign-out drops only its own login', async () => {
const session = (loginId, accessToken) => ({
userId: 'user-a', loginId, loginAt: 1, email: '[email protected]', accessToken, teamId: 'team_a',
diff --git a/tests/libfx-agent.test.mjs b/tests/libfx-agent.test.mjs
index 244b12d..260eb4f 100644
--- a/tests/libfx-agent.test.mjs
+++ b/tests/libfx-agent.test.mjs
@@ -60,6 +60,12 @@ globalThis.__fx0Agent = {
durableAgent({ toolHost: async () => { throw new Error('no tools here') } }, 'instructions')
const agentFetch = agentOptions.fetch
+test('screenshots in one response take turns: a chat has one capture open at a time', () => {
+ const screenshot = agentOptions.tools.find((tool) => tool.name === 'screenshot_design')
+ assert.equal(screenshot.idempotent, true)
+ assert.equal(screenshot.writes, true, 'a tool that writes never runs beside another call')
+})
+
test('a recovered sponsored chat uses its recorded trial funding instead of requiring a customer card', async (t) => {
const previous = { key: process.env.AI_GATEWAY_API_KEY, enabled: process.env.FX0_FREE_PROMPTS }
process.env.AI_GATEWAY_API_KEY = 'synthetic-server-key'
diff --git a/tests/route-security.test.mjs b/tests/route-security.test.mjs
index b7be32c..193bff2 100644
--- a/tests/route-security.test.mjs
+++ b/tests/route-security.test.mjs
@@ -113,6 +113,8 @@ const mocks = {
return { html: s.liveRevision === 0 ? null : '<!doctype html><html><head><title>private design</title></head><body>private artifact</body></html>', revision: s.liveRevision ?? 1 };
}
export async function currentSeq(id) { record('seq', id); return 0; }
+ export async function acknowledgeCapture(id) { record('capture-ack', id); return true; }
+ export async function answerCapture(id, cid, answer) { record('capture-answer', id); globalThis.securityTest.captureAnswer = answer; return true; }
`,
'lib/agent.ts': `
function effect(name, host) {
@@ -263,7 +265,7 @@ function request(path, method = 'GET', id = own, body, origin = process.env.APP_
return req
}
const routes = {}
-for (const name of ['answer', 'artifact', 'artifact-frame', 'artifact-image', 'cancel', 'debug', 'events',
+for (const name of ['answer', 'artifact', 'artifact-frame', 'capture', 'artifact-image', 'cancel', 'debug', 'events',
'credits', 'model', 'prompt', 'session', 'sessions', 'teams', 'warm']) {
routes[name] = await import(`../app/api/${name}/route.ts`)
}
@@ -421,6 +423,63 @@ test('session routes return indistinguishable 404 for unknown and foreign IDs be
}
})
+test('a tab acknowledges a capture and reports a known failure; an unknown reason is refused', async () => {
+ const s = reset()
+ const post = (query) => routes.capture.POST(new Request(`https://fx0.example/api/capture?session=${own}&cid=${'1'.repeat(8)}-1111-1111-1111-${'1'.repeat(12)}&${query}`, {
+ method: 'POST', headers: { origin: process.env.APP_URL },
+ }))
+ const ack = await post('ack=1')
+ assert.equal(ack.status, 200)
+ assert.deepEqual(await ack.json(), { ok: true })
+ assert.ok(s.trace.includes('capture-ack'))
+ const failed = await post('failed=too-large')
+ assert.deepEqual(await failed.json(), { ok: true })
+ assert.deepEqual(s.captureAnswer, { kind: 'failed', reason: 'too-large' })
+ assert.equal((await post('failed=__proto__')).status, 400)
+ assert.equal((await post('failed=anything')).status, 400)
+ const unsized = await post('w=1')
+ assert.equal(unsized.status, 411, 'a photo without a length is refused before it is read')
+})
+
+test('a photo reaches the waiting call; one the route refuses still answers it, with the reason', async () => {
+ const s = reset()
+ const PNG = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==', 'base64')
+ const upload = async (files, query = 'w=1280&h=800', length) => {
+ const form = new FormData()
+ for (const file of files) form.append('image', new Blob([file], { type: 'image/png' }))
+ const encoded = new Response(form)
+ const body = await encoded.arrayBuffer()
+ return routes.capture.POST(new Request(`https://fx0.example/api/capture?session=${own}&cid=${'2'.repeat(8)}-2222-2222-2222-${'2'.repeat(12)}&${query}`, {
+ method: 'POST', body,
+ headers: { origin: process.env.APP_URL, 'content-type': encoded.headers.get('content-type'), 'content-length': String(length ?? body.byteLength) },
+ }))
+ }
+ const accepted = await upload([PNG], 'w=1280&h=2400&truncated=1')
+ assert.deepEqual(await accepted.json(), { ok: true })
+ assert.equal(s.captureAnswer.kind, 'shot')
+ assert.deepEqual(s.captureAnswer.shot.images, [{ mimeType: 'image/png', data: PNG.toString('base64') }])
+ assert.deepEqual([s.captureAnswer.shot.width, s.captureAnswer.shot.height, s.captureAnswer.shot.truncated], [1280, 2400, true])
+ for (const [files, length, status, reason] of [
+ [[Buffer.from('not an image at all')], undefined, 415, 'refused'],
+ [Array(9).fill(PNG), undefined, 400, 'refused'],
+ [[PNG], 10 * 1024 * 1024, 413, 'too-large'],
+ ]) {
+ s.captureAnswer = null
+ assert.equal((await upload(files, undefined, length)).status, status)
+ assert.deepEqual(s.captureAnswer, { kind: 'failed', reason }, `a ${status} answers the call`)
+ }
+})
+
+test('a capture for an unknown or foreign chat is refused before its body or Redis', async () => {
+ for (const id of [foreign, fresh]) {
+ const s = reset()
+ const response = await routes.capture.POST(request('capture', 'POST', id))
+ assert.equal(response.status, 404)
+ assert.deepEqual(s.trace, ['auth', 'owner'])
+ assert.deepEqual(s.redisReads, [])
+ }
+})
+
test('the same owner can steer, enqueue or recover a chat after switching teams, with one owner check', async () => {
for (const [name, method, body] of sessionRoutes) {
const oldContext = reset()
@@ -440,7 +499,7 @@ test('the same owner can steer, enqueue or recover a chat after switching teams,
test('session routes and creation reject unverified team grants before runtime or writes', async () => {
for (const teamId of [undefined, 'user_a']) {
- for (const [name, method, body] of [...sessionRoutes, ['sessions', 'POST']]) {
+ for (const [name, method, body] of [...sessionRoutes, ['sessions', 'POST'], ['capture', 'POST']]) {
const s = reset({ session: session({ teamId }) })
const response = await routes[name][method](request(name, method, own, body))
assert.equal(response.status, 403, `${method} ${name}`)
@@ -598,8 +657,10 @@ test('the artifact preview paints no white of its own before the design does', a
reset()
const html = await (await routes['artifact-frame'].GET(request('artifact-frame', 'GET', own, undefined, 'null'))).text()
// The app's color scheme keeps the wrapper transparent over the app's background.
- assert.match(html, /:root\{color-scheme:light dark\}/)
- assert.doesNotMatch(html, /background:#fff/)
+ // The design (and the capture library) travel inside the wrapper's script; only its own markup counts.
+ const wrapper = html.slice(0, html.indexOf('<script>'))
+ assert.match(wrapper, /:root\{color-scheme:light dark\}/)
+ assert.doesNotMatch(wrapper, /background:#fff/)
})
test('the browser keeps a preview only when its URL names the revision that is live', async () => {
diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs
index 014d87e..6c1f0ac 100644
--- a/tests/runtime.test.mjs
+++ b/tests/runtime.test.mjs
@@ -7,11 +7,12 @@ import test from 'node:test'
// own integration test; the agent's fetch has tests/libfx-agent.test.mjs.
const journalNames = [
'answerActiveQuestion', 'appendEvent', 'bindLibfxSession', 'chatOfLibfxSession', 'claimSessionCleanup',
- 'clearActiveQuestion', 'currentSeq', 'emitLive', 'finishSessionCleanup', 'finishTurnWork', 'getActiveQuestion',
- 'getActiveTurn', 'nextQuestionId', 'projectEvents', 'projectedCursor', 'publishDesignArtifact', 'readDesignArtifact',
- 'readDesignAssets', 'readDesignMd', 'readEvents', 'recordDesignAssets', 'releaseDesignAssets', 'reserveDesignAssets',
- 'finishDesignAssetCleanup', 'pendingDesignAssetCleanups', 'queueDesignAssetCleanup',
- 'readSession', 'readSessionView', 'readTurnWork', 'releaseSessionCleanup', 'renewSessionCleanup',
+ 'awaitCapture', 'clearActiveQuestion', 'closeCapture', 'currentSeq', 'emitLive', 'finishDesignAssetCleanup',
+ 'finishSessionCleanup', 'finishTurnWork', 'getActiveQuestion', 'getActiveTurn', 'nextQuestionId', 'openCapture',
+ 'pendingDesignAssetCleanups', 'projectEvents', 'projectedCursor', 'publishDesignArtifact',
+ 'queueDesignAssetCleanup', 'readDesignArtifact', 'readDesignAssets', 'readDesignMd', 'readEvents',
+ 'readPreviewWaits', 'readSession', 'readSessionView', 'readTurnWork', 'recordDesignAssets', 'recordPreviewWait',
+ 'releaseDesignAssets', 'releaseSessionCleanup', 'renewSessionCleanup', 'reserveDesignAssets',
'setActiveQuestion', 'submitPrompt', 'takeAnswer', 'updateSession',
]
const dataUrl = (source) => `data:text/javascript,${encodeURIComponent(source)}`
@@ -449,11 +450,12 @@ test('the agent offers HTML and common tools only; a tool runs on the chat its t
const f = fixture(t)
runtime.agent()
const names = f.agentOptions.tools.map((tool) => tool.name)
- assert.deepEqual(names, ['read_design', 'publish_design', 'edit_design', 'web_fetch', 'ask_user_question', 'read_tool_result', 'capability_search', 'subagent', 'web_search'])
+ assert.deepEqual(names, ['read_design', 'publish_design', 'edit_design', 'screenshot_design', 'web_fetch', 'ask_user_question', 'read_tool_result', 'capability_search', 'subagent', 'web_search'])
const edit = f.agentOptions.tools.find((tool) => tool.name === 'edit_design')
assert.equal(edit.idempotent, false)
assert.equal(edit.writes, true)
assert.equal(f.agentOptions.tools.find((tool) => tool.name === 'read_design').idempotent, true)
+ assert.equal(f.agentOptions.tools.find((tool) => tool.name === 'screenshot_design').idempotent, true)
const ctx = { signal: new AbortController().signal, executionId: 'call-1', sessionId: 'wrun_1', context: { chatId: 'tool-chat', turnId: 4 } }
const result = await edit.execute({ edits: [{ old_string: 'old', new_string: 'new' }, { old_string: '<body>', new_string: '<body class="x">' }] }, ctx)
@@ -622,6 +624,20 @@ test('a turn\'s history and subagents keep the settings it started with after a
assert.equal(history.fast, false)
})
+test('the projector puts the preview wait of a screenshot call on its tool_end, and only there', async (t) => {
+ const f = fixture(t)
+ const entries = [
+ { type: 'tool_end', name: 'screenshot_design', toolCallId: 'shot', turnId: 1, at: 1 },
+ { type: 'tool_end', name: 'read_design', toolCallId: 'read', turnId: 1, at: 2 },
+ { type: 'tool_end', name: 'screenshot_design', toolCallId: 'unrecorded', turnId: 1, at: 3 },
+ ]
+ const asked = []
+ f.journal.readPreviewWaits = async (_id, ids) => { asked.push(...ids); return new Map([['shot', 515], ['read', 9]]) }
+ await projector.addPreviewWaits('chat', entries)
+ assert.deepEqual(asked, ['shot', 'unrecorded'], 'only screenshot calls are looked up')
+ assert.deepEqual(entries.map((entry) => entry.previewWaitMs), [515, undefined, undefined])
+})
+
test('the projector copies a turn once however many follow it, then records its history and design', async (t) => {
const f = fixture(t, { libfxSessionId: 'wrun_7', preferredModel: null, preferredEffort: null, preferredFast: null })
const host = boundHost('project-chat')
diff --git a/tests/transcript.test.mjs b/tests/transcript.test.mjs
index 4134ad6..3de16d6 100644
--- a/tests/transcript.test.mjs
+++ b/tests/transcript.test.mjs
@@ -84,3 +84,19 @@ test('a page may publish its checkpoint after responding without delaying the tr
await job()
assert.equal(published, true)
})
+
+test('a screenshot call reports its own time, not the time it waited for the preview to load', () => {
+ const view = { artifactRevision: 1 }
+ const state = compactTranscript([
+ { type: 'turn_start', id: 1, seq: 1, at: 0 },
+ // The capture event may land before the call's tool_start (the projector writes libfx's
+ // lines in batches); the wait travels on the tool_end instead.
+ { type: 'capture', turnId: 1, seq: 2, cid: 'c1', artifactRevision: 1, at: 1_009 },
+ { type: 'tool_start', turnId: 1, seq: 3, name: 'screenshot_design', at: 1_000 },
+ { type: 'tool_end', turnId: 1, seq: 4, name: 'screenshot_design', toolCallId: 'call_1', previewWaitMs: 515, content: 'Screenshot', at: 2_208 },
+ ], view)
+ const tool = state.entries.find((entry) => entry.type === 'tool')
+ assert.equal(tool.durationMs, 693)
+ assert.equal(tool.previewWaitMs, 515)
+ assert.equal(state.entries.filter((entry) => entry.type === 'tool').length, 1, 'capture events add no rows')
+})