3D engine that AI agents drive
Send raw video frames over a binary WebSocket and make raw capture the video default
Nguyen Xuan Anc4ee1129dd01
diff --git a/package-lock.json b/package-lock.jsonindex d1b94cb3..ceb680f7 100644--- a/package-lock.json+++ b/package-lock.json@@ -30,6 +30,7 @@"three-mesh-bvh": "0.9.16","vite": "8.3.5","wrangler": "4.156.0",+ "ws": "^8.21.0","zod": "4.6.7"},"engines": {diff --git a/package.json b/package.jsonindex 2ed1dd62..b6bb51ea 100644--- a/package.json+++ b/package.json@@ -124,6 +124,7 @@"smoke:agent-cloth": "node scripts/smoke-agent-cloth.mjs","smoke:agent-particles": "node scripts/smoke-agent-particles.mjs","smoke:frame-encoder": "node scripts/smoke-frame-encoder.mjs",+ "smoke:capture-parity": "node scripts/smoke-capture-parity.mjs","smoke:enhancements": "node scripts/smoke-enhancements.mjs","smoke:site": "node scripts/smoke-site.mjs","smoke:agent-review-close": "node scripts/smoke-agent-review-close.mjs",@@ -179,6 +180,7 @@"three-mesh-bvh": "0.9.16","vite": "8.3.5","wrangler": "4.156.0",+ "ws": "^8.21.0","zod": "4.6.7"}}diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex c057a035..c859039a 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -12,6 +12,7 @@ import { chooseDenoiser, finishWithOidn } from './agent-path-trace-denoise.mjs';import { createAgentAssetCatalog } from './agent-asset-catalog.mjs';import { createAgentRenderJobs, EXPORT_DIRECTORY_NAME } from './agent-render-jobs.mjs';import { createFrameEncoder } from './frame-encoder.mjs';+import { createFrameReceiver, installFrameReaderInPage } from './gpu-frame-reader.mjs';import { audioInputArguments, createAudioLibrary, measureStreamSeconds } from './job-audio.mjs';import { filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline } from '../src/film-timeline.js';import { isPreviewHost } from './preview-host.mjs';@@ -82,6 +83,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browlet phase = 'idle';let browserInstance;let httpServer;+ // The job that is capturing video frames, if any. Its socket path is secret and lives only as long as the job.+ let activeFrameReceiver;let page;let reviewPage;let cancelReview;@@ -230,6 +233,11 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browresponse.setHeader('Content-Type', request.url === '/' ? 'text/html; charset=utf-8' : 'application/javascript; charset=utf-8');response.end(request.url === '/' ? html : code);});+ httpServer.on('upgrade', (request, socket, head) => {+ const accepted = isPreviewHost(request.headers.host, httpServer.address().port)+ && activeFrameReceiver?.handleUpgrade(request, socket, head);+ if (!accepted) socket.destroy();+ });await new Promise((resolve, reject) => {httpServer.once('error', reject);httpServer.listen(0, '127.0.0.1', resolve);@@ -314,6 +322,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browasync function runJob({ jobId, directory, snapshot, settings, signal, progress }) {let jobPage;let encoder;+ let frameReceiver;let suspended = false;const sessionId = activeSessionId;const started = performance.now();@@ -348,8 +357,12 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browconst timeline = settings.timeline ? snapshot.recipe.timeline : undefined;const audio = settings.audio ? await prepareAudio(settings) : undefined;if (settings.kind === 'video') {+ // Video frames are read from the drawing buffer and sent over a socket on the preview's own port (issue #301).+ frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });+ activeFrameReceiver = frameReceiver;+ await installFrameReaderInPage(jobPage, `ws://127.0.0.1:${httpServer.address().port}${frameReceiver.path}`);encoder = createFrameEncoder({ executable: ffmpeg, output: join(directory, 'render.mp4.partial'),- fps: settings.fps, format: 'jpeg', signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',+ fps: settings.fps, format: 'rawvideo', width, height, signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [] });metrics.videoEncoder = settings.encoder === 'nvenc' ? 'h264_nvenc' : 'libx264';}@@ -388,14 +401,29 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browawait progress({ frames: 1, maximumPendingFrames: 1 });continue;}- const rendered = await jobPage.evaluate(input => window.__agentPreview.frame(input), frameInput);- metrics.renderSeconds += (performance.now() - renderStarted) / 1000;+ // A video frame is read in the same task as its draw, because the browser clears the drawing buffer after presenting.+ const rendered = await jobPage.evaluate(async ({ input, readFrame }) => {+ const drawStarted = performance.now();+ const result = await window.__agentPreview.frame(input);+ if (!readFrame) return result;+ window.__finishDraw();+ const drawMilliseconds = performance.now() - drawStarted;+ return { ...result, drawMilliseconds, rawFrame: await window.__readFrame() };+ }, { input: frameInput, readFrame: settings.kind === 'video' });+ const roundTripSeconds = (performance.now() - renderStarted) / 1000;+ const drawSeconds = rendered.rawFrame ? rendered.drawMilliseconds / 1000 : roundTripSeconds;+ metrics.renderSeconds += drawSeconds;+ metrics.captureSeconds += roundTripSeconds - drawSeconds;if (backend === 'cpu' && !/swiftshader/i.test(rendered.graphics)) throw new Error('CPU job did not select SwiftShader.');metrics.graphics = rendered.graphics;metrics.width = width;metrics.height = height;metrics.estimatedBytes = rendered.estimatedBytes;async function captureFrameBytes() {+ if (rendered.rawFrame) {+ signal.throwIfAborted();+ return frameReceiver.take({ width, height }, rendered.rawFrame);+ }const captureStarted = performance.now();const captured = await session.send('Page.captureScreenshot', { fromSurface: true,format: settings.kind === 'video' ? 'jpeg' : 'png',@@ -444,6 +472,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', browreturn { artifacts, metrics };} finally {signal.removeEventListener('abort', abort);+ activeFrameReceiver = undefined;+ frameReceiver?.close();try { await encoder?.close(); }finally {try {diff --git a/scripts/gpu-frame-reader.mjs b/scripts/gpu-frame-reader.mjsindex 8aefbe77..ddd7505a 100644--- a/scripts/gpu-frame-reader.mjs+++ b/scripts/gpu-frame-reader.mjs@@ -1,37 +1,49 @@// Raw video frames straight from the drawing buffer, shared by the MCP render job and scripts/render-media.mjs.// A DevTools screenshot makes the compositor encode a JPEG and send it as base64, 84 percent of a film job (issue #301).// A read of the drawing buffer skips both. It must run in the same task as the draw, because the browser clears the-// drawing buffer once it has presented the frame. The page then posts the bytes to a Node receiver as a binary body:-// returning a typed array from Playwright's evaluate costs about 50 ms for a 1280 by 720 frame, a post costs a few.+// drawing buffer once it has presented the frame. The page then sends the bytes to Node over one binary WebSocket and+// waits for a one byte acknowledgement, so one frame is in flight at a time and the encoder sets the pace.+// Returning a typed array from Playwright's evaluate costs about 50 ms for a 1280 by 720 frame. A fetch with a Blob body+// delivered 100 to 150 frames and then failed with "Failed to fetch", so a socket replaces it.import { randomBytes } from 'node:crypto';import { createServer } from 'node:http';+import { WebSocketServer } from 'ws';-/** Runs in the page. Defines window.__finishDraw() and window.__readFrame(url), which posts the bottom-up RGBA pixels. */-function installFrameReader() {+/** Runs in the page. Defines window.__finishDraw() and window.__readFrame(), which sends the bottom-up RGBA pixels. */+function installFrameReader(url) {let pixels;+ let socket;+ let pendingAcknowledgement;const canvasContext = () => document.querySelector('canvas').getContext('webgl2');// Waits for the GPU to finish the draw, so that draw time is not counted as read time.window.__finishDraw = () => canvasContext().finish();- window.__readFrame = async (url) => {+ window.__connectFrameSocket = () => new Promise((resolve, reject) => {+ socket = new WebSocket(url);+ socket.binaryType = 'arraybuffer';+ socket.onopen = () => resolve();+ socket.onerror = () => reject(new Error('The frame socket could not connect.'));+ socket.onclose = () => pendingAcknowledgement?.reject(new Error('The frame socket closed.'));+ socket.onmessage = () => pendingAcknowledgement?.resolve();+ });+ window.__readFrame = async () => {const context = canvasContext();const { drawingBufferWidth: width, drawingBufferHeight: height } = context;if (!pixels || pixels.length !== width * height * 4) pixels = new Uint8Array(width * height * 4);const readStarted = performance.now();context.readPixels(0, 0, width, height, context.RGBA, context.UNSIGNED_BYTE, pixels);const readMilliseconds = performance.now() - readStarted;- // A Blob body: Playwright keeps a copy of every typed-array body it sees (3.7 MB a frame) until the page closes.- // A connection that the server closed while idle fails the first post, so one retry follows.- let response;- try { response = await fetch(url, { method: 'POST', body: new Blob([pixels]) }); }- catch { response = await fetch(url, { method: 'POST', body: new Blob([pixels]) }); }- if (!response.ok) throw new Error(`The frame receiver answered ${response.status}.`);+ const acknowledged = new Promise((resolve, reject) => { pendingAcknowledgement = { resolve, reject }; });+ socket.send(pixels);+ await acknowledged;+ pendingAcknowledgement = undefined;return { width, height, readMilliseconds };};}-/** Call once per page before the first frame. */-export async function installFrameReaderInPage(page) {- await page.evaluate(installFrameReader);+/** Call once per page before the first frame. `url` is the receiver's socket address. */+export async function installFrameReaderInPage(page, url) {+ await page.evaluate(installFrameReader, url);+ await page.evaluate(() => window.__connectFrameSocket());}/** The ffmpeg input arguments for raw frames. GL rows run bottom-up, so the picture is flipped, and RGB converts with BT.709. */@@ -40,51 +52,29 @@ export function rawFrameInputArguments({ width, height, fps }) {filter: ['-vf', 'vflip,scale=out_color_matrix=bt709:out_range=tv'] };}-/** A slow frame on the CPU backend takes longer than the default five seconds a server keeps an idle connection. */-export const KEEP_ALIVE_MILLISECONDS = 120000;-/**- * Receives one posted frame at a time on a path nobody else knows. `handle(request, response)` returns true when it- * took the request. `crossOrigin` answers with the headers a page on another port needs.+ * Receives one frame at a time on a path nobody else knows. `handleUpgrade(request, socket, head)` returns true when it+ * took the connection, so a server that already serves the page can share its port.*/-export function createFrameReceiver({ maximumBytes, crossOrigin = false }) {+export function createFrameReceiver({ maximumBytes }) {const path = `/__frame/${randomBytes(12).toString('hex')}`;- // One buffer is reused for every frame, so a long film does not leave a buffer of garbage per frame for the collector.- const frameBuffer = Buffer.allocUnsafe(maximumBytes);+ const sockets = new WebSocketServer({ noServer: true, maxPayload: maximumBytes, perMessageDeflate: false });+ const acknowledgement = Buffer.from([1]);let received;- function handle(request, response) {- if (request.url !== path) return false;- if (crossOrigin) response.setHeader('Access-Control-Allow-Origin', '*');- if (request.method !== 'POST') {- response.writeHead(405);- response.end();- return true;- }- let total = 0;- let tooLarge = false;- request.on('data', chunk => {- if (tooLarge) return;- if (total + chunk.length > maximumBytes) {- tooLarge = true;- response.writeHead(413);- response.end();- return;- }- chunk.copy(frameBuffer, total);- total += chunk.length;- });- request.on('end', () => {- if (tooLarge) return;- received = frameBuffer.subarray(0, total);- response.writeHead(204);- response.end();+ sockets.on('connection', socket => {+ socket.on('message', (data, isBinary) => {+ if (isBinary) received = data;+ socket.send(acknowledgement);});- return true;- }+ });return Object.freeze({path,- handle,- /** The frame the page posted since the last call, checked against the size the encoder was started with. */+ handleUpgrade(request, socket, head) {+ if (request.url !== path) return false;+ sockets.handleUpgrade(request, socket, head, connection => sockets.emit('connection', connection, request));+ return true;+ },+ /** The frame the page sent since the last call, checked against the size the encoder was started with. */take({ width, height }, info) {if (info.width !== width || info.height !== height) {throw new Error(`The drawing buffer is ${info.width}x${info.height} but the video is ${width}x${height}.`);@@ -94,22 +84,26 @@ export function createFrameReceiver({ maximumBytes, crossOrigin = false }) {if (!frame || frame.length !== width * height * 4) throw new Error('The page did not deliver a complete frame.');return frame;},+ close() {+ for (const client of sockets.clients) client.terminate();+ sockets.close();+ },});}/** A loopback server that only receives frames, for pages that another server already serves. */export async function startFrameServer(receiver) {const server = createServer((request, response) => {- if (!receiver.handle(request, response)) {- response.writeHead(404);- response.end();- }+ response.writeHead(404);+ response.end();+ });+ server.on('upgrade', (request, socket, head) => {+ if (!receiver.handleUpgrade(request, socket, head)) socket.destroy();});await new Promise((resolve, reject) => {- server.keepAliveTimeout = KEEP_ALIVE_MILLISECONDS;server.once('error', reject);server.listen(0, '127.0.0.1', resolve);});- return { url: `http://127.0.0.1:${server.address().port}${receiver.path}`,- close: () => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }) };+ return { url: `ws://127.0.0.1:${server.address().port}${receiver.path}`,+ close: () => new Promise(resolve => { receiver.close(); server.closeAllConnections(); server.close(resolve); }) };}diff --git a/scripts/render-media.mjs b/scripts/render-media.mjsindex 3b002daa..ff1ca347 100644--- a/scripts/render-media.mjs+++ b/scripts/render-media.mjs@@ -53,7 +53,7 @@ export function parseRenderOptions(values) {const start = numberOption('start', 0, 0, 3600);const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);const assembly = numberOption('assembly', 0, 0, 1);- const captureFormat = options.capture ?? (kind === 'video' ? 'jpeg' : 'png');+ const captureFormat = options.capture ?? (kind === 'video' ? 'raw' : 'png');if (!['jpeg', 'png', 'raw'].includes(captureFormat)) throw new Error('Capture must be raw, jpeg or png.');if (captureFormat === 'raw' && kind !== 'video') throw new Error('Raw capture is for video only. Stills are PNG or JPEG screenshots.');const jpegQuality = numberOption('jpeg-quality', 95, 1, 100);@@ -193,15 +193,15 @@ export async function renderMedia(values = process.argv.slice(2)) {}, { memory, width, height, samples: backend === 'cpu' ? 1 : preset.samples, scene, assembly });const session = await page.context().newCDPSession(page);if (captureFormat === 'raw') {- await installFrameReaderInPage(page);- frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4, crossOrigin: true });+ frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });frameServer = await startFrameServer(frameReceiver);+ await installFrameReaderInPage(page, frameServer.url);}// A raw frame is read from the drawing buffer in the same task as the draw. The round trip beyond the draw is capture.const render = async (seconds) => {if (errors.length) throw new Error(`Browser render errors: ${errors.join('; ')}`);const renderStarted = performance.now();- const drawn = await page.evaluate(async ({ value, readFrame, receiverUrl }) => {+ const drawn = await page.evaluate(async ({ value, readFrame }) => {const drawStarted = performance.now();if (window.__scene) window.__scene.renderAt(value);else if (window.renderAt) window.renderAt(value);@@ -211,8 +211,8 @@ export async function renderMedia(values = process.argv.slice(2)) {if (glError !== 0) throw new Error(`WebGL render error ${glError}.`);if (readFrame) window.__finishDraw();const drawMilliseconds = performance.now() - drawStarted;- return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame(receiverUrl) : undefined };- }, { value: seconds, readFrame: captureFormat === 'raw', receiverUrl: frameServer?.url });+ return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined };+ }, { value: seconds, readFrame: captureFormat === 'raw' });const rendererPlan = drawn.rendererPlan;const roundTripSeconds = (performance.now() - renderStarted) / 1000;const drawSeconds = drawn.frame ? drawn.drawMilliseconds / 1000 : roundTripSeconds;diff --git a/scripts/smoke-capture-parity.mjs b/scripts/smoke-capture-parity.mjsnew file mode 100644index 00000000..678f774b--- /dev/null+++ b/scripts/smoke-capture-parity.mjs@@ -0,0 +1,30 @@+// A raw video frame must match the PNG seek of the same time. Both come from SwiftShader, so no GPU is needed (issue #301).+import assert from 'node:assert/strict';+import { spawnSync } from 'node:child_process';+import { mkdtemp, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';+import { renderMedia } from './render-media.mjs';++const MINIMUM_PSNR_DECIBELS = 30;+const FPS = 12;+const FRAME_INDEX = 6;+const ffmpeg = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffmpeg.exe') : 'ffmpeg';+const directory = await mkdtemp(join(tmpdir(), 'capture-parity-'));+try {+ const videoPath = join(directory, 'clip.mp4');+ const stillPath = join(directory, 'seek.png');+ const decodedPath = join(directory, 'decoded.png');+ const common = ['--scene', 'film', '--backend', 'cpu', '--preset', 'cpu'];+ await renderMedia([...common, '--kind', 'video', '--capture', 'raw', '--duration', '1', '--output', videoPath]);+ await renderMedia([...common, '--kind', 'image', '--time', String(FRAME_INDEX / FPS), '--output', stillPath]);+ const decode = spawnSync(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', '-i', videoPath,+ '-vf', `select=eq(n\\,${FRAME_INDEX})`, '-frames:v', '1', decodedPath]);+ assert.equal(decode.status, 0, String(decode.stderr));+ const compare = spawnSync(ffmpeg, ['-hide_banner', '-i', decodedPath, '-i', stillPath, '-lavfi', 'psnr', '-f', 'null', '-'], { encoding: 'utf8' });+ const decibels = Number(/average:(\S+)/.exec(compare.stderr)?.[1]);+ assert.ok(decibels >= MINIMUM_PSNR_DECIBELS, `the raw frame matches its PNG seek at ${decibels} dB, below ${MINIMUM_PSNR_DECIBELS} dB`);+ console.log(`raw capture parity ok: ${decibels.toFixed(1)} dB`);+} finally {+ await rm(directory, { recursive: true, force: true });+}diff --git a/tests/export-options.test.js b/tests/export-options.test.jsindex e460c793..46b1e4fc 100644--- a/tests/export-options.test.js+++ b/tests/export-options.test.js@@ -55,12 +55,12 @@ test('invalid export options fail before starting resources', () => {]) assert.throws(() => parseRenderOptions(args));});-test('video defaults to JPEG 95 and auto encoding while stills retain lossless PNG', () => {+test('video defaults to raw drawing buffer reads and auto encoding while stills retain lossless PNG', () => {const video = parseRenderOptions(['--kind', 'video']);- assert.equal(video.captureFormat, 'jpeg');+ assert.equal(video.captureFormat, 'raw');assert.equal(video.jpegQuality, 95);assert.throws(() => parseRenderOptions(['--capture', 'raw']));- assert.equal(parseRenderOptions(['--kind', 'video', '--capture', 'raw']).captureFormat, 'raw');+ assert.equal(parseRenderOptions(['--kind', 'video', '--capture', 'jpeg']).captureFormat, 'jpeg');assert.equal(video.encoderBackend, 'auto');assert.equal(parseRenderOptions([]).captureFormat, 'png');assert.equal(parseRenderOptions(['--kind', 'stills']).captureFormat, 'png');