Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Send raw video frames over a binary WebSocket and make raw capture the video default

Nguyen Xuan Anc4ee1129dd01
diff --git a/package-lock.json b/package-lock.json
index d1b94cb3..ceb680f7 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -30,6 +30,7 @@
"three-mesh-bvh": "0.9.16",
"vite": "8.3.5",
"wrangler": "4.156.0",
+ "ws": "^8.21.0",
"zod": "4.6.7"
},
"engines": {
diff --git a/package.json b/package.json
index 2ed1dd62..b6bb51ea 100644
--- a/package.json
+++ b/package.json
@@ -124,6 +124,7 @@
"smoke:agent-cloth": "node scripts/smoke-agent-cloth.mjs",
"smoke:agent-particles": "node scripts/smoke-agent-particles.mjs",
"smoke:frame-encoder": "node scripts/smoke-frame-encoder.mjs",
+ "smoke:capture-parity": "node scripts/smoke-capture-parity.mjs",
"smoke:enhancements": "node scripts/smoke-enhancements.mjs",
"smoke:site": "node scripts/smoke-site.mjs",
"smoke:agent-review-close": "node scripts/smoke-agent-review-close.mjs",
@@ -179,6 +180,7 @@
"three-mesh-bvh": "0.9.16",
"vite": "8.3.5",
"wrangler": "4.156.0",
+ "ws": "^8.21.0",
"zod": "4.6.7"
}
}
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index c057a035..c859039a 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -12,6 +12,7 @@ import { chooseDenoiser, finishWithOidn } from './agent-path-trace-denoise.mjs';
import { createAgentAssetCatalog } from './agent-asset-catalog.mjs';
import { createAgentRenderJobs, EXPORT_DIRECTORY_NAME } from './agent-render-jobs.mjs';
import { createFrameEncoder } from './frame-encoder.mjs';
+import { createFrameReceiver, installFrameReaderInPage } from './gpu-frame-reader.mjs';
import { audioInputArguments, createAudioLibrary, measureStreamSeconds } from './job-audio.mjs';
import { filmTimelineDuration, filmTimelineShotStarts, sampleFilmTimeline } from '../src/film-timeline.js';
import { isPreviewHost } from './preview-host.mjs';
@@ -82,6 +83,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
let phase = 'idle';
let browserInstance;
let httpServer;
+ // The job that is capturing video frames, if any. Its socket path is secret and lives only as long as the job.
+ let activeFrameReceiver;
let page;
let reviewPage;
let cancelReview;
@@ -230,6 +233,11 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
response.setHeader('Content-Type', request.url === '/' ? 'text/html; charset=utf-8' : 'application/javascript; charset=utf-8');
response.end(request.url === '/' ? html : code);
});
+ httpServer.on('upgrade', (request, socket, head) => {
+ const accepted = isPreviewHost(request.headers.host, httpServer.address().port)
+ && activeFrameReceiver?.handleUpgrade(request, socket, head);
+ if (!accepted) socket.destroy();
+ });
await new Promise((resolve, reject) => {
httpServer.once('error', reject);
httpServer.listen(0, '127.0.0.1', resolve);
@@ -314,6 +322,7 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
async function runJob({ jobId, directory, snapshot, settings, signal, progress }) {
let jobPage;
let encoder;
+ let frameReceiver;
let suspended = false;
const sessionId = activeSessionId;
const started = performance.now();
@@ -348,8 +357,12 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
const timeline = settings.timeline ? snapshot.recipe.timeline : undefined;
const audio = settings.audio ? await prepareAudio(settings) : undefined;
if (settings.kind === 'video') {
+ // Video frames are read from the drawing buffer and sent over a socket on the preview's own port (issue #301).
+ frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });
+ activeFrameReceiver = frameReceiver;
+ await installFrameReaderInPage(jobPage, `ws://127.0.0.1:${httpServer.address().port}${frameReceiver.path}`);
encoder = createFrameEncoder({ executable: ffmpeg, output: join(directory, 'render.mp4.partial'),
- fps: settings.fps, format: 'jpeg', signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',
+ fps: settings.fps, format: 'rawvideo', width, height, signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',
audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [] });
metrics.videoEncoder = settings.encoder === 'nvenc' ? 'h264_nvenc' : 'libx264';
}
@@ -388,14 +401,29 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
await progress({ frames: 1, maximumPendingFrames: 1 });
continue;
}
- const rendered = await jobPage.evaluate(input => window.__agentPreview.frame(input), frameInput);
- metrics.renderSeconds += (performance.now() - renderStarted) / 1000;
+ // A video frame is read in the same task as its draw, because the browser clears the drawing buffer after presenting.
+ const rendered = await jobPage.evaluate(async ({ input, readFrame }) => {
+ const drawStarted = performance.now();
+ const result = await window.__agentPreview.frame(input);
+ if (!readFrame) return result;
+ window.__finishDraw();
+ const drawMilliseconds = performance.now() - drawStarted;
+ return { ...result, drawMilliseconds, rawFrame: await window.__readFrame() };
+ }, { input: frameInput, readFrame: settings.kind === 'video' });
+ const roundTripSeconds = (performance.now() - renderStarted) / 1000;
+ const drawSeconds = rendered.rawFrame ? rendered.drawMilliseconds / 1000 : roundTripSeconds;
+ metrics.renderSeconds += drawSeconds;
+ metrics.captureSeconds += roundTripSeconds - drawSeconds;
if (backend === 'cpu' && !/swiftshader/i.test(rendered.graphics)) throw new Error('CPU job did not select SwiftShader.');
metrics.graphics = rendered.graphics;
metrics.width = width;
metrics.height = height;
metrics.estimatedBytes = rendered.estimatedBytes;
async function captureFrameBytes() {
+ if (rendered.rawFrame) {
+ signal.throwIfAborted();
+ return frameReceiver.take({ width, height }, rendered.rawFrame);
+ }
const captureStarted = performance.now();
const captured = await session.send('Page.captureScreenshot', { fromSurface: true,
format: settings.kind === 'video' ? 'jpeg' : 'png',
@@ -444,6 +472,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
return { artifacts, metrics };
} finally {
signal.removeEventListener('abort', abort);
+ activeFrameReceiver = undefined;
+ frameReceiver?.close();
try { await encoder?.close(); }
finally {
try {
diff --git a/scripts/gpu-frame-reader.mjs b/scripts/gpu-frame-reader.mjs
index 8aefbe77..ddd7505a 100644
--- a/scripts/gpu-frame-reader.mjs
+++ b/scripts/gpu-frame-reader.mjs
@@ -1,37 +1,49 @@
// Raw video frames straight from the drawing buffer, shared by the MCP render job and scripts/render-media.mjs.
// A DevTools screenshot makes the compositor encode a JPEG and send it as base64, 84 percent of a film job (issue #301).
// A read of the drawing buffer skips both. It must run in the same task as the draw, because the browser clears the
-// drawing buffer once it has presented the frame. The page then posts the bytes to a Node receiver as a binary body:
-// returning a typed array from Playwright's evaluate costs about 50 ms for a 1280 by 720 frame, a post costs a few.
+// drawing buffer once it has presented the frame. The page then sends the bytes to Node over one binary WebSocket and
+// waits for a one byte acknowledgement, so one frame is in flight at a time and the encoder sets the pace.
+// Returning a typed array from Playwright's evaluate costs about 50 ms for a 1280 by 720 frame. A fetch with a Blob body
+// delivered 100 to 150 frames and then failed with "Failed to fetch", so a socket replaces it.
import { randomBytes } from 'node:crypto';
import { createServer } from 'node:http';
+import { WebSocketServer } from 'ws';
-/** Runs in the page. Defines window.__finishDraw() and window.__readFrame(url), which posts the bottom-up RGBA pixels. */
-function installFrameReader() {
+/** Runs in the page. Defines window.__finishDraw() and window.__readFrame(), which sends the bottom-up RGBA pixels. */
+function installFrameReader(url) {
let pixels;
+ let socket;
+ let pendingAcknowledgement;
const canvasContext = () => document.querySelector('canvas').getContext('webgl2');
// Waits for the GPU to finish the draw, so that draw time is not counted as read time.
window.__finishDraw = () => canvasContext().finish();
- window.__readFrame = async (url) => {
+ window.__connectFrameSocket = () => new Promise((resolve, reject) => {
+ socket = new WebSocket(url);
+ socket.binaryType = 'arraybuffer';
+ socket.onopen = () => resolve();
+ socket.onerror = () => reject(new Error('The frame socket could not connect.'));
+ socket.onclose = () => pendingAcknowledgement?.reject(new Error('The frame socket closed.'));
+ socket.onmessage = () => pendingAcknowledgement?.resolve();
+ });
+ window.__readFrame = async () => {
const context = canvasContext();
const { drawingBufferWidth: width, drawingBufferHeight: height } = context;
if (!pixels || pixels.length !== width * height * 4) pixels = new Uint8Array(width * height * 4);
const readStarted = performance.now();
context.readPixels(0, 0, width, height, context.RGBA, context.UNSIGNED_BYTE, pixels);
const readMilliseconds = performance.now() - readStarted;
- // A Blob body: Playwright keeps a copy of every typed-array body it sees (3.7 MB a frame) until the page closes.
- // A connection that the server closed while idle fails the first post, so one retry follows.
- let response;
- try { response = await fetch(url, { method: 'POST', body: new Blob([pixels]) }); }
- catch { response = await fetch(url, { method: 'POST', body: new Blob([pixels]) }); }
- if (!response.ok) throw new Error(`The frame receiver answered ${response.status}.`);
+ const acknowledged = new Promise((resolve, reject) => { pendingAcknowledgement = { resolve, reject }; });
+ socket.send(pixels);
+ await acknowledged;
+ pendingAcknowledgement = undefined;
return { width, height, readMilliseconds };
};
}
-/** Call once per page before the first frame. */
-export async function installFrameReaderInPage(page) {
- await page.evaluate(installFrameReader);
+/** Call once per page before the first frame. `url` is the receiver's socket address. */
+export async function installFrameReaderInPage(page, url) {
+ await page.evaluate(installFrameReader, url);
+ await page.evaluate(() => window.__connectFrameSocket());
}
/** The ffmpeg input arguments for raw frames. GL rows run bottom-up, so the picture is flipped, and RGB converts with BT.709. */
@@ -40,51 +52,29 @@ export function rawFrameInputArguments({ width, height, fps }) {
filter: ['-vf', 'vflip,scale=out_color_matrix=bt709:out_range=tv'] };
}
-/** A slow frame on the CPU backend takes longer than the default five seconds a server keeps an idle connection. */
-export const KEEP_ALIVE_MILLISECONDS = 120000;
-
/**
- * Receives one posted frame at a time on a path nobody else knows. `handle(request, response)` returns true when it
- * took the request. `crossOrigin` answers with the headers a page on another port needs.
+ * Receives one frame at a time on a path nobody else knows. `handleUpgrade(request, socket, head)` returns true when it
+ * took the connection, so a server that already serves the page can share its port.
*/
-export function createFrameReceiver({ maximumBytes, crossOrigin = false }) {
+export function createFrameReceiver({ maximumBytes }) {
const path = `/__frame/${randomBytes(12).toString('hex')}`;
- // One buffer is reused for every frame, so a long film does not leave a buffer of garbage per frame for the collector.
- const frameBuffer = Buffer.allocUnsafe(maximumBytes);
+ const sockets = new WebSocketServer({ noServer: true, maxPayload: maximumBytes, perMessageDeflate: false });
+ const acknowledgement = Buffer.from([1]);
let received;
- function handle(request, response) {
- if (request.url !== path) return false;
- if (crossOrigin) response.setHeader('Access-Control-Allow-Origin', '*');
- if (request.method !== 'POST') {
- response.writeHead(405);
- response.end();
- return true;
- }
- let total = 0;
- let tooLarge = false;
- request.on('data', chunk => {
- if (tooLarge) return;
- if (total + chunk.length > maximumBytes) {
- tooLarge = true;
- response.writeHead(413);
- response.end();
- return;
- }
- chunk.copy(frameBuffer, total);
- total += chunk.length;
- });
- request.on('end', () => {
- if (tooLarge) return;
- received = frameBuffer.subarray(0, total);
- response.writeHead(204);
- response.end();
+ sockets.on('connection', socket => {
+ socket.on('message', (data, isBinary) => {
+ if (isBinary) received = data;
+ socket.send(acknowledgement);
});
- return true;
- }
+ });
return Object.freeze({
path,
- handle,
- /** The frame the page posted since the last call, checked against the size the encoder was started with. */
+ handleUpgrade(request, socket, head) {
+ if (request.url !== path) return false;
+ sockets.handleUpgrade(request, socket, head, connection => sockets.emit('connection', connection, request));
+ return true;
+ },
+ /** The frame the page sent since the last call, checked against the size the encoder was started with. */
take({ width, height }, info) {
if (info.width !== width || info.height !== height) {
throw new Error(`The drawing buffer is ${info.width}x${info.height} but the video is ${width}x${height}.`);
@@ -94,22 +84,26 @@ export function createFrameReceiver({ maximumBytes, crossOrigin = false }) {
if (!frame || frame.length !== width * height * 4) throw new Error('The page did not deliver a complete frame.');
return frame;
},
+ close() {
+ for (const client of sockets.clients) client.terminate();
+ sockets.close();
+ },
});
}
/** A loopback server that only receives frames, for pages that another server already serves. */
export async function startFrameServer(receiver) {
const server = createServer((request, response) => {
- if (!receiver.handle(request, response)) {
- response.writeHead(404);
- response.end();
- }
+ response.writeHead(404);
+ response.end();
+ });
+ server.on('upgrade', (request, socket, head) => {
+ if (!receiver.handleUpgrade(request, socket, head)) socket.destroy();
});
await new Promise((resolve, reject) => {
- server.keepAliveTimeout = KEEP_ALIVE_MILLISECONDS;
server.once('error', reject);
server.listen(0, '127.0.0.1', resolve);
});
- return { url: `http://127.0.0.1:${server.address().port}${receiver.path}`,
- close: () => new Promise(resolve => { server.closeAllConnections(); server.close(resolve); }) };
+ return { url: `ws://127.0.0.1:${server.address().port}${receiver.path}`,
+ close: () => new Promise(resolve => { receiver.close(); server.closeAllConnections(); server.close(resolve); }) };
}
diff --git a/scripts/render-media.mjs b/scripts/render-media.mjs
index 3b002daa..ff1ca347 100644
--- a/scripts/render-media.mjs
+++ b/scripts/render-media.mjs
@@ -53,7 +53,7 @@ export function parseRenderOptions(values) {
const start = numberOption('start', 0, 0, 3600);
const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);
const assembly = numberOption('assembly', 0, 0, 1);
- const captureFormat = options.capture ?? (kind === 'video' ? 'jpeg' : 'png');
+ const captureFormat = options.capture ?? (kind === 'video' ? 'raw' : 'png');
if (!['jpeg', 'png', 'raw'].includes(captureFormat)) throw new Error('Capture must be raw, jpeg or png.');
if (captureFormat === 'raw' && kind !== 'video') throw new Error('Raw capture is for video only. Stills are PNG or JPEG screenshots.');
const jpegQuality = numberOption('jpeg-quality', 95, 1, 100);
@@ -193,15 +193,15 @@ export async function renderMedia(values = process.argv.slice(2)) {
}, { memory, width, height, samples: backend === 'cpu' ? 1 : preset.samples, scene, assembly });
const session = await page.context().newCDPSession(page);
if (captureFormat === 'raw') {
- await installFrameReaderInPage(page);
- frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4, crossOrigin: true });
+ frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });
frameServer = await startFrameServer(frameReceiver);
+ await installFrameReaderInPage(page, frameServer.url);
}
// A raw frame is read from the drawing buffer in the same task as the draw. The round trip beyond the draw is capture.
const render = async (seconds) => {
if (errors.length) throw new Error(`Browser render errors: ${errors.join('; ')}`);
const renderStarted = performance.now();
- const drawn = await page.evaluate(async ({ value, readFrame, receiverUrl }) => {
+ const drawn = await page.evaluate(async ({ value, readFrame }) => {
const drawStarted = performance.now();
if (window.__scene) window.__scene.renderAt(value);
else if (window.renderAt) window.renderAt(value);
@@ -211,8 +211,8 @@ export async function renderMedia(values = process.argv.slice(2)) {
if (glError !== 0) throw new Error(`WebGL render error ${glError}.`);
if (readFrame) window.__finishDraw();
const drawMilliseconds = performance.now() - drawStarted;
- return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame(receiverUrl) : undefined };
- }, { value: seconds, readFrame: captureFormat === 'raw', receiverUrl: frameServer?.url });
+ return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined };
+ }, { value: seconds, readFrame: captureFormat === 'raw' });
const rendererPlan = drawn.rendererPlan;
const roundTripSeconds = (performance.now() - renderStarted) / 1000;
const drawSeconds = drawn.frame ? drawn.drawMilliseconds / 1000 : roundTripSeconds;
diff --git a/scripts/smoke-capture-parity.mjs b/scripts/smoke-capture-parity.mjs
new file mode 100644
index 00000000..678f774b
--- /dev/null
+++ b/scripts/smoke-capture-parity.mjs
@@ -0,0 +1,30 @@
+// A raw video frame must match the PNG seek of the same time. Both come from SwiftShader, so no GPU is needed (issue #301).
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { renderMedia } from './render-media.mjs';
+
+const MINIMUM_PSNR_DECIBELS = 30;
+const FPS = 12;
+const FRAME_INDEX = 6;
+const ffmpeg = process.platform === 'win32' ? resolve(process.env.USERPROFILE, '.local/bin/ffmpeg.exe') : 'ffmpeg';
+const directory = await mkdtemp(join(tmpdir(), 'capture-parity-'));
+try {
+ const videoPath = join(directory, 'clip.mp4');
+ const stillPath = join(directory, 'seek.png');
+ const decodedPath = join(directory, 'decoded.png');
+ const common = ['--scene', 'film', '--backend', 'cpu', '--preset', 'cpu'];
+ await renderMedia([...common, '--kind', 'video', '--capture', 'raw', '--duration', '1', '--output', videoPath]);
+ await renderMedia([...common, '--kind', 'image', '--time', String(FRAME_INDEX / FPS), '--output', stillPath]);
+ const decode = spawnSync(ffmpeg, ['-hide_banner', '-loglevel', 'error', '-y', '-i', videoPath,
+ '-vf', `select=eq(n\\,${FRAME_INDEX})`, '-frames:v', '1', decodedPath]);
+ assert.equal(decode.status, 0, String(decode.stderr));
+ const compare = spawnSync(ffmpeg, ['-hide_banner', '-i', decodedPath, '-i', stillPath, '-lavfi', 'psnr', '-f', 'null', '-'], { encoding: 'utf8' });
+ const decibels = Number(/average:(\S+)/.exec(compare.stderr)?.[1]);
+ assert.ok(decibels >= MINIMUM_PSNR_DECIBELS, `the raw frame matches its PNG seek at ${decibels} dB, below ${MINIMUM_PSNR_DECIBELS} dB`);
+ console.log(`raw capture parity ok: ${decibels.toFixed(1)} dB`);
+} finally {
+ await rm(directory, { recursive: true, force: true });
+}
diff --git a/tests/export-options.test.js b/tests/export-options.test.js
index e460c793..46b1e4fc 100644
--- a/tests/export-options.test.js
+++ b/tests/export-options.test.js
@@ -55,12 +55,12 @@ test('invalid export options fail before starting resources', () => {
]) assert.throws(() => parseRenderOptions(args));
});
-test('video defaults to JPEG 95 and auto encoding while stills retain lossless PNG', () => {
+test('video defaults to raw drawing buffer reads and auto encoding while stills retain lossless PNG', () => {
const video = parseRenderOptions(['--kind', 'video']);
- assert.equal(video.captureFormat, 'jpeg');
+ assert.equal(video.captureFormat, 'raw');
assert.equal(video.jpegQuality, 95);
assert.throws(() => parseRenderOptions(['--capture', 'raw']));
- assert.equal(parseRenderOptions(['--kind', 'video', '--capture', 'raw']).captureFormat, 'raw');
+ assert.equal(parseRenderOptions(['--kind', 'video', '--capture', 'jpeg']).captureFormat, 'jpeg');
assert.equal(video.encoderBackend, 'auto');
assert.equal(parseRenderOptions([]).captureFormat, 'png');
assert.equal(parseRenderOptions(['--kind', 'stills']).captureFormat, 'png');