3D engine that AI agents drive
Encode video frames in the page with WebCodecs and send only H.264 chunks (part of #301)
Nguyen Xuan An66f5276b0e85
diff --git a/scripts/frame-encoder.mjs b/scripts/frame-encoder.mjsindex c2641ee6..728233da 100644--- a/scripts/frame-encoder.mjs+++ b/scripts/frame-encoder.mjs@@ -11,7 +11,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, formataudioArguments = [], width, height, signal, maximumFrameBytes = 1024 * 1024,maximumOutputBytes = 128 * 1024 * 1024, timeoutMs = DEFAULT_ENCODER_TIMEOUT_MS } = {}) {if (typeof output !== 'string' || !output || !Number.isFinite(fps) || fps < 1 || fps > 60 ||- !['jpeg', 'png', 'rawvideo'].includes(format) || !['cpu', 'gpu'].includes(backend)) {+ !['jpeg', 'png', 'rawvideo', 'h264'].includes(format) || !['cpu', 'gpu'].includes(backend)) {throw new TypeError('Encoder requires an output path, valid fps, format and backend.');}for (const value of [maximumFrameBytes, maximumOutputBytes, timeoutMs]) {@@ -22,12 +22,15 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format}signal?.throwIfAborted();const raw = format === 'rawvideo' ? rawFrameInputArguments({ width, height, fps }) : undefined;- const codec = backend === 'gpu'+ const encoded = format === 'h264';+ const codec = encoded+ ? ['-c:v', 'copy']+ : backend === 'gpu'? ['-c:v', 'h264_nvenc', '-preset', 'p5', '-cq', '20']: ['-c:v', 'libx264', '-preset', 'fast', '-crf', '18', '-threads', '2'];- const argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),+ const argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(encoded ? ['-f', 'h264', '-framerate', String(fps)] : raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),'-i', 'pipe:0', ...audioArguments, ...(raw ? raw.filter : []), ...codec,- '-pix_fmt', 'yuv420p', '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709',+ ...(encoded ? [] : ['-pix_fmt', 'yuv420p']), '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709','-movflags', '+faststart', ...(audioArguments.length ? ['-c:a', 'aac', '-b:a', '192k', '-shortest'] : []),'-f', 'mp4', output];const child = spawn(executable, argumentsValue, { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });@@ -36,7 +39,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, formatlet busy = false;let phase = 'writing';let closePromise;- const metrics = { frames: 0, maximumPendingFrames: 0, backpressureSeconds: 0, finishSeconds: 0 };+ const metrics = { frames: 0, chunks: 0, maximumPendingFrames: 0, backpressureSeconds: 0, finishSeconds: 0 };const retainError = value => { diagnostics = (diagnostics + String(value)).slice(-8192); };child.stderr.on('data', retainError);child.on('error', error => { processError = error; retainError(error.message); });@@ -121,6 +124,14 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, formatmetrics.frames++;} finally { busy = false; }},+ /** Appends an encoded H.264 chunk that the page already produced. Chunks are small, so the pipe buffer absorbs them. */+ writeChunk(bytes) {+ if (!encoded) throw new Error('Chunks are for encoder format h264.');+ if (phase !== 'writing') throw new Error('Encoder accepts no chunks after finalization.');+ check();+ child.stdin.write(bytes);+ metrics.chunks++;+ },async finish() {if (phase !== 'writing' || busy) throw new Error('Finish requires an idle encoder that has not been finalized.');check();diff --git a/scripts/gpu-frame-reader.mjs b/scripts/gpu-frame-reader.mjsindex ddd7505a..db71690d 100644--- a/scripts/gpu-frame-reader.mjs+++ b/scripts/gpu-frame-reader.mjs@@ -61,8 +61,14 @@ export function createFrameReceiver({ maximumBytes }) {const sockets = new WebSocketServer({ noServer: true, maxPayload: maximumBytes, perMessageDeflate: false });const acknowledgement = Buffer.from([1]);let received;+ let chunkHandler;sockets.on('connection', socket => {socket.on('message', (data, isBinary) => {+ // Encoded chunks go straight to the handler and are not acknowledged. The page's closing text message is.+ if (isBinary && chunkHandler) {+ chunkHandler(data);+ return;+ }if (isBinary) received = data;socket.send(acknowledgement);});@@ -74,6 +80,10 @@ export function createFrameReceiver({ maximumBytes }) {sockets.handleUpgrade(request, socket, head, connection => sockets.emit('connection', connection, request));return true;},+ /** Sends each encoded video chunk from the page to `handler(bytes)` in order. */+ onChunk(handler) {+ chunkHandler = handler;+ },/** The frame the page sent since the last call, checked against the size the encoder was started with. */take({ width, height }, info) {if (info.width !== width || info.height !== height) {@@ -107,3 +117,76 @@ export async function startFrameServer(receiver) {return { url: `ws://127.0.0.1:${server.address().port}${receiver.path}`,close: () => new Promise(resolve => { receiver.close(); server.closeAllConnections(); server.close(resolve); }) };}++/**+ * Runs in the page. Defines window.__startVideoEncoder(), __encodeFrame() and __finishVideoEncoder().+ * The page encodes each frame with the browser's hardware H.264 encoder, so only the encoded chunks, a few tens of+ * kilobytes a frame, travel over the socket instead of 3.7 MB of pixels (issue #301).+ */+function installVideoEncoder(url) {+ const maximumQueuedFrames = 3;+ const maximumBufferedBytes = 16 * 1024 ** 2;+ let encoder;+ let socket;+ let encoderFailure;+ let wakeAcknowledgement;+ let settings;+ const pause = () => new Promise(resolve => setTimeout(resolve, 1));+ // Waits for the GPU to finish the draw, so that draw time is not counted as encode time.+ window.__finishDraw = () => document.querySelector('canvas').getContext('webgl2').finish();+ window.__startVideoEncoder = async ({ width, height, fps, quantizer }) => {+ if (typeof VideoEncoder === 'undefined' || typeof VideoFrame === 'undefined') return { supported: false, reason: 'WebCodecs is unavailable.' };+ const level = width * height > 1920 * 1088 ? '33' : '28';+ const config = { codec: `avc1.6400${level}`, width, height, framerate: fps, bitrate: 20e6, bitrateMode: 'quantizer',+ hardwareAcceleration: 'prefer-hardware', latencyMode: 'quality', avc: { format: 'annexb' } };+ const support = await VideoEncoder.isConfigSupported(config);+ if (!support.supported) return { supported: false, reason: 'The browser cannot encode this size with a hardware H.264 encoder.' };+ socket = new WebSocket(url);+ socket.binaryType = 'arraybuffer';+ await new Promise((resolve, reject) => {+ socket.onopen = () => resolve();+ socket.onerror = () => reject(new Error('The video socket could not connect.'));+ });+ socket.onmessage = () => wakeAcknowledgement?.();+ encoder = new VideoEncoder({+ output: chunk => {+ const bytes = new Uint8Array(chunk.byteLength);+ chunk.copyTo(bytes);+ socket.send(bytes);+ },+ error: error => { encoderFailure = error; },+ });+ encoder.configure(config);+ settings = { fps, quantizer, keyFrameInterval: fps * 2 };+ return { supported: true };+ };+ // Call in the same task as the draw: the browser clears the drawing buffer once it presents the frame.+ window.__encodeFrame = async index => {+ if (encoderFailure) throw new Error(`The page video encoder failed: ${encoderFailure.message}`);+ const started = performance.now();+ const frame = new VideoFrame(document.querySelector('canvas'), { timestamp: Math.round(index * 1e6 / settings.fps) });+ encoder.encode(frame, { keyFrame: index % settings.keyFrameInterval === 0, avc: { quantizer: settings.quantizer } });+ frame.close();+ const submitMilliseconds = performance.now() - started;+ while (encoder.encodeQueueSize > maximumQueuedFrames || socket.bufferedAmount > maximumBufferedBytes) await pause();+ return { submitMilliseconds, waitMilliseconds: performance.now() - started - submitMilliseconds };+ };+ window.__finishVideoEncoder = async () => {+ await encoder.flush();+ if (encoderFailure) throw new Error(`The page video encoder failed: ${encoderFailure.message}`);+ const acknowledged = new Promise(resolve => { wakeAcknowledgement = resolve; });+ socket.send('end');+ await acknowledged;+ encoder.close();+ socket.close();+ };+}++/** Starts the page's hardware H.264 encoder. Returns { supported, reason }: when it is not supported, use readback frames. */+export async function startVideoEncoderInPage(page, url, { width, height, fps, quantizer }) {+ await page.evaluate(installVideoEncoder, url);+ return page.evaluate(options => window.__startVideoEncoder(options), { width, height, fps, quantizer });+}++/** Quantizer that matches the quality of the NVENC constant quality 20 files this pipeline wrote before. */+export const DEFAULT_PAGE_ENCODER_QUANTIZER = 20;diff --git a/scripts/render-media.mjs b/scripts/render-media.mjsindex ff1ca347..9b8949a2 100644--- a/scripts/render-media.mjs+++ b/scripts/render-media.mjs@@ -8,7 +8,7 @@ import { resolve, dirname, basename } from 'node:path';import { FILM_DURATION } from '../src/film.js';import { RENDER_PRESETS } from '../src/budget.js';import { createFrameEncoder } from './frame-encoder.mjs';-import { createFrameReceiver, installFrameReaderInPage, startFrameServer } from './gpu-frame-reader.mjs';+import { DEFAULT_PAGE_ENCODER_QUANTIZER, createFrameReceiver, installFrameReaderInPage, startFrameServer, startVideoEncoderInPage } from './gpu-frame-reader.mjs';function argumentsFrom(values) {const result = {};@@ -53,9 +53,9 @@ export function parseRenderOptions(values) {const start = numberOption('start', 0, 0, 3600);const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);const assembly = numberOption('assembly', 0, 0, 1);- const captureFormat = options.capture ?? (kind === 'video' ? 'raw' : 'png');- if (!['jpeg', 'png', 'raw'].includes(captureFormat)) throw new Error('Capture must be raw, jpeg or png.');- if (captureFormat === 'raw' && kind !== 'video') throw new Error('Raw capture is for video only. Stills are PNG or JPEG screenshots.');+ const captureFormat = options.capture ?? (kind === 'video' ? 'auto' : 'png');+ if (!['auto', 'webcodecs', 'raw', 'jpeg', 'png'].includes(captureFormat)) throw new Error('Capture must be auto, webcodecs, raw, jpeg or png.');+ if (['auto', 'webcodecs', 'raw'].includes(captureFormat) && kind !== 'video') throw new Error('Auto, webcodecs and raw capture are for video only. Stills are PNG or JPEG screenshots.');const jpegQuality = numberOption('jpeg-quality', 95, 1, 100);if (!Number.isInteger(jpegQuality)) throw new Error('JPEG quality must be an integer.');const output = resolve(options.output ?? `artifacts/${scene}-${backend}.${kind === 'video' ? 'mp4' : captureFormat === 'jpeg' ? 'jpg' : 'png'}`);@@ -111,7 +111,8 @@ function freePort() {export async function renderMedia(values = process.argv.slice(2)) {const plan = parseRenderOptions(values);- const { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, time, start, memory, assembly, output, captureFormat, jpegQuality } = plan;+ const { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, time, start, memory, assembly, output, jpegQuality } = plan;+ let captureFormat = plan.captureFormat;let duration = plan.duration;if (options.asset) {const details = await stat(resolve(options.asset));@@ -192,16 +193,26 @@ export async function renderMedia(values = process.argv.slice(2)) {} else window.renderAt = (time) => window.__nature.renderAt(time);}, { memory, width, height, samples: backend === 'cpu' ? 1 : preset.samples, scene, assembly });const session = await page.context().newCDPSession(page);- if (captureFormat === 'raw') {+ if (['auto', 'webcodecs', 'raw'].includes(captureFormat)) {frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });frameServer = await startFrameServer(frameReceiver);- await installFrameReaderInPage(page, frameServer.url);}+ if (['auto', 'webcodecs'].includes(captureFormat)) {+ // The page encodes with the browser's hardware H.264 encoder and sends only the small encoded chunks. Software+ // rendering or a CPU encoder request keeps the readback path, which feeds ffmpeg's libx264.+ const wantsCpu = backend === 'cpu' || options.encoder === 'cpu';+ const started = wantsCpu ? { supported: false, reason: 'A CPU render or encoder was requested.' }+ : await startVideoEncoderInPage(page, frameServer.url, { width, height, fps, quantizer: DEFAULT_PAGE_ENCODER_QUANTIZER });+ if (started.supported) captureFormat = 'webcodecs';+ else if (captureFormat === 'webcodecs') throw new Error(`Capture webcodecs is not available: ${started.reason}`);+ else captureFormat = 'raw';+ }+ if (captureFormat === 'raw') await installFrameReaderInPage(page, frameServer.url);// A raw frame is read from the drawing buffer in the same task as the draw. The round trip beyond the draw is capture.- const render = async (seconds) => {+ const render = async (seconds, frameIndex) => {if (errors.length) throw new Error(`Browser render errors: ${errors.join('; ')}`);const renderStarted = performance.now();- const drawn = await page.evaluate(async ({ value, readFrame }) => {+ const drawn = await page.evaluate(async ({ value, readFrame, encodeIndex }) => {const drawStarted = performance.now();if (window.__scene) window.__scene.renderAt(value);else if (window.renderAt) window.renderAt(value);@@ -209,13 +220,14 @@ export async function renderMedia(values = process.argv.slice(2)) {const renderer = window.__scene?.renderer ?? window.__film?.renderer ?? window.__nature?.renderer ?? window.__studio?.renderer;const glError = renderer.context.getError();if (glError !== 0) throw new Error(`WebGL render error ${glError}.`);- if (readFrame) window.__finishDraw();+ if (readFrame || encodeIndex !== null) window.__finishDraw();const drawMilliseconds = performance.now() - drawStarted;- return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined };- }, { value: seconds, readFrame: captureFormat === 'raw' });+ return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined,+ encoded: encodeIndex !== null ? await window.__encodeFrame(encodeIndex) : undefined };+ }, { value: seconds, readFrame: captureFormat === 'raw', encodeIndex: captureFormat === 'webcodecs' ? frameIndex : null });const rendererPlan = drawn.rendererPlan;const roundTripSeconds = (performance.now() - renderStarted) / 1000;- const drawSeconds = drawn.frame ? drawn.drawMilliseconds / 1000 : roundTripSeconds;+ const drawSeconds = drawn.frame || drawn.encoded ? drawn.drawMilliseconds / 1000 : roundTripSeconds;stageTimings.renderSeconds += drawSeconds;stageTimings.captureSeconds += roundTripSeconds - drawSeconds;stageTimings.readPixelsSeconds += (drawn.frame?.readMilliseconds ?? 0) / 1000;@@ -225,10 +237,14 @@ export async function renderMedia(values = process.argv.slice(2)) {const nodeBytes = process.memoryUsage().rss;peakNodeBytes = Math.max(peakNodeBytes, nodeBytes);if (nodeBytes > 512 * 1024 ** 2) throw new Error('Capture process exceeded 512 MiB. Export stopped before buffering more frames.');- return drawn.frame;+ return drawn;};- async function capture(seconds) {- const rawFrame = await render(seconds);+ async function capture(seconds, frameIndex) {+ const { frame: rawFrame, encoded } = await render(seconds, frameIndex);+ if (encoded) {+ capturedFrames++;+ return undefined;+ }if (rawFrame) {capturedFrames++;const bytes = frameReceiver.take({ width, height }, rawFrame);@@ -251,15 +267,16 @@ export async function renderMedia(values = process.argv.slice(2)) {const audioPath = plan.audio ? resolve(plan.audio) : null;if (options.audio && !existsSync(audioPath)) throw new Error('The selected audio file was not found.');const audioArguments = audioPath && existsSync(audioPath) ? ['-ss', String(start), '-i', audioPath] : [];- encoder = createFrameEncoder({ executable: ffmpeg, output, fps, format: captureFormat === 'raw' ? 'rawvideo' : captureFormat,+ encoder = createFrameEncoder({ executable: ffmpeg, output, fps, format: captureFormat === 'raw' ? 'rawvideo' : captureFormat === 'webcodecs' ? 'h264' : captureFormat,width, height, backend: encoderBackend, audioArguments, maximumFrameBytes: 64 * 1024 * 1024,maximumOutputBytes: Number.MAX_SAFE_INTEGER });const frameCount = Math.ceil(duration * fps);+ if (captureFormat === 'webcodecs') frameReceiver.onChunk(chunk => encoder.writeChunk(chunk));const captureSequenceStarted = performance.now();for (let frame = 0; frame < frameCount; frame++) {- const image = await capture(start + frame / fps);+ const image = await capture(start + frame / fps, frame);maximumPendingFrames = Math.max(maximumPendingFrames, 1);- await encoder.writeFrame(image);+ if (image) await encoder.writeFrame(image);stageTimings.encoderBackpressureSeconds = encoder.metrics.backpressureSeconds;if (frame % 48 === 0) {const { usedSize } = await session.send('Runtime.getHeapUsage');@@ -272,6 +289,7 @@ export async function renderMedia(values = process.argv.slice(2)) {console.log(message);}}+ if (captureFormat === 'webcodecs') await page.evaluate(() => window.__finishVideoEncoder());const encoding = await encoder.finish();stageTimings.encoderFinishSeconds = encoding.finishSeconds;} else if (kind === 'stills') {@@ -295,7 +313,7 @@ export async function renderMedia(values = process.argv.slice(2)) {if (graphics.glError !== 0) throw new Error(`WebGL render error ${graphics.glError}.`);if (backend === 'cpu' && !graphics.gpu.toLowerCase().includes('swiftshader')) throw new Error('CPU backend did not select SwiftShader.');const elapsedSeconds = (performance.now() - started) / 1000;- const report = { scene, kind, backend, preset: presetName, aspect, start, encoder: encoderBackend, requestedEncoder: kind === 'video' ? plan.encoderBackend : null, captureFormat, jpegQuality: captureFormat === 'jpeg' ? jpegQuality : null, capturedFrames, stageTimings, effectiveFPS: capturedFrames / elapsedSeconds, speedRatio: kind === 'video' ? capturedFrames / fps / elapsedSeconds : null, assembly: scene === 'object' ? assembly : null, width, height, actualRenderSize, fps, duration: kind === 'video' ? duration : null, memoryBudgetMiB: memory, peakEstimatedRenderBytes, peakNodeBytes, peakBrowserHeap, maximumPendingFrames, firstFrameBytes, elapsedSeconds, graphics, output: basename(output), errors };+ const report = { scene, kind, backend, preset: presetName, aspect, start, encoder: captureFormat === 'webcodecs' ? 'browser-h264' : encoderBackend, requestedEncoder: kind === 'video' ? plan.encoderBackend : null, captureFormat, jpegQuality: captureFormat === 'jpeg' ? jpegQuality : null, capturedFrames, stageTimings, effectiveFPS: capturedFrames / elapsedSeconds, speedRatio: kind === 'video' ? capturedFrames / fps / elapsedSeconds : null, assembly: scene === 'object' ? assembly : null, width, height, actualRenderSize, fps, duration: kind === 'video' ? duration : null, memoryBudgetMiB: memory, peakEstimatedRenderBytes, peakNodeBytes, peakBrowserHeap, maximumPendingFrames, firstFrameBytes, elapsedSeconds, graphics, output: basename(output), errors };await writeFile(`${output}.json`, JSON.stringify(report, null, 2));console.log(JSON.stringify(report, null, 2));} finally {