Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Encode video frames in the page with WebCodecs and send only H.264 chunks (part of #301)

Nguyen Xuan An66f5276b0e85
diff --git a/scripts/frame-encoder.mjs b/scripts/frame-encoder.mjs
index c2641ee6..728233da 100644
--- a/scripts/frame-encoder.mjs
+++ b/scripts/frame-encoder.mjs
@@ -11,7 +11,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format
audioArguments = [], width, height, signal, maximumFrameBytes = 1024 * 1024,
maximumOutputBytes = 128 * 1024 * 1024, timeoutMs = DEFAULT_ENCODER_TIMEOUT_MS } = {}) {
if (typeof output !== 'string' || !output || !Number.isFinite(fps) || fps < 1 || fps > 60 ||
- !['jpeg', 'png', 'rawvideo'].includes(format) || !['cpu', 'gpu'].includes(backend)) {
+ !['jpeg', 'png', 'rawvideo', 'h264'].includes(format) || !['cpu', 'gpu'].includes(backend)) {
throw new TypeError('Encoder requires an output path, valid fps, format and backend.');
}
for (const value of [maximumFrameBytes, maximumOutputBytes, timeoutMs]) {
@@ -22,12 +22,15 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format
}
signal?.throwIfAborted();
const raw = format === 'rawvideo' ? rawFrameInputArguments({ width, height, fps }) : undefined;
- const codec = backend === 'gpu'
+ const encoded = format === 'h264';
+ const codec = encoded
+ ? ['-c:v', 'copy']
+ : backend === 'gpu'
? ['-c:v', 'h264_nvenc', '-preset', 'p5', '-cq', '20']
: ['-c:v', 'libx264', '-preset', 'fast', '-crf', '18', '-threads', '2'];
- const argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),
+ const argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(encoded ? ['-f', 'h264', '-framerate', String(fps)] : raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),
'-i', 'pipe:0', ...audioArguments, ...(raw ? raw.filter : []), ...codec,
- '-pix_fmt', 'yuv420p', '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709',
+ ...(encoded ? [] : ['-pix_fmt', 'yuv420p']), '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709',
'-movflags', '+faststart', ...(audioArguments.length ? ['-c:a', 'aac', '-b:a', '192k', '-shortest'] : []),
'-f', 'mp4', output];
const child = spawn(executable, argumentsValue, { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });
@@ -36,7 +39,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format
let busy = false;
let phase = 'writing';
let closePromise;
- const metrics = { frames: 0, maximumPendingFrames: 0, backpressureSeconds: 0, finishSeconds: 0 };
+ const metrics = { frames: 0, chunks: 0, maximumPendingFrames: 0, backpressureSeconds: 0, finishSeconds: 0 };
const retainError = value => { diagnostics = (diagnostics + String(value)).slice(-8192); };
child.stderr.on('data', retainError);
child.on('error', error => { processError = error; retainError(error.message); });
@@ -121,6 +124,14 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format
metrics.frames++;
} finally { busy = false; }
},
+ /** Appends an encoded H.264 chunk that the page already produced. Chunks are small, so the pipe buffer absorbs them. */
+ writeChunk(bytes) {
+ if (!encoded) throw new Error('Chunks are for encoder format h264.');
+ if (phase !== 'writing') throw new Error('Encoder accepts no chunks after finalization.');
+ check();
+ child.stdin.write(bytes);
+ metrics.chunks++;
+ },
async finish() {
if (phase !== 'writing' || busy) throw new Error('Finish requires an idle encoder that has not been finalized.');
check();
diff --git a/scripts/gpu-frame-reader.mjs b/scripts/gpu-frame-reader.mjs
index ddd7505a..db71690d 100644
--- a/scripts/gpu-frame-reader.mjs
+++ b/scripts/gpu-frame-reader.mjs
@@ -61,8 +61,14 @@ export function createFrameReceiver({ maximumBytes }) {
const sockets = new WebSocketServer({ noServer: true, maxPayload: maximumBytes, perMessageDeflate: false });
const acknowledgement = Buffer.from([1]);
let received;
+ let chunkHandler;
sockets.on('connection', socket => {
socket.on('message', (data, isBinary) => {
+ // Encoded chunks go straight to the handler and are not acknowledged. The page's closing text message is.
+ if (isBinary && chunkHandler) {
+ chunkHandler(data);
+ return;
+ }
if (isBinary) received = data;
socket.send(acknowledgement);
});
@@ -74,6 +80,10 @@ export function createFrameReceiver({ maximumBytes }) {
sockets.handleUpgrade(request, socket, head, connection => sockets.emit('connection', connection, request));
return true;
},
+ /** Sends each encoded video chunk from the page to `handler(bytes)` in order. */
+ onChunk(handler) {
+ chunkHandler = handler;
+ },
/** The frame the page sent since the last call, checked against the size the encoder was started with. */
take({ width, height }, info) {
if (info.width !== width || info.height !== height) {
@@ -107,3 +117,76 @@ export async function startFrameServer(receiver) {
return { url: `ws://127.0.0.1:${server.address().port}${receiver.path}`,
close: () => new Promise(resolve => { receiver.close(); server.closeAllConnections(); server.close(resolve); }) };
}
+
+/**
+ * Runs in the page. Defines window.__startVideoEncoder(), __encodeFrame() and __finishVideoEncoder().
+ * The page encodes each frame with the browser's hardware H.264 encoder, so only the encoded chunks, a few tens of
+ * kilobytes a frame, travel over the socket instead of 3.7 MB of pixels (issue #301).
+ */
+function installVideoEncoder(url) {
+ const maximumQueuedFrames = 3;
+ const maximumBufferedBytes = 16 * 1024 ** 2;
+ let encoder;
+ let socket;
+ let encoderFailure;
+ let wakeAcknowledgement;
+ let settings;
+ const pause = () => new Promise(resolve => setTimeout(resolve, 1));
+ // Waits for the GPU to finish the draw, so that draw time is not counted as encode time.
+ window.__finishDraw = () => document.querySelector('canvas').getContext('webgl2').finish();
+ window.__startVideoEncoder = async ({ width, height, fps, quantizer }) => {
+ if (typeof VideoEncoder === 'undefined' || typeof VideoFrame === 'undefined') return { supported: false, reason: 'WebCodecs is unavailable.' };
+ const level = width * height > 1920 * 1088 ? '33' : '28';
+ const config = { codec: `avc1.6400${level}`, width, height, framerate: fps, bitrate: 20e6, bitrateMode: 'quantizer',
+ hardwareAcceleration: 'prefer-hardware', latencyMode: 'quality', avc: { format: 'annexb' } };
+ const support = await VideoEncoder.isConfigSupported(config);
+ if (!support.supported) return { supported: false, reason: 'The browser cannot encode this size with a hardware H.264 encoder.' };
+ socket = new WebSocket(url);
+ socket.binaryType = 'arraybuffer';
+ await new Promise((resolve, reject) => {
+ socket.onopen = () => resolve();
+ socket.onerror = () => reject(new Error('The video socket could not connect.'));
+ });
+ socket.onmessage = () => wakeAcknowledgement?.();
+ encoder = new VideoEncoder({
+ output: chunk => {
+ const bytes = new Uint8Array(chunk.byteLength);
+ chunk.copyTo(bytes);
+ socket.send(bytes);
+ },
+ error: error => { encoderFailure = error; },
+ });
+ encoder.configure(config);
+ settings = { fps, quantizer, keyFrameInterval: fps * 2 };
+ return { supported: true };
+ };
+ // Call in the same task as the draw: the browser clears the drawing buffer once it presents the frame.
+ window.__encodeFrame = async index => {
+ if (encoderFailure) throw new Error(`The page video encoder failed: ${encoderFailure.message}`);
+ const started = performance.now();
+ const frame = new VideoFrame(document.querySelector('canvas'), { timestamp: Math.round(index * 1e6 / settings.fps) });
+ encoder.encode(frame, { keyFrame: index % settings.keyFrameInterval === 0, avc: { quantizer: settings.quantizer } });
+ frame.close();
+ const submitMilliseconds = performance.now() - started;
+ while (encoder.encodeQueueSize > maximumQueuedFrames || socket.bufferedAmount > maximumBufferedBytes) await pause();
+ return { submitMilliseconds, waitMilliseconds: performance.now() - started - submitMilliseconds };
+ };
+ window.__finishVideoEncoder = async () => {
+ await encoder.flush();
+ if (encoderFailure) throw new Error(`The page video encoder failed: ${encoderFailure.message}`);
+ const acknowledged = new Promise(resolve => { wakeAcknowledgement = resolve; });
+ socket.send('end');
+ await acknowledged;
+ encoder.close();
+ socket.close();
+ };
+}
+
+/** Starts the page's hardware H.264 encoder. Returns { supported, reason }: when it is not supported, use readback frames. */
+export async function startVideoEncoderInPage(page, url, { width, height, fps, quantizer }) {
+ await page.evaluate(installVideoEncoder, url);
+ return page.evaluate(options => window.__startVideoEncoder(options), { width, height, fps, quantizer });
+}
+
+/** Quantizer that matches the quality of the NVENC constant quality 20 files this pipeline wrote before. */
+export const DEFAULT_PAGE_ENCODER_QUANTIZER = 20;
diff --git a/scripts/render-media.mjs b/scripts/render-media.mjs
index ff1ca347..9b8949a2 100644
--- a/scripts/render-media.mjs
+++ b/scripts/render-media.mjs
@@ -8,7 +8,7 @@ import { resolve, dirname, basename } from 'node:path';
import { FILM_DURATION } from '../src/film.js';
import { RENDER_PRESETS } from '../src/budget.js';
import { createFrameEncoder } from './frame-encoder.mjs';
-import { createFrameReceiver, installFrameReaderInPage, startFrameServer } from './gpu-frame-reader.mjs';
+import { DEFAULT_PAGE_ENCODER_QUANTIZER, createFrameReceiver, installFrameReaderInPage, startFrameServer, startVideoEncoderInPage } from './gpu-frame-reader.mjs';
function argumentsFrom(values) {
const result = {};
@@ -53,9 +53,9 @@ export function parseRenderOptions(values) {
const start = numberOption('start', 0, 0, 3600);
const memory = numberOption('memory', preset.memoryBudgetMiB, 32, 1024);
const assembly = numberOption('assembly', 0, 0, 1);
- const captureFormat = options.capture ?? (kind === 'video' ? 'raw' : 'png');
- if (!['jpeg', 'png', 'raw'].includes(captureFormat)) throw new Error('Capture must be raw, jpeg or png.');
- if (captureFormat === 'raw' && kind !== 'video') throw new Error('Raw capture is for video only. Stills are PNG or JPEG screenshots.');
+ const captureFormat = options.capture ?? (kind === 'video' ? 'auto' : 'png');
+ if (!['auto', 'webcodecs', 'raw', 'jpeg', 'png'].includes(captureFormat)) throw new Error('Capture must be auto, webcodecs, raw, jpeg or png.');
+ if (['auto', 'webcodecs', 'raw'].includes(captureFormat) && kind !== 'video') throw new Error('Auto, webcodecs and raw capture are for video only. Stills are PNG or JPEG screenshots.');
const jpegQuality = numberOption('jpeg-quality', 95, 1, 100);
if (!Number.isInteger(jpegQuality)) throw new Error('JPEG quality must be an integer.');
const output = resolve(options.output ?? `artifacts/${scene}-${backend}.${kind === 'video' ? 'mp4' : captureFormat === 'jpeg' ? 'jpg' : 'png'}`);
@@ -111,7 +111,8 @@ function freePort() {
export async function renderMedia(values = process.argv.slice(2)) {
const plan = parseRenderOptions(values);
- const { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, time, start, memory, assembly, output, captureFormat, jpegQuality } = plan;
+ const { options, kind, backend, scene, presetName, preset, aspect, width, height, fps, time, start, memory, assembly, output, jpegQuality } = plan;
+ let captureFormat = plan.captureFormat;
let duration = plan.duration;
if (options.asset) {
const details = await stat(resolve(options.asset));
@@ -192,16 +193,26 @@ export async function renderMedia(values = process.argv.slice(2)) {
} else window.renderAt = (time) => window.__nature.renderAt(time);
}, { memory, width, height, samples: backend === 'cpu' ? 1 : preset.samples, scene, assembly });
const session = await page.context().newCDPSession(page);
- if (captureFormat === 'raw') {
+ if (['auto', 'webcodecs', 'raw'].includes(captureFormat)) {
frameReceiver = createFrameReceiver({ maximumBytes: width * height * 4 });
frameServer = await startFrameServer(frameReceiver);
- await installFrameReaderInPage(page, frameServer.url);
}
+ if (['auto', 'webcodecs'].includes(captureFormat)) {
+ // The page encodes with the browser's hardware H.264 encoder and sends only the small encoded chunks. Software
+ // rendering or a CPU encoder request keeps the readback path, which feeds ffmpeg's libx264.
+ const wantsCpu = backend === 'cpu' || options.encoder === 'cpu';
+ const started = wantsCpu ? { supported: false, reason: 'A CPU render or encoder was requested.' }
+ : await startVideoEncoderInPage(page, frameServer.url, { width, height, fps, quantizer: DEFAULT_PAGE_ENCODER_QUANTIZER });
+ if (started.supported) captureFormat = 'webcodecs';
+ else if (captureFormat === 'webcodecs') throw new Error(`Capture webcodecs is not available: ${started.reason}`);
+ else captureFormat = 'raw';
+ }
+ if (captureFormat === 'raw') await installFrameReaderInPage(page, frameServer.url);
// A raw frame is read from the drawing buffer in the same task as the draw. The round trip beyond the draw is capture.
- const render = async (seconds) => {
+ const render = async (seconds, frameIndex) => {
if (errors.length) throw new Error(`Browser render errors: ${errors.join('; ')}`);
const renderStarted = performance.now();
- const drawn = await page.evaluate(async ({ value, readFrame }) => {
+ const drawn = await page.evaluate(async ({ value, readFrame, encodeIndex }) => {
const drawStarted = performance.now();
if (window.__scene) window.__scene.renderAt(value);
else if (window.renderAt) window.renderAt(value);
@@ -209,13 +220,14 @@ export async function renderMedia(values = process.argv.slice(2)) {
const renderer = window.__scene?.renderer ?? window.__film?.renderer ?? window.__nature?.renderer ?? window.__studio?.renderer;
const glError = renderer.context.getError();
if (glError !== 0) throw new Error(`WebGL render error ${glError}.`);
- if (readFrame) window.__finishDraw();
+ if (readFrame || encodeIndex !== null) window.__finishDraw();
const drawMilliseconds = performance.now() - drawStarted;
- return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined };
- }, { value: seconds, readFrame: captureFormat === 'raw' });
+ return { rendererPlan: renderer.memoryPlan, drawMilliseconds, frame: readFrame ? await window.__readFrame() : undefined,
+ encoded: encodeIndex !== null ? await window.__encodeFrame(encodeIndex) : undefined };
+ }, { value: seconds, readFrame: captureFormat === 'raw', encodeIndex: captureFormat === 'webcodecs' ? frameIndex : null });
const rendererPlan = drawn.rendererPlan;
const roundTripSeconds = (performance.now() - renderStarted) / 1000;
- const drawSeconds = drawn.frame ? drawn.drawMilliseconds / 1000 : roundTripSeconds;
+ const drawSeconds = drawn.frame || drawn.encoded ? drawn.drawMilliseconds / 1000 : roundTripSeconds;
stageTimings.renderSeconds += drawSeconds;
stageTimings.captureSeconds += roundTripSeconds - drawSeconds;
stageTimings.readPixelsSeconds += (drawn.frame?.readMilliseconds ?? 0) / 1000;
@@ -225,10 +237,14 @@ export async function renderMedia(values = process.argv.slice(2)) {
const nodeBytes = process.memoryUsage().rss;
peakNodeBytes = Math.max(peakNodeBytes, nodeBytes);
if (nodeBytes > 512 * 1024 ** 2) throw new Error('Capture process exceeded 512 MiB. Export stopped before buffering more frames.');
- return drawn.frame;
+ return drawn;
};
- async function capture(seconds) {
- const rawFrame = await render(seconds);
+ async function capture(seconds, frameIndex) {
+ const { frame: rawFrame, encoded } = await render(seconds, frameIndex);
+ if (encoded) {
+ capturedFrames++;
+ return undefined;
+ }
if (rawFrame) {
capturedFrames++;
const bytes = frameReceiver.take({ width, height }, rawFrame);
@@ -251,15 +267,16 @@ export async function renderMedia(values = process.argv.slice(2)) {
const audioPath = plan.audio ? resolve(plan.audio) : null;
if (options.audio && !existsSync(audioPath)) throw new Error('The selected audio file was not found.');
const audioArguments = audioPath && existsSync(audioPath) ? ['-ss', String(start), '-i', audioPath] : [];
- encoder = createFrameEncoder({ executable: ffmpeg, output, fps, format: captureFormat === 'raw' ? 'rawvideo' : captureFormat,
+ encoder = createFrameEncoder({ executable: ffmpeg, output, fps, format: captureFormat === 'raw' ? 'rawvideo' : captureFormat === 'webcodecs' ? 'h264' : captureFormat,
width, height, backend: encoderBackend, audioArguments, maximumFrameBytes: 64 * 1024 * 1024,
maximumOutputBytes: Number.MAX_SAFE_INTEGER });
const frameCount = Math.ceil(duration * fps);
+ if (captureFormat === 'webcodecs') frameReceiver.onChunk(chunk => encoder.writeChunk(chunk));
const captureSequenceStarted = performance.now();
for (let frame = 0; frame < frameCount; frame++) {
- const image = await capture(start + frame / fps);
+ const image = await capture(start + frame / fps, frame);
maximumPendingFrames = Math.max(maximumPendingFrames, 1);
- await encoder.writeFrame(image);
+ if (image) await encoder.writeFrame(image);
stageTimings.encoderBackpressureSeconds = encoder.metrics.backpressureSeconds;
if (frame % 48 === 0) {
const { usedSize } = await session.send('Runtime.getHeapUsage');
@@ -272,6 +289,7 @@ export async function renderMedia(values = process.argv.slice(2)) {
console.log(message);
}
}
+ if (captureFormat === 'webcodecs') await page.evaluate(() => window.__finishVideoEncoder());
const encoding = await encoder.finish();
stageTimings.encoderFinishSeconds = encoding.finishSeconds;
} else if (kind === 'stills') {
@@ -295,7 +313,7 @@ export async function renderMedia(values = process.argv.slice(2)) {
if (graphics.glError !== 0) throw new Error(`WebGL render error ${graphics.glError}.`);
if (backend === 'cpu' && !graphics.gpu.toLowerCase().includes('swiftshader')) throw new Error('CPU backend did not select SwiftShader.');
const elapsedSeconds = (performance.now() - started) / 1000;
- const report = { scene, kind, backend, preset: presetName, aspect, start, encoder: encoderBackend, requestedEncoder: kind === 'video' ? plan.encoderBackend : null, captureFormat, jpegQuality: captureFormat === 'jpeg' ? jpegQuality : null, capturedFrames, stageTimings, effectiveFPS: capturedFrames / elapsedSeconds, speedRatio: kind === 'video' ? capturedFrames / fps / elapsedSeconds : null, assembly: scene === 'object' ? assembly : null, width, height, actualRenderSize, fps, duration: kind === 'video' ? duration : null, memoryBudgetMiB: memory, peakEstimatedRenderBytes, peakNodeBytes, peakBrowserHeap, maximumPendingFrames, firstFrameBytes, elapsedSeconds, graphics, output: basename(output), errors };
+ const report = { scene, kind, backend, preset: presetName, aspect, start, encoder: captureFormat === 'webcodecs' ? 'browser-h264' : encoderBackend, requestedEncoder: kind === 'video' ? plan.encoderBackend : null, captureFormat, jpegQuality: captureFormat === 'jpeg' ? jpegQuality : null, capturedFrames, stageTimings, effectiveFPS: capturedFrames / elapsedSeconds, speedRatio: kind === 'video' ? capturedFrames / fps / elapsedSeconds : null, assembly: scene === 'object' ? assembly : null, width, height, actualRenderSize, fps, duration: kind === 'video' ? duration : null, memoryBudgetMiB: memory, peakEstimatedRenderBytes, peakNodeBytes, peakBrowserHeap, maximumPendingFrames, firstFrameBytes, elapsedSeconds, graphics, output: basename(output), errors };
await writeFile(`${output}.json`, JSON.stringify(report, null, 2));
console.log(JSON.stringify(report, null, 2));
} finally {