3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import { spawn } from 'node:child_process';
import { lstat, realpath } from 'node:fs/promises';
import { isAbsolute, join, sep } from 'node:path';
import { AUDIO_FILE_PATTERN } from '../src/agent-scene-capabilities.js';
import { videoFrameCount } from '../src/film-timeline.js';
// Audio for render jobs: local files in one configured folder, chosen by plain file name. The agent never supplies a
// path, URL or ffmpeg argument; ffmpeg only receives the resolved path and a fixed protocol whitelist.
export const MAXIMUM_AUDIO_BYTES = 32 * 1024 ** 2;
const PROBE_TIMEOUT_MS = 30000;
// Every file is resampled to this rate: AAC adds one 1024-sample frame of delay, 21 ms here but 64 ms at 16 kHz.
const OUTPUT_SAMPLE_RATE = 48000;
// Decoded audio and video lengths of a finished file may differ by the encoder delay; more than this fails the job.
export const SYNC_TOLERANCE_SECONDS = 0.05;
/** Looks files up in `root`, refusing links and anything that resolves outside it. */
export function createAudioLibrary(root) {
if (typeof root !== 'string' || !isAbsolute(root)) throw new TypeError('Audio root must be an explicit absolute directory path.');
return Object.freeze({
async resolve(file) {
if (typeof file !== 'string' || !AUDIO_FILE_PATTERN.test(file)) throw new TypeError('Audio file name is not allowed.');
const path = join(root, file);
const information = await lstat(path).catch(() => { throw new RangeError(`Audio file ${file} is not in the audio folder.`); });
if (!information.isFile() || information.isSymbolicLink() || information.nlink > 1) throw new RangeError(`Audio file ${file} must be a regular file without links.`);
if (information.size < 1 || information.size > MAXIMUM_AUDIO_BYTES) throw new RangeError(`Audio file ${file} must be 1 byte to ${MAXIMUM_AUDIO_BYTES} bytes.`);
const rootReal = await realpath(root);
if (!(await realpath(path)).startsWith(rootReal + sep)) throw new RangeError('Audio file resolves outside the audio folder.');
return path;
},
});
}
/** Seek time as a plain decimal: String(1e-7) is '1e-7', which ffmpeg rejects as a duration. */
export function formatSeekSeconds(seconds) {
return seconds.toFixed(6);
}
/** Throws unless the sound still has enough left after its offset to cover the whole video. */
export function assertAudioCoversVideo(fileSeconds, offsetSeconds, videoSeconds, toleranceSeconds = SYNC_TOLERANCE_SECONDS) {
const availableSeconds = fileSeconds - offsetSeconds;
if (availableSeconds + toleranceSeconds < videoSeconds) {
throw new RangeError(`Audio has ${availableSeconds.toFixed(2)} s after its offset but the video lasts ${videoSeconds.toFixed(2)} s.`);
}
return availableSeconds;
}
// ffmpeg prints decoded time in hundredths of a second, so that is the finest slack a command line check can honour.
const COMMAND_LINE_TOLERANCE_SECONDS = 0.01;
/**
* The audio input arguments and frame count for `render-media`. Refuses a sound that ends before the picture, because the
* encoder's -shortest would cut frames. `measureAudioSeconds` is injected so tests need no ffmpeg.
*/
export async function planRenderMediaAudio({ audioPath, startSeconds, fps, durationSeconds, measureAudioSeconds }) {
const frameCount = videoFrameCount(fps, durationSeconds);
const audioSeconds = await measureAudioSeconds(audioPath);
try {
assertAudioCoversVideo(audioSeconds, startSeconds, frameCount / fps, COMMAND_LINE_TOLERANCE_SECONDS);
} catch (error) {
throw new RangeError(`${error.message} Shorten --duration, lower --start, or pass a longer sound with --audio (this run used ${audioPath}).`);
}
return { audioArguments: ['-ss', formatSeekSeconds(startSeconds), '-i', audioPath], frameCount };
}
/** Input arguments for the encoder: the file is read at `offset` seconds and may only be opened as a plain file. */
export function audioInputArguments(path, offsetSeconds) {
return ['-protocol_whitelist', 'file', '-ss', formatSeekSeconds(offsetSeconds), '-i', path, '-ar', String(OUTPUT_SAMPLE_RATE)];
}
function parseSeconds(text) {
const matches = [...text.matchAll(/time=(\d+):(\d+):(\d+(?:\.\d+)?)/g)];
const last = matches.at(-1);
if (!last) throw new Error('ffmpeg reported no decoded duration.');
return Number(last[1]) * 3600 + Number(last[2]) * 60 + Number(last[3]);
}
/** Decodes one stream to nowhere and reads how long it ran: `audio` or `video`. Decoded length, not container metadata. */
export function measureStreamSeconds(executable, path, stream, { format } = {}) {
const map = stream === 'audio' ? ['-map', '0:a:0', '-vn'] : ['-map', '0:v:0', '-an'];
const argumentsValue = ['-hide_banner', '-nostdin', '-protocol_whitelist', 'file', ...(format ? ['-f', format] : []), '-i', path, ...map, '-f', 'null', '-'];
return new Promise((resolve, reject) => {
const child = spawn(executable, argumentsValue, { windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] });
let diagnostics = '';
child.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-16384); });
const timer = setTimeout(() => child.kill(), PROBE_TIMEOUT_MS);
child.once('error', reject);
child.once('close', code => {
clearTimeout(timer);
if (code !== 0) return reject(new Error(`ffmpeg could not decode the ${stream} stream: ${diagnostics.slice(-300)}`));
try { resolve(parseSeconds(diagnostics)); } catch (error) { reject(error); }
});
});
}