3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
import test from 'node:test';
import assert from 'node:assert/strict';
import { spawnSync } from 'node:child_process';
import { mkdtemp, rm, readFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { audioTrimArguments, createFrameEncoder, frameEncoderArguments } from '../scripts/frame-encoder.mjs';
import { audioInputArguments, measureStreamSeconds } from '../scripts/job-audio.mjs';
import { assertOutputHasAudio, parseRenderOptions, renderEncoderOptions, stillPath, stillPaths } from '../scripts/render-media.mjs';
const ffmpegAvailable = spawnSync('ffmpeg', ['-version']).status === 0;
const SYNC_TOLERANCE_SECONDS = 0.05;
function runFfmpeg(argumentsValue) {
const result = spawnSync('ffmpeg', ['-hide_banner', '-loglevel', 'error', '-y', ...argumentsValue]);
assert.equal(result.status, 0, String(result.stderr));
}
async function makeInputs(directory, fps, frameCount) {
const sound = join(directory, 'sound.wav');
const annexB = join(directory, 'clip.h264');
runFfmpeg(['-f', 'lavfi', '-i', 'sine=d=3:r=48000', sound]);
runFfmpeg(['-f', 'lavfi', '-i', `testsrc=s=64x64:r=${fps}`, '-frames:v', String(frameCount), '-c:v', 'libx264', '-bf', '0', '-f', 'h264', annexB]);
return { sound, clip: await readFile(annexB) };
}
async function encodeWithAudio(directory, { fps, frameCount, audioCodec }) {
const { sound, clip } = await makeInputs(directory, fps, frameCount);
const output = join(directory, `${audioCodec}-${fps}.mp4`);
const encoder = createFrameEncoder({ output, fps, format: 'h264', audioCodec, frameCount,
audioArguments: audioInputArguments(sound, 0) });
encoder.writeChunk(clip);
await encoder.finish();
await encoder.close();
return output;
}
test('the audio trim pads with silence and cuts at the picture length', () => {
assert.deepEqual(audioTrimArguments('pcm', 1), ['-af', 'apad', '-c:a', 'pcm_s16le', '-t', '1.000000']);
assert.deepEqual(audioTrimArguments('aac', 24 / 12).slice(-2), ['-t', '2.000000']);
});
test('an encoder with audio refuses to start without a frame count', () => {
assert.throws(() => createFrameEncoder({ output: 'x.mp4', fps: 24, format: 'h264', audioArguments: ['-i', 'a.wav'] }), /frameCount/);
});
for (const [audioCodec, fps, frameCount] of [['aac', 24, 24], ['pcm', 24, 24], ['pcm', 12, 24], ['pcm', 24, 12]]) {
test(`${audioCodec} audio on the page-encoded H.264 copy path keeps the picture length (${frameCount} frames at ${fps} fps)`, { skip: !ffmpegAvailable }, async () => {
const directory = await mkdtemp(join(tmpdir(), 'crucible-1284-'));
try {
const output = await encodeWithAudio(directory, { fps, frameCount, audioCodec });
const audioSeconds = await measureStreamSeconds('ffmpeg', output, 'audio');
const videoSeconds = await measureStreamSeconds('ffmpeg', output, 'video');
assert.ok(Math.abs(audioSeconds - frameCount / fps) <= 0.011, `audio ${audioSeconds}`);
assert.ok(Math.abs(audioSeconds - videoSeconds) <= SYNC_TOLERANCE_SECONDS, `audio ${audioSeconds} video ${videoSeconds}`);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
}
test('a rendered video without an audio stream fails the render', { skip: !ffmpegAvailable }, async () => {
const directory = await mkdtemp(join(tmpdir(), 'crucible-1284-'));
try {
const silent = join(directory, 'silent.mp4');
runFfmpeg(['-f', 'lavfi', '-i', 'testsrc=s=64x64:r=24', '-frames:v', '24', '-pix_fmt', 'yuv420p', silent]);
await assert.rejects(assertOutputHasAudio('ffmpeg', silent), /no audio stream/);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
for (const [format, backend] of [['h264', 'cpu'], ['jpeg', 'cpu'], ['rawvideo', 'cpu'], ['jpeg', 'gpu']]) {
test(`the ${format} encoder (${backend}) pads the sound and cuts at the picture length without -shortest, with no ffmpeg needed`, () => {
const encoderArguments = frameEncoderArguments({ output: 'o.mp4', fps: 24, format, backend, audioArguments: ['-i', 'a.wav'],
audioCodec: 'aac', frameCount: 12, width: 64, height: 64 });
assert.ok(!encoderArguments.includes('-shortest'));
const padIndex = encoderArguments.indexOf('-af');
assert.equal(encoderArguments[padIndex + 1], 'apad');
assert.equal(encoderArguments[encoderArguments.indexOf('-t') + 1], '0.500000');
});
}
test('render-media hands the frame count to the encoder, which an audio render requires', () => {
const options = renderEncoderOptions({ ffmpeg: 'ffmpeg', output: 'o.mp4', fps: 24, captureFormat: 'webcodecs', width: 64, height: 64,
encoderBackend: 'cpu', audioArguments: ['-i', 'a.wav'], frameCount: 24 });
assert.equal(options.frameCount, 24);
assert.equal(options.format, 'h264');
assert.doesNotThrow(() => frameEncoderArguments({ ...options, audioCodec: 'aac' }));
});
test('a rendered audio length that differs from the picture fails the render', { skip: !ffmpegAvailable }, async () => {
const directory = await mkdtemp(join(tmpdir(), 'crucible-1284-'));
try {
const video = join(directory, 'long-audio.mp4');
runFfmpeg(['-f', 'lavfi', '-i', 'testsrc=s=64x64:r=24', '-f', 'lavfi', '-i', 'sine=d=1:r=48000', '-frames:v', '24', '-pix_fmt', 'yuv420p', '-t', '1', video]);
await assertOutputHasAudio('ffmpeg', video, 1);
await assert.rejects(assertOutputHasAudio('ffmpeg', video, 2), /must match within 50 ms/);
} finally {
await rm(directory, { recursive: true, force: true });
}
});
test('stills are distinct files frame-0 to frame-7 beside --output, however short the duration', () => {
const names = stillPaths(join('out', 's.png'), 8, 'png');
assert.equal(new Set(names).size, 8);
assert.match(names[0], /frame-0\.png$/);
assert.match(names[7], /frame-7\.png$/);
assert.match(stillPath(join('out', 's.jpg'), 3, 'jpeg'), /frame-3\.jpg$/);
});
test('a derived video dimension is rounded to even and a passed odd one is still refused', () => {
const derived = parseRenderOptions(['--kind', 'video', '--width', '1000']);
assert.deepEqual([derived.width, derived.height], [1000, 562]);
const portrait = parseRenderOptions(['--kind', 'video', '--width', '1000', '--aspect', 'portrait']);
assert.deepEqual([portrait.width, portrait.height], [1000, 1778]);
assert.throws(() => parseRenderOptions(['--kind', 'video', '--width', '1001', '--height', '564']), /even/);
assert.equal(parseRenderOptions(['--kind', 'image', '--width', '1000']).height, 563);
});