3D engine that AI agents drive
Let video jobs write PCM audio so joined parts have no dropout (closes #389)
Nguyen Xuan An960084cc1032
diff --git a/changes/389.md b/changes/389.mdnew file mode 100644index 00000000..b400103b--- /dev/null+++ b/changes/389.md@@ -0,0 +1 @@+- Fixed the 5 ms audio dropout at every join of a film rendered as several video jobs. A video job takes `audio.track: 'pcm'`, which writes lossless audio without AAC priming and padding, and `docs/agent-render-jobs.md` has the join recipe: copy the video parts, lay the audio once as AAC (#389).diff --git a/docs/agent-render-jobs.md b/docs/agent-render-jobs.mdindex 0c9fbcd3..ff20e5b5 100644--- a/docs/agent-render-jobs.md+++ b/docs/agent-render-jobs.md@@ -88,6 +88,16 @@ A job whose file is shorter than the video after its offset fails before renderi(`videoSeconds`, `audioSeconds`, `differenceSeconds`); a difference over 50 ms fails the job.ffmpeg only receives the resolved path with `-protocol_whitelist file`; no URL, argument or output path comes from the agent.+**Joining parts.** A film longer than one job (60 s, 900 frames) is rendered as several video jobs. Do not join AAC parts: each carries encoder priming and padding, which leaves a dropout of about 5 ms at every join (issue #389).+Give every part `audio: { file, offset, track: 'pcm' }` (`offset` = the part's start in the sound), which writes lossless audio with no priming. Then copy the video of the parts and lay the sound once:++```+ffmpeg -f concat -safe 0 -i parts.txt -c copy joined.mp4+ffmpeg -i joined.mp4 -map 0:v -map 0:a -c:v copy -c:a aac -b:a 192k film.mp4+```++Or skip the sound in the parts' join and lay the source file instead: `ffmpeg -i joined.mp4 -i score.wav -map 0:v -map 1:a -c:v copy -c:a aac -shortest film.mp4`.+**Encoder.** `settings.encoder` is `cpu` (libx264, default) or `nvenc` (h264_nvenc). A machine without NVENC fails the job with the ffmpeg error and never falls back.`npm run smoke:film-timeline` proves it through MCP on SwiftShader (`cpu` backend): 9 PNG seeks over 8 stills across three shots and a dip are identical to the byte,diff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjsindex 51d63d8b..6ed45a9c 100644--- a/scripts/agent-render-jobs.mjs+++ b/scripts/agent-render-jobs.mjs@@ -172,11 +172,12 @@ function timelineStartFrom(input, recipe) {function audioFrom(audio, kind) {if (kind !== 'video') throw new TypeError('audio needs kind video.');- objectKeys(audio, ['file', 'offset'], 'audio');+ objectKeys(audio, ['file', 'offset', 'track'], 'audio');+ if (audio.track !== undefined && !['aac', 'pcm'].includes(audio.track)) throw new TypeError('audio.track must be aac or pcm.');if (typeof audio.file !== 'string' || !AUDIO_FILE_PATTERN.test(audio.file)) {throw new TypeError('audio.file must be a file name in the configured audio folder ending in .wav, .mp3, .ogg or .flac.');}- return { file: audio.file, offset: boundedNumber(audio.offset ?? 0, 0, 3600, 'audio offset') };+ return { file: audio.file, offset: boundedNumber(audio.offset ?? 0, 0, 3600, 'audio offset'), track: audio.track ?? 'aac' };}function passesFrom(passes, kind) {diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjsindex 79676fb8..69a601aa 100644--- a/scripts/agent-runtime.mjs+++ b/scripts/agent-runtime.mjs@@ -408,7 +408,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow}encoder = createFrameEncoder({ executable: ffmpeg, output: join(directory, 'render.mp4.partial'),fps: settings.fps, format: pageEncodes ? 'h264' : 'jpeg', width, height, signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',- audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [] });+ audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [],+ audioCodec: settings.audio?.track === 'pcm' ? 'pcm' : 'aac' });if (pageEncodes) frameReceiver.onChunk(chunk => encoder.writeChunk(chunk));metrics.videoEncoder = pageEncodes ? 'browser-h264' : settings.encoder === 'nvenc' ? 'h264_nvenc' : 'libx264';}diff --git a/scripts/frame-encoder.mjs b/scripts/frame-encoder.mjsindex 728233da..b171a262 100644--- a/scripts/frame-encoder.mjs+++ b/scripts/frame-encoder.mjs@@ -6,9 +6,16 @@ import { rawFrameInputArguments } from './gpu-frame-reader.mjs';/** Drain and finalization wait this long: below-normal priority jobs on a busy machine need more than 5 s. */export const DEFAULT_ENCODER_TIMEOUT_MS = 30000;+/** AAC adds priming and padding at both ends of a part; PCM does not, so parts meant to be joined use PCM. */+export function audioCodecArguments(audioCodec) {+ if (audioCodec === 'aac') return ['-c:a', 'aac', '-b:a', '192k'];+ if (audioCodec === 'pcm') return ['-c:a', 'pcm_s16le'];+ throw new TypeError('Audio codec must be aac or pcm.');+}+/** One streamed frame at a time. The caller owns output publication and cleanup. */export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format = 'jpeg', backend = 'cpu',- audioArguments = [], width, height, signal, maximumFrameBytes = 1024 * 1024,+ audioArguments = [], audioCodec = 'aac', width, height, signal, maximumFrameBytes = 1024 * 1024,maximumOutputBytes = 128 * 1024 * 1024, timeoutMs = DEFAULT_ENCODER_TIMEOUT_MS } = {}) {if (typeof output !== 'string' || !output || !Number.isFinite(fps) || fps < 1 || fps > 60 ||!['jpeg', 'png', 'rawvideo', 'h264'].includes(format) || !['cpu', 'gpu'].includes(backend)) {@@ -31,7 +38,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, formatconst argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(encoded ? ['-f', 'h264', '-framerate', String(fps)] : raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),'-i', 'pipe:0', ...audioArguments, ...(raw ? raw.filter : []), ...codec,...(encoded ? [] : ['-pix_fmt', 'yuv420p']), '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709',- '-movflags', '+faststart', ...(audioArguments.length ? ['-c:a', 'aac', '-b:a', '192k', '-shortest'] : []),+ '-movflags', '+faststart', ...(audioArguments.length ? [...audioCodecArguments(audioCodec), '-shortest'] : []),'-f', 'mp4', output];const child = spawn(executable, argumentsValue, { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });let diagnostics = '';diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjsindex 5cfd2bec..f578c50c 100644--- a/scripts/mcp-server.mjs+++ b/scripts/mcp-server.mjs@@ -653,7 +653,7 @@ const renderSettings = z.strictObject({rendererTime: z.strictObject({ start: z.number().min(0).max(3600).optional(), rate: z.number().min(-16).max(16).optional() }).optional(),renderer: z.strictObject({ type: z.literal('pathTracer'), samples: z.number().optional(), bounces: z.number().optional(), denoise: z.union([z.boolean(), z.enum(['wavelet', 'oidn', 'none'])]).optional(), seed: z.number().optional(), maxSeconds: z.number().optional(), noiseThreshold: z.number().optional() }).optional().describe('image only: path tracer; see scene_capabilities pathTracer.'),timeline: z.strictObject({ start: z.number().min(0).max(60).optional() }).optional().describe('Play the recipe timeline from start (default 0); duration defaults to the rest of it. It owns the motion clock, so leave motion and animation out.'),- audio: z.strictObject({ file: z.string().max(68), offset: z.number().min(0).max(3600).optional() }).optional().describe('Video only: a file name in the --audio folder (.wav, .mp3, .ogg, .flac) mixed in from offset seconds.'),+ audio: z.strictObject({ file: z.string().max(68), offset: z.number().min(0).max(3600).optional(), track: z.enum(['aac', 'pcm']).optional() }).optional().describe('Video only: a file name in the --audio folder (.wav, .mp3, .ogg, .flac) mixed in from offset seconds. track pcm writes lossless audio for parts you join, then encode AAC once.'),encoder: z.enum(VIDEO_ENCODERS).optional().describe('Video only: cpu (default) or nvenc.'),passes: z.array(z.enum(RENDER_PASS_NAMES)).min(1).max(RENDER_PASS_NAMES.length).optional().describe('image and stills only: also write raw PNGs per frame for these passes; opaque geometry only.'),}).describe('Immutable snapshot render: one PNG, eight evenly spaced PNGs, or CPU H.264 MP4. Video dimensions must be even. Defaults: 24 fps, one second. Use motion.start for authored recipe tracks, animation for an imported clip, or both to layer them. Omitted motion holds time zero. Nonlooping samples must fit the authored duration.');diff --git a/tests/film-timeline.test.js b/tests/film-timeline.test.jsindex e6bb6ab5..10cf2262 100644--- a/tests/film-timeline.test.js+++ b/tests/film-timeline.test.js@@ -8,6 +8,7 @@ import { createAgentScene, AGENT_SCENE_CAPABILITIES } from '../src/agent-scene.jimport { filmTimelineDuration, filmTimelineShotStarts, normalizeFilmTimeline, sampleFilmTimeline } from '../src/film-timeline.js';import { createAgentRenderJobs } from '../scripts/agent-render-jobs.mjs';import { createAudioLibrary } from '../scripts/job-audio.mjs';+import { audioCodecArguments } from '../scripts/frame-encoder.mjs';function recipeWith(extra = {}) {return {@@ -157,11 +158,26 @@ test('encoder is a bounded enum and audio is a plain file name, both video only'await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', gain: 2 } })), /Unknown audio field/);await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', offset: -1 } })), /audio offset/);await assert.rejects(controller.submit(submission({ kind: 'image', audio: { file: 'score.wav' } })), /audio needs kind video/);+ for (const track of ['wav', 'AAC', '', null, 7, ['pcm']]) {+ await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', track } })), /audio.track/);+ }const accepted = await controller.submit(submission({ encoder: 'nvenc', audio: { file: 'score.wav', offset: 1.5 } }));- assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 1.5 });+ assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 1.5, track: 'aac' });assert.equal(controller.status(accepted.jobId).settings.encoder, 'nvenc');});+test('audio.track pcm is kept on the job for parts that will be joined', async context => {+ const controller = await controllerFor(context);+ const accepted = await controller.submit(submission({ audio: { file: 'score.wav', offset: 30, track: 'pcm' } }));+ assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 30, track: 'pcm' });+});++test('audio codec arguments: AAC by default, PCM for parts that will be joined, nothing else', () => {+ assert.deepEqual(audioCodecArguments('aac'), ['-c:a', 'aac', '-b:a', '192k']);+ assert.deepEqual(audioCodecArguments('pcm'), ['-c:a', 'pcm_s16le']);+ assert.throws(() => audioCodecArguments('libmp3lame'), /aac or pcm/);+});+test('the audio library resolves plain files only and refuses links, empty files and wrong names', async context => {const root = await mkdtemp(join(tmpdir(), '67js-audio-library-'));const outside = await mkdtemp(join(tmpdir(), '67js-audio-outside-'));