Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Let video jobs write PCM audio so joined parts have no dropout (closes #389)

Nguyen Xuan An960084cc1032
diff --git a/changes/389.md b/changes/389.md
new file mode 100644
index 00000000..b400103b
--- /dev/null
+++ b/changes/389.md
@@ -0,0 +1 @@
+- Fixed the 5 ms audio dropout at every join of a film rendered as several video jobs. A video job takes `audio.track: 'pcm'`, which writes lossless audio without AAC priming and padding, and `docs/agent-render-jobs.md` has the join recipe: copy the video parts, lay the audio once as AAC (#389).
diff --git a/docs/agent-render-jobs.md b/docs/agent-render-jobs.md
index 0c9fbcd3..ff20e5b5 100644
--- a/docs/agent-render-jobs.md
+++ b/docs/agent-render-jobs.md
@@ -88,6 +88,16 @@ A job whose file is shorter than the video after its offset fails before renderi
(`videoSeconds`, `audioSeconds`, `differenceSeconds`); a difference over 50 ms fails the job.
ffmpeg only receives the resolved path with `-protocol_whitelist file`; no URL, argument or output path comes from the agent.
+**Joining parts.** A film longer than one job (60 s, 900 frames) is rendered as several video jobs. Do not join AAC parts: each carries encoder priming and padding, which leaves a dropout of about 5 ms at every join (issue #389).
+Give every part `audio: { file, offset, track: 'pcm' }` (`offset` = the part's start in the sound), which writes lossless audio with no priming. Then copy the video of the parts and lay the sound once:
+
+```
+ffmpeg -f concat -safe 0 -i parts.txt -c copy joined.mp4
+ffmpeg -i joined.mp4 -map 0:v -map 0:a -c:v copy -c:a aac -b:a 192k film.mp4
+```
+
+Or skip the sound in the parts' join and lay the source file instead: `ffmpeg -i joined.mp4 -i score.wav -map 0:v -map 1:a -c:v copy -c:a aac -shortest film.mp4`.
+
**Encoder.** `settings.encoder` is `cpu` (libx264, default) or `nvenc` (h264_nvenc). A machine without NVENC fails the job with the ffmpeg error and never falls back.
`npm run smoke:film-timeline` proves it through MCP on SwiftShader (`cpu` backend): 9 PNG seeks over 8 stills across three shots and a dip are identical to the byte,
diff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjs
index 51d63d8b..6ed45a9c 100644
--- a/scripts/agent-render-jobs.mjs
+++ b/scripts/agent-render-jobs.mjs
@@ -172,11 +172,12 @@ function timelineStartFrom(input, recipe) {
function audioFrom(audio, kind) {
if (kind !== 'video') throw new TypeError('audio needs kind video.');
- objectKeys(audio, ['file', 'offset'], 'audio');
+ objectKeys(audio, ['file', 'offset', 'track'], 'audio');
+ if (audio.track !== undefined && !['aac', 'pcm'].includes(audio.track)) throw new TypeError('audio.track must be aac or pcm.');
if (typeof audio.file !== 'string' || !AUDIO_FILE_PATTERN.test(audio.file)) {
throw new TypeError('audio.file must be a file name in the configured audio folder ending in .wav, .mp3, .ogg or .flac.');
}
- return { file: audio.file, offset: boundedNumber(audio.offset ?? 0, 0, 3600, 'audio offset') };
+ return { file: audio.file, offset: boundedNumber(audio.offset ?? 0, 0, 3600, 'audio offset'), track: audio.track ?? 'aac' };
}
function passesFrom(passes, kind) {
diff --git a/scripts/agent-runtime.mjs b/scripts/agent-runtime.mjs
index 79676fb8..69a601aa 100644
--- a/scripts/agent-runtime.mjs
+++ b/scripts/agent-runtime.mjs
@@ -408,7 +408,8 @@ export function createAgentRuntime({ kernelWorkers = true, backend = 'gpu', brow
}
encoder = createFrameEncoder({ executable: ffmpeg, output: join(directory, 'render.mp4.partial'),
fps: settings.fps, format: pageEncodes ? 'h264' : 'jpeg', width, height, signal, maximumFrameBytes: MAXIMUM_JOB_FRAME_BYTES, backend: settings.encoder === 'nvenc' ? 'gpu' : 'cpu',
- audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [] });
+ audioArguments: settings.audio ? audioInputArguments(audio.path, settings.audio.offset) : [],
+ audioCodec: settings.audio?.track === 'pcm' ? 'pcm' : 'aac' });
if (pageEncodes) frameReceiver.onChunk(chunk => encoder.writeChunk(chunk));
metrics.videoEncoder = pageEncodes ? 'browser-h264' : settings.encoder === 'nvenc' ? 'h264_nvenc' : 'libx264';
}
diff --git a/scripts/frame-encoder.mjs b/scripts/frame-encoder.mjs
index 728233da..b171a262 100644
--- a/scripts/frame-encoder.mjs
+++ b/scripts/frame-encoder.mjs
@@ -6,9 +6,16 @@ import { rawFrameInputArguments } from './gpu-frame-reader.mjs';
/** Drain and finalization wait this long: below-normal priority jobs on a busy machine need more than 5 s. */
export const DEFAULT_ENCODER_TIMEOUT_MS = 30000;
+/** AAC adds priming and padding at both ends of a part; PCM does not, so parts meant to be joined use PCM. */
+export function audioCodecArguments(audioCodec) {
+ if (audioCodec === 'aac') return ['-c:a', 'aac', '-b:a', '192k'];
+ if (audioCodec === 'pcm') return ['-c:a', 'pcm_s16le'];
+ throw new TypeError('Audio codec must be aac or pcm.');
+}
+
/** One streamed frame at a time. The caller owns output publication and cleanup. */
export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format = 'jpeg', backend = 'cpu',
- audioArguments = [], width, height, signal, maximumFrameBytes = 1024 * 1024,
+ audioArguments = [], audioCodec = 'aac', width, height, signal, maximumFrameBytes = 1024 * 1024,
maximumOutputBytes = 128 * 1024 * 1024, timeoutMs = DEFAULT_ENCODER_TIMEOUT_MS } = {}) {
if (typeof output !== 'string' || !output || !Number.isFinite(fps) || fps < 1 || fps > 60 ||
!['jpeg', 'png', 'rawvideo', 'h264'].includes(format) || !['cpu', 'gpu'].includes(backend)) {
@@ -31,7 +38,7 @@ export function createFrameEncoder({ executable = 'ffmpeg', output, fps, format
const argumentsValue = ['-hide_banner', '-loglevel', 'error', '-y', ...(encoded ? ['-f', 'h264', '-framerate', String(fps)] : raw ? raw.input : ['-f', 'image2pipe', '-framerate', String(fps), '-vcodec', format === 'jpeg' ? 'mjpeg' : 'png']),
'-i', 'pipe:0', ...audioArguments, ...(raw ? raw.filter : []), ...codec,
...(encoded ? [] : ['-pix_fmt', 'yuv420p']), '-color_primaries', 'bt709', '-color_trc', 'bt709', '-colorspace', 'bt709',
- '-movflags', '+faststart', ...(audioArguments.length ? ['-c:a', 'aac', '-b:a', '192k', '-shortest'] : []),
+ '-movflags', '+faststart', ...(audioArguments.length ? [...audioCodecArguments(audioCodec), '-shortest'] : []),
'-f', 'mp4', output];
const child = spawn(executable, argumentsValue, { windowsHide: true, stdio: ['pipe', 'ignore', 'pipe'] });
let diagnostics = '';
diff --git a/scripts/mcp-server.mjs b/scripts/mcp-server.mjs
index 5cfd2bec..f578c50c 100644
--- a/scripts/mcp-server.mjs
+++ b/scripts/mcp-server.mjs
@@ -653,7 +653,7 @@ const renderSettings = z.strictObject({
rendererTime: z.strictObject({ start: z.number().min(0).max(3600).optional(), rate: z.number().min(-16).max(16).optional() }).optional(),
renderer: z.strictObject({ type: z.literal('pathTracer'), samples: z.number().optional(), bounces: z.number().optional(), denoise: z.union([z.boolean(), z.enum(['wavelet', 'oidn', 'none'])]).optional(), seed: z.number().optional(), maxSeconds: z.number().optional(), noiseThreshold: z.number().optional() }).optional().describe('image only: path tracer; see scene_capabilities pathTracer.'),
timeline: z.strictObject({ start: z.number().min(0).max(60).optional() }).optional().describe('Play the recipe timeline from start (default 0); duration defaults to the rest of it. It owns the motion clock, so leave motion and animation out.'),
- audio: z.strictObject({ file: z.string().max(68), offset: z.number().min(0).max(3600).optional() }).optional().describe('Video only: a file name in the --audio folder (.wav, .mp3, .ogg, .flac) mixed in from offset seconds.'),
+ audio: z.strictObject({ file: z.string().max(68), offset: z.number().min(0).max(3600).optional(), track: z.enum(['aac', 'pcm']).optional() }).optional().describe('Video only: a file name in the --audio folder (.wav, .mp3, .ogg, .flac) mixed in from offset seconds. track pcm writes lossless audio for parts you join, then encode AAC once.'),
encoder: z.enum(VIDEO_ENCODERS).optional().describe('Video only: cpu (default) or nvenc.'),
passes: z.array(z.enum(RENDER_PASS_NAMES)).min(1).max(RENDER_PASS_NAMES.length).optional().describe('image and stills only: also write raw PNGs per frame for these passes; opaque geometry only.'),
}).describe('Immutable snapshot render: one PNG, eight evenly spaced PNGs, or CPU H.264 MP4. Video dimensions must be even. Defaults: 24 fps, one second. Use motion.start for authored recipe tracks, animation for an imported clip, or both to layer them. Omitted motion holds time zero. Nonlooping samples must fit the authored duration.');
diff --git a/tests/film-timeline.test.js b/tests/film-timeline.test.js
index e6bb6ab5..10cf2262 100644
--- a/tests/film-timeline.test.js
+++ b/tests/film-timeline.test.js
@@ -8,6 +8,7 @@ import { createAgentScene, AGENT_SCENE_CAPABILITIES } from '../src/agent-scene.j
import { filmTimelineDuration, filmTimelineShotStarts, normalizeFilmTimeline, sampleFilmTimeline } from '../src/film-timeline.js';
import { createAgentRenderJobs } from '../scripts/agent-render-jobs.mjs';
import { createAudioLibrary } from '../scripts/job-audio.mjs';
+import { audioCodecArguments } from '../scripts/frame-encoder.mjs';
function recipeWith(extra = {}) {
return {
@@ -157,11 +158,26 @@ test('encoder is a bounded enum and audio is a plain file name, both video only'
await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', gain: 2 } })), /Unknown audio field/);
await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', offset: -1 } })), /audio offset/);
await assert.rejects(controller.submit(submission({ kind: 'image', audio: { file: 'score.wav' } })), /audio needs kind video/);
+ for (const track of ['wav', 'AAC', '', null, 7, ['pcm']]) {
+ await assert.rejects(controller.submit(submission({ audio: { file: 'score.wav', track } })), /audio.track/);
+ }
const accepted = await controller.submit(submission({ encoder: 'nvenc', audio: { file: 'score.wav', offset: 1.5 } }));
- assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 1.5 });
+ assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 1.5, track: 'aac' });
assert.equal(controller.status(accepted.jobId).settings.encoder, 'nvenc');
});
+test('audio.track pcm is kept on the job for parts that will be joined', async context => {
+ const controller = await controllerFor(context);
+ const accepted = await controller.submit(submission({ audio: { file: 'score.wav', offset: 30, track: 'pcm' } }));
+ assert.deepEqual(controller.status(accepted.jobId).settings.audio, { file: 'score.wav', offset: 30, track: 'pcm' });
+});
+
+test('audio codec arguments: AAC by default, PCM for parts that will be joined, nothing else', () => {
+ assert.deepEqual(audioCodecArguments('aac'), ['-c:a', 'aac', '-b:a', '192k']);
+ assert.deepEqual(audioCodecArguments('pcm'), ['-c:a', 'pcm_s16le']);
+ assert.throws(() => audioCodecArguments('libmp3lame'), /aac or pcm/);
+});
+
test('the audio library resolves plain files only and refuses links, empty files and wrong names', async context => {
const root = await mkdtemp(join(tmpdir(), '67js-audio-library-'));
const outside = await mkdtemp(join(tmpdir(), '67js-audio-outside-'));