Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Join clips whose sound ends before their picture

Claudeee5736283090
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.md
index f07daf5d8..447e00d6f 100644
--- a/docs/providers-and-models.md
+++ b/docs/providers-and-models.md
@@ -311,14 +311,19 @@ submitted. A longer video is passed as `scenes`: WebBrain fits each scene to the
clip lengths the model offers, splitting a scene that is longer than its longest
clip, generates up to three clips at a time with one retry each, and joins them
into one MP4 without re-encoding (`src/*/src/agent/video-join.js`, which parses
-the clips with the vendored mp4box.js). Videos are capped at 120 seconds and 24
-clips. When the estimate is longer than 120 seconds, WebBrain asks on its own
-card whether to generate the shorter version before any clip is submitted; the
-card never times out, and an unattended run generates nothing. Clips that
-cannot be joined this way (different encodings, edit lists or fragment times
-the joined file cannot reproduce, or more than 160 MB together) are kept as
-separate clips and shown in order. The model never sees the video, so the tool result
-gives it the video's real length (`durationSeconds`) and `clipCount` to report.
+the clips with the vendored mp4box.js). A clip's sound often ends a little
+before its picture or starts after it; with AAC-LC sound (the usual MP4 audio,
+which Grok Imagine returns) the joiner fills those gaps with silent frames, so
+the sound plays without gaps and each of its frames plays within half a frame of
+its time. Videos are capped at 120 seconds and 24 clips. When the estimate is
+longer than 120 seconds, WebBrain asks on its own card whether to generate the
+shorter version before any clip is submitted; the card never times out, and an
+unattended run generates nothing. Clips that cannot be joined this way
+(different encodings, edit lists or timing the joined file cannot reproduce, or
+more than 160 MB together) are kept as separate clips and shown in order, and
+the tool result says why (`joinError`), which traces show. The model never sees
+the video, so the tool result gives it the video's real length
+(`durationSeconds`) and `clipCount` to report.
The Act system prompt and planner direct the model to call `generate_image` when
the user asks to generate media. Provider results are treated as untrusted content.
diff --git a/src/chrome/src/agent/generative-media.js b/src/chrome/src/agent/generative-media.js
index 5beabe33a..d23251afb 100644
--- a/src/chrome/src/agent/generative-media.js
+++ b/src/chrome/src/agent/generative-media.js
@@ -455,8 +455,10 @@ function declinedVideoMessage({ answer, estimatedSeconds, totalSeconds }) {
}
// Joins a video's clips into one stored asset. Clips that cannot be joined
-// losslessly are stored one by one and shown in order.
+// losslessly are stored one by one and shown in order, and the result says why
+// (joinError) so a trace shows it.
async function storeVideoClips(clips, onProgress) {
+ let joinError;
if (clips.length > 1) {
try {
onProgress?.(`Joining ${clips.length} clips…`);
@@ -465,6 +467,7 @@ async function storeVideoClips(clips, onProgress) {
return { ...asset, durationSeconds: roundSeconds(joined.durationSeconds), clipCount: clips.length };
} catch (error) {
if (!(error instanceof VideoJoinError)) throw error;
+ joinError = error.message;
}
}
const stored = [];
@@ -474,7 +477,7 @@ async function storeVideoClips(clips, onProgress) {
}
const durationSeconds = roundSeconds(stored.reduce((sum, clip) => sum + (clip.durationSeconds || 0), 0));
if (stored.length === 1) return { ...stored[0], inlineMedia: true, durationSeconds, clipCount: 1 };
- return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false };
+ return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false, joinError };
}
/**
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index fda3224be..ae7cc41bf 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -22,6 +22,9 @@ export class VideoJoinError extends Error {
const incompatible = reason => new VideoJoinError(`The clips cannot be joined into one file: ${reason}.`, 'clips_incompatible');
const MEDIA_HEADER_TYPES = new Set(['vmhd', 'smhd', 'nmhd', 'sthd', 'hmhd']);
const MAX_UINT32 = 2 ** 32 - 1;
+// Silence fills gaps in the sound, as long as the clips run: clips that claim
+// to run for hours are not joined.
+const MAX_SECONDS = 3600;
function toArrayBuffer(input) {
if (input instanceof ArrayBuffer) return input;
@@ -77,6 +80,34 @@ function sampleDescriptionKey(bytes, stsd) {
return key;
}
+// A silent frame for AAC-LC (mono or stereo, 1024 samples), to fill gaps in a
+// clip's sound so it plays continuously: players such as Chrome time audio by
+// the samples they play, so a gap in timestamps would shift the sound against
+// the picture. The frame is one channel element with an empty spectrum
+// (max_sfb 0). Other codecs have none (null), and their tracks are joined as
+// video tracks are.
+const SILENT_AAC_FRAMES = { 1: Uint8Array.of(0x00, 0x00, 0x00, 0x07), 2: Uint8Array.of(0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e) };
+const AAC_SAMPLE_RATES = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
+
+function silentFrame(entry, timescale) {
+ if (entry?.type !== 'mp4a') return null;
+ const config = entry.esds?.esd?.descs?.find(desc => desc.tag === 4)?.descs?.find(desc => desc.tag === 5)?.data;
+ if (!config || config.length < 2) return null;
+ const objectType = config[0] >> 3;
+ const rate = AAC_SAMPLE_RATES[((config[0] & 7) << 1) | (config[1] >> 7)];
+ const channels = (config[1] >> 3) & 15;
+ // AAC-LC with 1024-sample frames and no core coder; only an explicit
+ // "no SBR" extension may follow.
+ if (objectType !== 2 || !rate || !SILENT_AAC_FRAMES[channels] || (config[1] & 0x06)) return null;
+ if (config.length > 2) {
+ const syncExtension = (config[2] << 3) | (config[3] >> 5);
+ const extensionType = config[3] & 0x1f;
+ if (config.length < 5 || syncExtension !== 0x2b7 || extensionType !== 5 || (config[4] & 0x80)) return null;
+ }
+ const duration = 1024 * timescale / rate;
+ return Number.isInteger(duration) ? { bytes: SILENT_AAC_FRAMES[channels], duration } : null;
+}
+
function readTrack(trak, movieTimescale, bytes) {
const mdia = trak?.mdia, minf = mdia?.minf, stbl = minf?.stbl;
const handler = mdia?.hdlr?.handler;
@@ -108,20 +139,29 @@ function readTrack(trak, movieTimescale, bytes) {
// Reordered frames (B-frames) are shown after they are decoded, so the
// track's picture ends at its last composition time, not its last decode time.
const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);
- // One edit that skips the codec's start-up delay is normal. Empty edits or
- // several edits describe a timeline these tables cannot carry over.
+ // One edit that skips the codec's start-up delay is normal, and so is an
+ // empty edit before it (a track that starts late). Other edit lists describe
+ // a timeline these tables cannot carry over.
const edits = trak.edts?.elst?.entries || [];
- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {
+ const empty = edits.length === 2 && edits[0].media_time === -1 ? edits[0] : null;
+ const media = edits[empty ? 1 : 0];
+ if (edits.length > (empty ? 2 : 1)
+ || (media && (media.media_time < 0 || media.media_rate_integer !== 1 || media.media_rate_fraction))) {
throw incompatible('a track has a complex edit list');
}
+ const ticks = movieTicks => Math.round(movieTicks * timescale / movieTimescale);
+ const delay = empty ? ticks(empty.segment_duration) : 0;
+ const editDuration = media && media.segment_duration > 0 ? ticks(media.segment_duration) : Infinity;
return {
handler,
timescale,
samples,
- hasEdit: edits.length > 0,
- editStart: edits.length ? edits[0].media_time : 0,
- editDuration: edits.length && edits[0].segment_duration > 0
- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,
+ hasEdit: !!media,
+ // The media time the movie shows at its start, before any lead.
+ editStart: (media ? media.media_time : 0) - delay,
+ editDuration,
+ editEnd: media ? media.media_time + editDuration : Infinity,
+ silence: handler === 'soun' ? silentFrame(stbl.stsd.entries[0], timescale) : null,
firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),
compositionEnd,
tkhd: trak.tkhd,
@@ -170,18 +210,23 @@ function parseClip(input) {
// - its first frame must decode without the previous clip's frames;
// - an edit list that hides whole frames before or after the picture keeps
// them in the file, where a join would play them over the next clip;
- // - a track whose first sample comes after the clip's start (by more than a
- // millisecond) would need a leading gap, which one edit cannot show;
- // - a track whose own edit ends before the picture (by more than one of its
- // frames) would play on to the next clip in the joined file.
+ // - a track whose first sample comes after the clip's start needs a leading
+ // gap: silent frames fill it where the codec allows, as long as the sound
+ // starts before the clip ends; otherwise one edit cannot show more than a
+ // millisecond of it;
+ // - a track whose own edit hides more than one of its frames at the end
+ // would play that sound on into the next clip in the joined file.
const pictureEnd = timing.mediaTime + presented;
- const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)
- && track.editDuration < Math.round(presented / timing.timescale * track.timescale)
- - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
+ const startsLate = track => {
+ const late = track.samples[0].dts - track.mediaTime;
+ return track.silence ? late >= presented / timing.timescale * track.timescale : late > track.timescale / 1000;
+ };
+ const endsEarly = track => track.compositionEnd - track.editEnd
+ > track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
: timing.samples.some(sample => sample.duration > 0
&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
- : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
+ : tracks.some(startsLate) ? 'has a track that starts after the clip does'
: tracks.some(endsEarly) ? 'ends a track early with an edit list'
: null;
return {
@@ -200,57 +245,113 @@ export function mp4DurationSeconds(input) {
try { return parseClip(input).duration; } catch { return null; }
}
-// Lays out one output track: each clip's samples in order, with the previous
-// clip's last sample lengthened or shortened so that this clip starts where
-// the timeline says. At each join the audio follows the picture: a later
-// clip's codec start-up frames are dropped, and so are the previous clip's
-// frames that would mostly play after this clip's audio begins (padding, or
-// audio longer than the picture). Video frames are never dropped.
+// Lays out one output track: each clip's samples in order, so that each clip
+// starts where the timeline says. At each join the audio follows the picture:
+// a later clip's codec start-up frames (those that would mostly play before
+// the clip starts) are dropped, and so are the previous clip's frames that
+// would mostly play after this clip's sound begins (padding, or sound longer
+// than the picture). Video frames are never dropped.
+//
+// Where the codec has a silent frame (AAC-LC), the sound is laid out as the
+// frames a player plays: each frame at its full length, one after another,
+// with silent frames wherever a clip's timeline runs ahead (sound shorter than
+// the picture, sound that starts late, gaps in a clip's own timing). Every
+// frame then plays within half a frame of its time, and the sound stays with
+// the picture in players that time audio by the samples they play. A clip's
+// start-up frames stay right before its first frame where there is room for
+// them, so that frame decodes as it was encoded. Otherwise the previous clip's
+// last sample is lengthened or shortened to meet the next clip, as it is for
+// video.
function joinTrack(clips, index, starts) {
const first = clips[0].tracks[index];
- // The output's decode times start at 0, wherever the first clip's start
- // (fragmented files can start at any decode time): its start in the output.
- const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);
+ const silence = clips.every(clip => clip.tracks[index].silence) ? first.silence : null;
+ const frame = silence?.duration;
const samples = [];
const chunks = [];
+ // The media time the output's edit starts at: where the first clip's start
+ // falls in the output (fragmented files can start at any decode time).
+ let mediaStart = 0;
+ let dts = 0;
clips.forEach((clip, clipIndex) => {
const track = clip.tracks[index];
const audio = track.handler === 'soun';
- const kept = audio && clipIndex > 0
- ? track.samples.filter(sample => sample.cts + sample.duration / 2 >= track.mediaTime)
- : track.samples;
- if (!kept.length) throw incompatible('a clip has an empty track');
- if (samples.length) {
- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;
+ const laidOut = audio && !!silence;
+ const startup = audio ? track.samples.findIndex(sample => sample.cts + sample.duration / 2 >= track.mediaTime) : 0;
+ if (startup < 0) throw incompatible('a clip has an empty track');
+ const kept = track.samples.slice(clipIndex > 0 || laidOut ? startup : 0);
+ // Where the clip's first kept sample goes, and how many frames (silence,
+ // then the clip's own start-up frames) come right before it.
+ let base = 0;
+ let gap = 0;
+ if (clipIndex === 0) {
+ if (laidOut) {
+ // Whole frames before a sound that starts late, enough to reach its start.
+ gap = Math.max(startup, Math.ceil((kept[0].cts - track.mediaTime) / frame));
+ base = gap * frame;
+ mediaStart = base - (kept[0].cts - track.mediaTime);
+ } else {
+ mediaStart = Math.max(0, track.mediaTime - track.samples[0].dts);
+ }
+ } else {
+ base = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;
const previous = chunks[chunks.length - 1];
while (audio && previous.count > 1
- && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {
+ && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= base) {
previous.count -= 1;
previous.size -= samples.pop().size;
}
const last = samples[samples.length - 1];
- // Decode times only move forward. A clip that must start decoding before
- // the previous clip's last frame (a longer composition delay) cannot
- // follow it without moving its picture.
- if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
- last.duration = target - last.dts;
+ if (laidOut) {
+ dts = last.dts + last.duration;
+ gap = Math.max(0, Math.round((base - dts) / frame));
+ } else {
+ // Decode times only move forward. A clip that must start decoding
+ // before the previous clip's last frame (a longer composition delay)
+ // cannot follow it without moving its picture.
+ if (base <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
+ last.duration = base - last.dts;
+ dts = base;
+ }
}
- let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;
- chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });
- const chunk = chunks[chunks.length - 1];
- for (const sample of kept) {
- samples.push({
- dts,
- duration: sample.duration,
- offset: sample.cts - sample.dts,
- size: sample.size,
- source: sample.offset,
- sync: sample.is_sync !== false,
- groups: track.groups.map(group => group.indices[sample.number] ?? 0),
- clipIndex,
- });
+ const chunk = { clipIndex, first: samples.length, count: 0, size: 0, offset: 0 };
+ chunks.push(chunk);
+ const push = sample => {
+ samples.push(sample);
+ chunk.count += 1;
chunk.size += sample.size;
dts += sample.duration;
+ };
+ const copy = (sample, duration) => push({
+ dts,
+ duration,
+ offset: sample.cts - sample.dts,
+ size: sample.size,
+ source: sample.offset,
+ sync: sample.is_sync !== false,
+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),
+ clipIndex,
+ });
+ const quiet = () => push({ dts, duration: frame, offset: 0, size: silence.bytes.length, data: silence.bytes,
+ sync: true, groups: first.groups.map(() => 0), clipIndex });
+ for (let k = gap; k > 0; k--) {
+ if (k <= startup) copy(track.samples[startup - k], frame);
+ else quiet();
+ }
+ // Where the clip ends in the output: its frames that would start later
+ // would play over the next clip, or after the video ends, so they are dropped.
+ const end = Math.round((starts[clipIndex] + clip.duration) * track.timescale) + mediaStart;
+ for (const [i, sample] of kept.entries()) {
+ if (laidOut) {
+ const target = base + sample.dts - kept[0].dts;
+ if (i > 0 && target >= end) break;
+ // Silence up to where this frame belongs.
+ for (let k = Math.round((target - dts) / frame); k > 0; k--) quiet();
+ }
+ // An AAC-LC frame plays 1024 samples, even a clip's last one that its
+ // sample table shortens to where the clip's sound ends; only the joined
+ // video's very last frame keeps its own length.
+ const final = clipIndex === clips.length - 1 && i === kept.length - 1;
+ copy(sample, laidOut && !final ? frame : sample.duration);
}
});
const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);
@@ -436,6 +537,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const starts = [];
let totalSeconds = 0;
for (const clip of clips) { starts.push(totalSeconds); totalSeconds += clip.duration; }
+ if (totalSeconds > MAX_SECONDS) throw new VideoJoinError('The joined video would be longer than an hour.', 'too_large');
const tracks = reference.tracks.map((_track, index) => joinTrack(clips, index, starts));
const payload = tracks.reduce((sum, track) => sum + track.chunks.reduce((size, chunk) => size + chunk.size, 0), 0);
const ftyp = reference.ftyp || (() => {
@@ -471,7 +573,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const source = clips[chunk.clipIndex].bytes;
for (let i = chunk.first; i < chunk.first + chunk.count; i++) {
const sample = track.samples[i];
- out.set(source.subarray(sample.source, sample.source + sample.size), position);
+ out.set(sample.data || source.subarray(sample.source, sample.source + sample.size), position);
position += sample.size;
}
}
diff --git a/src/firefox/src/agent/generative-media.js b/src/firefox/src/agent/generative-media.js
index 5beabe33a..d23251afb 100644
--- a/src/firefox/src/agent/generative-media.js
+++ b/src/firefox/src/agent/generative-media.js
@@ -455,8 +455,10 @@ function declinedVideoMessage({ answer, estimatedSeconds, totalSeconds }) {
}
// Joins a video's clips into one stored asset. Clips that cannot be joined
-// losslessly are stored one by one and shown in order.
+// losslessly are stored one by one and shown in order, and the result says why
+// (joinError) so a trace shows it.
async function storeVideoClips(clips, onProgress) {
+ let joinError;
if (clips.length > 1) {
try {
onProgress?.(`Joining ${clips.length} clips…`);
@@ -465,6 +467,7 @@ async function storeVideoClips(clips, onProgress) {
return { ...asset, durationSeconds: roundSeconds(joined.durationSeconds), clipCount: clips.length };
} catch (error) {
if (!(error instanceof VideoJoinError)) throw error;
+ joinError = error.message;
}
}
const stored = [];
@@ -474,7 +477,7 @@ async function storeVideoClips(clips, onProgress) {
}
const durationSeconds = roundSeconds(stored.reduce((sum, clip) => sum + (clip.durationSeconds || 0), 0));
if (stored.length === 1) return { ...stored[0], inlineMedia: true, durationSeconds, clipCount: 1 };
- return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false };
+ return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false, joinError };
}
/**
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index fda3224be..ae7cc41bf 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -22,6 +22,9 @@ export class VideoJoinError extends Error {
const incompatible = reason => new VideoJoinError(`The clips cannot be joined into one file: ${reason}.`, 'clips_incompatible');
const MEDIA_HEADER_TYPES = new Set(['vmhd', 'smhd', 'nmhd', 'sthd', 'hmhd']);
const MAX_UINT32 = 2 ** 32 - 1;
+// Silence fills gaps in the sound, as long as the clips run: clips that claim
+// to run for hours are not joined.
+const MAX_SECONDS = 3600;
function toArrayBuffer(input) {
if (input instanceof ArrayBuffer) return input;
@@ -77,6 +80,34 @@ function sampleDescriptionKey(bytes, stsd) {
return key;
}
+// A silent frame for AAC-LC (mono or stereo, 1024 samples), to fill gaps in a
+// clip's sound so it plays continuously: players such as Chrome time audio by
+// the samples they play, so a gap in timestamps would shift the sound against
+// the picture. The frame is one channel element with an empty spectrum
+// (max_sfb 0). Other codecs have none (null), and their tracks are joined as
+// video tracks are.
+const SILENT_AAC_FRAMES = { 1: Uint8Array.of(0x00, 0x00, 0x00, 0x07), 2: Uint8Array.of(0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e) };
+const AAC_SAMPLE_RATES = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
+
+function silentFrame(entry, timescale) {
+ if (entry?.type !== 'mp4a') return null;
+ const config = entry.esds?.esd?.descs?.find(desc => desc.tag === 4)?.descs?.find(desc => desc.tag === 5)?.data;
+ if (!config || config.length < 2) return null;
+ const objectType = config[0] >> 3;
+ const rate = AAC_SAMPLE_RATES[((config[0] & 7) << 1) | (config[1] >> 7)];
+ const channels = (config[1] >> 3) & 15;
+ // AAC-LC with 1024-sample frames and no core coder; only an explicit
+ // "no SBR" extension may follow.
+ if (objectType !== 2 || !rate || !SILENT_AAC_FRAMES[channels] || (config[1] & 0x06)) return null;
+ if (config.length > 2) {
+ const syncExtension = (config[2] << 3) | (config[3] >> 5);
+ const extensionType = config[3] & 0x1f;
+ if (config.length < 5 || syncExtension !== 0x2b7 || extensionType !== 5 || (config[4] & 0x80)) return null;
+ }
+ const duration = 1024 * timescale / rate;
+ return Number.isInteger(duration) ? { bytes: SILENT_AAC_FRAMES[channels], duration } : null;
+}
+
function readTrack(trak, movieTimescale, bytes) {
const mdia = trak?.mdia, minf = mdia?.minf, stbl = minf?.stbl;
const handler = mdia?.hdlr?.handler;
@@ -108,20 +139,29 @@ function readTrack(trak, movieTimescale, bytes) {
// Reordered frames (B-frames) are shown after they are decoded, so the
// track's picture ends at its last composition time, not its last decode time.
const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);
- // One edit that skips the codec's start-up delay is normal. Empty edits or
- // several edits describe a timeline these tables cannot carry over.
+ // One edit that skips the codec's start-up delay is normal, and so is an
+ // empty edit before it (a track that starts late). Other edit lists describe
+ // a timeline these tables cannot carry over.
const edits = trak.edts?.elst?.entries || [];
- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {
+ const empty = edits.length === 2 && edits[0].media_time === -1 ? edits[0] : null;
+ const media = edits[empty ? 1 : 0];
+ if (edits.length > (empty ? 2 : 1)
+ || (media && (media.media_time < 0 || media.media_rate_integer !== 1 || media.media_rate_fraction))) {
throw incompatible('a track has a complex edit list');
}
+ const ticks = movieTicks => Math.round(movieTicks * timescale / movieTimescale);
+ const delay = empty ? ticks(empty.segment_duration) : 0;
+ const editDuration = media && media.segment_duration > 0 ? ticks(media.segment_duration) : Infinity;
return {
handler,
timescale,
samples,
- hasEdit: edits.length > 0,
- editStart: edits.length ? edits[0].media_time : 0,
- editDuration: edits.length && edits[0].segment_duration > 0
- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,
+ hasEdit: !!media,
+ // The media time the movie shows at its start, before any lead.
+ editStart: (media ? media.media_time : 0) - delay,
+ editDuration,
+ editEnd: media ? media.media_time + editDuration : Infinity,
+ silence: handler === 'soun' ? silentFrame(stbl.stsd.entries[0], timescale) : null,
firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),
compositionEnd,
tkhd: trak.tkhd,
@@ -170,18 +210,23 @@ function parseClip(input) {
// - its first frame must decode without the previous clip's frames;
// - an edit list that hides whole frames before or after the picture keeps
// them in the file, where a join would play them over the next clip;
- // - a track whose first sample comes after the clip's start (by more than a
- // millisecond) would need a leading gap, which one edit cannot show;
- // - a track whose own edit ends before the picture (by more than one of its
- // frames) would play on to the next clip in the joined file.
+ // - a track whose first sample comes after the clip's start needs a leading
+ // gap: silent frames fill it where the codec allows, as long as the sound
+ // starts before the clip ends; otherwise one edit cannot show more than a
+ // millisecond of it;
+ // - a track whose own edit hides more than one of its frames at the end
+ // would play that sound on into the next clip in the joined file.
const pictureEnd = timing.mediaTime + presented;
- const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)
- && track.editDuration < Math.round(presented / timing.timescale * track.timescale)
- - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
+ const startsLate = track => {
+ const late = track.samples[0].dts - track.mediaTime;
+ return track.silence ? late >= presented / timing.timescale * track.timescale : late > track.timescale / 1000;
+ };
+ const endsEarly = track => track.compositionEnd - track.editEnd
+ > track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
: timing.samples.some(sample => sample.duration > 0
&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
- : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
+ : tracks.some(startsLate) ? 'has a track that starts after the clip does'
: tracks.some(endsEarly) ? 'ends a track early with an edit list'
: null;
return {
@@ -200,57 +245,113 @@ export function mp4DurationSeconds(input) {
try { return parseClip(input).duration; } catch { return null; }
}
-// Lays out one output track: each clip's samples in order, with the previous
-// clip's last sample lengthened or shortened so that this clip starts where
-// the timeline says. At each join the audio follows the picture: a later
-// clip's codec start-up frames are dropped, and so are the previous clip's
-// frames that would mostly play after this clip's audio begins (padding, or
-// audio longer than the picture). Video frames are never dropped.
+// Lays out one output track: each clip's samples in order, so that each clip
+// starts where the timeline says. At each join the audio follows the picture:
+// a later clip's codec start-up frames (those that would mostly play before
+// the clip starts) are dropped, and so are the previous clip's frames that
+// would mostly play after this clip's sound begins (padding, or sound longer
+// than the picture). Video frames are never dropped.
+//
+// Where the codec has a silent frame (AAC-LC), the sound is laid out as the
+// frames a player plays: each frame at its full length, one after another,
+// with silent frames wherever a clip's timeline runs ahead (sound shorter than
+// the picture, sound that starts late, gaps in a clip's own timing). Every
+// frame then plays within half a frame of its time, and the sound stays with
+// the picture in players that time audio by the samples they play. A clip's
+// start-up frames stay right before its first frame where there is room for
+// them, so that frame decodes as it was encoded. Otherwise the previous clip's
+// last sample is lengthened or shortened to meet the next clip, as it is for
+// video.
function joinTrack(clips, index, starts) {
const first = clips[0].tracks[index];
- // The output's decode times start at 0, wherever the first clip's start
- // (fragmented files can start at any decode time): its start in the output.
- const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);
+ const silence = clips.every(clip => clip.tracks[index].silence) ? first.silence : null;
+ const frame = silence?.duration;
const samples = [];
const chunks = [];
+ // The media time the output's edit starts at: where the first clip's start
+ // falls in the output (fragmented files can start at any decode time).
+ let mediaStart = 0;
+ let dts = 0;
clips.forEach((clip, clipIndex) => {
const track = clip.tracks[index];
const audio = track.handler === 'soun';
- const kept = audio && clipIndex > 0
- ? track.samples.filter(sample => sample.cts + sample.duration / 2 >= track.mediaTime)
- : track.samples;
- if (!kept.length) throw incompatible('a clip has an empty track');
- if (samples.length) {
- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;
+ const laidOut = audio && !!silence;
+ const startup = audio ? track.samples.findIndex(sample => sample.cts + sample.duration / 2 >= track.mediaTime) : 0;
+ if (startup < 0) throw incompatible('a clip has an empty track');
+ const kept = track.samples.slice(clipIndex > 0 || laidOut ? startup : 0);
+ // Where the clip's first kept sample goes, and how many frames (silence,
+ // then the clip's own start-up frames) come right before it.
+ let base = 0;
+ let gap = 0;
+ if (clipIndex === 0) {
+ if (laidOut) {
+ // Whole frames before a sound that starts late, enough to reach its start.
+ gap = Math.max(startup, Math.ceil((kept[0].cts - track.mediaTime) / frame));
+ base = gap * frame;
+ mediaStart = base - (kept[0].cts - track.mediaTime);
+ } else {
+ mediaStart = Math.max(0, track.mediaTime - track.samples[0].dts);
+ }
+ } else {
+ base = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;
const previous = chunks[chunks.length - 1];
while (audio && previous.count > 1
- && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {
+ && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= base) {
previous.count -= 1;
previous.size -= samples.pop().size;
}
const last = samples[samples.length - 1];
- // Decode times only move forward. A clip that must start decoding before
- // the previous clip's last frame (a longer composition delay) cannot
- // follow it without moving its picture.
- if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
- last.duration = target - last.dts;
+ if (laidOut) {
+ dts = last.dts + last.duration;
+ gap = Math.max(0, Math.round((base - dts) / frame));
+ } else {
+ // Decode times only move forward. A clip that must start decoding
+ // before the previous clip's last frame (a longer composition delay)
+ // cannot follow it without moving its picture.
+ if (base <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
+ last.duration = base - last.dts;
+ dts = base;
+ }
}
- let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;
- chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });
- const chunk = chunks[chunks.length - 1];
- for (const sample of kept) {
- samples.push({
- dts,
- duration: sample.duration,
- offset: sample.cts - sample.dts,
- size: sample.size,
- source: sample.offset,
- sync: sample.is_sync !== false,
- groups: track.groups.map(group => group.indices[sample.number] ?? 0),
- clipIndex,
- });
+ const chunk = { clipIndex, first: samples.length, count: 0, size: 0, offset: 0 };
+ chunks.push(chunk);
+ const push = sample => {
+ samples.push(sample);
+ chunk.count += 1;
chunk.size += sample.size;
dts += sample.duration;
+ };
+ const copy = (sample, duration) => push({
+ dts,
+ duration,
+ offset: sample.cts - sample.dts,
+ size: sample.size,
+ source: sample.offset,
+ sync: sample.is_sync !== false,
+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),
+ clipIndex,
+ });
+ const quiet = () => push({ dts, duration: frame, offset: 0, size: silence.bytes.length, data: silence.bytes,
+ sync: true, groups: first.groups.map(() => 0), clipIndex });
+ for (let k = gap; k > 0; k--) {
+ if (k <= startup) copy(track.samples[startup - k], frame);
+ else quiet();
+ }
+ // Where the clip ends in the output: its frames that would start later
+ // would play over the next clip, or after the video ends, so they are dropped.
+ const end = Math.round((starts[clipIndex] + clip.duration) * track.timescale) + mediaStart;
+ for (const [i, sample] of kept.entries()) {
+ if (laidOut) {
+ const target = base + sample.dts - kept[0].dts;
+ if (i > 0 && target >= end) break;
+ // Silence up to where this frame belongs.
+ for (let k = Math.round((target - dts) / frame); k > 0; k--) quiet();
+ }
+ // An AAC-LC frame plays 1024 samples, even a clip's last one that its
+ // sample table shortens to where the clip's sound ends; only the joined
+ // video's very last frame keeps its own length.
+ const final = clipIndex === clips.length - 1 && i === kept.length - 1;
+ copy(sample, laidOut && !final ? frame : sample.duration);
}
});
const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);
@@ -436,6 +537,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const starts = [];
let totalSeconds = 0;
for (const clip of clips) { starts.push(totalSeconds); totalSeconds += clip.duration; }
+ if (totalSeconds > MAX_SECONDS) throw new VideoJoinError('The joined video would be longer than an hour.', 'too_large');
const tracks = reference.tracks.map((_track, index) => joinTrack(clips, index, starts));
const payload = tracks.reduce((sum, track) => sum + track.chunks.reduce((size, chunk) => size + chunk.size, 0), 0);
const ftyp = reference.ftyp || (() => {
@@ -471,7 +573,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const source = clips[chunk.clipIndex].bytes;
for (let i = chunk.first; i < chunk.first + chunk.count; i++) {
const sample = track.samples[i];
- out.set(source.subarray(sample.source, sample.source + sample.size), position);
+ out.set(sample.data || source.subarray(sample.source, sample.source + sample.size), position);
position += sample.size;
}
}
diff --git a/test/fixtures/video/README.md b/test/fixtures/video/README.md
index c72773162..78d154b76 100644
--- a/test/fixtures/video/README.md
+++ b/test/fixtures/video/README.md
@@ -9,9 +9,13 @@ FFmpeg 6.1 from its built-in test sources:
| `h264-aac-fragmented-1.mp4`, `h264-aac-fragmented-2.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono | Fragmented MP4, no edit lists |
| `h264-aac-48px.mp4` | H.264 at 48×48 | AAC | MP4 (codec setup differs from the others) |
| `vp9-opus-1.mp4`, `vp9-opus-2.mp4` | VP9, 12 fps | Opus, 48 kHz mono | MP4 (royalty-free codecs, playable in Playwright's Chromium) |
+| `h264-aac-short-audio.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono, ends 0.25 s before the picture | MP4, faststart, edit lists |
+| `h264-aac-late-audio.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono, starts 0.25 s in (an empty edit) | MP4, faststart, edit lists |
+| `vp9-opus-late-audio.mp4` | VP9, 12 fps | Opus, 48 kHz mono, starts 0.25 s in (an empty edit) | MP4 |
```sh
-H="-c:v libx264 -preset veryfast -pix_fmt yuv420p -g 12 -bf 2 -c:a aac -b:a 24k -ac 1 -shortest"
+E="-c:v libx264 -preset veryfast -pix_fmt yuv420p -g 12 -bf 2 -c:a aac -b:a 24k -ac 1"
+H="$E -shortest"
ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=300:sample_rate=16000:duration=1" \
$H -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-1.mp4
ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=500:sample_rate=16000:duration=1" \
@@ -21,6 +25,14 @@ ffmpeg -f lavfi -i "testsrc2=size=64x64:rate=12:duration=1" -f lavfi -i "sine=fr
-fflags +bitexact -map_metadata -1 vp9-opus-1.mp4
ffmpeg -f lavfi -i "testsrc=size=48x48:rate=12:duration=1" -f lavfi -i "sine=frequency=300:sample_rate=16000:duration=1" \
$H -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-48px.mp4
+# Sound shorter than the picture, or starting after it: the picture keeps its full second.
+ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=900:sample_rate=16000:duration=0.75" \
+ $E -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-short-audio.mp4
+ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -itsoffset 0.25 -f lavfi -i "sine=frequency=1100:sample_rate=16000:duration=0.75" \
+ $E -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-late-audio.mp4
+ffmpeg -f lavfi -i "testsrc2=size=64x64:rate=12:duration=1" -itsoffset 0.25 -f lavfi -i "sine=frequency=1300:sample_rate=48000:duration=0.75" \
+ -c:v libvpx-vp9 -b:v 60k -g 12 -deadline realtime -cpu-used 8 -c:a libopus -b:a 16k -ac 1 \
+ -fflags +bitexact -map_metadata -1 vp9-opus-late-audio.mp4
```
The second clip of each pair uses a different tone (600, 1000 and 800 Hz), so its
diff --git a/test/fixtures/video/h264-aac-late-audio.mp4 b/test/fixtures/video/h264-aac-late-audio.mp4
new file mode 100644
index 000000000..108d18138
Binary files /dev/null and b/test/fixtures/video/h264-aac-late-audio.mp4 differ
diff --git a/test/fixtures/video/h264-aac-short-audio.mp4 b/test/fixtures/video/h264-aac-short-audio.mp4
new file mode 100644
index 000000000..0ba11d9f0
Binary files /dev/null and b/test/fixtures/video/h264-aac-short-audio.mp4 differ
diff --git a/test/fixtures/video/vp9-opus-late-audio.mp4 b/test/fixtures/video/vp9-opus-late-audio.mp4
new file mode 100644
index 000000000..83706bfda
Binary files /dev/null and b/test/fixtures/video/vp9-opus-late-audio.mp4 differ
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index 4dfa41857..f8b698643 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -43,6 +43,7 @@ const json = (body, status = 200, headers = {}) => new Response(JSON.stringify(b
const soon = { 'Retry-After': '0.001' };
const fixtures = Object.fromEntries(await Promise.all([
'h264-aac-1', 'h264-aac-2', 'h264-aac-48px', 'h264-aac-fragmented-1', 'h264-aac-fragmented-2', 'vp9-opus-1', 'vp9-opus-2',
+ 'h264-aac-short-audio', 'h264-aac-late-audio', 'vp9-opus-late-audio',
].map(async name => [name, new Uint8Array(await fs.readFile(new URL(`./fixtures/video/${name}.mp4`, import.meta.url)))])));
const { createFile, MP4BoxBuffer } = await import('../src/chrome/vendor/mp4box/mp4box.all.mjs');
@@ -58,6 +59,7 @@ const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(s
const boxBytes = (bytes, box) => bytes.subarray(box.start, box.start + box.size);
const videoOf = file => file.moov.traks[0];
+const audioOf = file => file.moov.traks[1];
// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of
// the copy and the parsed original.
@@ -188,14 +190,17 @@ for (const build of ['chrome', 'firefox']) {
const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);
assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));
assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });
- assert.equal(shown(output, audio).to, 2);
// Audio: the first clip's frames are all kept; the second clip's start-up
- // frame is dropped, the rest is copied as is, and its first real frame
- // plays exactly one second in, with its picture.
+ // frame is dropped and the rest is copied as is. Every frame but the last
+ // plays in full (1024 samples, as AAC frames do), so the second clip's first
+ // real frame plays within half a frame (512 samples) of one second in, with
+ // its picture, and the sound ends within half a frame of the picture.
const [firstAudio, secondAudio] = sources.map(source => sampleBytes(source, parse(source).moov.traks[1]));
assert.deepEqual(sampleBytes(bytes, audio).slice(0, 17), firstAudio);
assert.deepEqual(sampleBytes(bytes, audio).slice(17), secondAudio.slice(1));
- assert.equal(audio.samples[17].cts - audio.edts.elst.entries[0].media_time, 16000);
+ assert.ok(audio.samples.every((sample, i) => sample.duration === 1024 || i === audio.samples.length - 1));
+ assert.ok(Math.abs(audio.samples[17].cts - audio.edts.elst.entries[0].media_time - 16000) <= 512);
+ assert.ok(Math.abs(shown(output, audio).to - 2) <= 0.032);
// AAC's pre-roll for seeking (a 'roll' sample group) covers every frame.
const sourceRoll = parse(sources[0]).moov.traks[1].mdia.minf.stbl.sgpds[0];
assert.deepEqual(boxBytes(bytes, audio.mdia.minf.stbl.sgpds[0]), boxBytes(sources[0], sourceRoll));
@@ -205,16 +210,20 @@ for (const build of ['chrome', 'firefox']) {
test(`${build}: fragmented and VP9/Opus clips join too`, () => {
// [clips, codecs, audio frames kept from the first clip, start-up frames dropped
// from the second, how far into its first kept frame the second clip's sound
- // starts, the audio's pre-roll groups as runs of [frames, group]]
- for (const [names, codecs, kept, dropped, lead, roll] of [
+ // starts, the audio's pre-roll groups as runs of [frames, group], and how far
+ // (in samples) the sound may play from its time]
+ for (const [names, codecs, kept, dropped, lead, roll, slack] of [
// No edit lists: the picture starts after a B-frame delay (2048/12288 s),
- // so every track is shown from there and the movie starts at its first picture.
- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, []],
+ // so every track is shown from there and the movie starts at its first
+ // picture. The first audio frame (the encoder's start-up) lasts until
+ // then; it plays right before the sound, as in a file with an edit list.
+ // AAC frames play in full, each within half a frame of its time.
+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, [], 512],
// Opus: the first clip's short final frame would play over the second
// clip's first frame, whose first 312 samples are the decoder's start-up,
// so the short frame is dropped rather than squeezed. Each clip keeps its
// own pre-roll groups (none for its first four frames).
- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]]],
+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]], 0],
]) {
const label = names.join('+');
const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));
@@ -224,22 +233,102 @@ for (const build of ['chrome', 'firefox']) {
assert.deepEqual(output.moov.traks.map(trak => trak.mdia.minf.stbl.stsd.entries[0].type), codecs, label);
assert.equal(video.samples.length, 24, label);
assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 }, `${label}: the picture runs from 0 to the declared length`);
- assert.equal(shown(output, audio).to, 2, `${label}: and so does the sound`);
+ assert.ok(Math.abs(shown(output, audio).to - 2) <= slack / audio.mdia.mdhd.timescale, `${label}: and so does the sound`);
const [firstAudio, secondAudio] = names.map(name => parse(fixtures[name]).moov.traks[1]);
assert.deepEqual(sampleBytes(bytes, audio).slice(0, kept), sampleBytes(fixtures[names[0]], firstAudio).slice(0, kept), `${label}: the first clip's audio up to the join`);
assert.deepEqual(sampleBytes(bytes, audio).slice(kept), sampleBytes(fixtures[names[1]], secondAudio).slice(dropped), `${label}: the second clip's audio`);
- assert.equal(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time, audio.mdia.mdhd.timescale,
+ assert.ok(Math.abs(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time - audio.mdia.mdhd.timescale) <= slack,
`${label}: the second clip's sound starts one second in`);
const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));
assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);
assert.deepEqual((audio.mdia.minf.stbl.sbgps[0]?.entries || []).map(entry => [entry.sample_count, entry.group_description_index]), roll, `${label}: pre-roll groups`);
}
// Fragments can start at any decode time: five seconds later, the clips
- // join into the same file. A track that starts after the picture cannot join.
+ // join into the same file.
const fragmented = [fixtures['h264-aac-fragmented-1'], fixtures['h264-aac-fragmented-2']];
assert.equal(parse(laterFragments(fragmented[0], { vide: 5, soun: 5 })).moov.traks[0].samples[0].dts, 5 * 12288);
assert.deepEqual(join.joinMp4Clips(fragmented.map(clip => laterFragments(clip, { vide: 5, soun: 5 }))).bytes, join.joinMp4Clips(fragmented).bytes);
- assert.throws(() => join.joinMp4Clips([laterFragments(fragmented[0], { soun: 5 }), fragmented[1]]), error => error.code === 'clips_incompatible');
+ });
+
+ test(`${build}: sound shorter than its picture, or starting after it, is filled with silence`, () => {
+ // 'S' for AAC-LC's silent mono frame (an empty spectrum), '#' for sound.
+ const marks = (bytes, trak) => sampleBytes(bytes, trak)
+ .map(data => (data.length === 4 && [0, 0, 0, 7].every((value, i) => data[i] === value) ? 'S' : '#')).join('');
+ const joined = clips => {
+ const { bytes, durationSeconds } = join.joinMp4Clips(clips.map(clip => (typeof clip === 'string' ? fixtures[clip] : clip)));
+ const output = parse(bytes);
+ return { bytes, durationSeconds, output, audio: audioOf(output) };
+ };
+ // Where a joined frame plays, in seconds; the sound may start up to half a frame (32 ms) off.
+ const playsAt = (audio, index) => (audio.samples[index].cts - audio.edts.elst.entries[0].media_time) / audio.mdia.mdhd.timescale;
+ // A clip whose sound ends 0.25 s before its picture (as generated clips' sound
+ // often ends a little early): four frames carry it to the next clip, three
+ // silent ones and the next clip's start-up frame, which primes the decoder
+ // so that clip's first frame sounds as it was encoded.
+ let { bytes, durationSeconds, output, audio } = joined(['h264-aac-short-audio', 'h264-aac-2']);
+ assert.equal(durationSeconds, 2);
+ assert.deepEqual(shown(output, videoOf(output)), { from: 0, to: 2, movie: 2 });
+ assert.equal(marks(bytes, audio), `${'#'.repeat(13)}SSS${'#'.repeat(17)}`);
+ const nextClip = fixtures['h264-aac-2'];
+ assert.deepEqual(sampleBytes(bytes, audio).slice(16), sampleBytes(nextClip, parse(nextClip).moov.traks[1]));
+ assert.ok(Math.abs(playsAt(audio, 17) - 1) <= 0.032, `the second clip's sound starts at ${playsAt(audio, 17)} s`);
+ // Every frame but the video's last counts in full, so the sound plays
+ // without gaps and stays with the picture.
+ assert.ok(audio.samples.every((sample, i) => (sample.duration === 1024 || i === audio.samples.length - 1)
+ && (!i || sample.dts === audio.samples[i - 1].dts + 1024)));
+ // A clip whose sound starts 0.25 s in (an empty edit, then the encoder's
+ // start-up frame from 0.186 s): silence before it, in the middle of the video...
+ ({ bytes, audio } = joined(['h264-aac-1', 'h264-aac-late-audio']));
+ assert.equal(marks(bytes, audio), `${'#'.repeat(17)}SSS${'#'.repeat(13)}`);
+ assert.ok(Math.abs(playsAt(audio, 21) - 1.25) <= 0.032, `the late sound starts at ${playsAt(audio, 21)} s`);
+ // ...or at its start, where the silence is whole frames, enough to reach the
+ // sound, and the video's edit starts inside the first of them: the sound
+ // plays exactly on time. Starting at 0.15 s still takes three frames.
+ ({ bytes, audio } = joined(['h264-aac-late-audio', 'h264-aac-1']));
+ assert.equal(marks(bytes, audio), `SSS${'#'.repeat(29)}`);
+ assert.equal(playsAt(audio, 4), 0.25);
+ const sooner = patched('h264-aac-late-audio', (view, file) => {
+ const { start } = audioOf(file).edts.elst;
+ view.setUint32(start + 16, 150);
+ view.setUint32(start + 28, 850);
+ });
+ ({ bytes, audio } = joined([sooner, 'h264-aac-1']));
+ assert.equal(marks(bytes, audio), `SSS${'#'.repeat(29)}`);
+ assert.equal(playsAt(audio, 3), 0.15);
+ // A gap in a clip's own timing (one frame lasting three frames' time) is
+ // filled with silence too, so the sound after it stays on time. The two
+ // frames it pushes past the picture's end are dropped.
+ const gappy = patched('h264-aac-fragmented-2', (view, file) => {
+ const run = file.moofs[0].trafs.find(traf => traf.tfhd.track_id === audioOf(file).tkhd.track_id).truns[0];
+ assert.equal(run.flags, 0x301, 'each sample\'s duration and size follow the data offset');
+ view.setUint32(run.start + 20 + 8 * 8, 3 * 1024);
+ });
+ ({ bytes, audio } = joined(['h264-aac-fragmented-1', gappy]));
+ assert.equal(marks(bytes, audio), `${'#'.repeat(25)}SS${'#'.repeat(6)}`);
+ assert.ok(Math.abs(playsAt(audio, 27) - (1 + 10 * 1024 / 16000)) <= 0.032, `the sound after the gap starts at ${playsAt(audio, 27)} s`);
+ // Sound may start late as long as it starts before the picture ends.
+ const startingAt = milliseconds => patched('h264-aac-late-audio', (view, file) => view.setUint32(audioOf(file).edts.elst.start + 16, milliseconds));
+ assert.ok(joined(['h264-aac-1', startingAt(999)]).bytes);
+ assert.throws(() => joined(['h264-aac-1', startingAt(1000)]), error => error.message.includes('starts after the clip does'));
+ // Only AAC-LC has a silent frame here: Opus, or the same clips marked as
+ // HE-AAC, keep a clip whose sound starts late apart.
+ assert.throws(() => joined(['vp9-opus-1', 'vp9-opus-late-audio']), error => error.message.includes('starts after the clip does'));
+ const heAac = name => patched(name, (view, file) => {
+ const { esds } = audioOf(file).mdia.minf.stbl.stsd.entries[0];
+ const config = esds.esd.descs.find(desc => desc.tag === 4).descs.find(desc => desc.tag === 5).data;
+ const box = boxBytes(fixtures[name], esds);
+ const at = box.findIndex((_, i) => box[i - 1] === config.length && config.every((value, k) => box[i + k] === value));
+ view.setUint8(esds.start + at, (5 << 3) | (config[0] & 7));
+ });
+ assert.ok(joined([heAac('h264-aac-1'), heAac('h264-aac-2')]).bytes, 'HE-AAC clips that need no silence still join');
+ assert.throws(() => joined([heAac('h264-aac-1'), heAac('h264-aac-late-audio')]), error => error.message.includes('starts after the clip does'));
+ // So do AAC-LC clips whose timescale cannot count a frame in whole ticks.
+ const oddTimescale = name => patched(name, (view, file) => {
+ const { mdhd } = audioOf(file).mdia;
+ assert.equal(mdhd.version, 0);
+ view.setUint32(mdhd.start + 20, 15999);
+ });
+ assert.throws(() => joined([oddTimescale('h264-aac-1'), oddTimescale('h264-aac-late-audio')]), error => error.message.includes('starts after the clip does'));
});
test(`${build}: clips with different codec settings or tracks are not joined`, () => {
@@ -259,6 +348,18 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(join.mp4DurationSeconds(cut), null);
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');
+ // Nor are clips that claim to run for more than an hour (silence would fill
+ // their sound for as long).
+ const endless = patched('h264-aac-2', (view, file) => {
+ const video = videoOf(file);
+ const { stts } = video.mdia.minf.stbl;
+ assert.deepEqual([stts.sample_deltas.length, video.mdia.mdhd.version], [1, 0]);
+ view.setUint32(stts.start + 20, 1024 * 3700);
+ view.setUint32(video.mdia.mdhd.start + 24, 12 * 1024 * 3700);
+ view.setUint32(video.edts.elst.start + 16, 3700 * 1000);
+ });
+ assert.ok(join.mp4DurationSeconds(endless) > 3699);
+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], endless]), error => error.code === 'too_large' && /an hour/.test(error.message));
assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);
});
@@ -276,8 +377,16 @@ for (const build of ['chrome', 'firefox']) {
assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);
refused([fixtures['h264-aac-1'], edited({ milliseconds: 500 })], 'hides some of its frames');
refused([fixtures['h264-aac-1'], edited({ mediaTime: 3072 })], 'hides some of its frames');
- // Shortening the last frame alone is fine.
- assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);
+ // Shortening the last frame alone is fine. The sound past the shortened
+ // picture is dropped (a frame that would start after it, or play mostly
+ // after it), and the next clip's sound starts with its picture.
+ for (const milliseconds of [960, 980]) {
+ const shortened = join.joinMp4Clips([edited({ milliseconds }), fixtures['h264-aac-2']]);
+ assert.ok(Math.abs(shortened.durationSeconds - (1 + milliseconds / 1000)) < 0.001);
+ const sound = audioOf(parse(shortened.bytes));
+ assert.equal(sound.samples.length, 32, 'each clip keeps 16 frames');
+ assert.ok(Math.abs(sound.samples[16].cts - sound.edts.elst.entries[0].media_time - milliseconds * 16) <= 512);
+ }
// A one-second-longer composition delay keeps the picture where it was but
// starts decoding a second earlier: such a clip can open a video, not follow one.
const delayed = patched('h264-aac-2', (view, file) => {
@@ -298,15 +407,14 @@ for (const build of ['chrome', 'firefox']) {
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],
'does not start with a keyframe');
// Encrypted samples, samples kept in another file, and a different pre-roll.
- const audioOf = file => file.moov.traks[1];
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => 'enca'.split('').forEach((char, i) =>
view.setUint8(audioOf(file).mdia.minf.stbl.stsd.entries[0].start + 4 + i, char.charCodeAt(0))))], 'encrypted');
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint8(videoOf(file).mdia.minf.dinf.dref.entries[0].start + 11, 0))],
'in another file');
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(audioOf(file).mdia.minf.stbl.sgpds[0].start + 24, -2))],
'different codec settings');
- // Sound whose own edit ends half a second before the picture would play
- // on in a joined file; ending within a frame of it is fine.
+ // Sound whose own edit hides its last half second would play on in a
+ // joined file; hiding less than a frame of it is fine.
const soundEdit = milliseconds => patched('h264-aac-2', (view, file) => view.setUint32(audioOf(file).edts.elst.start + 16, milliseconds));
refused([fixtures['h264-aac-1'], soundEdit(500)], 'ends a track early');
assert.equal(join.joinMp4Clips([fixtures['h264-aac-1'], soundEdit(960)]).durationSeconds, 2);
@@ -446,6 +554,7 @@ for (const build of ['chrome', 'firefox']) {
const result = await generate({ prompt: 'Story', scenes: scenes(2, 10), estimated_duration: 20 }, { fetchImpl: api.fetchImpl });
assert.equal(result.success, true, result.error);
assert.equal(result.joined, false);
+ assert.match(result.joinError, /different codec settings/, 'the trace shows why');
assert.equal(result.clipCount, 2);
assert.equal(result.durationSeconds, 2);
assert.equal(result.mediaId, undefined);