Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Join clips whose sound ends before their picture
Claudeee5736283090
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex f07daf5d8..447e00d6f 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -311,14 +311,19 @@ submitted. A longer video is passed as `scenes`: WebBrain fits each scene to theclip lengths the model offers, splitting a scene that is longer than its longestclip, generates up to three clips at a time with one retry each, and joins theminto one MP4 without re-encoding (`src/*/src/agent/video-join.js`, which parses-the clips with the vendored mp4box.js). Videos are capped at 120 seconds and 24-clips. When the estimate is longer than 120 seconds, WebBrain asks on its own-card whether to generate the shorter version before any clip is submitted; the-card never times out, and an unattended run generates nothing. Clips that-cannot be joined this way (different encodings, edit lists or fragment times-the joined file cannot reproduce, or more than 160 MB together) are kept as-separate clips and shown in order. The model never sees the video, so the tool result-gives it the video's real length (`durationSeconds`) and `clipCount` to report.+the clips with the vendored mp4box.js). A clip's sound often ends a little+before its picture or starts after it; with AAC-LC sound (the usual MP4 audio,+which Grok Imagine returns) the joiner fills those gaps with silent frames, so+the sound plays without gaps and each of its frames plays within half a frame of+its time. Videos are capped at 120 seconds and 24 clips. When the estimate is+longer than 120 seconds, WebBrain asks on its own card whether to generate the+shorter version before any clip is submitted; the card never times out, and an+unattended run generates nothing. Clips that cannot be joined this way+(different encodings, edit lists or timing the joined file cannot reproduce, or+more than 160 MB together) are kept as separate clips and shown in order, and+the tool result says why (`joinError`), which traces show. The model never sees+the video, so the tool result gives it the video's real length+(`durationSeconds`) and `clipCount` to report.The Act system prompt and planner direct the model to call `generate_image` whenthe user asks to generate media. Provider results are treated as untrusted content.diff --git a/src/chrome/src/agent/generative-media.js b/src/chrome/src/agent/generative-media.jsindex 5beabe33a..d23251afb 100644--- a/src/chrome/src/agent/generative-media.js+++ b/src/chrome/src/agent/generative-media.js@@ -455,8 +455,10 @@ function declinedVideoMessage({ answer, estimatedSeconds, totalSeconds }) {}// Joins a video's clips into one stored asset. Clips that cannot be joined-// losslessly are stored one by one and shown in order.+// losslessly are stored one by one and shown in order, and the result says why+// (joinError) so a trace shows it.async function storeVideoClips(clips, onProgress) {+ let joinError;if (clips.length > 1) {try {onProgress?.(`Joining ${clips.length} clips…`);@@ -465,6 +467,7 @@ async function storeVideoClips(clips, onProgress) {return { ...asset, durationSeconds: roundSeconds(joined.durationSeconds), clipCount: clips.length };} catch (error) {if (!(error instanceof VideoJoinError)) throw error;+ joinError = error.message;}}const stored = [];@@ -474,7 +477,7 @@ async function storeVideoClips(clips, onProgress) {}const durationSeconds = roundSeconds(stored.reduce((sum, clip) => sum + (clip.durationSeconds || 0), 0));if (stored.length === 1) return { ...stored[0], inlineMedia: true, durationSeconds, clipCount: 1 };- return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false };+ return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false, joinError };}/**diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex fda3224be..ae7cc41bf 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -22,6 +22,9 @@ export class VideoJoinError extends Error {const incompatible = reason => new VideoJoinError(`The clips cannot be joined into one file: ${reason}.`, 'clips_incompatible');const MEDIA_HEADER_TYPES = new Set(['vmhd', 'smhd', 'nmhd', 'sthd', 'hmhd']);const MAX_UINT32 = 2 ** 32 - 1;+// Silence fills gaps in the sound, as long as the clips run: clips that claim+// to run for hours are not joined.+const MAX_SECONDS = 3600;function toArrayBuffer(input) {if (input instanceof ArrayBuffer) return input;@@ -77,6 +80,34 @@ function sampleDescriptionKey(bytes, stsd) {return key;}+// A silent frame for AAC-LC (mono or stereo, 1024 samples), to fill gaps in a+// clip's sound so it plays continuously: players such as Chrome time audio by+// the samples they play, so a gap in timestamps would shift the sound against+// the picture. The frame is one channel element with an empty spectrum+// (max_sfb 0). Other codecs have none (null), and their tracks are joined as+// video tracks are.+const SILENT_AAC_FRAMES = { 1: Uint8Array.of(0x00, 0x00, 0x00, 0x07), 2: Uint8Array.of(0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e) };+const AAC_SAMPLE_RATES = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];++function silentFrame(entry, timescale) {+ if (entry?.type !== 'mp4a') return null;+ const config = entry.esds?.esd?.descs?.find(desc => desc.tag === 4)?.descs?.find(desc => desc.tag === 5)?.data;+ if (!config || config.length < 2) return null;+ const objectType = config[0] >> 3;+ const rate = AAC_SAMPLE_RATES[((config[0] & 7) << 1) | (config[1] >> 7)];+ const channels = (config[1] >> 3) & 15;+ // AAC-LC with 1024-sample frames and no core coder; only an explicit+ // "no SBR" extension may follow.+ if (objectType !== 2 || !rate || !SILENT_AAC_FRAMES[channels] || (config[1] & 0x06)) return null;+ if (config.length > 2) {+ const syncExtension = (config[2] << 3) | (config[3] >> 5);+ const extensionType = config[3] & 0x1f;+ if (config.length < 5 || syncExtension !== 0x2b7 || extensionType !== 5 || (config[4] & 0x80)) return null;+ }+ const duration = 1024 * timescale / rate;+ return Number.isInteger(duration) ? { bytes: SILENT_AAC_FRAMES[channels], duration } : null;+}+function readTrack(trak, movieTimescale, bytes) {const mdia = trak?.mdia, minf = mdia?.minf, stbl = minf?.stbl;const handler = mdia?.hdlr?.handler;@@ -108,20 +139,29 @@ function readTrack(trak, movieTimescale, bytes) {// Reordered frames (B-frames) are shown after they are decoded, so the// track's picture ends at its last composition time, not its last decode time.const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);- // One edit that skips the codec's start-up delay is normal. Empty edits or- // several edits describe a timeline these tables cannot carry over.+ // One edit that skips the codec's start-up delay is normal, and so is an+ // empty edit before it (a track that starts late). Other edit lists describe+ // a timeline these tables cannot carry over.const edits = trak.edts?.elst?.entries || [];- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {+ const empty = edits.length === 2 && edits[0].media_time === -1 ? edits[0] : null;+ const media = edits[empty ? 1 : 0];+ if (edits.length > (empty ? 2 : 1)+ || (media && (media.media_time < 0 || media.media_rate_integer !== 1 || media.media_rate_fraction))) {throw incompatible('a track has a complex edit list');}+ const ticks = movieTicks => Math.round(movieTicks * timescale / movieTimescale);+ const delay = empty ? ticks(empty.segment_duration) : 0;+ const editDuration = media && media.segment_duration > 0 ? ticks(media.segment_duration) : Infinity;return {handler,timescale,samples,- hasEdit: edits.length > 0,- editStart: edits.length ? edits[0].media_time : 0,- editDuration: edits.length && edits[0].segment_duration > 0- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,+ hasEdit: !!media,+ // The media time the movie shows at its start, before any lead.+ editStart: (media ? media.media_time : 0) - delay,+ editDuration,+ editEnd: media ? media.media_time + editDuration : Infinity,+ silence: handler === 'soun' ? silentFrame(stbl.stsd.entries[0], timescale) : null,firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),compositionEnd,tkhd: trak.tkhd,@@ -170,18 +210,23 @@ function parseClip(input) {// - its first frame must decode without the previous clip's frames;// - an edit list that hides whole frames before or after the picture keeps// them in the file, where a join would play them over the next clip;- // - a track whose first sample comes after the clip's start (by more than a- // millisecond) would need a leading gap, which one edit cannot show;- // - a track whose own edit ends before the picture (by more than one of its- // frames) would play on to the next clip in the joined file.+ // - a track whose first sample comes after the clip's start needs a leading+ // gap: silent frames fill it where the codec allows, as long as the sound+ // starts before the clip ends; otherwise one edit cannot show more than a+ // millisecond of it;+ // - a track whose own edit hides more than one of its frames at the end+ // would play that sound on into the next clip in the joined file.const pictureEnd = timing.mediaTime + presented;- const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)- && track.editDuration < Math.round(presented / timing.timescale * track.timescale)- - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);+ const startsLate = track => {+ const late = track.samples[0].dts - track.mediaTime;+ return track.silence ? late >= presented / timing.timescale * track.timescale : late > track.timescale / 1000;+ };+ const endsEarly = track => track.compositionEnd - track.editEnd+ > track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe': timing.samples.some(sample => sample.duration > 0&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'- : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'+ : tracks.some(startsLate) ? 'has a track that starts after the clip does': tracks.some(endsEarly) ? 'ends a track early with an edit list': null;return {@@ -200,57 +245,113 @@ export function mp4DurationSeconds(input) {try { return parseClip(input).duration; } catch { return null; }}-// Lays out one output track: each clip's samples in order, with the previous-// clip's last sample lengthened or shortened so that this clip starts where-// the timeline says. At each join the audio follows the picture: a later-// clip's codec start-up frames are dropped, and so are the previous clip's-// frames that would mostly play after this clip's audio begins (padding, or-// audio longer than the picture). Video frames are never dropped.+// Lays out one output track: each clip's samples in order, so that each clip+// starts where the timeline says. At each join the audio follows the picture:+// a later clip's codec start-up frames (those that would mostly play before+// the clip starts) are dropped, and so are the previous clip's frames that+// would mostly play after this clip's sound begins (padding, or sound longer+// than the picture). Video frames are never dropped.+//+// Where the codec has a silent frame (AAC-LC), the sound is laid out as the+// frames a player plays: each frame at its full length, one after another,+// with silent frames wherever a clip's timeline runs ahead (sound shorter than+// the picture, sound that starts late, gaps in a clip's own timing). Every+// frame then plays within half a frame of its time, and the sound stays with+// the picture in players that time audio by the samples they play. A clip's+// start-up frames stay right before its first frame where there is room for+// them, so that frame decodes as it was encoded. Otherwise the previous clip's+// last sample is lengthened or shortened to meet the next clip, as it is for+// video.function joinTrack(clips, index, starts) {const first = clips[0].tracks[index];- // The output's decode times start at 0, wherever the first clip's start- // (fragmented files can start at any decode time): its start in the output.- const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);+ const silence = clips.every(clip => clip.tracks[index].silence) ? first.silence : null;+ const frame = silence?.duration;const samples = [];const chunks = [];+ // The media time the output's edit starts at: where the first clip's start+ // falls in the output (fragmented files can start at any decode time).+ let mediaStart = 0;+ let dts = 0;clips.forEach((clip, clipIndex) => {const track = clip.tracks[index];const audio = track.handler === 'soun';- const kept = audio && clipIndex > 0- ? track.samples.filter(sample => sample.cts + sample.duration / 2 >= track.mediaTime)- : track.samples;- if (!kept.length) throw incompatible('a clip has an empty track');- if (samples.length) {- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;+ const laidOut = audio && !!silence;+ const startup = audio ? track.samples.findIndex(sample => sample.cts + sample.duration / 2 >= track.mediaTime) : 0;+ if (startup < 0) throw incompatible('a clip has an empty track');+ const kept = track.samples.slice(clipIndex > 0 || laidOut ? startup : 0);+ // Where the clip's first kept sample goes, and how many frames (silence,+ // then the clip's own start-up frames) come right before it.+ let base = 0;+ let gap = 0;+ if (clipIndex === 0) {+ if (laidOut) {+ // Whole frames before a sound that starts late, enough to reach its start.+ gap = Math.max(startup, Math.ceil((kept[0].cts - track.mediaTime) / frame));+ base = gap * frame;+ mediaStart = base - (kept[0].cts - track.mediaTime);+ } else {+ mediaStart = Math.max(0, track.mediaTime - track.samples[0].dts);+ }+ } else {+ base = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;const previous = chunks[chunks.length - 1];while (audio && previous.count > 1- && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {+ && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= base) {previous.count -= 1;previous.size -= samples.pop().size;}const last = samples[samples.length - 1];- // Decode times only move forward. A clip that must start decoding before- // the previous clip's last frame (a longer composition delay) cannot- // follow it without moving its picture.- if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');- last.duration = target - last.dts;+ if (laidOut) {+ dts = last.dts + last.duration;+ gap = Math.max(0, Math.round((base - dts) / frame));+ } else {+ // Decode times only move forward. A clip that must start decoding+ // before the previous clip's last frame (a longer composition delay)+ // cannot follow it without moving its picture.+ if (base <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');+ last.duration = base - last.dts;+ dts = base;+ }}- let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;- chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });- const chunk = chunks[chunks.length - 1];- for (const sample of kept) {- samples.push({- dts,- duration: sample.duration,- offset: sample.cts - sample.dts,- size: sample.size,- source: sample.offset,- sync: sample.is_sync !== false,- groups: track.groups.map(group => group.indices[sample.number] ?? 0),- clipIndex,- });+ const chunk = { clipIndex, first: samples.length, count: 0, size: 0, offset: 0 };+ chunks.push(chunk);+ const push = sample => {+ samples.push(sample);+ chunk.count += 1;chunk.size += sample.size;dts += sample.duration;+ };+ const copy = (sample, duration) => push({+ dts,+ duration,+ offset: sample.cts - sample.dts,+ size: sample.size,+ source: sample.offset,+ sync: sample.is_sync !== false,+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),+ clipIndex,+ });+ const quiet = () => push({ dts, duration: frame, offset: 0, size: silence.bytes.length, data: silence.bytes,+ sync: true, groups: first.groups.map(() => 0), clipIndex });+ for (let k = gap; k > 0; k--) {+ if (k <= startup) copy(track.samples[startup - k], frame);+ else quiet();+ }+ // Where the clip ends in the output: its frames that would start later+ // would play over the next clip, or after the video ends, so they are dropped.+ const end = Math.round((starts[clipIndex] + clip.duration) * track.timescale) + mediaStart;+ for (const [i, sample] of kept.entries()) {+ if (laidOut) {+ const target = base + sample.dts - kept[0].dts;+ if (i > 0 && target >= end) break;+ // Silence up to where this frame belongs.+ for (let k = Math.round((target - dts) / frame); k > 0; k--) quiet();+ }+ // An AAC-LC frame plays 1024 samples, even a clip's last one that its+ // sample table shortens to where the clip's sound ends; only the joined+ // video's very last frame keeps its own length.+ const final = clipIndex === clips.length - 1 && i === kept.length - 1;+ copy(sample, laidOut && !final ? frame : sample.duration);}});const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);@@ -436,6 +537,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const starts = [];let totalSeconds = 0;for (const clip of clips) { starts.push(totalSeconds); totalSeconds += clip.duration; }+ if (totalSeconds > MAX_SECONDS) throw new VideoJoinError('The joined video would be longer than an hour.', 'too_large');const tracks = reference.tracks.map((_track, index) => joinTrack(clips, index, starts));const payload = tracks.reduce((sum, track) => sum + track.chunks.reduce((size, chunk) => size + chunk.size, 0), 0);const ftyp = reference.ftyp || (() => {@@ -471,7 +573,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const source = clips[chunk.clipIndex].bytes;for (let i = chunk.first; i < chunk.first + chunk.count; i++) {const sample = track.samples[i];- out.set(source.subarray(sample.source, sample.source + sample.size), position);+ out.set(sample.data || source.subarray(sample.source, sample.source + sample.size), position);position += sample.size;}}diff --git a/src/firefox/src/agent/generative-media.js b/src/firefox/src/agent/generative-media.jsindex 5beabe33a..d23251afb 100644--- a/src/firefox/src/agent/generative-media.js+++ b/src/firefox/src/agent/generative-media.js@@ -455,8 +455,10 @@ function declinedVideoMessage({ answer, estimatedSeconds, totalSeconds }) {}// Joins a video's clips into one stored asset. Clips that cannot be joined-// losslessly are stored one by one and shown in order.+// losslessly are stored one by one and shown in order, and the result says why+// (joinError) so a trace shows it.async function storeVideoClips(clips, onProgress) {+ let joinError;if (clips.length > 1) {try {onProgress?.(`Joining ${clips.length} clips…`);@@ -465,6 +467,7 @@ async function storeVideoClips(clips, onProgress) {return { ...asset, durationSeconds: roundSeconds(joined.durationSeconds), clipCount: clips.length };} catch (error) {if (!(error instanceof VideoJoinError)) throw error;+ joinError = error.message;}}const stored = [];@@ -474,7 +477,7 @@ async function storeVideoClips(clips, onProgress) {}const durationSeconds = roundSeconds(stored.reduce((sum, clip) => sum + (clip.durationSeconds || 0), 0));if (stored.length === 1) return { ...stored[0], inlineMedia: true, durationSeconds, clipCount: 1 };- return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false };+ return { inlineMedia: true, mimeType: 'video/mp4', clips: stored, clipCount: stored.length, durationSeconds, joined: false, joinError };}/**diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex fda3224be..ae7cc41bf 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -22,6 +22,9 @@ export class VideoJoinError extends Error {const incompatible = reason => new VideoJoinError(`The clips cannot be joined into one file: ${reason}.`, 'clips_incompatible');const MEDIA_HEADER_TYPES = new Set(['vmhd', 'smhd', 'nmhd', 'sthd', 'hmhd']);const MAX_UINT32 = 2 ** 32 - 1;+// Silence fills gaps in the sound, as long as the clips run: clips that claim+// to run for hours are not joined.+const MAX_SECONDS = 3600;function toArrayBuffer(input) {if (input instanceof ArrayBuffer) return input;@@ -77,6 +80,34 @@ function sampleDescriptionKey(bytes, stsd) {return key;}+// A silent frame for AAC-LC (mono or stereo, 1024 samples), to fill gaps in a+// clip's sound so it plays continuously: players such as Chrome time audio by+// the samples they play, so a gap in timestamps would shift the sound against+// the picture. The frame is one channel element with an empty spectrum+// (max_sfb 0). Other codecs have none (null), and their tracks are joined as+// video tracks are.+const SILENT_AAC_FRAMES = { 1: Uint8Array.of(0x00, 0x00, 0x00, 0x07), 2: Uint8Array.of(0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e) };+const AAC_SAMPLE_RATES = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];++function silentFrame(entry, timescale) {+ if (entry?.type !== 'mp4a') return null;+ const config = entry.esds?.esd?.descs?.find(desc => desc.tag === 4)?.descs?.find(desc => desc.tag === 5)?.data;+ if (!config || config.length < 2) return null;+ const objectType = config[0] >> 3;+ const rate = AAC_SAMPLE_RATES[((config[0] & 7) << 1) | (config[1] >> 7)];+ const channels = (config[1] >> 3) & 15;+ // AAC-LC with 1024-sample frames and no core coder; only an explicit+ // "no SBR" extension may follow.+ if (objectType !== 2 || !rate || !SILENT_AAC_FRAMES[channels] || (config[1] & 0x06)) return null;+ if (config.length > 2) {+ const syncExtension = (config[2] << 3) | (config[3] >> 5);+ const extensionType = config[3] & 0x1f;+ if (config.length < 5 || syncExtension !== 0x2b7 || extensionType !== 5 || (config[4] & 0x80)) return null;+ }+ const duration = 1024 * timescale / rate;+ return Number.isInteger(duration) ? { bytes: SILENT_AAC_FRAMES[channels], duration } : null;+}+function readTrack(trak, movieTimescale, bytes) {const mdia = trak?.mdia, minf = mdia?.minf, stbl = minf?.stbl;const handler = mdia?.hdlr?.handler;@@ -108,20 +139,29 @@ function readTrack(trak, movieTimescale, bytes) {// Reordered frames (B-frames) are shown after they are decoded, so the// track's picture ends at its last composition time, not its last decode time.const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);- // One edit that skips the codec's start-up delay is normal. Empty edits or- // several edits describe a timeline these tables cannot carry over.+ // One edit that skips the codec's start-up delay is normal, and so is an+ // empty edit before it (a track that starts late). Other edit lists describe+ // a timeline these tables cannot carry over.const edits = trak.edts?.elst?.entries || [];- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {+ const empty = edits.length === 2 && edits[0].media_time === -1 ? edits[0] : null;+ const media = edits[empty ? 1 : 0];+ if (edits.length > (empty ? 2 : 1)+ || (media && (media.media_time < 0 || media.media_rate_integer !== 1 || media.media_rate_fraction))) {throw incompatible('a track has a complex edit list');}+ const ticks = movieTicks => Math.round(movieTicks * timescale / movieTimescale);+ const delay = empty ? ticks(empty.segment_duration) : 0;+ const editDuration = media && media.segment_duration > 0 ? ticks(media.segment_duration) : Infinity;return {handler,timescale,samples,- hasEdit: edits.length > 0,- editStart: edits.length ? edits[0].media_time : 0,- editDuration: edits.length && edits[0].segment_duration > 0- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,+ hasEdit: !!media,+ // The media time the movie shows at its start, before any lead.+ editStart: (media ? media.media_time : 0) - delay,+ editDuration,+ editEnd: media ? media.media_time + editDuration : Infinity,+ silence: handler === 'soun' ? silentFrame(stbl.stsd.entries[0], timescale) : null,firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),compositionEnd,tkhd: trak.tkhd,@@ -170,18 +210,23 @@ function parseClip(input) {// - its first frame must decode without the previous clip's frames;// - an edit list that hides whole frames before or after the picture keeps// them in the file, where a join would play them over the next clip;- // - a track whose first sample comes after the clip's start (by more than a- // millisecond) would need a leading gap, which one edit cannot show;- // - a track whose own edit ends before the picture (by more than one of its- // frames) would play on to the next clip in the joined file.+ // - a track whose first sample comes after the clip's start needs a leading+ // gap: silent frames fill it where the codec allows, as long as the sound+ // starts before the clip ends; otherwise one edit cannot show more than a+ // millisecond of it;+ // - a track whose own edit hides more than one of its frames at the end+ // would play that sound on into the next clip in the joined file.const pictureEnd = timing.mediaTime + presented;- const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)- && track.editDuration < Math.round(presented / timing.timescale * track.timescale)- - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);+ const startsLate = track => {+ const late = track.samples[0].dts - track.mediaTime;+ return track.silence ? late >= presented / timing.timescale * track.timescale : late > track.timescale / 1000;+ };+ const endsEarly = track => track.compositionEnd - track.editEnd+ > track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe': timing.samples.some(sample => sample.duration > 0&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'- : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'+ : tracks.some(startsLate) ? 'has a track that starts after the clip does': tracks.some(endsEarly) ? 'ends a track early with an edit list': null;return {@@ -200,57 +245,113 @@ export function mp4DurationSeconds(input) {try { return parseClip(input).duration; } catch { return null; }}-// Lays out one output track: each clip's samples in order, with the previous-// clip's last sample lengthened or shortened so that this clip starts where-// the timeline says. At each join the audio follows the picture: a later-// clip's codec start-up frames are dropped, and so are the previous clip's-// frames that would mostly play after this clip's audio begins (padding, or-// audio longer than the picture). Video frames are never dropped.+// Lays out one output track: each clip's samples in order, so that each clip+// starts where the timeline says. At each join the audio follows the picture:+// a later clip's codec start-up frames (those that would mostly play before+// the clip starts) are dropped, and so are the previous clip's frames that+// would mostly play after this clip's sound begins (padding, or sound longer+// than the picture). Video frames are never dropped.+//+// Where the codec has a silent frame (AAC-LC), the sound is laid out as the+// frames a player plays: each frame at its full length, one after another,+// with silent frames wherever a clip's timeline runs ahead (sound shorter than+// the picture, sound that starts late, gaps in a clip's own timing). Every+// frame then plays within half a frame of its time, and the sound stays with+// the picture in players that time audio by the samples they play. A clip's+// start-up frames stay right before its first frame where there is room for+// them, so that frame decodes as it was encoded. Otherwise the previous clip's+// last sample is lengthened or shortened to meet the next clip, as it is for+// video.function joinTrack(clips, index, starts) {const first = clips[0].tracks[index];- // The output's decode times start at 0, wherever the first clip's start- // (fragmented files can start at any decode time): its start in the output.- const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);+ const silence = clips.every(clip => clip.tracks[index].silence) ? first.silence : null;+ const frame = silence?.duration;const samples = [];const chunks = [];+ // The media time the output's edit starts at: where the first clip's start+ // falls in the output (fragmented files can start at any decode time).+ let mediaStart = 0;+ let dts = 0;clips.forEach((clip, clipIndex) => {const track = clip.tracks[index];const audio = track.handler === 'soun';- const kept = audio && clipIndex > 0- ? track.samples.filter(sample => sample.cts + sample.duration / 2 >= track.mediaTime)- : track.samples;- if (!kept.length) throw incompatible('a clip has an empty track');- if (samples.length) {- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;+ const laidOut = audio && !!silence;+ const startup = audio ? track.samples.findIndex(sample => sample.cts + sample.duration / 2 >= track.mediaTime) : 0;+ if (startup < 0) throw incompatible('a clip has an empty track');+ const kept = track.samples.slice(clipIndex > 0 || laidOut ? startup : 0);+ // Where the clip's first kept sample goes, and how many frames (silence,+ // then the clip's own start-up frames) come right before it.+ let base = 0;+ let gap = 0;+ if (clipIndex === 0) {+ if (laidOut) {+ // Whole frames before a sound that starts late, enough to reach its start.+ gap = Math.max(startup, Math.ceil((kept[0].cts - track.mediaTime) / frame));+ base = gap * frame;+ mediaStart = base - (kept[0].cts - track.mediaTime);+ } else {+ mediaStart = Math.max(0, track.mediaTime - track.samples[0].dts);+ }+ } else {+ base = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;const previous = chunks[chunks.length - 1];while (audio && previous.count > 1- && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {+ && samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= base) {previous.count -= 1;previous.size -= samples.pop().size;}const last = samples[samples.length - 1];- // Decode times only move forward. A clip that must start decoding before- // the previous clip's last frame (a longer composition delay) cannot- // follow it without moving its picture.- if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');- last.duration = target - last.dts;+ if (laidOut) {+ dts = last.dts + last.duration;+ gap = Math.max(0, Math.round((base - dts) / frame));+ } else {+ // Decode times only move forward. A clip that must start decoding+ // before the previous clip's last frame (a longer composition delay)+ // cannot follow it without moving its picture.+ if (base <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');+ last.duration = base - last.dts;+ dts = base;+ }}- let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;- chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });- const chunk = chunks[chunks.length - 1];- for (const sample of kept) {- samples.push({- dts,- duration: sample.duration,- offset: sample.cts - sample.dts,- size: sample.size,- source: sample.offset,- sync: sample.is_sync !== false,- groups: track.groups.map(group => group.indices[sample.number] ?? 0),- clipIndex,- });+ const chunk = { clipIndex, first: samples.length, count: 0, size: 0, offset: 0 };+ chunks.push(chunk);+ const push = sample => {+ samples.push(sample);+ chunk.count += 1;chunk.size += sample.size;dts += sample.duration;+ };+ const copy = (sample, duration) => push({+ dts,+ duration,+ offset: sample.cts - sample.dts,+ size: sample.size,+ source: sample.offset,+ sync: sample.is_sync !== false,+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),+ clipIndex,+ });+ const quiet = () => push({ dts, duration: frame, offset: 0, size: silence.bytes.length, data: silence.bytes,+ sync: true, groups: first.groups.map(() => 0), clipIndex });+ for (let k = gap; k > 0; k--) {+ if (k <= startup) copy(track.samples[startup - k], frame);+ else quiet();+ }+ // Where the clip ends in the output: its frames that would start later+ // would play over the next clip, or after the video ends, so they are dropped.+ const end = Math.round((starts[clipIndex] + clip.duration) * track.timescale) + mediaStart;+ for (const [i, sample] of kept.entries()) {+ if (laidOut) {+ const target = base + sample.dts - kept[0].dts;+ if (i > 0 && target >= end) break;+ // Silence up to where this frame belongs.+ for (let k = Math.round((target - dts) / frame); k > 0; k--) quiet();+ }+ // An AAC-LC frame plays 1024 samples, even a clip's last one that its+ // sample table shortens to where the clip's sound ends; only the joined+ // video's very last frame keeps its own length.+ const final = clipIndex === clips.length - 1 && i === kept.length - 1;+ copy(sample, laidOut && !final ? frame : sample.duration);}});const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);@@ -436,6 +537,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const starts = [];let totalSeconds = 0;for (const clip of clips) { starts.push(totalSeconds); totalSeconds += clip.duration; }+ if (totalSeconds > MAX_SECONDS) throw new VideoJoinError('The joined video would be longer than an hour.', 'too_large');const tracks = reference.tracks.map((_track, index) => joinTrack(clips, index, starts));const payload = tracks.reduce((sum, track) => sum + track.chunks.reduce((size, chunk) => size + chunk.size, 0), 0);const ftyp = reference.ftyp || (() => {@@ -471,7 +573,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const source = clips[chunk.clipIndex].bytes;for (let i = chunk.first; i < chunk.first + chunk.count; i++) {const sample = track.samples[i];- out.set(source.subarray(sample.source, sample.source + sample.size), position);+ out.set(sample.data || source.subarray(sample.source, sample.source + sample.size), position);position += sample.size;}}diff --git a/test/fixtures/video/README.md b/test/fixtures/video/README.mdindex c72773162..78d154b76 100644--- a/test/fixtures/video/README.md+++ b/test/fixtures/video/README.md@@ -9,9 +9,13 @@ FFmpeg 6.1 from its built-in test sources:| `h264-aac-fragmented-1.mp4`, `h264-aac-fragmented-2.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono | Fragmented MP4, no edit lists || `h264-aac-48px.mp4` | H.264 at 48×48 | AAC | MP4 (codec setup differs from the others) || `vp9-opus-1.mp4`, `vp9-opus-2.mp4` | VP9, 12 fps | Opus, 48 kHz mono | MP4 (royalty-free codecs, playable in Playwright's Chromium) |+| `h264-aac-short-audio.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono, ends 0.25 s before the picture | MP4, faststart, edit lists |+| `h264-aac-late-audio.mp4` | H.264, 12 fps, 2 B-frames | AAC, 16 kHz mono, starts 0.25 s in (an empty edit) | MP4, faststart, edit lists |+| `vp9-opus-late-audio.mp4` | VP9, 12 fps | Opus, 48 kHz mono, starts 0.25 s in (an empty edit) | MP4 |```sh-H="-c:v libx264 -preset veryfast -pix_fmt yuv420p -g 12 -bf 2 -c:a aac -b:a 24k -ac 1 -shortest"+E="-c:v libx264 -preset veryfast -pix_fmt yuv420p -g 12 -bf 2 -c:a aac -b:a 24k -ac 1"+H="$E -shortest"ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=300:sample_rate=16000:duration=1" \$H -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-1.mp4ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=500:sample_rate=16000:duration=1" \@@ -21,6 +25,14 @@ ffmpeg -f lavfi -i "testsrc2=size=64x64:rate=12:duration=1" -f lavfi -i "sine=fr-fflags +bitexact -map_metadata -1 vp9-opus-1.mp4ffmpeg -f lavfi -i "testsrc=size=48x48:rate=12:duration=1" -f lavfi -i "sine=frequency=300:sample_rate=16000:duration=1" \$H -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-48px.mp4+# Sound shorter than the picture, or starting after it: the picture keeps its full second.+ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -f lavfi -i "sine=frequency=900:sample_rate=16000:duration=0.75" \+ $E -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-short-audio.mp4+ffmpeg -f lavfi -i "testsrc=size=64x64:rate=12:duration=1" -itsoffset 0.25 -f lavfi -i "sine=frequency=1100:sample_rate=16000:duration=0.75" \+ $E -movflags +faststart -fflags +bitexact -map_metadata -1 h264-aac-late-audio.mp4+ffmpeg -f lavfi -i "testsrc2=size=64x64:rate=12:duration=1" -itsoffset 0.25 -f lavfi -i "sine=frequency=1300:sample_rate=48000:duration=0.75" \+ -c:v libvpx-vp9 -b:v 60k -g 12 -deadline realtime -cpu-used 8 -c:a libopus -b:a 16k -ac 1 \+ -fflags +bitexact -map_metadata -1 vp9-opus-late-audio.mp4```The second clip of each pair uses a different tone (600, 1000 and 800 Hz), so itsdiff --git a/test/fixtures/video/h264-aac-late-audio.mp4 b/test/fixtures/video/h264-aac-late-audio.mp4new file mode 100644index 000000000..108d18138Binary files /dev/null and b/test/fixtures/video/h264-aac-late-audio.mp4 differdiff --git a/test/fixtures/video/h264-aac-short-audio.mp4 b/test/fixtures/video/h264-aac-short-audio.mp4new file mode 100644index 000000000..0ba11d9f0Binary files /dev/null and b/test/fixtures/video/h264-aac-short-audio.mp4 differdiff --git a/test/fixtures/video/vp9-opus-late-audio.mp4 b/test/fixtures/video/vp9-opus-late-audio.mp4new file mode 100644index 000000000..83706bfdaBinary files /dev/null and b/test/fixtures/video/vp9-opus-late-audio.mp4 differdiff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 4dfa41857..f8b698643 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -43,6 +43,7 @@ const json = (body, status = 200, headers = {}) => new Response(JSON.stringify(bconst soon = { 'Retry-After': '0.001' };const fixtures = Object.fromEntries(await Promise.all(['h264-aac-1', 'h264-aac-2', 'h264-aac-48px', 'h264-aac-fragmented-1', 'h264-aac-fragmented-2', 'vp9-opus-1', 'vp9-opus-2',+ 'h264-aac-short-audio', 'h264-aac-late-audio', 'vp9-opus-late-audio',].map(async name => [name, new Uint8Array(await fs.readFile(new URL(`./fixtures/video/${name}.mp4`, import.meta.url)))])));const { createFile, MP4BoxBuffer } = await import('../src/chrome/vendor/mp4box/mp4box.all.mjs');@@ -58,6 +59,7 @@ const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sconst boxBytes = (bytes, box) => bytes.subarray(box.start, box.start + box.size);const videoOf = file => file.moov.traks[0];+const audioOf = file => file.moov.traks[1];// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of// the copy and the parsed original.@@ -188,14 +190,17 @@ for (const build of ['chrome', 'firefox']) {const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });- assert.equal(shown(output, audio).to, 2);// Audio: the first clip's frames are all kept; the second clip's start-up- // frame is dropped, the rest is copied as is, and its first real frame- // plays exactly one second in, with its picture.+ // frame is dropped and the rest is copied as is. Every frame but the last+ // plays in full (1024 samples, as AAC frames do), so the second clip's first+ // real frame plays within half a frame (512 samples) of one second in, with+ // its picture, and the sound ends within half a frame of the picture.const [firstAudio, secondAudio] = sources.map(source => sampleBytes(source, parse(source).moov.traks[1]));assert.deepEqual(sampleBytes(bytes, audio).slice(0, 17), firstAudio);assert.deepEqual(sampleBytes(bytes, audio).slice(17), secondAudio.slice(1));- assert.equal(audio.samples[17].cts - audio.edts.elst.entries[0].media_time, 16000);+ assert.ok(audio.samples.every((sample, i) => sample.duration === 1024 || i === audio.samples.length - 1));+ assert.ok(Math.abs(audio.samples[17].cts - audio.edts.elst.entries[0].media_time - 16000) <= 512);+ assert.ok(Math.abs(shown(output, audio).to - 2) <= 0.032);// AAC's pre-roll for seeking (a 'roll' sample group) covers every frame.const sourceRoll = parse(sources[0]).moov.traks[1].mdia.minf.stbl.sgpds[0];assert.deepEqual(boxBytes(bytes, audio.mdia.minf.stbl.sgpds[0]), boxBytes(sources[0], sourceRoll));@@ -205,16 +210,20 @@ for (const build of ['chrome', 'firefox']) {test(`${build}: fragmented and VP9/Opus clips join too`, () => {// [clips, codecs, audio frames kept from the first clip, start-up frames dropped// from the second, how far into its first kept frame the second clip's sound- // starts, the audio's pre-roll groups as runs of [frames, group]]- for (const [names, codecs, kept, dropped, lead, roll] of [+ // starts, the audio's pre-roll groups as runs of [frames, group], and how far+ // (in samples) the sound may play from its time]+ for (const [names, codecs, kept, dropped, lead, roll, slack] of [// No edit lists: the picture starts after a B-frame delay (2048/12288 s),- // so every track is shown from there and the movie starts at its first picture.- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, []],+ // so every track is shown from there and the movie starts at its first+ // picture. The first audio frame (the encoder's start-up) lasts until+ // then; it plays right before the sound, as in a file with an edit list.+ // AAC frames play in full, each within half a frame of its time.+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, [], 512],// Opus: the first clip's short final frame would play over the second// clip's first frame, whose first 312 samples are the decoder's start-up,// so the short frame is dropped rather than squeezed. Each clip keeps its// own pre-roll groups (none for its first four frames).- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]]],+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]], 0],]) {const label = names.join('+');const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));@@ -224,22 +233,102 @@ for (const build of ['chrome', 'firefox']) {assert.deepEqual(output.moov.traks.map(trak => trak.mdia.minf.stbl.stsd.entries[0].type), codecs, label);assert.equal(video.samples.length, 24, label);assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 }, `${label}: the picture runs from 0 to the declared length`);- assert.equal(shown(output, audio).to, 2, `${label}: and so does the sound`);+ assert.ok(Math.abs(shown(output, audio).to - 2) <= slack / audio.mdia.mdhd.timescale, `${label}: and so does the sound`);const [firstAudio, secondAudio] = names.map(name => parse(fixtures[name]).moov.traks[1]);assert.deepEqual(sampleBytes(bytes, audio).slice(0, kept), sampleBytes(fixtures[names[0]], firstAudio).slice(0, kept), `${label}: the first clip's audio up to the join`);assert.deepEqual(sampleBytes(bytes, audio).slice(kept), sampleBytes(fixtures[names[1]], secondAudio).slice(dropped), `${label}: the second clip's audio`);- assert.equal(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time, audio.mdia.mdhd.timescale,+ assert.ok(Math.abs(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time - audio.mdia.mdhd.timescale) <= slack,`${label}: the second clip's sound starts one second in`);const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);assert.deepEqual((audio.mdia.minf.stbl.sbgps[0]?.entries || []).map(entry => [entry.sample_count, entry.group_description_index]), roll, `${label}: pre-roll groups`);}// Fragments can start at any decode time: five seconds later, the clips- // join into the same file. A track that starts after the picture cannot join.+ // join into the same file.const fragmented = [fixtures['h264-aac-fragmented-1'], fixtures['h264-aac-fragmented-2']];assert.equal(parse(laterFragments(fragmented[0], { vide: 5, soun: 5 })).moov.traks[0].samples[0].dts, 5 * 12288);assert.deepEqual(join.joinMp4Clips(fragmented.map(clip => laterFragments(clip, { vide: 5, soun: 5 }))).bytes, join.joinMp4Clips(fragmented).bytes);- assert.throws(() => join.joinMp4Clips([laterFragments(fragmented[0], { soun: 5 }), fragmented[1]]), error => error.code === 'clips_incompatible');+ });++ test(`${build}: sound shorter than its picture, or starting after it, is filled with silence`, () => {+ // 'S' for AAC-LC's silent mono frame (an empty spectrum), '#' for sound.+ const marks = (bytes, trak) => sampleBytes(bytes, trak)+ .map(data => (data.length === 4 && [0, 0, 0, 7].every((value, i) => data[i] === value) ? 'S' : '#')).join('');+ const joined = clips => {+ const { bytes, durationSeconds } = join.joinMp4Clips(clips.map(clip => (typeof clip === 'string' ? fixtures[clip] : clip)));+ const output = parse(bytes);+ return { bytes, durationSeconds, output, audio: audioOf(output) };+ };+ // Where a joined frame plays, in seconds; the sound may start up to half a frame (32 ms) off.+ const playsAt = (audio, index) => (audio.samples[index].cts - audio.edts.elst.entries[0].media_time) / audio.mdia.mdhd.timescale;+ // A clip whose sound ends 0.25 s before its picture (as generated clips' sound+ // often ends a little early): four frames carry it to the next clip, three+ // silent ones and the next clip's start-up frame, which primes the decoder+ // so that clip's first frame sounds as it was encoded.+ let { bytes, durationSeconds, output, audio } = joined(['h264-aac-short-audio', 'h264-aac-2']);+ assert.equal(durationSeconds, 2);+ assert.deepEqual(shown(output, videoOf(output)), { from: 0, to: 2, movie: 2 });+ assert.equal(marks(bytes, audio), `${'#'.repeat(13)}SSS${'#'.repeat(17)}`);+ const nextClip = fixtures['h264-aac-2'];+ assert.deepEqual(sampleBytes(bytes, audio).slice(16), sampleBytes(nextClip, parse(nextClip).moov.traks[1]));+ assert.ok(Math.abs(playsAt(audio, 17) - 1) <= 0.032, `the second clip's sound starts at ${playsAt(audio, 17)} s`);+ // Every frame but the video's last counts in full, so the sound plays+ // without gaps and stays with the picture.+ assert.ok(audio.samples.every((sample, i) => (sample.duration === 1024 || i === audio.samples.length - 1)+ && (!i || sample.dts === audio.samples[i - 1].dts + 1024)));+ // A clip whose sound starts 0.25 s in (an empty edit, then the encoder's+ // start-up frame from 0.186 s): silence before it, in the middle of the video...+ ({ bytes, audio } = joined(['h264-aac-1', 'h264-aac-late-audio']));+ assert.equal(marks(bytes, audio), `${'#'.repeat(17)}SSS${'#'.repeat(13)}`);+ assert.ok(Math.abs(playsAt(audio, 21) - 1.25) <= 0.032, `the late sound starts at ${playsAt(audio, 21)} s`);+ // ...or at its start, where the silence is whole frames, enough to reach the+ // sound, and the video's edit starts inside the first of them: the sound+ // plays exactly on time. Starting at 0.15 s still takes three frames.+ ({ bytes, audio } = joined(['h264-aac-late-audio', 'h264-aac-1']));+ assert.equal(marks(bytes, audio), `SSS${'#'.repeat(29)}`);+ assert.equal(playsAt(audio, 4), 0.25);+ const sooner = patched('h264-aac-late-audio', (view, file) => {+ const { start } = audioOf(file).edts.elst;+ view.setUint32(start + 16, 150);+ view.setUint32(start + 28, 850);+ });+ ({ bytes, audio } = joined([sooner, 'h264-aac-1']));+ assert.equal(marks(bytes, audio), `SSS${'#'.repeat(29)}`);+ assert.equal(playsAt(audio, 3), 0.15);+ // A gap in a clip's own timing (one frame lasting three frames' time) is+ // filled with silence too, so the sound after it stays on time. The two+ // frames it pushes past the picture's end are dropped.+ const gappy = patched('h264-aac-fragmented-2', (view, file) => {+ const run = file.moofs[0].trafs.find(traf => traf.tfhd.track_id === audioOf(file).tkhd.track_id).truns[0];+ assert.equal(run.flags, 0x301, 'each sample\'s duration and size follow the data offset');+ view.setUint32(run.start + 20 + 8 * 8, 3 * 1024);+ });+ ({ bytes, audio } = joined(['h264-aac-fragmented-1', gappy]));+ assert.equal(marks(bytes, audio), `${'#'.repeat(25)}SS${'#'.repeat(6)}`);+ assert.ok(Math.abs(playsAt(audio, 27) - (1 + 10 * 1024 / 16000)) <= 0.032, `the sound after the gap starts at ${playsAt(audio, 27)} s`);+ // Sound may start late as long as it starts before the picture ends.+ const startingAt = milliseconds => patched('h264-aac-late-audio', (view, file) => view.setUint32(audioOf(file).edts.elst.start + 16, milliseconds));+ assert.ok(joined(['h264-aac-1', startingAt(999)]).bytes);+ assert.throws(() => joined(['h264-aac-1', startingAt(1000)]), error => error.message.includes('starts after the clip does'));+ // Only AAC-LC has a silent frame here: Opus, or the same clips marked as+ // HE-AAC, keep a clip whose sound starts late apart.+ assert.throws(() => joined(['vp9-opus-1', 'vp9-opus-late-audio']), error => error.message.includes('starts after the clip does'));+ const heAac = name => patched(name, (view, file) => {+ const { esds } = audioOf(file).mdia.minf.stbl.stsd.entries[0];+ const config = esds.esd.descs.find(desc => desc.tag === 4).descs.find(desc => desc.tag === 5).data;+ const box = boxBytes(fixtures[name], esds);+ const at = box.findIndex((_, i) => box[i - 1] === config.length && config.every((value, k) => box[i + k] === value));+ view.setUint8(esds.start + at, (5 << 3) | (config[0] & 7));+ });+ assert.ok(joined([heAac('h264-aac-1'), heAac('h264-aac-2')]).bytes, 'HE-AAC clips that need no silence still join');+ assert.throws(() => joined([heAac('h264-aac-1'), heAac('h264-aac-late-audio')]), error => error.message.includes('starts after the clip does'));+ // So do AAC-LC clips whose timescale cannot count a frame in whole ticks.+ const oddTimescale = name => patched(name, (view, file) => {+ const { mdhd } = audioOf(file).mdia;+ assert.equal(mdhd.version, 0);+ view.setUint32(mdhd.start + 20, 15999);+ });+ assert.throws(() => joined([oddTimescale('h264-aac-1'), oddTimescale('h264-aac-late-audio')]), error => error.message.includes('starts after the clip does'));});test(`${build}: clips with different codec settings or tracks are not joined`, () => {@@ -259,6 +348,18 @@ for (const build of ['chrome', 'firefox']) {assert.equal(join.mp4DurationSeconds(cut), null);assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');+ // Nor are clips that claim to run for more than an hour (silence would fill+ // their sound for as long).+ const endless = patched('h264-aac-2', (view, file) => {+ const video = videoOf(file);+ const { stts } = video.mdia.minf.stbl;+ assert.deepEqual([stts.sample_deltas.length, video.mdia.mdhd.version], [1, 0]);+ view.setUint32(stts.start + 20, 1024 * 3700);+ view.setUint32(video.mdia.mdhd.start + 24, 12 * 1024 * 3700);+ view.setUint32(video.edts.elst.start + 16, 3700 * 1000);+ });+ assert.ok(join.mp4DurationSeconds(endless) > 3699);+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], endless]), error => error.code === 'too_large' && /an hour/.test(error.message));assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);});@@ -276,8 +377,16 @@ for (const build of ['chrome', 'firefox']) {assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);refused([fixtures['h264-aac-1'], edited({ milliseconds: 500 })], 'hides some of its frames');refused([fixtures['h264-aac-1'], edited({ mediaTime: 3072 })], 'hides some of its frames');- // Shortening the last frame alone is fine.- assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);+ // Shortening the last frame alone is fine. The sound past the shortened+ // picture is dropped (a frame that would start after it, or play mostly+ // after it), and the next clip's sound starts with its picture.+ for (const milliseconds of [960, 980]) {+ const shortened = join.joinMp4Clips([edited({ milliseconds }), fixtures['h264-aac-2']]);+ assert.ok(Math.abs(shortened.durationSeconds - (1 + milliseconds / 1000)) < 0.001);+ const sound = audioOf(parse(shortened.bytes));+ assert.equal(sound.samples.length, 32, 'each clip keeps 16 frames');+ assert.ok(Math.abs(sound.samples[16].cts - sound.edts.elst.entries[0].media_time - milliseconds * 16) <= 512);+ }// A one-second-longer composition delay keeps the picture where it was but// starts decoding a second earlier: such a clip can open a video, not follow one.const delayed = patched('h264-aac-2', (view, file) => {@@ -298,15 +407,14 @@ for (const build of ['chrome', 'firefox']) {refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],'does not start with a keyframe');// Encrypted samples, samples kept in another file, and a different pre-roll.- const audioOf = file => file.moov.traks[1];refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => 'enca'.split('').forEach((char, i) =>view.setUint8(audioOf(file).mdia.minf.stbl.stsd.entries[0].start + 4 + i, char.charCodeAt(0))))], 'encrypted');refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint8(videoOf(file).mdia.minf.dinf.dref.entries[0].start + 11, 0))],'in another file');refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(audioOf(file).mdia.minf.stbl.sgpds[0].start + 24, -2))],'different codec settings');- // Sound whose own edit ends half a second before the picture would play- // on in a joined file; ending within a frame of it is fine.+ // Sound whose own edit hides its last half second would play on in a+ // joined file; hiding less than a frame of it is fine.const soundEdit = milliseconds => patched('h264-aac-2', (view, file) => view.setUint32(audioOf(file).edts.elst.start + 16, milliseconds));refused([fixtures['h264-aac-1'], soundEdit(500)], 'ends a track early');assert.equal(join.joinMp4Clips([fixtures['h264-aac-1'], soundEdit(960)]).durationSeconds, 2);@@ -446,6 +554,7 @@ for (const build of ['chrome', 'firefox']) {const result = await generate({ prompt: 'Story', scenes: scenes(2, 10), estimated_duration: 20 }, { fetchImpl: api.fetchImpl });assert.equal(result.success, true, result.error);assert.equal(result.joined, false);+ assert.match(result.joinError, /different codec settings/, 'the trace shows why');assert.equal(result.clipCount, 2);assert.equal(result.durationSeconds, 2);assert.equal(result.mediaId, undefined);