Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Join fragments that start at a later decode time
Claude12f680f452c9
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex a61319228..f07daf5d8 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -315,8 +315,9 @@ the clips with the vendored mp4box.js). Videos are capped at 120 seconds and 24clips. When the estimate is longer than 120 seconds, WebBrain asks on its owncard whether to generate the shorter version before any clip is submitted; thecard never times out, and an unattended run generates nothing. Clips that-cannot be joined this way (different encodings, edit lists that hide frames, or-more than 160 MB together) are kept as separate clips and shown in order. The model never sees the video, so the tool result+cannot be joined this way (different encodings, edit lists or fragment times+the joined file cannot reproduce, or more than 160 MB together) are kept as+separate clips and shown in order. The model never sees the video, so the tool resultgives it the video's real length (`durationSeconds`) and `clipCount` to report.The Act system prompt and planner direct the model to call `generate_image` whendiff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex cdac51c24..2d59c80f3 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -150,6 +150,9 @@ function parseClip(input) {const pictureEnd = timing.mediaTime + presented;const hidesFrames = timing.samples.some(sample => sample.duration > 0&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));+ // A track whose first sample comes after the clip's start (more than a+ // millisecond) would have to begin with a gap, which one edit cannot show.+ const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);return {bytes,movieTimescale,@@ -158,6 +161,7 @@ function parseClip(input) {tracks,duration: presented / timing.timescale,hidesFrames,+ startsLate,};}@@ -174,6 +178,9 @@ export function mp4DurationSeconds(input) {// audio longer than the picture). Video frames are never dropped.function joinTrack(clips, index, starts) {const first = clips[0].tracks[index];+ // The output's decode times start at 0, wherever the first clip's start+ // (fragmented files can start at any decode time): its start in the output.+ const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);const samples = [];const chunks = [];clips.forEach((clip, clipIndex) => {@@ -184,7 +191,7 @@ function joinTrack(clips, index, starts) {: track.samples;if (!kept.length) throw incompatible('a clip has an empty track');if (samples.length) {- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + first.mediaTime;+ const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;const previous = chunks[chunks.length - 1];while (audio && previous.count > 1&& samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {@@ -212,7 +219,7 @@ function joinTrack(clips, index, starts) {}});const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);- return { first, samples, chunks, mediaDuration };+ return { first, samples, chunks, mediaDuration, mediaStart };}class Writer {@@ -330,7 +337,7 @@ function writeMovie(clips, tracks, totalSeconds) {// Every track is shown from its start, so the movie runs from 0 to// exactly its declared duration.writer.box('edts', () => writer.full('elst', 0, 0, () => {- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);+ writer.u32(1); writer.u32(movieDuration); writer.i32(track.mediaStart); writer.i16(1); writer.i16(0);}));writer.box('mdia', () => {writer.full('mdhd', 0, 0, () => {@@ -361,6 +368,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');const clips = inputs.map(parseClip);if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');+ if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');const reference = clips[0];for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex cdac51c24..2d59c80f3 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -150,6 +150,9 @@ function parseClip(input) {const pictureEnd = timing.mediaTime + presented;const hidesFrames = timing.samples.some(sample => sample.duration > 0&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));+ // A track whose first sample comes after the clip's start (more than a+ // millisecond) would have to begin with a gap, which one edit cannot show.+ const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);return {bytes,movieTimescale,@@ -158,6 +161,7 @@ function parseClip(input) {tracks,duration: presented / timing.timescale,hidesFrames,+ startsLate,};}@@ -174,6 +178,9 @@ export function mp4DurationSeconds(input) {// audio longer than the picture). Video frames are never dropped.function joinTrack(clips, index, starts) {const first = clips[0].tracks[index];+ // The output's decode times start at 0, wherever the first clip's start+ // (fragmented files can start at any decode time): its start in the output.+ const mediaStart = Math.max(0, first.mediaTime - first.samples[0].dts);const samples = [];const chunks = [];clips.forEach((clip, clipIndex) => {@@ -184,7 +191,7 @@ function joinTrack(clips, index, starts) {: track.samples;if (!kept.length) throw incompatible('a clip has an empty track');if (samples.length) {- const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + first.mediaTime;+ const target = Math.round(starts[clipIndex] * track.timescale) + (kept[0].dts - track.mediaTime) + mediaStart;const previous = chunks[chunks.length - 1];while (audio && previous.count > 1&& samples[samples.length - 1].dts + samples[samples.length - 1].duration / 2 >= target) {@@ -212,7 +219,7 @@ function joinTrack(clips, index, starts) {}});const mediaDuration = samples.reduce((sum, sample) => sum + sample.duration, 0);- return { first, samples, chunks, mediaDuration };+ return { first, samples, chunks, mediaDuration, mediaStart };}class Writer {@@ -330,7 +337,7 @@ function writeMovie(clips, tracks, totalSeconds) {// Every track is shown from its start, so the movie runs from 0 to// exactly its declared duration.writer.box('edts', () => writer.full('elst', 0, 0, () => {- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);+ writer.u32(1); writer.u32(movieDuration); writer.i32(track.mediaStart); writer.i16(1); writer.i16(0);}));writer.box('mdia', () => {writer.full('mdhd', 0, 0, () => {@@ -361,6 +368,7 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');const clips = inputs.map(parseClip);if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');+ if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');const reference = clips[0];for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 47d0b8cee..483643662 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -56,6 +56,22 @@ function parse(bytes) {const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));+// A fragmented clip whose fragments start later: each track's decode times+// moved by `seconds` ({ vide, soun }).+function laterFragments(bytes, seconds) {+ const file = parse(bytes);+ const copy = bytes.slice();+ const view = new DataView(copy.buffer);+ const traks = new Map(file.moov.traks.map(trak => [trak.tkhd.track_id, trak]));+ for (const traf of file.moofs.flatMap(moof => moof.trafs)) {+ const trak = traks.get(traf.tfhd.track_id);+ const add = Math.round((seconds[trak.mdia.hdlr.handler] || 0) * trak.mdia.mdhd.timescale);+ if (traf.tfdt.version === 1) view.setBigUint64(traf.tfdt.start + 12, BigInt(traf.tfdt.baseMediaDecodeTime + add));+ else view.setUint32(traf.tfdt.start + 12, traf.tfdt.baseMediaDecodeTime + add);+ }+ return copy;+}+// Where a track's samples are shown on the movie's timeline, in seconds: its// edit names the first media time shown.function shown(file, trak) {@@ -200,6 +216,12 @@ for (const build of ['chrome', 'firefox']) {const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);}+ // Fragments can start at any decode time: five seconds later, the clips+ // join into the same file. A track that starts after the picture cannot join.+ const fragmented = [fixtures['h264-aac-fragmented-1'], fixtures['h264-aac-fragmented-2']];+ assert.equal(parse(laterFragments(fragmented[0], { vide: 5, soun: 5 })).moov.traks[0].samples[0].dts, 5 * 12288);+ assert.deepEqual(join.joinMp4Clips(fragmented.map(clip => laterFragments(clip, { vide: 5, soun: 5 }))).bytes, join.joinMp4Clips(fragmented).bytes);+ assert.throws(() => join.joinMp4Clips([laterFragments(fragmented[0], { soun: 5 }), fragmented[1]]), error => error.code === 'clips_incompatible');});test(`${build}: clips with different codec settings or tracks are not joined`, () => {