Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep clips whose edit lists hide frames as separate clips

Claude1b85febd9486
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.md
index 61747ed9d..a61319228 100644
--- a/docs/providers-and-models.md
+++ b/docs/providers-and-models.md
@@ -314,9 +314,9 @@ into one MP4 without re-encoding (`src/*/src/agent/video-join.js`, which parses
the clips with the vendored mp4box.js). Videos are capped at 120 seconds and 24
clips. When the estimate is longer than 120 seconds, WebBrain asks on its own
card whether to generate the shorter version before any clip is submitted; the
-card never times out, and an unattended run generates nothing. Clips whose
-encodings differ, or that would join into more than 160 MB, are kept as separate
-clips and shown in order. The model never sees the video, so the tool result
+card never times out, and an unattended run generates nothing. Clips that
+cannot be joined this way (different encodings, edit lists that hide frames, or
+more than 160 MB together) are kept as separate clips and shown in order. The model never sees the video, so the tool result
gives it the video's real length (`durationSeconds`) and `clipCount` to report.
The Act system prompt and planner direct the model to call `generate_image` when
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 6163edfe0..cdac51c24 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -145,6 +145,11 @@ function parseClip(input) {
const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
+ // An edit list that hides whole frames before or after the picture keeps
+ // them in the file; joined, they would play over the neighbouring clip.
+ const pictureEnd = timing.mediaTime + presented;
+ const hidesFrames = timing.samples.some(sample => sample.duration > 0
+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));
return {
bytes,
movieTimescale,
@@ -152,6 +157,7 @@ function parseClip(input) {
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
duration: presented / timing.timescale,
+ hidesFrames,
};
}
@@ -354,6 +360,7 @@ function writeMovie(clips, tracks, totalSeconds) {
export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');
const clips = inputs.map(parseClip);
+ if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');
const reference = clips[0];
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 6163edfe0..cdac51c24 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -145,6 +145,11 @@ function parseClip(input) {
const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
+ // An edit list that hides whole frames before or after the picture keeps
+ // them in the file; joined, they would play over the neighbouring clip.
+ const pictureEnd = timing.mediaTime + presented;
+ const hidesFrames = timing.samples.some(sample => sample.duration > 0
+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));
return {
bytes,
movieTimescale,
@@ -152,6 +157,7 @@ function parseClip(input) {
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
duration: presented / timing.timescale,
+ hidesFrames,
};
}
@@ -354,6 +360,7 @@ function writeMovie(clips, tracks, totalSeconds) {
export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');
const clips = inputs.map(parseClip);
+ if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');
const reference = clips[0];
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index a0f134d8f..47d0b8cee 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -206,6 +206,22 @@ for (const build of ['chrome', 'firefox']) {
for (const pair of [['h264-aac-1', 'h264-aac-48px'], ['h264-aac-1', 'vp9-opus-1']]) {
assert.throws(() => join.joinMp4Clips(pair.map(name => fixtures[name])), error => error.code === 'clips_incompatible', pair.join('+'));
}
+ // An edit list that hides whole frames (here the video's last half second,
+ // or its first frame) leaves them in the file, where a join would play them.
+ const videoEdit = parse(fixtures['h264-aac-1']).moov.traks[0].edts.elst;
+ assert.equal(videoEdit.version, 0);
+ const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => {
+ const copy = fixtures['h264-aac-1'].slice();
+ new DataView(copy.buffer).setUint32(videoEdit.start + 16, milliseconds);
+ new DataView(copy.buffer).setInt32(videoEdit.start + 20, mediaTime);
+ return copy;
+ };
+ assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);
+ for (const clip of [edited({ milliseconds: 500 }), edited({ mediaTime: 3072 })]) {
+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], clip]), error => error.code === 'clips_incompatible');
+ }
+ // Shortening the last frame alone is fine.
+ assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');
assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);