Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep clips whose sound edit ends early apart

Claude0eba6b4cbf97
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 578607804..fda3224be 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -171,13 +171,19 @@ function parseClip(input) {
// - an edit list that hides whole frames before or after the picture keeps
// them in the file, where a join would play them over the next clip;
// - a track whose first sample comes after the clip's start (by more than a
- // millisecond) would need a leading gap, which one edit cannot show.
+ // millisecond) would need a leading gap, which one edit cannot show;
+ // - a track whose own edit ends before the picture (by more than one of its
+ // frames) would play on to the next clip in the joined file.
const pictureEnd = timing.mediaTime + presented;
+ const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)
+ && track.editDuration < Math.round(presented / timing.timescale * track.timescale)
+ - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
: timing.samples.some(sample => sample.duration > 0
&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
: tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
- : null;
+ : tracks.some(endsEarly) ? 'ends a track early with an edit list'
+ : null;
return {
bytes,
movieTimescale,
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 578607804..fda3224be 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -171,13 +171,19 @@ function parseClip(input) {
// - an edit list that hides whole frames before or after the picture keeps
// them in the file, where a join would play them over the next clip;
// - a track whose first sample comes after the clip's start (by more than a
- // millisecond) would need a leading gap, which one edit cannot show.
+ // millisecond) would need a leading gap, which one edit cannot show;
+ // - a track whose own edit ends before the picture (by more than one of its
+ // frames) would play on to the next clip in the joined file.
const pictureEnd = timing.mediaTime + presented;
+ const endsEarly = track => track !== timing && Number.isFinite(track.editDuration)
+ && track.editDuration < Math.round(presented / timing.timescale * track.timescale)
+ - track.samples.reduce((longest, sample) => Math.max(longest, sample.duration), 0);
const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
: timing.samples.some(sample => sample.duration > 0
&& (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
: tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
- : null;
+ : tracks.some(endsEarly) ? 'ends a track early with an edit list'
+ : null;
return {
bytes,
movieTimescale,
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index 751f1a17b..4dfa41857 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -305,6 +305,11 @@ for (const build of ['chrome', 'firefox']) {
'in another file');
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(audioOf(file).mdia.minf.stbl.sgpds[0].start + 24, -2))],
'different codec settings');
+ // Sound whose own edit ends half a second before the picture would play
+ // on in a joined file; ending within a frame of it is fine.
+ const soundEdit = milliseconds => patched('h264-aac-2', (view, file) => view.setUint32(audioOf(file).edts.elst.start + 16, milliseconds));
+ refused([fixtures['h264-aac-1'], soundEdit(500)], 'ends a track early');
+ assert.equal(join.joinMp4Clips([fixtures['h264-aac-1'], soundEdit(960)]).durationSeconds, 2);
});
test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {