Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Refuse joins whose decode timelines cannot meet

Claude94cfa924e575
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 2d59c80f3..96b58f7bd 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -92,7 +92,7 @@ function readTrack(trak, movieTimescale, bytes) {
// One edit that skips the codec's start-up delay is normal. Empty edits or
// several edits describe a timeline these tables cannot carry over.
const edits = trak.edts?.elst?.entries || [];
- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {
+ if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {
throw incompatible('a track has a complex edit list');
}
return {
@@ -145,14 +145,18 @@ function parseClip(input) {
const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
- // An edit list that hides whole frames before or after the picture keeps
- // them in the file; joined, they would play over the neighbouring clip.
+ // What would keep this clip from being laid end to end with others:
+ // - its first frame must decode without the previous clip's frames;
+ // - an edit list that hides whole frames before or after the picture keeps
+ // them in the file, where a join would play them over the next clip;
+ // - a track whose first sample comes after the clip's start (by more than a
+ // millisecond) would need a leading gap, which one edit cannot show.
const pictureEnd = timing.mediaTime + presented;
- const hidesFrames = timing.samples.some(sample => sample.duration > 0
- && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));
- // A track whose first sample comes after the clip's start (more than a
- // millisecond) would have to begin with a gap, which one edit cannot show.
- const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);
+ const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
+ : timing.samples.some(sample => sample.duration > 0
+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
+ : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
+ : null;
return {
bytes,
movieTimescale,
@@ -160,8 +164,7 @@ function parseClip(input) {
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
duration: presented / timing.timescale,
- hidesFrames,
- startsLate,
+ problem,
};
}
@@ -199,7 +202,11 @@ function joinTrack(clips, index, starts) {
previous.size -= samples.pop().size;
}
const last = samples[samples.length - 1];
- last.duration = Math.max(1, target - last.dts);
+ // Decode times only move forward. A clip that must start decoding before
+ // the previous clip's last frame (a longer composition delay) cannot
+ // follow it without moving its picture.
+ if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
+ last.duration = target - last.dts;
}
let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;
chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });
@@ -367,8 +374,8 @@ function writeMovie(clips, tracks, totalSeconds) {
export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');
const clips = inputs.map(parseClip);
- if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');
- if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');
+ const problem = clips.find(clip => clip.problem)?.problem;
+ if (problem) throw incompatible(`a clip ${problem}`);
const reference = clips[0];
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 2d59c80f3..96b58f7bd 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -92,7 +92,7 @@ function readTrack(trak, movieTimescale, bytes) {
// One edit that skips the codec's start-up delay is normal. Empty edits or
// several edits describe a timeline these tables cannot carry over.
const edits = trak.edts?.elst?.entries || [];
- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {
+ if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {
throw incompatible('a track has a complex edit list');
}
return {
@@ -145,14 +145,18 @@ function parseClip(input) {
const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
- // An edit list that hides whole frames before or after the picture keeps
- // them in the file; joined, they would play over the neighbouring clip.
+ // What would keep this clip from being laid end to end with others:
+ // - its first frame must decode without the previous clip's frames;
+ // - an edit list that hides whole frames before or after the picture keeps
+ // them in the file, where a join would play them over the next clip;
+ // - a track whose first sample comes after the clip's start (by more than a
+ // millisecond) would need a leading gap, which one edit cannot show.
const pictureEnd = timing.mediaTime + presented;
- const hidesFrames = timing.samples.some(sample => sample.duration > 0
- && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));
- // A track whose first sample comes after the clip's start (more than a
- // millisecond) would have to begin with a gap, which one edit cannot show.
- const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);
+ const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'
+ : timing.samples.some(sample => sample.duration > 0
+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'
+ : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'
+ : null;
return {
bytes,
movieTimescale,
@@ -160,8 +164,7 @@ function parseClip(input) {
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
duration: presented / timing.timescale,
- hidesFrames,
- startsLate,
+ problem,
};
}
@@ -199,7 +202,11 @@ function joinTrack(clips, index, starts) {
previous.size -= samples.pop().size;
}
const last = samples[samples.length - 1];
- last.duration = Math.max(1, target - last.dts);
+ // Decode times only move forward. A clip that must start decoding before
+ // the previous clip's last frame (a longer composition delay) cannot
+ // follow it without moving its picture.
+ if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');
+ last.duration = target - last.dts;
}
let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;
chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });
@@ -367,8 +374,8 @@ function writeMovie(clips, tracks, totalSeconds) {
export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');
const clips = inputs.map(parseClip);
- if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');
- if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');
+ const problem = clips.find(clip => clip.problem)?.problem;
+ if (problem) throw incompatible(`a clip ${problem}`);
const reference = clips[0];
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index 483643662..735d6e924 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -56,6 +56,14 @@ function parse(bytes) {
const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));
+// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of
+// the copy and the parsed original.
+function patched(name, edit) {
+ const copy = fixtures[name].slice();
+ edit(new DataView(copy.buffer), parse(fixtures[name]));
+ return copy;
+}
+
// A fragmented clip whose fragments start later: each track's decode times
// moved by `seconds` ({ vide, soun }).
function laterFragments(bytes, seconds) {
@@ -228,27 +236,49 @@ for (const build of ['chrome', 'firefox']) {
for (const pair of [['h264-aac-1', 'h264-aac-48px'], ['h264-aac-1', 'vp9-opus-1']]) {
assert.throws(() => join.joinMp4Clips(pair.map(name => fixtures[name])), error => error.code === 'clips_incompatible', pair.join('+'));
}
- // An edit list that hides whole frames (here the video's last half second,
- // or its first frame) leaves them in the file, where a join would play them.
- const videoEdit = parse(fixtures['h264-aac-1']).moov.traks[0].edts.elst;
- assert.equal(videoEdit.version, 0);
- const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => {
- const copy = fixtures['h264-aac-1'].slice();
- new DataView(copy.buffer).setUint32(videoEdit.start + 16, milliseconds);
- new DataView(copy.buffer).setInt32(videoEdit.start + 20, mediaTime);
- return copy;
- };
- assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);
- for (const clip of [edited({ milliseconds: 500 }), edited({ mediaTime: 3072 })]) {
- assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], clip]), error => error.code === 'clips_incompatible');
- }
- // Shortening the last frame alone is fine.
- assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');
assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);
});
+ test(`${build}: clips whose timing cannot be laid end to end are kept apart`, () => {
+ const refused = (clips, reason) => assert.throws(() => join.joinMp4Clips(clips),
+ error => error.code === 'clips_incompatible' && error.message.includes(reason), reason);
+ const videoOf = file => file.moov.traks[0];
+ const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => patched('h264-aac-1', (view, file) => {
+ const elst = videoOf(file).edts.elst;
+ assert.equal(elst.version, 0);
+ view.setUint32(elst.start + 16, milliseconds);
+ view.setInt32(elst.start + 20, mediaTime);
+ });
+ // An edit list that hides whole frames (the last half second, or the first
+ // frame) leaves them in the file, where a join would play them.
+ assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);
+ refused([fixtures['h264-aac-1'], edited({ milliseconds: 500 })], 'hides some of its frames');
+ refused([fixtures['h264-aac-1'], edited({ mediaTime: 3072 })], 'hides some of its frames');
+ // Shortening the last frame alone is fine.
+ assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);
+ // A one-second-longer composition delay keeps the picture where it was but
+ // starts decoding a second earlier: such a clip can open a video, not follow one.
+ const delayed = patched('h264-aac-2', (view, file) => {
+ const ctts = videoOf(file).mdia.minf.stbl.ctts;
+ ctts.sample_offsets.forEach((offset, i) => view.setUint32(ctts.start + 16 + i * 8 + 4, offset + 12288));
+ view.setInt32(videoOf(file).edts.elst.start + 20, videoOf(file).edts.elst.entries[0].media_time + 12288);
+ });
+ assert.equal(join.mp4DurationSeconds(delayed), 1);
+ refused([fixtures['h264-aac-1'], delayed], 'start decoding before the previous one ends');
+ const output = parse(join.joinMp4Clips([delayed, fixtures['h264-aac-1']]).bytes);
+ const video = videoOf(output);
+ const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);
+ assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));
+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });
+ // An edit that plays at another speed, and a clip that does not open on a keyframe.
+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(videoOf(file).edts.elst.start + 26, 0x4000))],
+ 'complex edit list');
+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],
+ 'does not start with a keyframe');
+ });
+
test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {
const { clips, total } = media.planVideoClips([{ prompt: 'Opening', duration: 12 }, { prompt: 'Long walk', duration: 20 }], [5, 10, 15]);
assert.deepEqual(clips, [