Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Refuse joins whose decode timelines cannot meet
Claude94cfa924e575
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex 2d59c80f3..96b58f7bd 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -92,7 +92,7 @@ function readTrack(trak, movieTimescale, bytes) {// One edit that skips the codec's start-up delay is normal. Empty edits or// several edits describe a timeline these tables cannot carry over.const edits = trak.edts?.elst?.entries || [];- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {+ if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {throw incompatible('a track has a complex edit list');}return {@@ -145,14 +145,18 @@ function parseClip(input) {const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));- // An edit list that hides whole frames before or after the picture keeps- // them in the file; joined, they would play over the neighbouring clip.+ // What would keep this clip from being laid end to end with others:+ // - its first frame must decode without the previous clip's frames;+ // - an edit list that hides whole frames before or after the picture keeps+ // them in the file, where a join would play them over the next clip;+ // - a track whose first sample comes after the clip's start (by more than a+ // millisecond) would need a leading gap, which one edit cannot show.const pictureEnd = timing.mediaTime + presented;- const hidesFrames = timing.samples.some(sample => sample.duration > 0- && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));- // A track whose first sample comes after the clip's start (more than a- // millisecond) would have to begin with a gap, which one edit cannot show.- const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);+ const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'+ : timing.samples.some(sample => sample.duration > 0+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'+ : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'+ : null;return {bytes,movieTimescale,@@ -160,8 +164,7 @@ function parseClip(input) {ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),tracks,duration: presented / timing.timescale,- hidesFrames,- startsLate,+ problem,};}@@ -199,7 +202,11 @@ function joinTrack(clips, index, starts) {previous.size -= samples.pop().size;}const last = samples[samples.length - 1];- last.duration = Math.max(1, target - last.dts);+ // Decode times only move forward. A clip that must start decoding before+ // the previous clip's last frame (a longer composition delay) cannot+ // follow it without moving its picture.+ if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');+ last.duration = target - last.dts;}let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });@@ -367,8 +374,8 @@ function writeMovie(clips, tracks, totalSeconds) {export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');const clips = inputs.map(parseClip);- if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');- if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');+ const problem = clips.find(clip => clip.problem)?.problem;+ if (problem) throw incompatible(`a clip ${problem}`);const reference = clips[0];for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex 2d59c80f3..96b58f7bd 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -92,7 +92,7 @@ function readTrack(trak, movieTimescale, bytes) {// One edit that skips the codec's start-up delay is normal. Empty edits or// several edits describe a timeline these tables cannot carry over.const edits = trak.edts?.elst?.entries || [];- if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {+ if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1 || edit.media_rate_fraction)) {throw incompatible('a track has a complex edit list');}return {@@ -145,14 +145,18 @@ function parseClip(input) {const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));- // An edit list that hides whole frames before or after the picture keeps- // them in the file; joined, they would play over the neighbouring clip.+ // What would keep this clip from being laid end to end with others:+ // - its first frame must decode without the previous clip's frames;+ // - an edit list that hides whole frames before or after the picture keeps+ // them in the file, where a join would play them over the next clip;+ // - a track whose first sample comes after the clip's start (by more than a+ // millisecond) would need a leading gap, which one edit cannot show.const pictureEnd = timing.mediaTime + presented;- const hidesFrames = timing.samples.some(sample => sample.duration > 0- && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd));- // A track whose first sample comes after the clip's start (more than a- // millisecond) would have to begin with a gap, which one edit cannot show.- const startsLate = tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000);+ const problem = timing.samples[0].is_sync === false ? 'does not start with a keyframe'+ : timing.samples.some(sample => sample.duration > 0+ && (sample.cts + sample.duration <= timing.mediaTime || sample.cts >= pictureEnd)) ? 'hides some of its frames with an edit list'+ : tracks.some(track => track.samples[0].dts - track.mediaTime > track.timescale / 1000) ? 'has a track that starts after the clip does'+ : null;return {bytes,movieTimescale,@@ -160,8 +164,7 @@ function parseClip(input) {ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),tracks,duration: presented / timing.timescale,- hidesFrames,- startsLate,+ problem,};}@@ -199,7 +202,11 @@ function joinTrack(clips, index, starts) {previous.size -= samples.pop().size;}const last = samples[samples.length - 1];- last.duration = Math.max(1, target - last.dts);+ // Decode times only move forward. A clip that must start decoding before+ // the previous clip's last frame (a longer composition delay) cannot+ // follow it without moving its picture.+ if (target <= last.dts) throw incompatible('a clip would start decoding before the previous one ends');+ last.duration = target - last.dts;}let dts = samples.length ? samples[samples.length - 1].dts + samples[samples.length - 1].duration : 0;chunks.push({ clipIndex, first: samples.length, count: kept.length, size: 0, offset: 0 });@@ -367,8 +374,8 @@ function writeMovie(clips, tracks, totalSeconds) {export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (!Array.isArray(inputs) || !inputs.length) throw new VideoJoinError('No clips to join.', 'not_mp4');const clips = inputs.map(parseClip);- if (clips.some(clip => clip.hidesFrames)) throw incompatible('a clip hides some of its frames with an edit list');- if (clips.some(clip => clip.startsLate)) throw incompatible('a clip has a track that starts after the clip does');+ const problem = clips.find(clip => clip.problem)?.problem;+ if (problem) throw incompatible(`a clip ${problem}`);const reference = clips[0];for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 483643662..735d6e924 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -56,6 +56,14 @@ function parse(bytes) {const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));+// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of+// the copy and the parsed original.+function patched(name, edit) {+ const copy = fixtures[name].slice();+ edit(new DataView(copy.buffer), parse(fixtures[name]));+ return copy;+}+// A fragmented clip whose fragments start later: each track's decode times// moved by `seconds` ({ vide, soun }).function laterFragments(bytes, seconds) {@@ -228,27 +236,49 @@ for (const build of ['chrome', 'firefox']) {for (const pair of [['h264-aac-1', 'h264-aac-48px'], ['h264-aac-1', 'vp9-opus-1']]) {assert.throws(() => join.joinMp4Clips(pair.map(name => fixtures[name])), error => error.code === 'clips_incompatible', pair.join('+'));}- // An edit list that hides whole frames (here the video's last half second,- // or its first frame) leaves them in the file, where a join would play them.- const videoEdit = parse(fixtures['h264-aac-1']).moov.traks[0].edts.elst;- assert.equal(videoEdit.version, 0);- const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => {- const copy = fixtures['h264-aac-1'].slice();- new DataView(copy.buffer).setUint32(videoEdit.start + 16, milliseconds);- new DataView(copy.buffer).setInt32(videoEdit.start + 20, mediaTime);- return copy;- };- assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);- for (const clip of [edited({ milliseconds: 500 }), edited({ mediaTime: 3072 })]) {- assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], clip]), error => error.code === 'clips_incompatible');- }- // Shortening the last frame alone is fine.- assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);});+ test(`${build}: clips whose timing cannot be laid end to end are kept apart`, () => {+ const refused = (clips, reason) => assert.throws(() => join.joinMp4Clips(clips),+ error => error.code === 'clips_incompatible' && error.message.includes(reason), reason);+ const videoOf = file => file.moov.traks[0];+ const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => patched('h264-aac-1', (view, file) => {+ const elst = videoOf(file).edts.elst;+ assert.equal(elst.version, 0);+ view.setUint32(elst.start + 16, milliseconds);+ view.setInt32(elst.start + 20, mediaTime);+ });+ // An edit list that hides whole frames (the last half second, or the first+ // frame) leaves them in the file, where a join would play them.+ assert.equal(join.mp4DurationSeconds(edited({ milliseconds: 500 })), 0.5);+ refused([fixtures['h264-aac-1'], edited({ milliseconds: 500 })], 'hides some of its frames');+ refused([fixtures['h264-aac-1'], edited({ mediaTime: 3072 })], 'hides some of its frames');+ // Shortening the last frame alone is fine.+ assert.ok(Math.abs(join.joinMp4Clips([edited({ milliseconds: 960 }), fixtures['h264-aac-2']]).durationSeconds - 1.96) < 0.001);+ // A one-second-longer composition delay keeps the picture where it was but+ // starts decoding a second earlier: such a clip can open a video, not follow one.+ const delayed = patched('h264-aac-2', (view, file) => {+ const ctts = videoOf(file).mdia.minf.stbl.ctts;+ ctts.sample_offsets.forEach((offset, i) => view.setUint32(ctts.start + 16 + i * 8 + 4, offset + 12288));+ view.setInt32(videoOf(file).edts.elst.start + 20, videoOf(file).edts.elst.entries[0].media_time + 12288);+ });+ assert.equal(join.mp4DurationSeconds(delayed), 1);+ refused([fixtures['h264-aac-1'], delayed], 'start decoding before the previous one ends');+ const output = parse(join.joinMp4Clips([delayed, fixtures['h264-aac-1']]).bytes);+ const video = videoOf(output);+ const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);+ assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });+ // An edit that plays at another speed, and a clip that does not open on a keyframe.+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(videoOf(file).edts.elst.start + 26, 0x4000))],+ 'complex edit list');+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],+ 'does not start with a keyframe');+ });+test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {const { clips, total } = media.planVideoClips([{ prompt: 'Opening', duration: 12 }, { prompt: 'Long walk', duration: 20 }], [5, 10, 15]);assert.deepEqual(clips, [