Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Refuse joins of clips shown at other sizes or angles, or cut short
Claudeddee5cb0a3b3
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex 96b58f7bd..598bd4837 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -83,6 +83,9 @@ function readTrack(trak, movieTimescale, bytes) {const timescale = mdia?.mdhd?.timescale;const samples = Array.isArray(trak?.samples) ? trak.samples : [];if (!stbl?.stsd || !(timescale > 0) || !samples.length) throw incompatible('a track has no samples');+ if (samples.some(sample => sample.offset + sample.size > bytes.length)) {+ throw new VideoJoinError('The clip is incomplete: its samples run past the end of the file.', 'not_mp4');+ }if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {throw incompatible('a track switches codec settings');}@@ -377,6 +380,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const problem = clips.find(clip => clip.problem)?.problem;if (problem) throw incompatible(`a clip ${problem}`);const reference = clips[0];+ // The joined file shows every clip at the first clip's size and angle.+ const placement = (matrix, tkhd) => JSON.stringify([Array.from(matrix || []), tkhd?.width, tkhd?.height, Array.from(tkhd?.matrix || [])]);for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');clip.tracks.forEach((track, index) => {@@ -384,6 +389,9 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {throw incompatible('the clips use different codec settings');}+ if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {+ throw incompatible('the clips are shown at different sizes or angles');+ }});}const starts = [];diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex 96b58f7bd..598bd4837 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -83,6 +83,9 @@ function readTrack(trak, movieTimescale, bytes) {const timescale = mdia?.mdhd?.timescale;const samples = Array.isArray(trak?.samples) ? trak.samples : [];if (!stbl?.stsd || !(timescale > 0) || !samples.length) throw incompatible('a track has no samples');+ if (samples.some(sample => sample.offset + sample.size > bytes.length)) {+ throw new VideoJoinError('The clip is incomplete: its samples run past the end of the file.', 'not_mp4');+ }if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {throw incompatible('a track switches codec settings');}@@ -377,6 +380,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {const problem = clips.find(clip => clip.problem)?.problem;if (problem) throw incompatible(`a clip ${problem}`);const reference = clips[0];+ // The joined file shows every clip at the first clip's size and angle.+ const placement = (matrix, tkhd) => JSON.stringify([Array.from(matrix || []), tkhd?.width, tkhd?.height, Array.from(tkhd?.matrix || [])]);for (const clip of clips.slice(1)) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');clip.tracks.forEach((track, index) => {@@ -384,6 +389,9 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {throw incompatible('the clips use different codec settings');}+ if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {+ throw incompatible('the clips are shown at different sizes or angles');+ }});}const starts = [];diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 735d6e924..de89ebb11 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -56,6 +56,8 @@ function parse(bytes) {const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));+const videoOf = file => file.moov.traks[0];+// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of// the copy and the parsed original.function patched(name, edit) {@@ -236,6 +238,17 @@ for (const build of ['chrome', 'firefox']) {for (const pair of [['h264-aac-1', 'h264-aac-48px'], ['h264-aac-1', 'vp9-opus-1']]) {assert.throws(() => join.joinMp4Clips(pair.map(name => fixtures[name])), error => error.code === 'clips_incompatible', pair.join('+'));}+ // The joined file shows every clip at the first clip's size and angle.+ for (const placed of [+ patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).tkhd.start + 84, 128 << 16)),+ patched('h264-aac-2', (view, file) => [0, 0x10000, 0, -0x10000, 0].forEach((value, i) => view.setInt32(videoOf(file).tkhd.start + 48 + i * 4, value))),+ ]) {+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], placed]), error => error.message.includes('different sizes or angles'));+ }+ // A clip cut short in the download is not read past its end.+ const cut = fixtures['h264-aac-2'].slice(0, -100);+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], cut]), error => error.code === 'not_mp4' && /incomplete/.test(error.message));+ assert.equal(join.mp4DurationSeconds(cut), null);assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);@@ -244,7 +257,6 @@ for (const build of ['chrome', 'firefox']) {test(`${build}: clips whose timing cannot be laid end to end are kept apart`, () => {const refused = (clips, reason) => assert.throws(() => join.joinMp4Clips(clips),error => error.code === 'clips_incompatible' && error.message.includes(reason), reason);- const videoOf = file => file.moov.traks[0];const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => patched('h264-aac-1', (view, file) => {const elst = videoOf(file).edts.elst;assert.equal(elst.version, 0);