Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Refuse joins of clips shown at other sizes or angles, or cut short

Claudeddee5cb0a3b3
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 96b58f7bd..598bd4837 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -83,6 +83,9 @@ function readTrack(trak, movieTimescale, bytes) {
const timescale = mdia?.mdhd?.timescale;
const samples = Array.isArray(trak?.samples) ? trak.samples : [];
if (!stbl?.stsd || !(timescale > 0) || !samples.length) throw incompatible('a track has no samples');
+ if (samples.some(sample => sample.offset + sample.size > bytes.length)) {
+ throw new VideoJoinError('The clip is incomplete: its samples run past the end of the file.', 'not_mp4');
+ }
if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {
throw incompatible('a track switches codec settings');
}
@@ -377,6 +380,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const problem = clips.find(clip => clip.problem)?.problem;
if (problem) throw incompatible(`a clip ${problem}`);
const reference = clips[0];
+ // The joined file shows every clip at the first clip's size and angle.
+ const placement = (matrix, tkhd) => JSON.stringify([Array.from(matrix || []), tkhd?.width, tkhd?.height, Array.from(tkhd?.matrix || [])]);
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
clip.tracks.forEach((track, index) => {
@@ -384,6 +389,9 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {
throw incompatible('the clips use different codec settings');
}
+ if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {
+ throw incompatible('the clips are shown at different sizes or angles');
+ }
});
}
const starts = [];
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 96b58f7bd..598bd4837 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -83,6 +83,9 @@ function readTrack(trak, movieTimescale, bytes) {
const timescale = mdia?.mdhd?.timescale;
const samples = Array.isArray(trak?.samples) ? trak.samples : [];
if (!stbl?.stsd || !(timescale > 0) || !samples.length) throw incompatible('a track has no samples');
+ if (samples.some(sample => sample.offset + sample.size > bytes.length)) {
+ throw new VideoJoinError('The clip is incomplete: its samples run past the end of the file.', 'not_mp4');
+ }
if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {
throw incompatible('a track switches codec settings');
}
@@ -377,6 +380,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
const problem = clips.find(clip => clip.problem)?.problem;
if (problem) throw incompatible(`a clip ${problem}`);
const reference = clips[0];
+ // The joined file shows every clip at the first clip's size and angle.
+ const placement = (matrix, tkhd) => JSON.stringify([Array.from(matrix || []), tkhd?.width, tkhd?.height, Array.from(tkhd?.matrix || [])]);
for (const clip of clips.slice(1)) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
clip.tracks.forEach((track, index) => {
@@ -384,6 +389,9 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {
throw incompatible('the clips use different codec settings');
}
+ if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {
+ throw incompatible('the clips are shown at different sizes or angles');
+ }
});
}
const starts = [];
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index 735d6e924..de89ebb11 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -56,6 +56,8 @@ function parse(bytes) {
const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));
+const videoOf = file => file.moov.traks[0];
+
// A fixture with some of its bytes rewritten: `edit(view, file)` gets a view of
// the copy and the parsed original.
function patched(name, edit) {
@@ -236,6 +238,17 @@ for (const build of ['chrome', 'firefox']) {
for (const pair of [['h264-aac-1', 'h264-aac-48px'], ['h264-aac-1', 'vp9-opus-1']]) {
assert.throws(() => join.joinMp4Clips(pair.map(name => fixtures[name])), error => error.code === 'clips_incompatible', pair.join('+'));
}
+ // The joined file shows every clip at the first clip's size and angle.
+ for (const placed of [
+ patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).tkhd.start + 84, 128 << 16)),
+ patched('h264-aac-2', (view, file) => [0, 0x10000, 0, -0x10000, 0].forEach((value, i) => view.setInt32(videoOf(file).tkhd.start + 48 + i * 4, value))),
+ ]) {
+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], placed]), error => error.message.includes('different sizes or angles'));
+ }
+ // A clip cut short in the download is not read past its end.
+ const cut = fixtures['h264-aac-2'].slice(0, -100);
+ assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], cut]), error => error.code === 'not_mp4' && /incomplete/.test(error.message));
+ assert.equal(join.mp4DurationSeconds(cut), null);
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], new Uint8Array([0, 0, 0, 8, 102, 114, 101, 101])]), error => error.code === 'not_mp4');
assert.throws(() => join.joinMp4Clips([fixtures['h264-aac-1'], fixtures['h264-aac-2']], { maxBytes: 1000 }), error => error.code === 'too_large');
assert.equal(join.mp4DurationSeconds(new Uint8Array([1, 2, 3])), null);
@@ -244,7 +257,6 @@ for (const build of ['chrome', 'firefox']) {
test(`${build}: clips whose timing cannot be laid end to end are kept apart`, () => {
const refused = (clips, reason) => assert.throws(() => join.joinMp4Clips(clips),
error => error.code === 'clips_incompatible' && error.message.includes(reason), reason);
- const videoOf = file => file.moov.traks[0];
const edited = ({ milliseconds = 1000, mediaTime = 2048 }) => patched('h264-aac-1', (view, file) => {
const elst = videoOf(file).edts.elst;
assert.equal(elst.version, 0);