Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Keep audio pre-roll groups in joined videos

Claude5c00f74a22a0
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 598bd4837..578607804 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -89,6 +89,22 @@ function readTrack(trak, movieTimescale, bytes) {
if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {
throw incompatible('a track switches codec settings');
}
+ // Encrypted samples need their per-sample keys and IVs, which are not
+ // carried over; samples kept in another file cannot be copied from this one.
+ if (/^enc[a-z]$/.test(stbl.stsd.entries[0]?.type || '')) throw incompatible('a track is encrypted');
+ if ((minf.dinf?.dref?.entries || []).some(entry => !(entry.flags & 1))) throw incompatible('a track keeps its samples in another file');
+ // Sample groups, such as an audio codec's pre-roll for seeking: each
+ // sample's group in this clip, and the groups' description box.
+ const groups = (stbl.sbgps || []).flatMap(sbgp => {
+ const description = rawBox(bytes, (stbl.sgpds || []).find(sgpd => sgpd.grouping_type === sbgp.grouping_type));
+ if (!description) return [];
+ const indices = new Array(samples.length).fill(0);
+ let next = 0;
+ for (const entry of sbgp.entries || []) {
+ for (let n = 0; n < entry.sample_count && next < indices.length; n++) indices[next++] = entry.group_description_index;
+ }
+ return [{ type: sbgp.grouping_type, parameter: sbgp.grouping_type_parameter || 0, indices, description }];
+ });
// Reordered frames (B-frames) are shown after they are decoded, so the
// track's picture ends at its last composition time, not its last decode time.
const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);
@@ -112,6 +128,8 @@ function readTrack(trak, movieTimescale, bytes) {
language: mdia.mdhd.language,
stsd: rawBox(bytes, stbl.stsd),
stsdKey: sampleDescriptionKey(bytes, stbl.stsd),
+ groups,
+ groupsKey: JSON.stringify(groups.map(group => [group.type, group.parameter, Array.from(group.description)])),
hdlr: rawBox(bytes, mdia.hdlr),
mediaHeader: rawBox(bytes, (minf.boxes || []).find(box => MEDIA_HEADER_TYPES.has(box.type))),
dinf: rawBox(bytes, (minf.boxes || []).find(box => box.type === 'dinf')),
@@ -222,6 +240,7 @@ function joinTrack(clips, index, starts) {
size: sample.size,
source: sample.offset,
sync: sample.is_sync !== false,
+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),
clipIndex,
});
chunk.size += sample.size;
@@ -315,6 +334,19 @@ function writeSampleTable(writer, track) {
for (const sample of samples) writer.u32(sample.size);
});
writer.full('stco', 0, 0, () => { writer.u32(chunks.length); for (const chunk of chunks) writer.u32(chunk.offset); });
+ const described = new Set();
+ first.groups.forEach((group, index) => {
+ // One description box per grouping type, however many sbgp boxes use it.
+ if (!described.has(group.type)) writer.bytes(group.description);
+ described.add(group.type);
+ const members = runs(samples.map(sample => sample.groups[index]));
+ writer.full('sbgp', group.parameter ? 1 : 0, 0, () => {
+ writer.type(group.type);
+ if (group.parameter) writer.u32(group.parameter);
+ writer.u32(members.length);
+ for (const run of members) { writer.u32(run.count); writer.u32(run.value); }
+ });
+ });
});
}
@@ -386,7 +418,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
clip.tracks.forEach((track, index) => {
const expected = reference.tracks[index];
- if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {
+ if (track.handler !== expected.handler || track.timescale !== expected.timescale
+ || !sameBytes(track.stsdKey, expected.stsdKey) || track.groupsKey !== expected.groupsKey) {
throw incompatible('the clips use different codec settings');
}
if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 598bd4837..578607804 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -89,6 +89,22 @@ function readTrack(trak, movieTimescale, bytes) {
if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {
throw incompatible('a track switches codec settings');
}
+ // Encrypted samples need their per-sample keys and IVs, which are not
+ // carried over; samples kept in another file cannot be copied from this one.
+ if (/^enc[a-z]$/.test(stbl.stsd.entries[0]?.type || '')) throw incompatible('a track is encrypted');
+ if ((minf.dinf?.dref?.entries || []).some(entry => !(entry.flags & 1))) throw incompatible('a track keeps its samples in another file');
+ // Sample groups, such as an audio codec's pre-roll for seeking: each
+ // sample's group in this clip, and the groups' description box.
+ const groups = (stbl.sbgps || []).flatMap(sbgp => {
+ const description = rawBox(bytes, (stbl.sgpds || []).find(sgpd => sgpd.grouping_type === sbgp.grouping_type));
+ if (!description) return [];
+ const indices = new Array(samples.length).fill(0);
+ let next = 0;
+ for (const entry of sbgp.entries || []) {
+ for (let n = 0; n < entry.sample_count && next < indices.length; n++) indices[next++] = entry.group_description_index;
+ }
+ return [{ type: sbgp.grouping_type, parameter: sbgp.grouping_type_parameter || 0, indices, description }];
+ });
// Reordered frames (B-frames) are shown after they are decoded, so the
// track's picture ends at its last composition time, not its last decode time.
const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);
@@ -112,6 +128,8 @@ function readTrack(trak, movieTimescale, bytes) {
language: mdia.mdhd.language,
stsd: rawBox(bytes, stbl.stsd),
stsdKey: sampleDescriptionKey(bytes, stbl.stsd),
+ groups,
+ groupsKey: JSON.stringify(groups.map(group => [group.type, group.parameter, Array.from(group.description)])),
hdlr: rawBox(bytes, mdia.hdlr),
mediaHeader: rawBox(bytes, (minf.boxes || []).find(box => MEDIA_HEADER_TYPES.has(box.type))),
dinf: rawBox(bytes, (minf.boxes || []).find(box => box.type === 'dinf')),
@@ -222,6 +240,7 @@ function joinTrack(clips, index, starts) {
size: sample.size,
source: sample.offset,
sync: sample.is_sync !== false,
+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),
clipIndex,
});
chunk.size += sample.size;
@@ -315,6 +334,19 @@ function writeSampleTable(writer, track) {
for (const sample of samples) writer.u32(sample.size);
});
writer.full('stco', 0, 0, () => { writer.u32(chunks.length); for (const chunk of chunks) writer.u32(chunk.offset); });
+ const described = new Set();
+ first.groups.forEach((group, index) => {
+ // One description box per grouping type, however many sbgp boxes use it.
+ if (!described.has(group.type)) writer.bytes(group.description);
+ described.add(group.type);
+ const members = runs(samples.map(sample => sample.groups[index]));
+ writer.full('sbgp', group.parameter ? 1 : 0, 0, () => {
+ writer.type(group.type);
+ if (group.parameter) writer.u32(group.parameter);
+ writer.u32(members.length);
+ for (const run of members) { writer.u32(run.count); writer.u32(run.value); }
+ });
+ });
});
}
@@ -386,7 +418,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {
if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');
clip.tracks.forEach((track, index) => {
const expected = reference.tracks[index];
- if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {
+ if (track.handler !== expected.handler || track.timescale !== expected.timescale
+ || !sameBytes(track.stsdKey, expected.stsdKey) || track.groupsKey !== expected.groupsKey) {
throw incompatible('the clips use different codec settings');
}
if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index de89ebb11..751f1a17b 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -55,6 +55,7 @@ function parse(bytes) {
}
const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));
+const boxBytes = (bytes, box) => bytes.subarray(box.start, box.start + box.size);
const videoOf = file => file.moov.traks[0];
@@ -195,19 +196,25 @@ for (const build of ['chrome', 'firefox']) {
assert.deepEqual(sampleBytes(bytes, audio).slice(0, 17), firstAudio);
assert.deepEqual(sampleBytes(bytes, audio).slice(17), secondAudio.slice(1));
assert.equal(audio.samples[17].cts - audio.edts.elst.entries[0].media_time, 16000);
+ // AAC's pre-roll for seeking (a 'roll' sample group) covers every frame.
+ const sourceRoll = parse(sources[0]).moov.traks[1].mdia.minf.stbl.sgpds[0];
+ assert.deepEqual(boxBytes(bytes, audio.mdia.minf.stbl.sgpds[0]), boxBytes(sources[0], sourceRoll));
+ assert.deepEqual(audio.mdia.minf.stbl.sbgps.map(sbgp => sbgp.entries), [[{ sample_count: 33, group_description_index: 1 }]]);
});
test(`${build}: fragmented and VP9/Opus clips join too`, () => {
// [clips, codecs, audio frames kept from the first clip, start-up frames dropped
- // from the second, how far into its first kept frame the second clip's sound starts]
- for (const [names, codecs, kept, dropped, lead] of [
+ // from the second, how far into its first kept frame the second clip's sound
+ // starts, the audio's pre-roll groups as runs of [frames, group]]
+ for (const [names, codecs, kept, dropped, lead, roll] of [
// No edit lists: the picture starts after a B-frame delay (2048/12288 s),
// so every track is shown from there and the movie starts at its first picture.
- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0],
+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, []],
// Opus: the first clip's short final frame would play over the second
// clip's first frame, whose first 312 samples are the decoder's start-up,
- // so the short frame is dropped rather than squeezed.
- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312],
+ // so the short frame is dropped rather than squeezed. Each clip keeps its
+ // own pre-roll groups (none for its first four frames).
+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]]],
]) {
const label = names.join('+');
const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));
@@ -225,6 +232,7 @@ for (const build of ['chrome', 'firefox']) {
`${label}: the second clip's sound starts one second in`);
const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));
assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);
+ assert.deepEqual((audio.mdia.minf.stbl.sbgps[0]?.entries || []).map(entry => [entry.sample_count, entry.group_description_index]), roll, `${label}: pre-roll groups`);
}
// Fragments can start at any decode time: five seconds later, the clips
// join into the same file. A track that starts after the picture cannot join.
@@ -289,6 +297,14 @@ for (const build of ['chrome', 'firefox']) {
'complex edit list');
refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],
'does not start with a keyframe');
+ // Encrypted samples, samples kept in another file, and a different pre-roll.
+ const audioOf = file => file.moov.traks[1];
+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => 'enca'.split('').forEach((char, i) =>
+ view.setUint8(audioOf(file).mdia.minf.stbl.stsd.entries[0].start + 4 + i, char.charCodeAt(0))))], 'encrypted');
+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint8(videoOf(file).mdia.minf.dinf.dref.entries[0].start + 11, 0))],
+ 'in another file');
+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(audioOf(file).mdia.minf.stbl.sgpds[0].start + 24, -2))],
+ 'different codec settings');
});
test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {