Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Keep audio pre-roll groups in joined videos
Claude5c00f74a22a0
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex 598bd4837..578607804 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -89,6 +89,22 @@ function readTrack(trak, movieTimescale, bytes) {if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {throw incompatible('a track switches codec settings');}+ // Encrypted samples need their per-sample keys and IVs, which are not+ // carried over; samples kept in another file cannot be copied from this one.+ if (/^enc[a-z]$/.test(stbl.stsd.entries[0]?.type || '')) throw incompatible('a track is encrypted');+ if ((minf.dinf?.dref?.entries || []).some(entry => !(entry.flags & 1))) throw incompatible('a track keeps its samples in another file');+ // Sample groups, such as an audio codec's pre-roll for seeking: each+ // sample's group in this clip, and the groups' description box.+ const groups = (stbl.sbgps || []).flatMap(sbgp => {+ const description = rawBox(bytes, (stbl.sgpds || []).find(sgpd => sgpd.grouping_type === sbgp.grouping_type));+ if (!description) return [];+ const indices = new Array(samples.length).fill(0);+ let next = 0;+ for (const entry of sbgp.entries || []) {+ for (let n = 0; n < entry.sample_count && next < indices.length; n++) indices[next++] = entry.group_description_index;+ }+ return [{ type: sbgp.grouping_type, parameter: sbgp.grouping_type_parameter || 0, indices, description }];+ });// Reordered frames (B-frames) are shown after they are decoded, so the// track's picture ends at its last composition time, not its last decode time.const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);@@ -112,6 +128,8 @@ function readTrack(trak, movieTimescale, bytes) {language: mdia.mdhd.language,stsd: rawBox(bytes, stbl.stsd),stsdKey: sampleDescriptionKey(bytes, stbl.stsd),+ groups,+ groupsKey: JSON.stringify(groups.map(group => [group.type, group.parameter, Array.from(group.description)])),hdlr: rawBox(bytes, mdia.hdlr),mediaHeader: rawBox(bytes, (minf.boxes || []).find(box => MEDIA_HEADER_TYPES.has(box.type))),dinf: rawBox(bytes, (minf.boxes || []).find(box => box.type === 'dinf')),@@ -222,6 +240,7 @@ function joinTrack(clips, index, starts) {size: sample.size,source: sample.offset,sync: sample.is_sync !== false,+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),clipIndex,});chunk.size += sample.size;@@ -315,6 +334,19 @@ function writeSampleTable(writer, track) {for (const sample of samples) writer.u32(sample.size);});writer.full('stco', 0, 0, () => { writer.u32(chunks.length); for (const chunk of chunks) writer.u32(chunk.offset); });+ const described = new Set();+ first.groups.forEach((group, index) => {+ // One description box per grouping type, however many sbgp boxes use it.+ if (!described.has(group.type)) writer.bytes(group.description);+ described.add(group.type);+ const members = runs(samples.map(sample => sample.groups[index]));+ writer.full('sbgp', group.parameter ? 1 : 0, 0, () => {+ writer.type(group.type);+ if (group.parameter) writer.u32(group.parameter);+ writer.u32(members.length);+ for (const run of members) { writer.u32(run.count); writer.u32(run.value); }+ });+ });});}@@ -386,7 +418,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');clip.tracks.forEach((track, index) => {const expected = reference.tracks[index];- if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {+ if (track.handler !== expected.handler || track.timescale !== expected.timescale+ || !sameBytes(track.stsdKey, expected.stsdKey) || track.groupsKey !== expected.groupsKey) {throw incompatible('the clips use different codec settings');}if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex 598bd4837..578607804 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -89,6 +89,22 @@ function readTrack(trak, movieTimescale, bytes) {if ((stbl.stsd.entries?.length || 0) !== 1 || samples.some(sample => sample.description_index !== 0)) {throw incompatible('a track switches codec settings');}+ // Encrypted samples need their per-sample keys and IVs, which are not+ // carried over; samples kept in another file cannot be copied from this one.+ if (/^enc[a-z]$/.test(stbl.stsd.entries[0]?.type || '')) throw incompatible('a track is encrypted');+ if ((minf.dinf?.dref?.entries || []).some(entry => !(entry.flags & 1))) throw incompatible('a track keeps its samples in another file');+ // Sample groups, such as an audio codec's pre-roll for seeking: each+ // sample's group in this clip, and the groups' description box.+ const groups = (stbl.sbgps || []).flatMap(sbgp => {+ const description = rawBox(bytes, (stbl.sgpds || []).find(sgpd => sgpd.grouping_type === sbgp.grouping_type));+ if (!description) return [];+ const indices = new Array(samples.length).fill(0);+ let next = 0;+ for (const entry of sbgp.entries || []) {+ for (let n = 0; n < entry.sample_count && next < indices.length; n++) indices[next++] = entry.group_description_index;+ }+ return [{ type: sbgp.grouping_type, parameter: sbgp.grouping_type_parameter || 0, indices, description }];+ });// Reordered frames (B-frames) are shown after they are decoded, so the// track's picture ends at its last composition time, not its last decode time.const compositionEnd = samples.reduce((end, sample) => Math.max(end, sample.cts + sample.duration), 0);@@ -112,6 +128,8 @@ function readTrack(trak, movieTimescale, bytes) {language: mdia.mdhd.language,stsd: rawBox(bytes, stbl.stsd),stsdKey: sampleDescriptionKey(bytes, stbl.stsd),+ groups,+ groupsKey: JSON.stringify(groups.map(group => [group.type, group.parameter, Array.from(group.description)])),hdlr: rawBox(bytes, mdia.hdlr),mediaHeader: rawBox(bytes, (minf.boxes || []).find(box => MEDIA_HEADER_TYPES.has(box.type))),dinf: rawBox(bytes, (minf.boxes || []).find(box => box.type === 'dinf')),@@ -222,6 +240,7 @@ function joinTrack(clips, index, starts) {size: sample.size,source: sample.offset,sync: sample.is_sync !== false,+ groups: track.groups.map(group => group.indices[sample.number] ?? 0),clipIndex,});chunk.size += sample.size;@@ -315,6 +334,19 @@ function writeSampleTable(writer, track) {for (const sample of samples) writer.u32(sample.size);});writer.full('stco', 0, 0, () => { writer.u32(chunks.length); for (const chunk of chunks) writer.u32(chunk.offset); });+ const described = new Set();+ first.groups.forEach((group, index) => {+ // One description box per grouping type, however many sbgp boxes use it.+ if (!described.has(group.type)) writer.bytes(group.description);+ described.add(group.type);+ const members = runs(samples.map(sample => sample.groups[index]));+ writer.full('sbgp', group.parameter ? 1 : 0, 0, () => {+ writer.type(group.type);+ if (group.parameter) writer.u32(group.parameter);+ writer.u32(members.length);+ for (const run of members) { writer.u32(run.count); writer.u32(run.value); }+ });+ });});}@@ -386,7 +418,8 @@ export function joinMp4Clips(inputs, { maxBytes = Infinity } = {}) {if (clip.tracks.length !== reference.tracks.length) throw incompatible('the clips have different tracks');clip.tracks.forEach((track, index) => {const expected = reference.tracks[index];- if (track.handler !== expected.handler || track.timescale !== expected.timescale || !sameBytes(track.stsdKey, expected.stsdKey)) {+ if (track.handler !== expected.handler || track.timescale !== expected.timescale+ || !sameBytes(track.stsdKey, expected.stsdKey) || track.groupsKey !== expected.groupsKey) {throw incompatible('the clips use different codec settings');}if (placement(clip.matrix, track.tkhd) !== placement(reference.matrix, expected.tkhd)) {diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex de89ebb11..751f1a17b 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -55,6 +55,7 @@ function parse(bytes) {}const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));+const boxBytes = (bytes, box) => bytes.subarray(box.start, box.start + box.size);const videoOf = file => file.moov.traks[0];@@ -195,19 +196,25 @@ for (const build of ['chrome', 'firefox']) {assert.deepEqual(sampleBytes(bytes, audio).slice(0, 17), firstAudio);assert.deepEqual(sampleBytes(bytes, audio).slice(17), secondAudio.slice(1));assert.equal(audio.samples[17].cts - audio.edts.elst.entries[0].media_time, 16000);+ // AAC's pre-roll for seeking (a 'roll' sample group) covers every frame.+ const sourceRoll = parse(sources[0]).moov.traks[1].mdia.minf.stbl.sgpds[0];+ assert.deepEqual(boxBytes(bytes, audio.mdia.minf.stbl.sgpds[0]), boxBytes(sources[0], sourceRoll));+ assert.deepEqual(audio.mdia.minf.stbl.sbgps.map(sbgp => sbgp.entries), [[{ sample_count: 33, group_description_index: 1 }]]);});test(`${build}: fragmented and VP9/Opus clips join too`, () => {// [clips, codecs, audio frames kept from the first clip, start-up frames dropped- // from the second, how far into its first kept frame the second clip's sound starts]- for (const [names, codecs, kept, dropped, lead] of [+ // from the second, how far into its first kept frame the second clip's sound+ // starts, the audio's pre-roll groups as runs of [frames, group]]+ for (const [names, codecs, kept, dropped, lead, roll] of [// No edit lists: the picture starts after a B-frame delay (2048/12288 s),// so every track is shown from there and the movie starts at its first picture.- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0],+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0, []],// Opus: the first clip's short final frame would play over the second// clip's first frame, whose first 312 samples are the decoder's start-up,- // so the short frame is dropped rather than squeezed.- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312],+ // so the short frame is dropped rather than squeezed. Each clip keeps its+ // own pre-roll groups (none for its first four frames).+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312, [[4, 0], [46, 1], [4, 0], [47, 1]]],]) {const label = names.join('+');const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));@@ -225,6 +232,7 @@ for (const build of ['chrome', 'firefox']) {`${label}: the second clip's sound starts one second in`);const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);+ assert.deepEqual((audio.mdia.minf.stbl.sbgps[0]?.entries || []).map(entry => [entry.sample_count, entry.group_description_index]), roll, `${label}: pre-roll groups`);}// Fragments can start at any decode time: five seconds later, the clips// join into the same file. A track that starts after the picture cannot join.@@ -289,6 +297,14 @@ for (const build of ['chrome', 'firefox']) {'complex edit list');refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint32(videoOf(file).mdia.minf.stbl.stss.start + 16, 2))],'does not start with a keyframe');+ // Encrypted samples, samples kept in another file, and a different pre-roll.+ const audioOf = file => file.moov.traks[1];+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => 'enca'.split('').forEach((char, i) =>+ view.setUint8(audioOf(file).mdia.minf.stbl.stsd.entries[0].start + 4 + i, char.charCodeAt(0))))], 'encrypted');+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setUint8(videoOf(file).mdia.minf.dinf.dref.entries[0].start + 11, 0))],+ 'in another file');+ refused([fixtures['h264-aac-1'], patched('h264-aac-2', (view, file) => view.setInt16(audioOf(file).mdia.minf.stbl.sgpds[0].start + 24, -2))],+ 'different codec settings');});test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {