Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

Start joined videos at their first picture

Claudeb2f7c524bad9
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.js
index 5e430fb98..6163edfe0 100644
--- a/src/chrome/src/agent/video-join.js
+++ b/src/chrome/src/agent/video-join.js
@@ -95,18 +95,16 @@ function readTrack(trak, movieTimescale, bytes) {
if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {
throw incompatible('a track has a complex edit list');
}
- // Without an edit the track is shown from its first composition time
- // (fragmented files from some encoders start video after a B-frame delay).
- const mediaTime = edits.length ? edits[0].media_time : samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity);
- const editDuration = edits.length && edits[0].segment_duration > 0
- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : compositionEnd - mediaTime;
return {
handler,
timescale,
samples,
- mediaTime,
- presented: Math.max(0, Math.min(editDuration, compositionEnd - mediaTime)),
hasEdit: edits.length > 0,
+ editStart: edits.length ? edits[0].media_time : 0,
+ editDuration: edits.length && edits[0].segment_duration > 0
+ ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,
+ firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),
+ compositionEnd,
tkhd: trak.tkhd,
language: mdia.mdhd.language,
stsd: rawBox(bytes, stbl.stsd),
@@ -140,13 +138,20 @@ function parseClip(input) {
.sort((a, b) => order[a.handler] - order[b.handler]);
if (!tracks.length) throw incompatible('a clip has no video or audio track');
const timing = tracks[0];
+ // Where each track's picture or sound starts (mediaTime, in its own units).
+ // Without edit lists a clip shows nothing before its first picture (some
+ // encoders' fragmented files start video after a B-frame delay), so every
+ // track starts that much later, in step with the picture.
+ const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
+ for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
+ const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
return {
bytes,
movieTimescale,
matrix: moov.mvhd.matrix,
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
- duration: timing.presented / timing.timescale,
+ duration: presented / timing.timescale,
};
}
@@ -316,11 +321,11 @@ function writeMovie(clips, tracks, totalSeconds) {
writeMatrix(writer, first.tkhd?.matrix);
writer.u32(first.tkhd?.width >>> 0 || 0); writer.u32(first.tkhd?.height >>> 0 || 0);
});
- if (first.hasEdit) {
- writer.box('edts', () => writer.full('elst', 0, 0, () => {
- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);
- }));
- }
+ // Every track is shown from its start, so the movie runs from 0 to
+ // exactly its declared duration.
+ writer.box('edts', () => writer.full('elst', 0, 0, () => {
+ writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);
+ }));
writer.box('mdia', () => {
writer.full('mdhd', 0, 0, () => {
writer.u32(0); writer.u32(0);
diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.js
index 5e430fb98..6163edfe0 100644
--- a/src/firefox/src/agent/video-join.js
+++ b/src/firefox/src/agent/video-join.js
@@ -95,18 +95,16 @@ function readTrack(trak, movieTimescale, bytes) {
if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {
throw incompatible('a track has a complex edit list');
}
- // Without an edit the track is shown from its first composition time
- // (fragmented files from some encoders start video after a B-frame delay).
- const mediaTime = edits.length ? edits[0].media_time : samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity);
- const editDuration = edits.length && edits[0].segment_duration > 0
- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : compositionEnd - mediaTime;
return {
handler,
timescale,
samples,
- mediaTime,
- presented: Math.max(0, Math.min(editDuration, compositionEnd - mediaTime)),
hasEdit: edits.length > 0,
+ editStart: edits.length ? edits[0].media_time : 0,
+ editDuration: edits.length && edits[0].segment_duration > 0
+ ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,
+ firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),
+ compositionEnd,
tkhd: trak.tkhd,
language: mdia.mdhd.language,
stsd: rawBox(bytes, stbl.stsd),
@@ -140,13 +138,20 @@ function parseClip(input) {
.sort((a, b) => order[a.handler] - order[b.handler]);
if (!tracks.length) throw incompatible('a clip has no video or audio track');
const timing = tracks[0];
+ // Where each track's picture or sound starts (mediaTime, in its own units).
+ // Without edit lists a clip shows nothing before its first picture (some
+ // encoders' fragmented files start video after a B-frame delay), so every
+ // track starts that much later, in step with the picture.
+ const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;
+ for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);
+ const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));
return {
bytes,
movieTimescale,
matrix: moov.mvhd.matrix,
ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),
tracks,
- duration: timing.presented / timing.timescale,
+ duration: presented / timing.timescale,
};
}
@@ -316,11 +321,11 @@ function writeMovie(clips, tracks, totalSeconds) {
writeMatrix(writer, first.tkhd?.matrix);
writer.u32(first.tkhd?.width >>> 0 || 0); writer.u32(first.tkhd?.height >>> 0 || 0);
});
- if (first.hasEdit) {
- writer.box('edts', () => writer.full('elst', 0, 0, () => {
- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);
- }));
- }
+ // Every track is shown from its start, so the movie runs from 0 to
+ // exactly its declared duration.
+ writer.box('edts', () => writer.full('elst', 0, 0, () => {
+ writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);
+ }));
writer.box('mdia', () => {
writer.full('mdhd', 0, 0, () => {
writer.u32(0); writer.u32(0);
diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjs
index 87e54d0c7..a0f134d8f 100644
--- a/test/generative-media-video.mjs
+++ b/test/generative-media-video.mjs
@@ -56,6 +56,18 @@ function parse(bytes) {
const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));
+// Where a track's samples are shown on the movie's timeline, in seconds: its
+// edit names the first media time shown.
+function shown(file, trak) {
+ const timescale = trak.mdia.mdhd.timescale;
+ const start = trak.edts?.elst?.entries?.[0]?.media_time ?? 0;
+ return {
+ from: (Math.min(...trak.samples.map(sample => sample.cts)) - start) / timescale,
+ to: (Math.max(...trak.samples.map(sample => sample.cts + sample.duration)) - start) / timescale,
+ movie: file.moov.mvhd.duration / file.moov.mvhd.timescale,
+ };
+}
+
// OpenRouter's video API: a catalog entry, jobs that complete on the first
// poll, and clip bytes chosen per job.
function videoApi({ entry, clip = () => fixtures['h264-aac-1'], fail = () => false }) {
@@ -144,9 +156,12 @@ for (const build of ['chrome', 'firefox']) {
const joinedFrames = sampleBytes(bytes, video);
assert.equal(joinedFrames.length, 24);
joinedFrames.forEach((frame, index) => assert.deepEqual(frame, sourceFrames[index], `frame ${index}`));
- // The second clip's frames are shown right after the first clip's.
- const shown = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);
- assert.deepEqual(shown.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));
+ // The second clip's frames are shown right after the first clip's, and the
+ // movie runs from its first picture to its declared length.
+ const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);
+ assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));
+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });
+ assert.equal(shown(output, audio).to, 2);
// Audio: the first clip's frames are all kept; the second clip's start-up
// frame is dropped, the rest is copied as is, and its first real frame
// plays exactly one second in, with its picture.
@@ -157,13 +172,16 @@ for (const build of ['chrome', 'firefox']) {
});
test(`${build}: fragmented and VP9/Opus clips join too`, () => {
- // [clips, codecs, audio frames kept from the first clip, where the second clip's audio starts]
- for (const [names, codecs, seam, seamTime] of [
- // No edit lists: the first clip's last kept frame is lengthened to meet the second.
- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 14, 16000],
+ // [clips, codecs, audio frames kept from the first clip, start-up frames dropped
+ // from the second, how far into its first kept frame the second clip's sound starts]
+ for (const [names, codecs, kept, dropped, lead] of [
+ // No edit lists: the picture starts after a B-frame delay (2048/12288 s),
+ // so every track is shown from there and the movie starts at its first picture.
+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0],
// Opus: the first clip's short final frame would play over the second
- // clip's first frame, so it is dropped rather than squeezed.
- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 48000],
+ // clip's first frame, whose first 312 samples are the decoder's start-up,
+ // so the short frame is dropped rather than squeezed.
+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312],
]) {
const label = names.join('+');
const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));
@@ -172,11 +190,13 @@ for (const build of ['chrome', 'firefox']) {
const [video, audio] = output.moov.traks;
assert.deepEqual(output.moov.traks.map(trak => trak.mdia.minf.stbl.stsd.entries[0].type), codecs, label);
assert.equal(video.samples.length, 24, label);
- const firstAudio = sampleBytes(fixtures[names[0]], parse(fixtures[names[0]]).moov.traks[1]);
- const secondAudio = parse(fixtures[names[1]]).moov.traks[1];
- assert.deepEqual(sampleBytes(bytes, audio).slice(0, seam), firstAudio.slice(0, seam), `${label}: the first clip's audio up to the seam`);
- assert.deepEqual(sampleBytes(bytes, audio).slice(seam), sampleBytes(fixtures[names[1]], secondAudio), `${label}: the second clip's audio, whole`);
- assert.equal(audio.samples[seam].cts, seamTime, `${label}: the second clip's audio starts one second in`);
+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 }, `${label}: the picture runs from 0 to the declared length`);
+ assert.equal(shown(output, audio).to, 2, `${label}: and so does the sound`);
+ const [firstAudio, secondAudio] = names.map(name => parse(fixtures[name]).moov.traks[1]);
+ assert.deepEqual(sampleBytes(bytes, audio).slice(0, kept), sampleBytes(fixtures[names[0]], firstAudio).slice(0, kept), `${label}: the first clip's audio up to the join`);
+ assert.deepEqual(sampleBytes(bytes, audio).slice(kept), sampleBytes(fixtures[names[1]], secondAudio).slice(dropped), `${label}: the second clip's audio`);
+ assert.equal(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time, audio.mdia.mdhd.timescale,
+ `${label}: the second clip's sound starts one second in`);
const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));
assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);
}