Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Start joined videos at their first picture
Claudeb2f7c524bad9
diff --git a/src/chrome/src/agent/video-join.js b/src/chrome/src/agent/video-join.jsindex 5e430fb98..6163edfe0 100644--- a/src/chrome/src/agent/video-join.js+++ b/src/chrome/src/agent/video-join.js@@ -95,18 +95,16 @@ function readTrack(trak, movieTimescale, bytes) {if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {throw incompatible('a track has a complex edit list');}- // Without an edit the track is shown from its first composition time- // (fragmented files from some encoders start video after a B-frame delay).- const mediaTime = edits.length ? edits[0].media_time : samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity);- const editDuration = edits.length && edits[0].segment_duration > 0- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : compositionEnd - mediaTime;return {handler,timescale,samples,- mediaTime,- presented: Math.max(0, Math.min(editDuration, compositionEnd - mediaTime)),hasEdit: edits.length > 0,+ editStart: edits.length ? edits[0].media_time : 0,+ editDuration: edits.length && edits[0].segment_duration > 0+ ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,+ firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),+ compositionEnd,tkhd: trak.tkhd,language: mdia.mdhd.language,stsd: rawBox(bytes, stbl.stsd),@@ -140,13 +138,20 @@ function parseClip(input) {.sort((a, b) => order[a.handler] - order[b.handler]);if (!tracks.length) throw incompatible('a clip has no video or audio track');const timing = tracks[0];+ // Where each track's picture or sound starts (mediaTime, in its own units).+ // Without edit lists a clip shows nothing before its first picture (some+ // encoders' fragmented files start video after a B-frame delay), so every+ // track starts that much later, in step with the picture.+ const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;+ for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);+ const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));return {bytes,movieTimescale,matrix: moov.mvhd.matrix,ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),tracks,- duration: timing.presented / timing.timescale,+ duration: presented / timing.timescale,};}@@ -316,11 +321,11 @@ function writeMovie(clips, tracks, totalSeconds) {writeMatrix(writer, first.tkhd?.matrix);writer.u32(first.tkhd?.width >>> 0 || 0); writer.u32(first.tkhd?.height >>> 0 || 0);});- if (first.hasEdit) {- writer.box('edts', () => writer.full('elst', 0, 0, () => {- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);- }));- }+ // Every track is shown from its start, so the movie runs from 0 to+ // exactly its declared duration.+ writer.box('edts', () => writer.full('elst', 0, 0, () => {+ writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);+ }));writer.box('mdia', () => {writer.full('mdhd', 0, 0, () => {writer.u32(0); writer.u32(0);diff --git a/src/firefox/src/agent/video-join.js b/src/firefox/src/agent/video-join.jsindex 5e430fb98..6163edfe0 100644--- a/src/firefox/src/agent/video-join.js+++ b/src/firefox/src/agent/video-join.js@@ -95,18 +95,16 @@ function readTrack(trak, movieTimescale, bytes) {if (edits.length > 1 || edits.some(edit => edit.media_time < 0 || edit.media_rate_integer !== 1)) {throw incompatible('a track has a complex edit list');}- // Without an edit the track is shown from its first composition time- // (fragmented files from some encoders start video after a B-frame delay).- const mediaTime = edits.length ? edits[0].media_time : samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity);- const editDuration = edits.length && edits[0].segment_duration > 0- ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : compositionEnd - mediaTime;return {handler,timescale,samples,- mediaTime,- presented: Math.max(0, Math.min(editDuration, compositionEnd - mediaTime)),hasEdit: edits.length > 0,+ editStart: edits.length ? edits[0].media_time : 0,+ editDuration: edits.length && edits[0].segment_duration > 0+ ? Math.round(edits[0].segment_duration * timescale / movieTimescale) : Infinity,+ firstComposition: samples.reduce((min, sample) => Math.min(min, sample.cts), Infinity),+ compositionEnd,tkhd: trak.tkhd,language: mdia.mdhd.language,stsd: rawBox(bytes, stbl.stsd),@@ -140,13 +138,20 @@ function parseClip(input) {.sort((a, b) => order[a.handler] - order[b.handler]);if (!tracks.length) throw incompatible('a clip has no video or audio track');const timing = tracks[0];+ // Where each track's picture or sound starts (mediaTime, in its own units).+ // Without edit lists a clip shows nothing before its first picture (some+ // encoders' fragmented files start video after a B-frame delay), so every+ // track starts that much later, in step with the picture.+ const lead = timing.hasEdit ? 0 : timing.firstComposition / timing.timescale;+ for (const track of tracks) track.mediaTime = track.editStart + Math.round(lead * track.timescale);+ const presented = Math.max(0, Math.min(timing.editDuration, timing.compositionEnd - timing.mediaTime));return {bytes,movieTimescale,matrix: moov.mvhd.matrix,ftyp: rawBox(bytes, (file.boxes || []).find(box => box.type === 'ftyp')),tracks,- duration: timing.presented / timing.timescale,+ duration: presented / timing.timescale,};}@@ -316,11 +321,11 @@ function writeMovie(clips, tracks, totalSeconds) {writeMatrix(writer, first.tkhd?.matrix);writer.u32(first.tkhd?.width >>> 0 || 0); writer.u32(first.tkhd?.height >>> 0 || 0);});- if (first.hasEdit) {- writer.box('edts', () => writer.full('elst', 0, 0, () => {- writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);- }));- }+ // Every track is shown from its start, so the movie runs from 0 to+ // exactly its declared duration.+ writer.box('edts', () => writer.full('elst', 0, 0, () => {+ writer.u32(1); writer.u32(movieDuration); writer.i32(first.mediaTime); writer.i16(1); writer.i16(0);+ }));writer.box('mdia', () => {writer.full('mdhd', 0, 0, () => {writer.u32(0); writer.u32(0);diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 87e54d0c7..a0f134d8f 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -56,6 +56,18 @@ function parse(bytes) {const sampleBytes = (bytes, trak) => trak.samples.map(sample => bytes.subarray(sample.offset, sample.offset + sample.size));+// Where a track's samples are shown on the movie's timeline, in seconds: its+// edit names the first media time shown.+function shown(file, trak) {+ const timescale = trak.mdia.mdhd.timescale;+ const start = trak.edts?.elst?.entries?.[0]?.media_time ?? 0;+ return {+ from: (Math.min(...trak.samples.map(sample => sample.cts)) - start) / timescale,+ to: (Math.max(...trak.samples.map(sample => sample.cts + sample.duration)) - start) / timescale,+ movie: file.moov.mvhd.duration / file.moov.mvhd.timescale,+ };+}+// OpenRouter's video API: a catalog entry, jobs that complete on the first// poll, and clip bytes chosen per job.function videoApi({ entry, clip = () => fixtures['h264-aac-1'], fail = () => false }) {@@ -144,9 +156,12 @@ for (const build of ['chrome', 'firefox']) {const joinedFrames = sampleBytes(bytes, video);assert.equal(joinedFrames.length, 24);joinedFrames.forEach((frame, index) => assert.deepEqual(frame, sourceFrames[index], `frame ${index}`));- // The second clip's frames are shown right after the first clip's.- const shown = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);- assert.deepEqual(shown.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));+ // The second clip's frames are shown right after the first clip's, and the+ // movie runs from its first picture to its declared length.+ const times = video.samples.map(sample => sample.cts - video.edts.elst.entries[0].media_time).sort((a, b) => a - b);+ assert.deepEqual(times.map(value => value / 1024), Array.from({ length: 24 }, (_, i) => i));+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 });+ assert.equal(shown(output, audio).to, 2);// Audio: the first clip's frames are all kept; the second clip's start-up// frame is dropped, the rest is copied as is, and its first real frame// plays exactly one second in, with its picture.@@ -157,13 +172,16 @@ for (const build of ['chrome', 'firefox']) {});test(`${build}: fragmented and VP9/Opus clips join too`, () => {- // [clips, codecs, audio frames kept from the first clip, where the second clip's audio starts]- for (const [names, codecs, seam, seamTime] of [- // No edit lists: the first clip's last kept frame is lengthened to meet the second.- [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 14, 16000],+ // [clips, codecs, audio frames kept from the first clip, start-up frames dropped+ // from the second, how far into its first kept frame the second clip's sound starts]+ for (const [names, codecs, kept, dropped, lead] of [+ // No edit lists: the picture starts after a B-frame delay (2048/12288 s),+ // so every track is shown from there and the movie starts at its first picture.+ [['h264-aac-fragmented-1', 'h264-aac-fragmented-2'], ['avc1', 'mp4a'], 17, 1, 0],// Opus: the first clip's short final frame would play over the second- // clip's first frame, so it is dropped rather than squeezed.- [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 48000],+ // clip's first frame, whose first 312 samples are the decoder's start-up,+ // so the short frame is dropped rather than squeezed.+ [['vp9-opus-1', 'vp9-opus-2'], ['vp09', 'Opus'], 50, 0, 312],]) {const label = names.join('+');const { bytes, durationSeconds } = join.joinMp4Clips(names.map(name => fixtures[name]));@@ -172,11 +190,13 @@ for (const build of ['chrome', 'firefox']) {const [video, audio] = output.moov.traks;assert.deepEqual(output.moov.traks.map(trak => trak.mdia.minf.stbl.stsd.entries[0].type), codecs, label);assert.equal(video.samples.length, 24, label);- const firstAudio = sampleBytes(fixtures[names[0]], parse(fixtures[names[0]]).moov.traks[1]);- const secondAudio = parse(fixtures[names[1]]).moov.traks[1];- assert.deepEqual(sampleBytes(bytes, audio).slice(0, seam), firstAudio.slice(0, seam), `${label}: the first clip's audio up to the seam`);- assert.deepEqual(sampleBytes(bytes, audio).slice(seam), sampleBytes(fixtures[names[1]], secondAudio), `${label}: the second clip's audio, whole`);- assert.equal(audio.samples[seam].cts, seamTime, `${label}: the second clip's audio starts one second in`);+ assert.deepEqual(shown(output, video), { from: 0, to: 2, movie: 2 }, `${label}: the picture runs from 0 to the declared length`);+ assert.equal(shown(output, audio).to, 2, `${label}: and so does the sound`);+ const [firstAudio, secondAudio] = names.map(name => parse(fixtures[name]).moov.traks[1]);+ assert.deepEqual(sampleBytes(bytes, audio).slice(0, kept), sampleBytes(fixtures[names[0]], firstAudio).slice(0, kept), `${label}: the first clip's audio up to the join`);+ assert.deepEqual(sampleBytes(bytes, audio).slice(kept), sampleBytes(fixtures[names[1]], secondAudio).slice(dropped), `${label}: the second clip's audio`);+ assert.equal(audio.samples[kept].cts + lead - audio.edts.elst.entries[0].media_time, audio.mdia.mdhd.timescale,+ `${label}: the second clip's sound starts one second in`);const shortest = Math.min(...secondAudio.samples.map(sample => sample.duration));assert.ok(audio.samples.every(sample => sample.duration >= shortest), `${label}: no frame is squeezed`);}