Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Stop scene numbers and split scenes from repeating on screen
Claude36e07e6a0da8
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex 883eb1a23..1ac20c5a7 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -307,10 +307,11 @@ estimates how long the requested video should run (`estimated_duration`). Asingle clip can set `duration`, `aspect_ratio` and `resolution`, which arechecked against the model's catalog entry (`supported_durations`,`supported_aspect_ratios`, `supported_resolutions`) before anything is-submitted. A longer video is passed as `scenes`: WebBrain fits each scene to the-clip lengths the model offers, splitting a scene that is longer than its longest-clip, generates up to three clips at a time with one retry each, and joins them-into one MP4 without re-encoding (`src/*/src/agent/video-join.js`, which parses+submitted. A longer video is passed as `scenes`: WebBrain makes one clip per+scene at a length the model offers (a scene longer than its longest clip is+refused, so the agent splits it into scenes with their own prompts), generates+up to three clips at a time with one retry each, and joins them into one MP4+without re-encoding (`src/*/src/agent/video-join.js`, which parsesthe clips with the vendored mp4box.js). A clip's sound often ends a littlebefore its picture or starts after it; with AAC-LC sound (the usual MP4 audio,which Grok Imagine returns) the joiner fills those gaps with silent frames, sodiff --git a/src/chrome/skills/video-scenes.md b/src/chrome/skills/video-scenes.mdindex a3683e892..0e6c1bc69 100644--- a/src/chrome/skills/video-scenes.md+++ b/src/chrome/skills/video-scenes.md@@ -22,6 +22,7 @@ Use this skill before calling `generate_image` with `scenes`. The video model ma## Each scene- One action a camera could film in the clip's few seconds. Call the characters by name; do not describe them again.+- Each scene is one clip, so keep it within the model's longest clip. WebBrain refuses a longer scene and names the limit; split it into scenes with their own prompts.- Keep what changed earlier: a door opened in one scene is still open in the next.- Show the idea happening. Video models cannot draw diagrams, arrows, equations, charts or readable text, so do not ask for them. To explain something, show it: repeat the experiment, put the outcomes side by side, or move in close on the mechanism.- When the idea needs words, give the scene one short narrator line in quotes, about a dozen words for an 8-second clip.diff --git a/src/chrome/src/agent/generative-media.js b/src/chrome/src/agent/generative-media.jsindex d23251afb..b5ff0460c 100644--- a/src/chrome/src/agent/generative-media.js+++ b/src/chrome/src/agent/generative-media.js@@ -161,27 +161,24 @@ function videoShape(entry, video) {}/**- * The clips for a video described as scenes. A scene longer than the model's- * longest clip becomes several clips, and every clip length rounds up to one- * the model offers. Refuses more than `maxSeconds` in total or `maxClips`.+ * The clips for a video described as scenes, one per scene, each length+ * rounded up to one the model offers. A scene longer than the model's longest+ * clip is refused rather than split: its parts would get the same prompt, and+ * the model would make the same clip twice. Refuses more than `maxSeconds` in+ * total or `maxClips`.*/export function planVideoClips(scenes, durations, { maxSeconds = MAX_VIDEO_SECONDS, maxClips = MAX_VIDEO_CLIPS } = {}) {if (!durations?.length) {throw new Error('This video model does not list the clip lengths it supports, so WebBrain cannot make a video from scenes with it. Generate one clip instead.');}const longest = durations[durations.length - 1];- const clips = [];- scenes.forEach((scene, index) => {+ const clips = scenes.map((scene, index) => {const prompt = String(scene?.prompt || '').trim();if (!prompt) throw new Error(`Scene ${index + 1} needs a prompt.`);- const parts = Math.ceil(scene.duration / longest);- const each = Math.ceil(scene.duration / parts);- for (let part = 1; part <= parts; part++) {- clips.push({- prompt: parts > 1 ? `${prompt} (part ${part} of ${parts})` : prompt,- duration: durations.find(value => value >= each) ?? longest,- });+ if (scene.duration > longest) {+ throw new Error(`Scene ${index + 1} is ${scene.duration} s, and this model makes clips of up to ${longest} s. Split it into scenes of up to ${longest} s, each with its own prompt for its part, so no clip repeats another.`);}+ return { prompt, duration: durations.find(value => value >= scene.duration) ?? longest };});const total = clips.reduce((sum, clip) => sum + clip.duration, 0);if (clips.length > maxClips) {@@ -210,8 +207,9 @@ function planVideo(prompt, video, entry) {total,estimated: video.estimated_duration,shape,- // Every clip carries the shared look so the scenes match.- clips: clips.map((clip, index) => ({ ...clip, prompt: `${prompt}\n\nScene ${index + 1} of ${clips.length}: ${clip.prompt}` })),+ // Every clip carries the shared look so the scenes match. No scene+ // numbers: video models draw them on screen as titles.+ clips: clips.map(clip => ({ ...clip, prompt: `${prompt}\n\n${clip.prompt}` })),};}const longest = durations?.[durations.length - 1];diff --git a/src/firefox/skills/video-scenes.md b/src/firefox/skills/video-scenes.mdindex a3683e892..0e6c1bc69 100644--- a/src/firefox/skills/video-scenes.md+++ b/src/firefox/skills/video-scenes.md@@ -22,6 +22,7 @@ Use this skill before calling `generate_image` with `scenes`. The video model ma## Each scene- One action a camera could film in the clip's few seconds. Call the characters by name; do not describe them again.+- Each scene is one clip, so keep it within the model's longest clip. WebBrain refuses a longer scene and names the limit; split it into scenes with their own prompts.- Keep what changed earlier: a door opened in one scene is still open in the next.- Show the idea happening. Video models cannot draw diagrams, arrows, equations, charts or readable text, so do not ask for them. To explain something, show it: repeat the experiment, put the outcomes side by side, or move in close on the mechanism.- When the idea needs words, give the scene one short narrator line in quotes, about a dozen words for an 8-second clip.diff --git a/src/firefox/src/agent/generative-media.js b/src/firefox/src/agent/generative-media.jsindex d23251afb..b5ff0460c 100644--- a/src/firefox/src/agent/generative-media.js+++ b/src/firefox/src/agent/generative-media.js@@ -161,27 +161,24 @@ function videoShape(entry, video) {}/**- * The clips for a video described as scenes. A scene longer than the model's- * longest clip becomes several clips, and every clip length rounds up to one- * the model offers. Refuses more than `maxSeconds` in total or `maxClips`.+ * The clips for a video described as scenes, one per scene, each length+ * rounded up to one the model offers. A scene longer than the model's longest+ * clip is refused rather than split: its parts would get the same prompt, and+ * the model would make the same clip twice. Refuses more than `maxSeconds` in+ * total or `maxClips`.*/export function planVideoClips(scenes, durations, { maxSeconds = MAX_VIDEO_SECONDS, maxClips = MAX_VIDEO_CLIPS } = {}) {if (!durations?.length) {throw new Error('This video model does not list the clip lengths it supports, so WebBrain cannot make a video from scenes with it. Generate one clip instead.');}const longest = durations[durations.length - 1];- const clips = [];- scenes.forEach((scene, index) => {+ const clips = scenes.map((scene, index) => {const prompt = String(scene?.prompt || '').trim();if (!prompt) throw new Error(`Scene ${index + 1} needs a prompt.`);- const parts = Math.ceil(scene.duration / longest);- const each = Math.ceil(scene.duration / parts);- for (let part = 1; part <= parts; part++) {- clips.push({- prompt: parts > 1 ? `${prompt} (part ${part} of ${parts})` : prompt,- duration: durations.find(value => value >= each) ?? longest,- });+ if (scene.duration > longest) {+ throw new Error(`Scene ${index + 1} is ${scene.duration} s, and this model makes clips of up to ${longest} s. Split it into scenes of up to ${longest} s, each with its own prompt for its part, so no clip repeats another.`);}+ return { prompt, duration: durations.find(value => value >= scene.duration) ?? longest };});const total = clips.reduce((sum, clip) => sum + clip.duration, 0);if (clips.length > maxClips) {@@ -210,8 +207,9 @@ function planVideo(prompt, video, entry) {total,estimated: video.estimated_duration,shape,- // Every clip carries the shared look so the scenes match.- clips: clips.map((clip, index) => ({ ...clip, prompt: `${prompt}\n\nScene ${index + 1} of ${clips.length}: ${clip.prompt}` })),+ // Every clip carries the shared look so the scenes match. No scene+ // numbers: video models draw them on screen as titles.+ clips: clips.map(clip => ({ ...clip, prompt: `${prompt}\n\n${clip.prompt}` })),};}const longest = durations?.[durations.length - 1];diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 82e057a68..8c2ec039d 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -422,13 +422,13 @@ for (const build of ['chrome', 'firefox']) {});test(`${build}: scenes fit the model's clip lengths and stay within two minutes`, () => {- const { clips, total } = media.planVideoClips([{ prompt: 'Opening', duration: 12 }, { prompt: 'Long walk', duration: 20 }], [5, 10, 15]);- assert.deepEqual(clips, [- { prompt: 'Opening', duration: 15 },- { prompt: 'Long walk (part 1 of 2)', duration: 10 },- { prompt: 'Long walk (part 2 of 2)', duration: 10 },- ]);- assert.equal(total, 35);+ const { clips, total } = media.planVideoClips([{ prompt: 'Opening', duration: 12 }, { prompt: 'Long walk', duration: 4 }], [5, 10, 15]);+ assert.deepEqual(clips, [{ prompt: 'Opening', duration: 15 }, { prompt: 'Long walk', duration: 5 }]);+ assert.equal(total, 20);+ // A scene longer than the longest clip is the agent's to split: halves of+ // one prompt would be the same clip twice.+ assert.throws(() => media.planVideoClips([{ prompt: 'Opening', duration: 12 }, { prompt: 'Long walk', duration: 20 }], [5, 10, 15]),+ /^Error: Scene 2 is 20 s, and this model makes clips of up to 15 s\. Split it into scenes of up to 15 s, each with its own prompt/);assert.equal(media.planVideoClips(scenes(8, 15), grok.supported_durations).total, 120);assert.throws(() => media.planVideoClips(scenes(9, 15), grok.supported_durations), /up to 120 s/);assert.throws(() => media.planVideoClips(scenes(25, 4), grok.supported_durations), /at most 24/);@@ -479,7 +479,8 @@ for (const build of ['chrome', 'firefox']) {assert.ok(api.calls.maxInFlight <= 3, `at most three clips at a time (saw ${api.calls.maxInFlight})`);api.calls.submits.forEach((body, index) => {assert.equal(body.duration, 15);- assert.equal(body.prompt, `Flat motion graphics, navy background\n\nScene ${index + 1} of 4: Scene idea ${index + 1}`);+ // The shared look, then the scene, with no scene number for the model to draw.+ assert.equal(body.prompt, `Flat motion graphics, navy background\n\nScene idea ${index + 1}`);});assert.equal(result.clipCount, 4);assert.equal(result.durationSeconds, 4, 'the real length of the joined clips');@@ -521,7 +522,7 @@ for (const build of ['chrome', 'firefox']) {const recovered = await generate({ prompt: 'Story', scenes: scenes(3, 10), estimated_duration: 30 }, { fetchImpl: retried.fetchImpl });assert.equal(recovered.success, true, recovered.error);assert.equal(retried.calls.submits.length, 4, 'three clips and one retry');- const failing = videoApi({ entry: grok, fail: (_index, body) => body.prompt.includes('Scene 2 of 3') });+ const failing = videoApi({ entry: grok, fail: (_index, body) => body.prompt.endsWith('Scene idea 2') });const stored = assets.size;const failed = await generate({ prompt: 'Story', scenes: scenes(3, 10), estimated_duration: 30 }, { fetchImpl: failing.fetchImpl });assert.equal(failed.success, false);