Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Add a Video scenes skill for multi-clip videos
Claude7eb048a04315
diff --git a/docs/architecture.md b/docs/architecture.mdindex e946e2b02..8f4a3b162 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -630,8 +630,8 @@ Chrome CSS patch records include the top-level `documentId` and a patch-specificSettings -> Skills stores enabled skills in `customSkills` (`chrome.storage.local`or `browser.storage.local`). On startup, `background.js` loads packaged defaultskills from `skills/*`, adds any missing default (currently FreeSkillz.xyz, the-email verification-code helper, and Humanizer), and refreshes an-existing+email verification-code helper, Humanizer, the WordPress and CMS skills, and+Video scenes), and refreshes an existingbuilt-in skill record when the packaged copy changes. If the user removes adefault skill, its removal tombstone prevents it from being silently re-added;new default IDs can still be migrated into existing installations.diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex 447e00d6f..883eb1a23 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -323,7 +323,10 @@ unattended run generates nothing. Clips that cannot be joined this waymore than 160 MB together) are kept as separate clips and shown in order, andthe tool result says why (`joinError`), which traces show. The model never seesthe video, so the tool result gives it the video's real length-(`durationSeconds`) and `clipCount` to report.+(`durationSeconds`) and `clipCount` to report. Before writing scenes, the agent can load the default Video scenes+skill (`skills/video-scenes.md`), which asks for a shared cast and set sheet in+`prompt`, one filmable action per scene, and demonstrations instead of diagrams+or on-screen text; it is loaded only for such videos.The Act system prompt and planner direct the model to call `generate_image` whenthe user asks to generate media. Provider results are treated as untrusted content.diff --git a/src/chrome/skills/video-scenes.md b/src/chrome/skills/video-scenes.mdnew file mode 100644index 000000000..a3683e892--- /dev/null+++ b/src/chrome/skills/video-scenes.md@@ -0,0 +1,33 @@+# Video scenes++```webbrain-skill+{+ "summary": "Write the scenes of a video made of several generated clips (generate_image with scenes) so the clips match and show the idea.",+ "modes": ["act"],+ "intents": ["generate_video", "explainer_video", "video_story", "how_to_video"]+}+```++Use this skill before calling `generate_image` with `scenes`. The video model makes each scene as a separate clip from its text alone, without seeing the other clips; WebBrain then joins the clips into one video.++## The shared prompt++`prompt` is repeated in front of every scene, so it is the only thing the clips share. Anything it leaves open will change from clip to clip. Write it as a short cast and set sheet:++- each recurring character: a name, age, hair, and clothes with their colours;+- the setting and its fixed layout: what is where, and its colours;+- the style, lighting and camera;+- the sound: natural sound, or a narrator with a described voice. No music unless the user asks for it, since every clip would get different music.++## Each scene++- One action a camera could film in the clip's few seconds. Call the characters by name; do not describe them again.+- Keep what changed earlier: a door opened in one scene is still open in the next.+- Show the idea happening. Video models cannot draw diagrams, arrows, equations, charts or readable text, so do not ask for them. To explain something, show it: repeat the experiment, put the outcomes side by side, or move in close on the mechanism.+- When the idea needs words, give the scene one short narrator line in quotes, about a dozen words for an 8-second clip.++## Explainers++- Take the explanation from the page or the user's request. Do not add facts, numbers or steps that are not there.+- Build the video around the one demonstration that makes the idea click: set it up, show it, and end on what it means.+- Three to five scenes are usually enough.diff --git a/src/chrome/src/agent/skills.js b/src/chrome/src/agent/skills.jsindex 18a764bb8..ad7bb018d 100644--- a/src/chrome/src/agent/skills.js+++ b/src/chrome/src/agent/skills.js@@ -121,6 +121,11 @@ export const PACKAGED_SKILL_SOURCES = Object.freeze([name: 'Sanity content (API-first)',path: 'skills/cms-sanity.md',}),+ Object.freeze({+ id: 'video-scenes',+ name: 'Video scenes',+ path: 'skills/video-scenes.md',+ }),]);export const DEFAULT_SKILL_SOURCES = Object.freeze(PACKAGED_SKILL_SOURCES.filter((source) => [@@ -137,6 +142,7 @@ export const DEFAULT_SKILL_SOURCES = Object.freeze('cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',].includes(source.id)));diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.jsindex 1bce230f4..c6dbe86b2 100644--- a/src/chrome/src/agent/tools.js+++ b/src/chrome/src/agent/tools.js@@ -1499,7 +1499,7 @@ export const AGENT_TOOLS = [resolution: { type: 'string', description: 'Video only, for example "720p".' },scenes: {type: 'array',- description: 'Video only: the scenes of a video longer than one clip, in order, at most 120 seconds in total.',+ description: 'Video only: the scenes of a video longer than one clip, in order, at most 120 seconds in total. Load the video-scenes skill first when it is listed.',items: {type: 'object',properties: {diff --git a/src/firefox/skills/video-scenes.md b/src/firefox/skills/video-scenes.mdnew file mode 100644index 000000000..a3683e892--- /dev/null+++ b/src/firefox/skills/video-scenes.md@@ -0,0 +1,33 @@+# Video scenes++```webbrain-skill+{+ "summary": "Write the scenes of a video made of several generated clips (generate_image with scenes) so the clips match and show the idea.",+ "modes": ["act"],+ "intents": ["generate_video", "explainer_video", "video_story", "how_to_video"]+}+```++Use this skill before calling `generate_image` with `scenes`. The video model makes each scene as a separate clip from its text alone, without seeing the other clips; WebBrain then joins the clips into one video.++## The shared prompt++`prompt` is repeated in front of every scene, so it is the only thing the clips share. Anything it leaves open will change from clip to clip. Write it as a short cast and set sheet:++- each recurring character: a name, age, hair, and clothes with their colours;+- the setting and its fixed layout: what is where, and its colours;+- the style, lighting and camera;+- the sound: natural sound, or a narrator with a described voice. No music unless the user asks for it, since every clip would get different music.++## Each scene++- One action a camera could film in the clip's few seconds. Call the characters by name; do not describe them again.+- Keep what changed earlier: a door opened in one scene is still open in the next.+- Show the idea happening. Video models cannot draw diagrams, arrows, equations, charts or readable text, so do not ask for them. To explain something, show it: repeat the experiment, put the outcomes side by side, or move in close on the mechanism.+- When the idea needs words, give the scene one short narrator line in quotes, about a dozen words for an 8-second clip.++## Explainers++- Take the explanation from the page or the user's request. Do not add facts, numbers or steps that are not there.+- Build the video around the one demonstration that makes the idea click: set it up, show it, and end on what it means.+- Three to five scenes are usually enough.diff --git a/src/firefox/src/agent/skills.js b/src/firefox/src/agent/skills.jsindex f198ede7c..f33ffa7e9 100644--- a/src/firefox/src/agent/skills.js+++ b/src/firefox/src/agent/skills.js@@ -121,6 +121,11 @@ export const PACKAGED_SKILL_SOURCES = Object.freeze([name: 'Sanity content (API-first)',path: 'skills/cms-sanity.md',}),+ Object.freeze({+ id: 'video-scenes',+ name: 'Video scenes',+ path: 'skills/video-scenes.md',+ }),]);export const DEFAULT_SKILL_SOURCES = Object.freeze(PACKAGED_SKILL_SOURCES.filter((source) => [@@ -137,6 +142,7 @@ export const DEFAULT_SKILL_SOURCES = Object.freeze('cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',].includes(source.id)));diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.jsindex 1ecbd5c2d..789f44e8f 100644--- a/src/firefox/src/agent/tools.js+++ b/src/firefox/src/agent/tools.js@@ -1354,7 +1354,7 @@ export const AGENT_TOOLS = [resolution: { type: 'string', description: 'Video only, for example "720p".' },scenes: {type: 'array',- description: 'Video only: the scenes of a video longer than one clip, in order, at most 120 seconds in total.',+ description: 'Video only: the scenes of a video longer than one clip, in order, at most 120 seconds in total. Load the video-scenes skill first when it is listed.',items: {type: 'object',properties: {diff --git a/test/run.js b/test/run.jsindex 67e181866..65dd51913 100644--- a/test/run.js+++ b/test/run.js@@ -32832,6 +32832,7 @@ test('every bundled skill declares its canonical semantic intents', () => {'humanizer': ['email_reply', 'draft_message', 'compose_prose', 'rewrite_text', 'humanize_writing', 'reply_to_thread'],'phonr-calls': ['outbound_phone_call', 'phone_inquiry', 'phone_call_status', 'phone_call_result', 'phone_call_recording', 'stop_phone_call'],'wordpress-rest-api': ['wordpress_content', 'wordpress_publish', 'wordpress_draft', 'wordpress_taxonomy'],+ 'video-scenes': ['generate_video', 'explainer_video', 'video_story', 'how_to_video'],...Object.fromEntries(['ghost', 'drupal', 'joomla', 'webflow', 'shopify', 'wix', 'strapi', 'contentful', 'sanity'].map(cms => [`cms-${cms}`, ['cms_content_management', `${cms}_content`]])),};@@ -32895,6 +32896,35 @@ test('WordPress skill loads on demand in Act/Dev Mid/Full and cannot restore itsassert.equal(contents[0], contents[1]);});+test('Video scenes skill is a default that loads only when a video needs it', () => {+ const contents = [];+ for (const [label, prefix, AgentClass, sources, defaults] of [+ ['chrome', 'src/chrome', AgentCh, PACKAGED_SKILL_SOURCES_CH, DEFAULT_SKILL_SOURCES_CH],+ ['firefox', 'src/firefox', AgentFx, PACKAGED_SKILL_SOURCES_FX, DEFAULT_SKILL_SOURCES_FX],+ ]) {+ const source = sources.find(s => s.id === 'video-scenes');+ assert.ok(defaults.some(s => s.id === source?.id), `${label}: missing default`);+ const content = fs.readFileSync(path.join(ROOT, prefix, source.path), 'utf8');+ contents.push(content);+ for (const mode of ['ask', 'act']) {+ const agent = new AgentClass({ getActive: () => ({ promptTier: 'full' }) });+ const tabId = 5216;+ agent.setCustomSkills([{ ...source, content, sourceType: 'built-in', sourceUrl: source.path }]);+ agent.conversationModes.set(tabId, mode);+ agent.conversations.set(tabId, [{ role: 'system', content: agent._buildSystemPrompt(mode, tabId) }]);+ // Only the catalog line is offered up front; the body arrives when loaded.+ assert.doesNotMatch(agent.conversations.get(tabId)[0].content, /cast and set sheet/, `${label}: ${mode} preloaded the body`);+ const loader = agent._skillLoaderDefinition(mode, 'full');+ assert.equal(!!loader, mode === 'act', `${label}: ${mode} catalog`);+ if (loader) assert.match(loader.function.description, /video-scenes — Video scenes: .*generate_image with scenes/);+ assert.equal(agent._loadSkillForRun(tabId, { skill_id: source.id }).success, mode === 'act', `${label}: ${mode} load`);+ assert.equal(agent.conversations.get(tabId)[0].content.includes('cast and set sheet'), mode === 'act', `${label}: ${mode} body after load`);+ assert.deepEqual(agent._skillToolDefinitions(tabId, mode, 'full'), [], 'instruction-only skill added tools');+ }+ }+ assert.equal(contents[0], contents[1]);+});+test('WordPress default seeding respects stored removal on startup and restart', async () => {for (const prefix of ['src/chrome', 'src/firefox']) {const mod = await import(pathToFileURL(path.join(ROOT, prefix, 'src/agent/skills.js')).href);@@ -117904,6 +117934,7 @@ test('settings exposes custom skills tab and packaged skills resource directory''cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',]);assert.deepEqual(PACKAGED_SKILL_SOURCES_FX.map((skill) => skill.id), ['freeskillz-xyz',@@ -117927,6 +117958,7 @@ test('settings exposes custom skills tab and packaged skills resource directory''cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',]);assert.deepEqual(DEFAULT_SKILL_SOURCES_CH.map((skill) => skill.id), ['freeskillz-xyz',@@ -117942,6 +117974,7 @@ test('settings exposes custom skills tab and packaged skills resource directory''cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',]);assert.deepEqual(DEFAULT_SKILL_SOURCES_FX.map((skill) => skill.id), ['freeskillz-xyz',@@ -117957,6 +117990,7 @@ test('settings exposes custom skills tab and packaged skills resource directory''cms-strapi','cms-contentful','cms-sanity',+ 'video-scenes',]);assert.equal(DEFAULT_SKILL_SOURCES_CH.some((skill) => skill.id === 'turkish-deasciifier'), false, 'chrome: Turkish deasciifier must remain opt-in');assert.equal(DEFAULT_SKILL_SOURCES_FX.some((skill) => skill.id === 'turkish-deasciifier'), false, 'firefox: Turkish deasciifier must remain opt-in');