Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
Make scene videos with fal.ai and Comfy Router video models
Claude098e75cbd556
diff --git a/docs/providers-and-models.md b/docs/providers-and-models.mdindex 9d8ff11ee..8a650c752 100644--- a/docs/providers-and-models.md+++ b/docs/providers-and-models.md@@ -285,8 +285,8 @@ they want get a video from it, as before.For Comfy Router models with native inputs other than `prompt`, paste their inputJSON in **Image model input JSON** or **Video model input JSON** and place`{{prompt}}` in the text field(s). Other values, such as duration or resolution,-are preserved. Without a placeholder, WebBrain supplies the tool prompt as-`prompt`. Refer to the model's+are preserved unless the agent sets that option for the clip. Without a+placeholder, WebBrain supplies the tool prompt as `prompt`. Refer to the model's[input schema](https://docs.comfy.org/development/comfy-router/schemas).For local ComfyUI, start the server, export a working workflow **in API format**,@@ -313,16 +313,27 @@ can charge for a partner generation already in flight. Inline media is limited tas media assets, so their previews and save links survive panel reopening withoutputting base64 bytes in the run journal or model context.-With an OpenRouter video model, a video can be longer than one clip. The model-estimates how long the requested video should run (`estimated_duration`). A-single clip can set `duration`, `aspect_ratio` and `resolution`, which are-checked against the model's catalog entry (`supported_durations`,-`supported_aspect_ratios`, `supported_resolutions`) before anything is-submitted. A longer video is passed as `scenes`: WebBrain makes one clip per+With a fal.ai, OpenRouter or Comfy Router video model, a video can be longer+than one clip. The model estimates how long the requested video should run+(`estimated_duration`). A single clip can set `duration`, `aspect_ratio` and+`resolution`, which are checked against what the video model accepts before+anything is submitted: OpenRouter's catalog entry (`supported_durations`,+`supported_aspect_ratios`, `supported_resolutions`), or the OpenAPI schema+fal.ai (`api.fal.ai/v1/models?endpoint_id=…&expand=openapi-3.0`) or Comfy+Router (`/v2/models/{provider}/{model}/openapi.json`) publishes for the model,+read once per call that sets any of them. From a schema, WebBrain takes the inputs the model's request+body has, its listed clip lengths (sent as listed, such as `6`, `"6"` or+`"8s"`) or the whole seconds of its length range (sent as numbers), and its+listed aspect ratios and resolutions (matched without regard to case and sent as+listed, such as `768P`). Without a schema, a single clip's options are sent as+given and scenes are refused, since the clip lengths are unknown. A local+ComfyUI workflow sets its own length and shape, so these options are refused+there. A longer video is passed as `scenes`: WebBrain makes one clip perscene at a length the model offers (a scene longer than its longest clip isrefused, so the agent splits it into scenes with their own prompts), generatesup to three clips at a time with one retry each (none when the provider refuses-the request, such as for lack of credit), and joins them into one MP4+the request, such as for lack of credit, or returns a clip that is not an MP4),+downloads hosted clips without the API key, and joins them into one MP4without re-encoding (`src/*/src/agent/video-join.js`, which parsesthe clips with the vendored mp4box.js). A clip's sound often ends a littlebefore its picture or starts after it; with AAC-LC sound (the usual MP4 audio,diff --git a/src/chrome/src/agent/fal-media.js b/src/chrome/src/agent/fal-media.jsindex 30d55a57d..963b6a1e4 100644--- a/src/chrome/src/agent/fal-media.js+++ b/src/chrome/src/agent/fal-media.js@@ -127,12 +127,17 @@ async function cancelFalRequest(cancelUrl, headers, fetchImpl) {}}-/** Run a queued generation: submit, poll, then fetch the result. */+/**+ * Run a queued generation: submit, poll, then fetch the result. `input` adds+ * model inputs (such as a video's duration) to the prompt.+ */export async function runFalGeneration({prompt,config,+ input = {},fetchImpl = fetch,timeoutMs = FAL_STATUS_TIMEOUT_MS,+ pollMs = FAL_STATUS_POLL_INTERVAL_MS,signal = null,}) {const model = normalizeFalModelId(config?.model);@@ -151,7 +156,7 @@ export async function runFalGeneration({method: 'POST',redirect: 'error', credentials: 'omit',headers,- body: JSON.stringify({ prompt: text }),+ body: JSON.stringify({ prompt: text, ...input }),signal: operation.signal,});if (!submitRes.ok) {@@ -177,7 +182,7 @@ export async function runFalGeneration({let status = 'IN_QUEUE';while (true) {- await abortableDelay(FAL_STATUS_POLL_INTERVAL_MS, operation.signal);+ await abortableDelay(pollMs, operation.signal);const statusRes = await fetchImpl(statusUrl, { headers, signal: operation.signal, redirect: 'error', credentials: 'omit' });if (!statusRes.ok) throw new Error(`fal.ai status check failed (HTTP ${statusRes.status}).`);diff --git a/src/chrome/src/agent/generative-media.js b/src/chrome/src/agent/generative-media.jsindex d70f34e72..d621b4fbd 100644--- a/src/chrome/src/agent/generative-media.js+++ b/src/chrome/src/agent/generative-media.js@@ -188,21 +188,126 @@ function clipDurations(entry) {return durations.length ? durations : null;}-// aspect_ratio and resolution, checked against the model's catalog entry so an-// unsupported value fails before a paid submission.+// aspect_ratio and resolution, checked against what the model accepts so an+// unsupported value fails before a paid submission. A value matches without+// regard to case and is sent as the model lists it ("768p" as "768P").function videoShape(entry, video) {const shape = {};for (const [name, key] of [['aspect_ratio', 'supported_aspect_ratios'], ['resolution', 'supported_resolutions']]) {if (video?.[name] === undefined) continue;+ if (entry?.inputs && !entry.inputs.includes(name)) throw new Error(`This video model does not take ${name}.`);const supported = supportedValues(entry, key);- if (supported && !supported.some(value => String(value) === String(video[name]))) {- throw new Error(`This video model does not support ${name} "${video[name]}". Supported: ${supported.join(', ')}.`);- }- shape[name] = String(video[name]);+ const wanted = String(video[name]);+ const match = supported ? supported.find(value => String(value).toLowerCase() === wanted.toLowerCase()) : wanted;+ if (match === undefined) throw new Error(`This video model does not support ${name} "${wanted}". Supported: ${supported.join(', ')}.`);+ shape[name] = match;}return shape;}+// The whole seconds a numeric schema allows: within its bounds, exclusive or+// not, and its whole-number multipleOf; at most an hour. None without an upper bound.+function wholeSeconds(form) {+ const types = [].concat(form.type ?? []);+ if (types.length && !types.some(type => type === 'integer' || type === 'number')) return [];+ const highs = [];+ const lows = [1];+ if (Number.isFinite(form.maximum)) highs.push(form.exclusiveMaximum === true ? Math.ceil(form.maximum) - 1 : Math.floor(form.maximum));+ if (Number.isFinite(form.exclusiveMaximum)) highs.push(Math.ceil(form.exclusiveMaximum) - 1);+ if (Number.isFinite(form.minimum)) lows.push(form.exclusiveMinimum === true ? Math.floor(form.minimum) + 1 : Math.ceil(form.minimum));+ if (Number.isFinite(form.exclusiveMinimum)) lows.push(Math.floor(form.exclusiveMinimum) + 1);+ if (!highs.length) return [];+ const step = Number.isInteger(form.multipleOf) && form.multipleOf > 0 ? form.multipleOf : 1;+ const seconds = [];+ for (let value = Math.max(...lows), longest = Math.min(...highs, 3600); value <= longest; value++) {+ if (value % step === 0) seconds.push(value);+ }+ return seconds;+}++/**+ * What a video model accepts, read from the OpenAPI document fal.ai and Comfy+ * Router publish for each model, in the form of OpenRouter's catalog entries+ * (supported_durations, supported_aspect_ratios, supported_resolutions), with+ * `inputs`, the names its request body takes, and `encodeDuration`, which+ * writes a clip length the way the model takes it. A model's own values are+ * sent as it lists them (6, "6", "8s"); a length within a range, as a number.+ * Null when the document describes no request body.+ */+export function videoInputCapabilities(openapi) {+ const schemas = openapi?.components?.schemas || {};+ const resolve = (node, depth = 0) => {+ if (!node || typeof node !== 'object' || depth > 8) return null;+ return typeof node.$ref === 'string' ? resolve(schemas[node.$ref.split('/').pop()], depth + 1) : node;+ };+ // A body's properties by name, with those of its allOf, anyOf and oneOf+ // parts; a name in more than one part takes any of their forms.+ const properties = (node, merged = new Map(), depth = 0) => {+ const body = resolve(node);+ if (!body || depth > 8) return merged;+ const add = (name, schema) => merged.set(name, merged.has(name) ? { anyOf: [merged.get(name), schema] } : schema);+ if (body.properties && typeof body.properties === 'object' && !Array.isArray(body.properties)) {+ for (const [name, schema] of Object.entries(body.properties)) add(name, schema);+ }+ for (const key of ['allOf', 'anyOf', 'oneOf']) {+ for (const part of Array.isArray(body[key]) ? body[key] : []) {+ for (const [name, schema] of properties(part, new Map(), depth + 1)) add(name, schema);+ }+ }+ return merged;+ };+ // The generation request's body: the one that takes a prompt, or else the+ // first (some models take the prompt inside another field).+ const bodies = Object.values(openapi?.paths || {})+ .map(operations => properties(operations?.post?.requestBody?.content?.['application/json']?.schema))+ .filter(body => body.size);+ const input = bodies.find(body => body.has('prompt')) || bodies[0];+ if (!input) return null;+ // Every form a property can take: its allOf, anyOf and oneOf branches.+ const forms = (node, depth = 0) => {+ const resolved = resolve(node);+ const branches = ['allOf', 'anyOf', 'oneOf'].flatMap(key => (Array.isArray(resolved?.[key]) ? resolved[key] : []));+ if (branches.length && depth < 8) return branches.flatMap(branch => forms(branch, depth + 1));+ return resolved ? [resolved] : [];+ };+ const listedValues = form => [...(Array.isArray(form.enum) ? form.enum : []), ...(form.const !== undefined ? [form.const] : [])]+ .filter(value => typeof value === 'string' || typeof value === 'number');+ const listed = name => {+ const values = new Map();+ for (const value of forms(input.get(name)).flatMap(listedValues)) if (!values.has(String(value))) values.set(String(value), value);+ return values.size ? [...values.values()] : null;+ };+ const entry = { inputs: [...input.keys()] };+ // Clip lengths in whole seconds, each with the value the model takes for it.+ const lengths = new Map();+ const durationForms = forms(input.get('duration'));+ for (const value of durationForms.flatMap(listedValues)) {+ // Listed lengths: 6, "6", "6.0" or "8s". Other values, such as "auto", are not lengths.+ const match = /^(\d+)(?:\.0+)?\s*s?$/i.exec(String(value).trim());+ const seconds = match ? Number(match[1]) : 0;+ if (seconds > 0 && !lengths.has(seconds)) lengths.set(seconds, value);+ }+ for (const form of durationForms) {+ if (listedValues(form).length) continue;+ for (const seconds of wholeSeconds(form)) if (!lengths.has(seconds)) lengths.set(seconds, seconds);+ }+ if (lengths.size) {+ entry.supported_durations = [...lengths.keys()].sort((a, b) => a - b);+ entry.encodeDuration = seconds => lengths.get(seconds);+ }+ const aspectRatios = listed('aspect_ratio');+ if (aspectRatios) entry.supported_aspect_ratios = aspectRatios;+ const resolutions = listed('resolution');+ if (resolutions) entry.supported_resolutions = resolutions;+ return entry;+}++// A clip's length as the model takes it, ready to merge into its request.+function clipLength(entry, clip) {+ if (clip.duration === undefined) return {};+ return { duration: entry?.encodeDuration ? entry.encodeDuration(clip.duration) : clip.duration };+}+/*** The clips for a video described as scenes, one per scene, each length* rounded up to one the model offers. A scene longer than the model's longest@@ -262,6 +367,7 @@ function planVideo(prompt, video, entry) {}let duration;if (video?.duration !== undefined) {+ if (entry?.inputs && !entry.inputs.includes('duration')) throw new Error('This video model does not take duration.');if (durations && !durations.includes(video.duration)) {throw new Error(`This video model does not make ${video.duration} s clips. Supported: ${durations.join(', ')} s.`);}@@ -301,19 +407,43 @@ function dataUrlBytes(url) {return { bytes: Uint8Array.from(binary, char => char.charCodeAt(0)), mime: match[1] };}+// A failure that trying the clip again would repeat, such as a model that+// makes another format.+const lasting = message => Object.assign(new Error(message), { retry: false });+// A clip that will be joined is kept as bytes, not as a data URL.async function videoBytes(response, label) {const mime = (response.headers?.get?.('Content-Type') || '').split(';')[0].trim();if (response.ok && mime === 'video/mp4') {const bytes = new Uint8Array(await response.arrayBuffer());- if (bytes.length > MAX_INLINE_BYTES) throw new Error('A generated clip exceeds the 20 MB inline limit.');+ if (bytes.length > MAX_INLINE_BYTES) throw lasting('A generated clip exceeds the 20 MB inline limit.');return bytes;}const decoded = dataUrlBytes(await mediaResponse(response, label));- if (decoded?.mime !== 'video/mp4') throw new Error(`${label} did not return an MP4 video.`);+ if (decoded?.mime !== 'video/mp4') throw lasting(`${label} did not return an MP4 video.`);return decoded.bytes;}+// A finished clip as MP4 bytes to join: inline media is decoded, a hosted file+// downloaded within the deadline. No key is sent with the download, so+// redirects are allowed.+async function downloadClip(url, { fetchImpl, signal, timeoutMs, label }) {+ let bytes = dataUrlBytes(url)?.bytes;+ if (!bytes) {+ if (!safeMediaUrl(url)) throw lasting(`${label} returned an unsupported media URL.`);+ const operation = operationSignal(signal, timeoutMs);+ try {+ const res = await fetchImpl(url, { signal: operation.signal, credentials: 'omit' });+ if (!res.ok) throw Object.assign(new Error(`${label} clip download failed (HTTP ${res.status}).`), { status: res.status });+ bytes = new Uint8Array(await res.arrayBuffer());+ } catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+ }+ if (bytes.length > MAX_JOINED_VIDEO_BYTES) throw lasting('A generated clip is larger than WebBrain can keep in the chat.');+ if (mp4DurationSeconds(bytes) === null) throw lasting(`${label} did not return an MP4 video.`);+ return bytes;+}+// Runs `task` for each item, `limit` at a time. The first failure stops the// others; Stop aborts them all with its own reason.async function mapWithConcurrency(items, limit, signal, task) {@@ -339,40 +469,20 @@ async function mapWithConcurrency(items, limit, signal, task) {} finally { signal?.removeEventListener('abort', onAbort); }}-async function runOpenRouter({ text, media, config, video, fetchImpl, signal, timeoutMs, pollMs, onProgress, confirmVideoLength }) {- const headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' };- // One bounded operation: its own deadline, aborted by Stop as well.- const bounded = async (parent, task) => {- const operation = operationSignal(parent, timeoutMs);- const request = (url, init = {}) => {- operation.signal.throwIfAborted();- return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });- };- try { return await task(request, operation.signal); }- catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }- finally { operation.dispose(); }- };- const base = 'https://openrouter.ai/api/v1/videos';- let plan = null;- let model = '';- const single = await bounded(signal, async (request, jobSignal) => {- const target = await openRouterTarget(config, media, openRouterCatalogs(headers, request));- const { kind, entry } = target;- model = target.model;- if (kind === 'images') {- const res = await request('https://openrouter.ai/api/v1/images', { method: 'POST', headers, body: JSON.stringify({ model, prompt: text }) });- return { url: await mediaResponse(res, 'OpenRouter generation'), model };- }- plan = planVideo(text, video, entry);- if (plan.scenes) return null;- const res = await openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, plan.clips[0], plan.shape) });- return { url: await mediaResponse(res, 'OpenRouter video download'), model };- });- if (single) return single;+/**+ * Makes a video with a provider's video model, the same way for every+ * provider: one clip (`single` returns { url }), or the clips of its scenes+ * (`clip` returns each one's MP4 bytes, to be joined), three at a time with+ * one retry each. `entry` is what the model accepts. A video longer than+ * WebBrain makes is generated only if the user agrees on its card.+ */+async function runVideo({ model, entry, text, video, signal, onProgress, confirmVideoLength, single, clip }) {+ const plan = planVideo(text, video, entry);+ if (!plan.scenes) return { ...(await single(plan.clips[0], plan.shape)), model };- // A video longer than WebBrain makes: the user decides on its card whether a- // shorter version is worth generating, before anything is submitted. The- // card is outside the generation deadline.+ // The user decides on the card whether a shorter version is worth+ // generating, before anything is submitted. The card is outside the+ // generation deadline.const lengthDetails = { estimatedSeconds: plan.estimated, totalSeconds: plan.total, clipCount: plan.clips.length, maxSeconds: MAX_VIDEO_SECONDS };if (plan.estimated > MAX_VIDEO_SECONDS) {const answer = typeof confirmVideoLength === 'function' ? await confirmVideoLength(lengthDetails) : 'unavailable';@@ -381,20 +491,19 @@ async function runOpenRouter({ text, media, config, video, fetchImpl, signal, ti}onProgress?.(`Generating ${plan.clips.length} video clips…`);let ready = 0;- const clips = await mapWithConcurrency(plan.clips, VIDEO_CLIP_CONCURRENCY, signal, async (clip, index, groupSignal) => {+ const clips = await mapWithConcurrency(plan.clips, VIDEO_CLIP_CONCURRENCY, signal, async (item, index, groupSignal) => {for (let attempt = 1; ; attempt++) {try {- const bytes = await bounded(groupSignal, async (request, jobSignal) => videoBytes(- await openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, clip, plan.shape) }),- 'OpenRouter video download'));+ const bytes = await clip(item, plan.shape, groupSignal);ready += 1;onProgress?.(`Video clips: ${ready} of ${plan.clips.length} ready`);return bytes;} catch (error) {// One retry per clip; a second failure, Stop or another clip's failure// ends the video. A request the provider refused (a 4xx other than a- // timeout or rate limit, such as no credit left) would fail again.- const refused = error?.status >= 400 && error.status < 500 && ![408, 429].includes(error.status);+ // timeout or rate limit, such as no credit left) would fail again, as+ // would a clip in another format.+ const refused = error?.retry === false || (error?.status >= 400 && error.status < 500 && ![408, 429].includes(error.status));if (groupSignal.aborted || attempt >= 2 || refused) {throw groupSignal.aborted ? (groupSignal.reason ?? error) : new Error(`Clip ${index + 1} of ${plan.clips.length} failed: ${error.message}`);}@@ -404,6 +513,107 @@ async function runOpenRouter({ text, media, config, video, fetchImpl, signal, tireturn { clips, model, ...lengthDetails };}+async function runOpenRouter({ text, media, config, video, fetchImpl, signal, timeoutMs, pollMs, onProgress, confirmVideoLength }) {+ const headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' };+ // One bounded operation: its own deadline, aborted by Stop as well.+ const bounded = async (parent, task) => {+ const operation = operationSignal(parent, timeoutMs);+ const request = (url, init = {}) => {+ operation.signal.throwIfAborted();+ return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ };+ try { return await task(request, operation.signal); }+ catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+ };+ const base = 'https://openrouter.ai/api/v1/videos';+ const target = await bounded(signal, request => openRouterTarget(config, media, openRouterCatalogs(headers, request)));+ const { model } = target;+ if (target.kind === 'images') {+ return bounded(signal, async request => {+ const res = await request('https://openrouter.ai/api/v1/images', { method: 'POST', headers, body: JSON.stringify({ model, prompt: text }) });+ return { url: await mediaResponse(res, 'OpenRouter generation'), model };+ });+ }+ const job = (request, jobSignal, clip, shape) => openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, clip, shape) });+ return runVideo({+ model, entry: target.entry, text, video, signal, onProgress, confirmVideoLength,+ single: (clip, shape) => bounded(signal, async (request, jobSignal) => ({ url: await mediaResponse(await job(request, jobSignal, clip, shape), 'OpenRouter video download') })),+ clip: (clip, shape, groupSignal) => bounded(groupSignal, async (request, jobSignal) => videoBytes(await job(request, jobSignal, clip, shape), 'OpenRouter video download')),+ });+}++// The OpenAPI document of a fal.ai or Comfy Router model, read within the+// generation's deadline (at most 30 s). Null when none is published for it.+async function modelSchema(url, headers, label, fetchImpl, signal, timeoutMs) {+ const operation = operationSignal(signal, Math.min(timeoutMs, 30000));+ try {+ const response = await fetchImpl(url, { headers, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ return response.status === 404 ? null : await jsonResponse(response, label);+ } catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+}++// What a fal.ai video model accepts ({} when fal.ai publishes no schema for it).+async function falVideoEntry(model, apiKey, fetchImpl, signal, timeoutMs) {+ const query = new URLSearchParams({ endpoint_id: model, expand: 'openapi-3.0' });+ const listing = await modelSchema(`https://api.fal.ai/v1/models?${query}`, { Authorization: `Key ${apiKey}` }, 'fal.ai model schema', fetchImpl, signal, timeoutMs);+ const openapi = (Array.isArray(listing?.models) ? listing.models : []).find(item => item?.endpoint_id === model)?.openapi;+ return videoInputCapabilities(openapi) || {};+}++// What a Comfy Router video model accepts ({} when Comfy Router publishes no schema for it).+async function comfyRouterVideoEntry(model, apiKey, fetchImpl, signal, timeoutMs) {+ const openapi = await modelSchema(`https://api.comfy.org/v2/models/${model}/openapi.json`, { 'X-API-Key': apiKey }, 'Comfy Router model schema', fetchImpl, signal, timeoutMs);+ return videoInputCapabilities(openapi) || {};+}++// The saved model input with the prompt bound into it ({{prompt}}), or added as `prompt`.+function comfyRouterBody(parameters, prompt) {+ return JSON.stringify(parameters).includes('{{prompt}}') ? bindMediaPrompt(parameters, prompt) : { ...parameters, prompt };+}++// Submits one Comfy Router request, polls it and returns its media URL (hosted+// or inline). A failure or Stop before completion cancels the request.+async function comfyRouterJob({ model, apiKey, body, fetchImpl, signal, timeoutMs, pollMs }) {+ const operation = operationSignal(signal, timeoutMs);+ const request = (url, init = {}) => {+ operation.signal.throwIfAborted();+ return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ };+ const base = `https://api.comfy.org/v2/models/${model}/requests`;+ const headers = { 'X-API-Key': apiKey, 'Content-Type': 'application/json' };+ let cancelUrl = '';+ let completed = false;+ try {+ const queued = await jsonResponse(await request(base, {+ method: 'POST', headers: { ...headers, 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(body),+ }), 'Comfy Router submit');+ if (!/^[A-Za-z0-9_-]+$/.test(queued.request_id || '')) throw new Error('Comfy Router returned an invalid request ID.');+ // Build authenticated URLs ourselves; never send the key to returned URLs.+ const resultUrl = `${base}/${queued.request_id}`;+ cancelUrl = `${resultUrl}/cancel`;+ let waitMs = pollMs;+ while (true) {+ await delay(waitMs, operation.signal);+ const res = await request(`${resultUrl}/status`, { headers });+ const status = await jsonResponse(res, 'Comfy Router status');+ waitMs = pollDelay(res, pollMs);+ if (status.status === 'COMPLETED') {+ const result = await request(resultUrl, { headers });+ if (result.status === 202) { waitMs = pollDelay(result, pollMs); continue; }+ const url = await mediaResponse(result, 'Comfy Router generation');+ completed = true;+ return url;+ }+ if (!['IN_QUEUE', 'IN_PROGRESS'].includes(status.status)) throw new Error('Comfy Router returned an unknown queue status.');+ }+ } catch (error) {+ if (!completed && cancelUrl) await cancelRequest(cancelUrl, { method: 'PUT', headers }, fetchImpl);+ throw operation.signal.aborted ? operation.signal.reason : error;+ } finally { operation.dispose(); }+}+/*** Generates one image or video. `media` is 'image' or 'video', choosing the* provider's image or video model; video options imply 'video'. Without@@ -418,8 +628,8 @@ export async function runMediaGeneration({ prompt, config, media = null, video =if (media === 'image' && video) {throw new Error('Video length, aspect ratio, resolution and scenes are for videos. Set media to "video" or leave these out.');}- if (video && provider !== 'openrouter') {- throw new Error(`Video length, aspect ratio, resolution and scenes work with OpenRouter video models. Omit them to use the ${MEDIA_PROVIDERS[provider].label} video model's defaults.`);+ if (video && provider === 'comfyui') {+ throw new Error('Video length, aspect ratio, resolution and scenes do not work with a local ComfyUI workflow, which sets them itself. Omit them.');}const kind = media || (video ? 'video' : null);const pollMs = pollIntervalMs ?? (provider === 'openrouter' ? 30000 : 2000);@@ -428,7 +638,32 @@ export async function runMediaGeneration({ prompt, config, media = null, video =}const slot = mediaSlot(config, kind || 'image');const { model } = slot;- if (provider === 'fal') return runFalGeneration({ prompt: text, config: { ...config, model }, fetchImpl, signal, timeoutMs });+ const videoRun = { model, text, video, signal, onProgress, confirmVideoLength };+ if (provider === 'fal') {+ const job = (prompt, input, jobSignal) => runFalGeneration({ prompt, config: { ...config, model }, input, fetchImpl, signal: jobSignal, timeoutMs, pollMs });+ if (kind !== 'video') return job(text, {}, signal);+ // The model's schema says what it accepts; a plain clip needs none of it.+ const entry = video ? await falVideoEntry(model, config.apiKey, fetchImpl, signal, timeoutMs) : {};+ return runVideo({+ ...videoRun, entry,+ single: async (clip, shape) => ({ url: (await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, signal)).url }),+ clip: async (clip, shape, groupSignal) => downloadClip((await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, groupSignal)).url,+ { fetchImpl, signal: groupSignal, timeoutMs, label: 'fal.ai' }),+ });+ }+ if (provider === 'comfyrouter') {+ const parameters = parseMediaJson(slot.parameters || '{}', kind === 'video' ? 'Video model input' : 'Image model input');+ // The clip's own length and shape replace those in the saved model input.+ const job = (prompt, input, jobSignal) => comfyRouterJob({ model, apiKey: config.apiKey, body: { ...comfyRouterBody(parameters, prompt), ...input }, fetchImpl, signal: jobSignal, timeoutMs, pollMs });+ if (kind !== 'video') return { url: await job(text, {}, signal), model };+ const entry = video ? await comfyRouterVideoEntry(model, config.apiKey, fetchImpl, signal, timeoutMs) : {};+ return runVideo({+ ...videoRun, entry,+ single: async (clip, shape) => ({ url: await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, signal) }),+ clip: async (clip, shape, groupSignal) => downloadClip(await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, groupSignal),+ { fetchImpl, signal: groupSignal, timeoutMs, label: 'Comfy Router' }),+ });+ }const operation = operationSignal(signal, timeoutMs);const request = (url, init = {}) => {operation.signal.throwIfAborted();@@ -437,35 +672,6 @@ export async function runMediaGeneration({ prompt, config, media = null, video =let cancelUrl = '', cancelInit;let completed = false;try {- if (provider === 'comfyrouter') {- const base = `https://api.comfy.org/v2/models/${model}/requests`;- const headers = { 'X-API-Key': config.apiKey, 'Content-Type': 'application/json' };- const parameters = parseMediaJson(slot.parameters || '{}', kind === 'video' ? 'Video model input' : 'Image model input');- const usesTemplate = JSON.stringify(parameters).includes('{{prompt}}');- const body = usesTemplate ? bindMediaPrompt(parameters, text) : { ...parameters, prompt: text };- const queued = await jsonResponse(await request(base, {- method: 'POST', headers: { ...headers, 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(body),- }), 'Comfy Router submit');- if (!/^[A-Za-z0-9_-]+$/.test(queued.request_id || '')) throw new Error('Comfy Router returned an invalid request ID.');- // Build authenticated URLs ourselves; never send the key to returned URLs.- const resultUrl = `${base}/${queued.request_id}`;- cancelUrl = `${resultUrl}/cancel`; cancelInit = { method: 'PUT', headers };- let waitMs = pollMs;- while (true) {- await delay(waitMs, operation.signal);- const res = await request(`${resultUrl}/status`, { headers });- const status = await jsonResponse(res, 'Comfy Router status');- waitMs = pollDelay(res, pollMs);- if (status.status === 'COMPLETED') {- const result = await request(resultUrl, { headers });- if (result.status === 202) { waitMs = pollDelay(result, pollMs); continue; }- const url = await mediaResponse(result, 'Comfy Router generation');- completed = true;- return { url, model };- }- if (!['IN_QUEUE', 'IN_PROGRESS'].includes(status.status)) throw new Error('Comfy Router returned an unknown queue status.');- }- }const base = comfyBaseUrl(config.baseUrl);const headers = { 'Content-Type': 'application/json' };const workflow = bindMediaPrompt(parseMediaJson(slot.workflow, 'ComfyUI workflow'), text);@@ -552,7 +758,7 @@ export async function generateImage(args, options = {}) {const provider = mediaProvider(config);const result = await runMediaGeneration({prompt: args?.prompt, media: args?.media ?? null, video: videoArguments(args), config, fetchImpl, signal: options.signal,- onProgress: options.onProgress, confirmVideoLength: options.confirmVideoLength,+ pollIntervalMs: options.pollIntervalMs, onProgress: options.onProgress, confirmVideoLength: options.confirmVideoLength,});if (result.declined) {return { success: false, declined: true, provider, model: result.model, error: declinedVideoMessage(result) };diff --git a/src/chrome/src/agent/tools.js b/src/chrome/src/agent/tools.jsindex 6e3fd8341..eb45bf407 100644--- a/src/chrome/src/agent/tools.js+++ b/src/chrome/src/agent/tools.js@@ -1488,7 +1488,7 @@ export const AGENT_TOOLS = [type: 'function',function: {name: 'generate_image',- description: 'Generate an image, or a video with media "video", from a text prompt using the image or video model the user configured (Providers → Generative). Supports fal.ai, OpenRouter, Comfy Router and local ComfyUI workflows; generation may take several minutes. Returns a media URL or inline media displayed in the chat. With an OpenRouter video model, estimate how long the requested video should run (estimated_duration): one clip can set duration, aspect_ratio and resolution; a longer video passes scenes, which WebBrain fits to the model\'s clip lengths and joins into one video of up to 120 s, asking the user first when the estimate is longer. Not available in Ask mode.',+ description: 'Generate an image, or a video with media "video", from a text prompt using the image or video model the user configured (Providers → Generative). Supports fal.ai, OpenRouter, Comfy Router and local ComfyUI workflows; generation may take several minutes. Returns a media URL or inline media displayed in the chat. For a video, estimate how long the requested video should run (estimated_duration): one clip can set duration, aspect_ratio and resolution; a longer video passes scenes, which WebBrain fits to the video model\'s clip lengths and joins into one video of up to 120 s, asking the user first when the estimate is longer. Not available in Ask mode.',parameters: {type: 'object',properties: {diff --git a/src/firefox/src/agent/fal-media.js b/src/firefox/src/agent/fal-media.jsindex 30d55a57d..963b6a1e4 100644--- a/src/firefox/src/agent/fal-media.js+++ b/src/firefox/src/agent/fal-media.js@@ -127,12 +127,17 @@ async function cancelFalRequest(cancelUrl, headers, fetchImpl) {}}-/** Run a queued generation: submit, poll, then fetch the result. */+/**+ * Run a queued generation: submit, poll, then fetch the result. `input` adds+ * model inputs (such as a video's duration) to the prompt.+ */export async function runFalGeneration({prompt,config,+ input = {},fetchImpl = fetch,timeoutMs = FAL_STATUS_TIMEOUT_MS,+ pollMs = FAL_STATUS_POLL_INTERVAL_MS,signal = null,}) {const model = normalizeFalModelId(config?.model);@@ -151,7 +156,7 @@ export async function runFalGeneration({method: 'POST',redirect: 'error', credentials: 'omit',headers,- body: JSON.stringify({ prompt: text }),+ body: JSON.stringify({ prompt: text, ...input }),signal: operation.signal,});if (!submitRes.ok) {@@ -177,7 +182,7 @@ export async function runFalGeneration({let status = 'IN_QUEUE';while (true) {- await abortableDelay(FAL_STATUS_POLL_INTERVAL_MS, operation.signal);+ await abortableDelay(pollMs, operation.signal);const statusRes = await fetchImpl(statusUrl, { headers, signal: operation.signal, redirect: 'error', credentials: 'omit' });if (!statusRes.ok) throw new Error(`fal.ai status check failed (HTTP ${statusRes.status}).`);diff --git a/src/firefox/src/agent/generative-media.js b/src/firefox/src/agent/generative-media.jsindex d70f34e72..d621b4fbd 100644--- a/src/firefox/src/agent/generative-media.js+++ b/src/firefox/src/agent/generative-media.js@@ -188,21 +188,126 @@ function clipDurations(entry) {return durations.length ? durations : null;}-// aspect_ratio and resolution, checked against the model's catalog entry so an-// unsupported value fails before a paid submission.+// aspect_ratio and resolution, checked against what the model accepts so an+// unsupported value fails before a paid submission. A value matches without+// regard to case and is sent as the model lists it ("768p" as "768P").function videoShape(entry, video) {const shape = {};for (const [name, key] of [['aspect_ratio', 'supported_aspect_ratios'], ['resolution', 'supported_resolutions']]) {if (video?.[name] === undefined) continue;+ if (entry?.inputs && !entry.inputs.includes(name)) throw new Error(`This video model does not take ${name}.`);const supported = supportedValues(entry, key);- if (supported && !supported.some(value => String(value) === String(video[name]))) {- throw new Error(`This video model does not support ${name} "${video[name]}". Supported: ${supported.join(', ')}.`);- }- shape[name] = String(video[name]);+ const wanted = String(video[name]);+ const match = supported ? supported.find(value => String(value).toLowerCase() === wanted.toLowerCase()) : wanted;+ if (match === undefined) throw new Error(`This video model does not support ${name} "${wanted}". Supported: ${supported.join(', ')}.`);+ shape[name] = match;}return shape;}+// The whole seconds a numeric schema allows: within its bounds, exclusive or+// not, and its whole-number multipleOf; at most an hour. None without an upper bound.+function wholeSeconds(form) {+ const types = [].concat(form.type ?? []);+ if (types.length && !types.some(type => type === 'integer' || type === 'number')) return [];+ const highs = [];+ const lows = [1];+ if (Number.isFinite(form.maximum)) highs.push(form.exclusiveMaximum === true ? Math.ceil(form.maximum) - 1 : Math.floor(form.maximum));+ if (Number.isFinite(form.exclusiveMaximum)) highs.push(Math.ceil(form.exclusiveMaximum) - 1);+ if (Number.isFinite(form.minimum)) lows.push(form.exclusiveMinimum === true ? Math.floor(form.minimum) + 1 : Math.ceil(form.minimum));+ if (Number.isFinite(form.exclusiveMinimum)) lows.push(Math.floor(form.exclusiveMinimum) + 1);+ if (!highs.length) return [];+ const step = Number.isInteger(form.multipleOf) && form.multipleOf > 0 ? form.multipleOf : 1;+ const seconds = [];+ for (let value = Math.max(...lows), longest = Math.min(...highs, 3600); value <= longest; value++) {+ if (value % step === 0) seconds.push(value);+ }+ return seconds;+}++/**+ * What a video model accepts, read from the OpenAPI document fal.ai and Comfy+ * Router publish for each model, in the form of OpenRouter's catalog entries+ * (supported_durations, supported_aspect_ratios, supported_resolutions), with+ * `inputs`, the names its request body takes, and `encodeDuration`, which+ * writes a clip length the way the model takes it. A model's own values are+ * sent as it lists them (6, "6", "8s"); a length within a range, as a number.+ * Null when the document describes no request body.+ */+export function videoInputCapabilities(openapi) {+ const schemas = openapi?.components?.schemas || {};+ const resolve = (node, depth = 0) => {+ if (!node || typeof node !== 'object' || depth > 8) return null;+ return typeof node.$ref === 'string' ? resolve(schemas[node.$ref.split('/').pop()], depth + 1) : node;+ };+ // A body's properties by name, with those of its allOf, anyOf and oneOf+ // parts; a name in more than one part takes any of their forms.+ const properties = (node, merged = new Map(), depth = 0) => {+ const body = resolve(node);+ if (!body || depth > 8) return merged;+ const add = (name, schema) => merged.set(name, merged.has(name) ? { anyOf: [merged.get(name), schema] } : schema);+ if (body.properties && typeof body.properties === 'object' && !Array.isArray(body.properties)) {+ for (const [name, schema] of Object.entries(body.properties)) add(name, schema);+ }+ for (const key of ['allOf', 'anyOf', 'oneOf']) {+ for (const part of Array.isArray(body[key]) ? body[key] : []) {+ for (const [name, schema] of properties(part, new Map(), depth + 1)) add(name, schema);+ }+ }+ return merged;+ };+ // The generation request's body: the one that takes a prompt, or else the+ // first (some models take the prompt inside another field).+ const bodies = Object.values(openapi?.paths || {})+ .map(operations => properties(operations?.post?.requestBody?.content?.['application/json']?.schema))+ .filter(body => body.size);+ const input = bodies.find(body => body.has('prompt')) || bodies[0];+ if (!input) return null;+ // Every form a property can take: its allOf, anyOf and oneOf branches.+ const forms = (node, depth = 0) => {+ const resolved = resolve(node);+ const branches = ['allOf', 'anyOf', 'oneOf'].flatMap(key => (Array.isArray(resolved?.[key]) ? resolved[key] : []));+ if (branches.length && depth < 8) return branches.flatMap(branch => forms(branch, depth + 1));+ return resolved ? [resolved] : [];+ };+ const listedValues = form => [...(Array.isArray(form.enum) ? form.enum : []), ...(form.const !== undefined ? [form.const] : [])]+ .filter(value => typeof value === 'string' || typeof value === 'number');+ const listed = name => {+ const values = new Map();+ for (const value of forms(input.get(name)).flatMap(listedValues)) if (!values.has(String(value))) values.set(String(value), value);+ return values.size ? [...values.values()] : null;+ };+ const entry = { inputs: [...input.keys()] };+ // Clip lengths in whole seconds, each with the value the model takes for it.+ const lengths = new Map();+ const durationForms = forms(input.get('duration'));+ for (const value of durationForms.flatMap(listedValues)) {+ // Listed lengths: 6, "6", "6.0" or "8s". Other values, such as "auto", are not lengths.+ const match = /^(\d+)(?:\.0+)?\s*s?$/i.exec(String(value).trim());+ const seconds = match ? Number(match[1]) : 0;+ if (seconds > 0 && !lengths.has(seconds)) lengths.set(seconds, value);+ }+ for (const form of durationForms) {+ if (listedValues(form).length) continue;+ for (const seconds of wholeSeconds(form)) if (!lengths.has(seconds)) lengths.set(seconds, seconds);+ }+ if (lengths.size) {+ entry.supported_durations = [...lengths.keys()].sort((a, b) => a - b);+ entry.encodeDuration = seconds => lengths.get(seconds);+ }+ const aspectRatios = listed('aspect_ratio');+ if (aspectRatios) entry.supported_aspect_ratios = aspectRatios;+ const resolutions = listed('resolution');+ if (resolutions) entry.supported_resolutions = resolutions;+ return entry;+}++// A clip's length as the model takes it, ready to merge into its request.+function clipLength(entry, clip) {+ if (clip.duration === undefined) return {};+ return { duration: entry?.encodeDuration ? entry.encodeDuration(clip.duration) : clip.duration };+}+/*** The clips for a video described as scenes, one per scene, each length* rounded up to one the model offers. A scene longer than the model's longest@@ -262,6 +367,7 @@ function planVideo(prompt, video, entry) {}let duration;if (video?.duration !== undefined) {+ if (entry?.inputs && !entry.inputs.includes('duration')) throw new Error('This video model does not take duration.');if (durations && !durations.includes(video.duration)) {throw new Error(`This video model does not make ${video.duration} s clips. Supported: ${durations.join(', ')} s.`);}@@ -301,19 +407,43 @@ function dataUrlBytes(url) {return { bytes: Uint8Array.from(binary, char => char.charCodeAt(0)), mime: match[1] };}+// A failure that trying the clip again would repeat, such as a model that+// makes another format.+const lasting = message => Object.assign(new Error(message), { retry: false });+// A clip that will be joined is kept as bytes, not as a data URL.async function videoBytes(response, label) {const mime = (response.headers?.get?.('Content-Type') || '').split(';')[0].trim();if (response.ok && mime === 'video/mp4') {const bytes = new Uint8Array(await response.arrayBuffer());- if (bytes.length > MAX_INLINE_BYTES) throw new Error('A generated clip exceeds the 20 MB inline limit.');+ if (bytes.length > MAX_INLINE_BYTES) throw lasting('A generated clip exceeds the 20 MB inline limit.');return bytes;}const decoded = dataUrlBytes(await mediaResponse(response, label));- if (decoded?.mime !== 'video/mp4') throw new Error(`${label} did not return an MP4 video.`);+ if (decoded?.mime !== 'video/mp4') throw lasting(`${label} did not return an MP4 video.`);return decoded.bytes;}+// A finished clip as MP4 bytes to join: inline media is decoded, a hosted file+// downloaded within the deadline. No key is sent with the download, so+// redirects are allowed.+async function downloadClip(url, { fetchImpl, signal, timeoutMs, label }) {+ let bytes = dataUrlBytes(url)?.bytes;+ if (!bytes) {+ if (!safeMediaUrl(url)) throw lasting(`${label} returned an unsupported media URL.`);+ const operation = operationSignal(signal, timeoutMs);+ try {+ const res = await fetchImpl(url, { signal: operation.signal, credentials: 'omit' });+ if (!res.ok) throw Object.assign(new Error(`${label} clip download failed (HTTP ${res.status}).`), { status: res.status });+ bytes = new Uint8Array(await res.arrayBuffer());+ } catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+ }+ if (bytes.length > MAX_JOINED_VIDEO_BYTES) throw lasting('A generated clip is larger than WebBrain can keep in the chat.');+ if (mp4DurationSeconds(bytes) === null) throw lasting(`${label} did not return an MP4 video.`);+ return bytes;+}+// Runs `task` for each item, `limit` at a time. The first failure stops the// others; Stop aborts them all with its own reason.async function mapWithConcurrency(items, limit, signal, task) {@@ -339,40 +469,20 @@ async function mapWithConcurrency(items, limit, signal, task) {} finally { signal?.removeEventListener('abort', onAbort); }}-async function runOpenRouter({ text, media, config, video, fetchImpl, signal, timeoutMs, pollMs, onProgress, confirmVideoLength }) {- const headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' };- // One bounded operation: its own deadline, aborted by Stop as well.- const bounded = async (parent, task) => {- const operation = operationSignal(parent, timeoutMs);- const request = (url, init = {}) => {- operation.signal.throwIfAborted();- return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });- };- try { return await task(request, operation.signal); }- catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }- finally { operation.dispose(); }- };- const base = 'https://openrouter.ai/api/v1/videos';- let plan = null;- let model = '';- const single = await bounded(signal, async (request, jobSignal) => {- const target = await openRouterTarget(config, media, openRouterCatalogs(headers, request));- const { kind, entry } = target;- model = target.model;- if (kind === 'images') {- const res = await request('https://openrouter.ai/api/v1/images', { method: 'POST', headers, body: JSON.stringify({ model, prompt: text }) });- return { url: await mediaResponse(res, 'OpenRouter generation'), model };- }- plan = planVideo(text, video, entry);- if (plan.scenes) return null;- const res = await openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, plan.clips[0], plan.shape) });- return { url: await mediaResponse(res, 'OpenRouter video download'), model };- });- if (single) return single;+/**+ * Makes a video with a provider's video model, the same way for every+ * provider: one clip (`single` returns { url }), or the clips of its scenes+ * (`clip` returns each one's MP4 bytes, to be joined), three at a time with+ * one retry each. `entry` is what the model accepts. A video longer than+ * WebBrain makes is generated only if the user agrees on its card.+ */+async function runVideo({ model, entry, text, video, signal, onProgress, confirmVideoLength, single, clip }) {+ const plan = planVideo(text, video, entry);+ if (!plan.scenes) return { ...(await single(plan.clips[0], plan.shape)), model };- // A video longer than WebBrain makes: the user decides on its card whether a- // shorter version is worth generating, before anything is submitted. The- // card is outside the generation deadline.+ // The user decides on the card whether a shorter version is worth+ // generating, before anything is submitted. The card is outside the+ // generation deadline.const lengthDetails = { estimatedSeconds: plan.estimated, totalSeconds: plan.total, clipCount: plan.clips.length, maxSeconds: MAX_VIDEO_SECONDS };if (plan.estimated > MAX_VIDEO_SECONDS) {const answer = typeof confirmVideoLength === 'function' ? await confirmVideoLength(lengthDetails) : 'unavailable';@@ -381,20 +491,19 @@ async function runOpenRouter({ text, media, config, video, fetchImpl, signal, ti}onProgress?.(`Generating ${plan.clips.length} video clips…`);let ready = 0;- const clips = await mapWithConcurrency(plan.clips, VIDEO_CLIP_CONCURRENCY, signal, async (clip, index, groupSignal) => {+ const clips = await mapWithConcurrency(plan.clips, VIDEO_CLIP_CONCURRENCY, signal, async (item, index, groupSignal) => {for (let attempt = 1; ; attempt++) {try {- const bytes = await bounded(groupSignal, async (request, jobSignal) => videoBytes(- await openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, clip, plan.shape) }),- 'OpenRouter video download'));+ const bytes = await clip(item, plan.shape, groupSignal);ready += 1;onProgress?.(`Video clips: ${ready} of ${plan.clips.length} ready`);return bytes;} catch (error) {// One retry per clip; a second failure, Stop or another clip's failure// ends the video. A request the provider refused (a 4xx other than a- // timeout or rate limit, such as no credit left) would fail again.- const refused = error?.status >= 400 && error.status < 500 && ![408, 429].includes(error.status);+ // timeout or rate limit, such as no credit left) would fail again, as+ // would a clip in another format.+ const refused = error?.retry === false || (error?.status >= 400 && error.status < 500 && ![408, 429].includes(error.status));if (groupSignal.aborted || attempt >= 2 || refused) {throw groupSignal.aborted ? (groupSignal.reason ?? error) : new Error(`Clip ${index + 1} of ${plan.clips.length} failed: ${error.message}`);}@@ -404,6 +513,107 @@ async function runOpenRouter({ text, media, config, video, fetchImpl, signal, tireturn { clips, model, ...lengthDetails };}+async function runOpenRouter({ text, media, config, video, fetchImpl, signal, timeoutMs, pollMs, onProgress, confirmVideoLength }) {+ const headers = { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json' };+ // One bounded operation: its own deadline, aborted by Stop as well.+ const bounded = async (parent, task) => {+ const operation = operationSignal(parent, timeoutMs);+ const request = (url, init = {}) => {+ operation.signal.throwIfAborted();+ return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ };+ try { return await task(request, operation.signal); }+ catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+ };+ const base = 'https://openrouter.ai/api/v1/videos';+ const target = await bounded(signal, request => openRouterTarget(config, media, openRouterCatalogs(headers, request)));+ const { model } = target;+ if (target.kind === 'images') {+ return bounded(signal, async request => {+ const res = await request('https://openrouter.ai/api/v1/images', { method: 'POST', headers, body: JSON.stringify({ model, prompt: text }) });+ return { url: await mediaResponse(res, 'OpenRouter generation'), model };+ });+ }+ const job = (request, jobSignal, clip, shape) => openRouterVideoJob({ base, headers, request, signal: jobSignal, pollMs, body: videoRequestBody(model, clip, shape) });+ return runVideo({+ model, entry: target.entry, text, video, signal, onProgress, confirmVideoLength,+ single: (clip, shape) => bounded(signal, async (request, jobSignal) => ({ url: await mediaResponse(await job(request, jobSignal, clip, shape), 'OpenRouter video download') })),+ clip: (clip, shape, groupSignal) => bounded(groupSignal, async (request, jobSignal) => videoBytes(await job(request, jobSignal, clip, shape), 'OpenRouter video download')),+ });+}++// The OpenAPI document of a fal.ai or Comfy Router model, read within the+// generation's deadline (at most 30 s). Null when none is published for it.+async function modelSchema(url, headers, label, fetchImpl, signal, timeoutMs) {+ const operation = operationSignal(signal, Math.min(timeoutMs, 30000));+ try {+ const response = await fetchImpl(url, { headers, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ return response.status === 404 ? null : await jsonResponse(response, label);+ } catch (error) { throw operation.signal.aborted ? operation.signal.reason : error; }+ finally { operation.dispose(); }+}++// What a fal.ai video model accepts ({} when fal.ai publishes no schema for it).+async function falVideoEntry(model, apiKey, fetchImpl, signal, timeoutMs) {+ const query = new URLSearchParams({ endpoint_id: model, expand: 'openapi-3.0' });+ const listing = await modelSchema(`https://api.fal.ai/v1/models?${query}`, { Authorization: `Key ${apiKey}` }, 'fal.ai model schema', fetchImpl, signal, timeoutMs);+ const openapi = (Array.isArray(listing?.models) ? listing.models : []).find(item => item?.endpoint_id === model)?.openapi;+ return videoInputCapabilities(openapi) || {};+}++// What a Comfy Router video model accepts ({} when Comfy Router publishes no schema for it).+async function comfyRouterVideoEntry(model, apiKey, fetchImpl, signal, timeoutMs) {+ const openapi = await modelSchema(`https://api.comfy.org/v2/models/${model}/openapi.json`, { 'X-API-Key': apiKey }, 'Comfy Router model schema', fetchImpl, signal, timeoutMs);+ return videoInputCapabilities(openapi) || {};+}++// The saved model input with the prompt bound into it ({{prompt}}), or added as `prompt`.+function comfyRouterBody(parameters, prompt) {+ return JSON.stringify(parameters).includes('{{prompt}}') ? bindMediaPrompt(parameters, prompt) : { ...parameters, prompt };+}++// Submits one Comfy Router request, polls it and returns its media URL (hosted+// or inline). A failure or Stop before completion cancels the request.+async function comfyRouterJob({ model, apiKey, body, fetchImpl, signal, timeoutMs, pollMs }) {+ const operation = operationSignal(signal, timeoutMs);+ const request = (url, init = {}) => {+ operation.signal.throwIfAborted();+ return fetchImpl(url, { ...init, signal: operation.signal, redirect: 'error', credentials: 'omit' });+ };+ const base = `https://api.comfy.org/v2/models/${model}/requests`;+ const headers = { 'X-API-Key': apiKey, 'Content-Type': 'application/json' };+ let cancelUrl = '';+ let completed = false;+ try {+ const queued = await jsonResponse(await request(base, {+ method: 'POST', headers: { ...headers, 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(body),+ }), 'Comfy Router submit');+ if (!/^[A-Za-z0-9_-]+$/.test(queued.request_id || '')) throw new Error('Comfy Router returned an invalid request ID.');+ // Build authenticated URLs ourselves; never send the key to returned URLs.+ const resultUrl = `${base}/${queued.request_id}`;+ cancelUrl = `${resultUrl}/cancel`;+ let waitMs = pollMs;+ while (true) {+ await delay(waitMs, operation.signal);+ const res = await request(`${resultUrl}/status`, { headers });+ const status = await jsonResponse(res, 'Comfy Router status');+ waitMs = pollDelay(res, pollMs);+ if (status.status === 'COMPLETED') {+ const result = await request(resultUrl, { headers });+ if (result.status === 202) { waitMs = pollDelay(result, pollMs); continue; }+ const url = await mediaResponse(result, 'Comfy Router generation');+ completed = true;+ return url;+ }+ if (!['IN_QUEUE', 'IN_PROGRESS'].includes(status.status)) throw new Error('Comfy Router returned an unknown queue status.');+ }+ } catch (error) {+ if (!completed && cancelUrl) await cancelRequest(cancelUrl, { method: 'PUT', headers }, fetchImpl);+ throw operation.signal.aborted ? operation.signal.reason : error;+ } finally { operation.dispose(); }+}+/*** Generates one image or video. `media` is 'image' or 'video', choosing the* provider's image or video model; video options imply 'video'. Without@@ -418,8 +628,8 @@ export async function runMediaGeneration({ prompt, config, media = null, video =if (media === 'image' && video) {throw new Error('Video length, aspect ratio, resolution and scenes are for videos. Set media to "video" or leave these out.');}- if (video && provider !== 'openrouter') {- throw new Error(`Video length, aspect ratio, resolution and scenes work with OpenRouter video models. Omit them to use the ${MEDIA_PROVIDERS[provider].label} video model's defaults.`);+ if (video && provider === 'comfyui') {+ throw new Error('Video length, aspect ratio, resolution and scenes do not work with a local ComfyUI workflow, which sets them itself. Omit them.');}const kind = media || (video ? 'video' : null);const pollMs = pollIntervalMs ?? (provider === 'openrouter' ? 30000 : 2000);@@ -428,7 +638,32 @@ export async function runMediaGeneration({ prompt, config, media = null, video =}const slot = mediaSlot(config, kind || 'image');const { model } = slot;- if (provider === 'fal') return runFalGeneration({ prompt: text, config: { ...config, model }, fetchImpl, signal, timeoutMs });+ const videoRun = { model, text, video, signal, onProgress, confirmVideoLength };+ if (provider === 'fal') {+ const job = (prompt, input, jobSignal) => runFalGeneration({ prompt, config: { ...config, model }, input, fetchImpl, signal: jobSignal, timeoutMs, pollMs });+ if (kind !== 'video') return job(text, {}, signal);+ // The model's schema says what it accepts; a plain clip needs none of it.+ const entry = video ? await falVideoEntry(model, config.apiKey, fetchImpl, signal, timeoutMs) : {};+ return runVideo({+ ...videoRun, entry,+ single: async (clip, shape) => ({ url: (await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, signal)).url }),+ clip: async (clip, shape, groupSignal) => downloadClip((await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, groupSignal)).url,+ { fetchImpl, signal: groupSignal, timeoutMs, label: 'fal.ai' }),+ });+ }+ if (provider === 'comfyrouter') {+ const parameters = parseMediaJson(slot.parameters || '{}', kind === 'video' ? 'Video model input' : 'Image model input');+ // The clip's own length and shape replace those in the saved model input.+ const job = (prompt, input, jobSignal) => comfyRouterJob({ model, apiKey: config.apiKey, body: { ...comfyRouterBody(parameters, prompt), ...input }, fetchImpl, signal: jobSignal, timeoutMs, pollMs });+ if (kind !== 'video') return { url: await job(text, {}, signal), model };+ const entry = video ? await comfyRouterVideoEntry(model, config.apiKey, fetchImpl, signal, timeoutMs) : {};+ return runVideo({+ ...videoRun, entry,+ single: async (clip, shape) => ({ url: await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, signal) }),+ clip: async (clip, shape, groupSignal) => downloadClip(await job(clip.prompt, { ...clipLength(entry, clip), ...shape }, groupSignal),+ { fetchImpl, signal: groupSignal, timeoutMs, label: 'Comfy Router' }),+ });+ }const operation = operationSignal(signal, timeoutMs);const request = (url, init = {}) => {operation.signal.throwIfAborted();@@ -437,35 +672,6 @@ export async function runMediaGeneration({ prompt, config, media = null, video =let cancelUrl = '', cancelInit;let completed = false;try {- if (provider === 'comfyrouter') {- const base = `https://api.comfy.org/v2/models/${model}/requests`;- const headers = { 'X-API-Key': config.apiKey, 'Content-Type': 'application/json' };- const parameters = parseMediaJson(slot.parameters || '{}', kind === 'video' ? 'Video model input' : 'Image model input');- const usesTemplate = JSON.stringify(parameters).includes('{{prompt}}');- const body = usesTemplate ? bindMediaPrompt(parameters, text) : { ...parameters, prompt: text };- const queued = await jsonResponse(await request(base, {- method: 'POST', headers: { ...headers, 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(body),- }), 'Comfy Router submit');- if (!/^[A-Za-z0-9_-]+$/.test(queued.request_id || '')) throw new Error('Comfy Router returned an invalid request ID.');- // Build authenticated URLs ourselves; never send the key to returned URLs.- const resultUrl = `${base}/${queued.request_id}`;- cancelUrl = `${resultUrl}/cancel`; cancelInit = { method: 'PUT', headers };- let waitMs = pollMs;- while (true) {- await delay(waitMs, operation.signal);- const res = await request(`${resultUrl}/status`, { headers });- const status = await jsonResponse(res, 'Comfy Router status');- waitMs = pollDelay(res, pollMs);- if (status.status === 'COMPLETED') {- const result = await request(resultUrl, { headers });- if (result.status === 202) { waitMs = pollDelay(result, pollMs); continue; }- const url = await mediaResponse(result, 'Comfy Router generation');- completed = true;- return { url, model };- }- if (!['IN_QUEUE', 'IN_PROGRESS'].includes(status.status)) throw new Error('Comfy Router returned an unknown queue status.');- }- }const base = comfyBaseUrl(config.baseUrl);const headers = { 'Content-Type': 'application/json' };const workflow = bindMediaPrompt(parseMediaJson(slot.workflow, 'ComfyUI workflow'), text);@@ -552,7 +758,7 @@ export async function generateImage(args, options = {}) {const provider = mediaProvider(config);const result = await runMediaGeneration({prompt: args?.prompt, media: args?.media ?? null, video: videoArguments(args), config, fetchImpl, signal: options.signal,- onProgress: options.onProgress, confirmVideoLength: options.confirmVideoLength,+ pollIntervalMs: options.pollIntervalMs, onProgress: options.onProgress, confirmVideoLength: options.confirmVideoLength,});if (result.declined) {return { success: false, declined: true, provider, model: result.model, error: declinedVideoMessage(result) };diff --git a/src/firefox/src/agent/tools.js b/src/firefox/src/agent/tools.jsindex 1ef7700f3..41efc0213 100644--- a/src/firefox/src/agent/tools.js+++ b/src/firefox/src/agent/tools.js@@ -1343,7 +1343,7 @@ export const AGENT_TOOLS = [type: 'function',function: {name: 'generate_image',- description: 'Generate an image, or a video with media "video", from a text prompt using the image or video model the user configured (Providers → Generative). Supports fal.ai, OpenRouter, Comfy Router and local ComfyUI workflows; generation may take several minutes. Returns a media URL or inline media displayed in the chat. With an OpenRouter video model, estimate how long the requested video should run (estimated_duration): one clip can set duration, aspect_ratio and resolution; a longer video passes scenes, which WebBrain fits to the model\'s clip lengths and joins into one video of up to 120 s, asking the user first when the estimate is longer. Not available in Ask mode.',+ description: 'Generate an image, or a video with media "video", from a text prompt using the image or video model the user configured (Providers → Generative). Supports fal.ai, OpenRouter, Comfy Router and local ComfyUI workflows; generation may take several minutes. Returns a media URL or inline media displayed in the chat. For a video, estimate how long the requested video should run (estimated_duration): one clip can set duration, aspect_ratio and resolution; a longer video passes scenes, which WebBrain fits to the video model\'s clip lengths and joins into one video of up to 120 s, asking the user first when the estimate is longer. Not available in Ask mode.',parameters: {type: 'object',properties: {diff --git a/test/generative-media-video.mjs b/test/generative-media-video.mjsindex 7dc889a33..cabf76c01 100644--- a/test/generative-media-video.mjs+++ b/test/generative-media-video.mjs@@ -129,6 +129,113 @@ function videoApi({ entry, clip = () => fixtures['h264-aac-1'], fail = () => falreturn { fetchImpl, calls };}+// The OpenAPI document fal.ai publishes for a model, as its model listing+// returns it: the request body behind a $ref.+const falSchema = input => ({+ openapi: '3.0.4',+ paths: {+ '/model/requests/{request_id}/status': { get: {} },+ '/model': { post: { requestBody: { content: { 'application/json': { schema: { $ref: '#/components/schemas/Input' } } } } } },+ },+ components: { schemas: { Input: { type: 'object', required: ['prompt'], properties: { prompt: { type: 'string' }, ...input } } } },+});+// MiniMax H3 Max on fal.ai: any length up to 15 s; resolutions in capitals.+const h3Max = { id: 'minimax/h3-max/text-to-video', schema: falSchema({+ duration: { type: 'number', minimum: 0.92, maximum: 15, default: 5 },+ aspect_ratio: { type: 'string', enum: ['21:9', '16:9', '4:3', '1:1', '3:4', '9:16'] },+ resolution: { type: 'string', enum: ['480P', '768P', '1080P'] },+}) };+// Hailuo 02 on fal.ai: clips of 6 or 10 s, given as text.+const hailuo = { id: 'fal-ai/minimax/hailuo-02/standard/text-to-video', schema: falSchema({ duration: { type: 'string', enum: ['6', '10'], default: '6' } }) };+const falConfig = (model, settings = {}) => ({ provider: 'fal', apiKey: 'fal-key', model: 'fal-ai/flux/schnell', videoModel: model.id, ...settings });++// fal.ai: the model's schema (`schema`: a document, null when fal.ai has+// none for it, or the HTTP status of the read), a queue whose jobs complete on+// the first poll, and hosted clips. `fail` refuses a submission: true for an+// overloaded provider (503), or the HTTP status to answer with.+function falApi({ model, schema, clip = () => fixtures['h264-aac-1'], fail = () => false }) {+ const calls = { schemas: 0, submits: [], downloads: [], cancels: [] };+ const fetchImpl = async (url, init = {}) => {+ if (url.startsWith('https://v3b.fal.media/')) {+ // A hosted clip is downloaded without the key.+ assert.equal(init.headers, undefined);+ assert.equal(init.credentials, 'omit');+ const index = Number(/clip-(\d+)\.mp4$/.exec(url)[1]);+ calls.downloads.push(index);+ return new Response(clip(index, calls.submits[index]), { headers: { 'Content-Type': 'video/mp4' } });+ }+ assert.equal(init.headers?.Authorization, 'Key fal-key');+ if (init.method === 'PUT') { calls.cancels.push(url); return json({}); }+ assert.equal(init.redirect, 'error');+ if (url.startsWith('https://api.fal.ai/v1/models?')) {+ calls.schemas += 1;+ const query = new URL(url).searchParams;+ assert.deepEqual([query.get('endpoint_id'), query.get('expand')], [model, 'openapi-3.0']);+ if (typeof schema === 'number') return json({ error: { type: 'not_found', message: 'Endpoint(s) not found' } }, schema);+ return json({ models: [{ endpoint_id: model, openapi: schema ?? { error: { code: 'expansion_failed' } } }] });+ }+ if (init.method === 'POST') {+ assert.equal(url, `https://queue.fal.run/${model}`);+ const body = JSON.parse(init.body);+ const index = calls.submits.push(body) - 1;+ const failure = fail(index, body);+ if (failure) return json({ detail: failure === 402 ? 'Exhausted balance' : 'Overloaded' }, failure === true ? 503 : failure);+ const job = `https://queue.fal.run/${model}/requests/req-${index}`;+ return json({ request_id: `req-${index}`, status_url: `${job}/status`, response_url: job, cancel_url: `${job}/cancel` });+ }+ const [, index, status] = /\/requests\/req-(\d+)(\/status)?$/.exec(url);+ return status ? json({ status: 'COMPLETED' }) : json({ video: { url: `https://v3b.fal.media/files/b/clip-${index}.mp4` } });+ };+ return { fetchImpl, calls };+}++// Luma Ray 2 on Comfy Router: the body schema inline, its lengths "5s" and "9s".+const rayTwo = { id: 'luma/ray-2', schema: {+ openapi: '3.0.2',+ paths: { '/v2/models/luma/ray-2': { post: { requestBody: { content: { 'application/json': { schema: {+ type: 'object', required: ['prompt'],+ properties: {+ prompt: { type: 'string' }, model: { type: 'string' }, loop: { type: 'boolean' }, duration: { $ref: '#/components/schemas/Duration' },+ resolution: { anyOf: [{ type: 'string', enum: ['540p', '720p', '1080p', '4k'] }, { type: 'string' }] }, aspect_ratio: { type: 'string', enum: ['16:9', '9:16'] },+ },+ } } } } } } },+ components: { schemas: { Duration: { anyOf: [{ type: 'string', enum: ['5s', '9s'] }, { type: 'string' }] } } },+} };++// Comfy Router: the model's schema (a document, or the HTTP status of the+// read), queued requests that complete on the first poll, and hosted clips.+function comfyRouterApi({ model, schema, clip = () => fixtures['h264-aac-1'], fail = () => false }) {+ const calls = { schemas: 0, submits: [], downloads: [], cancels: [] };+ const base = `https://api.comfy.org/v2/models/${model}`;+ const fetchImpl = async (url, init = {}) => {+ if (url.startsWith('https://storage.comfy.example/')) {+ assert.equal(init.headers, undefined);+ assert.equal(init.credentials, 'omit');+ const index = Number(/clip-(\d+)\.mp4$/.exec(url)[1]);+ calls.downloads.push(index);+ return new Response(clip(index, calls.submits[index]), { headers: { 'Content-Type': 'video/mp4' } });+ }+ assert.equal(init.headers?.['X-API-Key'], 'comfy-key');+ assert.equal(init.redirect, 'error');+ if (url === `${base}/openapi.json`) {+ calls.schemas += 1;+ return typeof schema === 'number' ? json({ detail: 'No schema', error_type: 'model_not_found' }, schema) : json(schema);+ }+ if (init.method === 'PUT') { calls.cancels.push(url); return json({}); }+ if (init.method === 'POST') {+ assert.equal(url, `${base}/requests`);+ const body = JSON.parse(init.body);+ const index = calls.submits.push(body) - 1;+ const failure = fail(index, body);+ if (failure) return json({ detail: 'Insufficient credits', error_type: 'insufficient_credits' }, failure === true ? 503 : failure);+ return json({ request_id: `req-${index}` }, 201);+ }+ const [, index, status] = /\/requests\/req-(\d+)(\/status)?$/.exec(url);+ return status ? json({ status: 'COMPLETED' }) : json({ assets: { video: `https://storage.comfy.example/clip-${index}.mp4` } });+ };+ return { fetchImpl, calls };+}+const grok = { id: 'x-ai/grok-imagine-video-1.5-lite', supported_durations: Array.from({ length: 15 }, (_, i) => i + 1),supported_resolutions: ['480p', '720p', '1080p'], supported_aspect_ratios: ['16:9', '9:16', '1:1'] };const config = { provider: 'openrouter', apiKey: 'or-key', model: grok.id };@@ -473,8 +580,10 @@ for (const build of ['chrome', 'firefox']) {/image-model makes images/);await assert.rejects(media.runMediaGeneration({ prompt: 'x', media: 'image', video: { duration: 5 }, config, fetchImpl: async url => assert.fail(url) }),/are for videos/);- await assert.rejects(media.runMediaGeneration({ prompt: 'x', video: { duration: 5 }, config: { apiKey: 'k', model: 'fal-ai/flux/schnell' },- fetchImpl: async url => assert.fail(url) }), /OpenRouter video models/);+ // A local ComfyUI workflow sets its own length and shape.+ const workflow = { '6': { class_type: 'CLIPTextEncode', inputs: { text: '{{prompt}}' } } };+ await assert.rejects(media.runMediaGeneration({ prompt: 'x', video: { duration: 5 }, config: { provider: 'comfyui', workflow },+ fetchImpl: async url => assert.fail(url) }), /do not work with a local ComfyUI workflow/);});test(`${build}: a video of two minutes or less generates without asking`, async () => {@@ -570,6 +679,169 @@ for (const build of ['chrome', 'firefox']) {assert.equal(assets.size, stored);});+ test(`${build}: fal.ai scenes are made one clip each and joined into one video`, async () => {+ const api = falApi({ model: h3Max.id, schema: h3Max.schema });+ const progress = [];+ const result = await generate({+ prompt: 'Flat motion graphics, navy background', media: 'video', scenes: scenes(3, 10), estimated_duration: 30,+ aspect_ratio: '16:9', resolution: '768p',+ }, { config: falConfig(h3Max), fetchImpl: api.fetchImpl, pollIntervalMs: 1, onProgress: message => progress.push(message) });+ assert.equal(result.success, true, result.error);+ assert.equal(api.calls.schemas, 1, 'the schema is read once per video');+ // Each clip as the model takes it: its length as a number, the resolution as the model writes it.+ assert.deepEqual(api.calls.submits, [1, 2, 3].map(scene => ({+ prompt: `Flat motion graphics, navy background\n\nScene idea ${scene}`, duration: 10, aspect_ratio: '16:9', resolution: '768P',+ })));+ assert.deepEqual(api.calls.downloads.sort(), [0, 1, 2]);+ assert.deepEqual([result.provider, result.model, result.clipCount, result.durationSeconds, result.mimeType], ['fal', h3Max.id, 3, 3, 'video/mp4']);+ assert.equal(join.mp4DurationSeconds(await storedBytes(result.mediaId)), 3, 'one joined asset');+ assert.deepEqual(progress, ['Generating 3 video clips…', 'Video clips: 1 of 3 ready', 'Video clips: 2 of 3 ready',+ 'Video clips: 3 of 3 ready', 'Joining 3 clips…']);+ // One clip without options needs no schema and stays a hosted video.+ const plain = falApi({ model: h3Max.id, schema: h3Max.schema });+ const single = await generate({ prompt: 'A red apple', media: 'video' }, { config: falConfig(h3Max), fetchImpl: plain.fetchImpl, pollIntervalMs: 1 });+ assert.equal(single.success, true, single.error);+ assert.deepEqual([plain.calls.schemas, plain.calls.submits, plain.calls.downloads], [0, [{ prompt: 'A red apple' }], []]);+ assert.equal(single.url, 'https://v3b.fal.media/files/b/clip-0.mp4');+ });++ test(`${build}: fal.ai lengths are sent the way the model lists them`, async () => {+ const run = async (model, video) => {+ const api = falApi({ model: model.id, schema: model.schema });+ const result = await media.runMediaGeneration({ prompt: 'Story', video, config: falConfig(model), fetchImpl: api.fetchImpl, pollIntervalMs: 1 });+ return { result, durations: api.calls.submits.map(body => body.duration) };+ };+ // Scenes round up to the model's clip lengths: Hailuo 02 makes 6 and 10 s clips.+ const scenesRun = await run(hailuo, { scenes: [{ prompt: 'One', duration: 5 }, { prompt: 'Two', duration: 8 }], estimated_duration: 13 });+ assert.deepEqual([scenesRun.durations, scenesRun.result.totalSeconds, scenesRun.result.clips.length], [['6', '10'], 16, 2]);+ assert.deepEqual((await run(hailuo, { duration: 10 })).durations, ['10']);+ assert.deepEqual((await run(hailuo, { estimated_duration: 7 })).durations, ['10']);+ const ltx = { id: 'fal-ai/ltx-2.3/text-to-video', schema: falSchema({ duration: { type: 'integer', enum: [6, 8, 10] } }) };+ assert.deepEqual((await run(ltx, { scenes: [{ prompt: 'One', duration: 7 }], estimated_duration: 7 })).durations, [8]);+ });++ test(`${build}: fal.ai and Comfy Router refuse what the model's schema rules out, before submitting`, async () => {+ const pro = { id: 'fal-ai/minimax/hailuo-02/pro/text-to-video', schema: falSchema({ prompt_optimizer: { type: 'boolean' } }) };+ for (const [model, video, pattern] of [+ [hailuo, { duration: 7 }, /does not make 7 s clips\. Supported: 6, 10 s/],+ [hailuo, { duration: 16 }, /needs more than one clip: this model makes clips of up to 10 s/],+ [hailuo, { scenes: [{ prompt: 'Long', duration: 12 }], estimated_duration: 12 }, /Scene 1 is 12 s, and this model makes clips of up to 10 s/],+ [h3Max, { resolution: '4K' }, /does not support resolution "4K"\. Supported: 480P, 768P, 1080P/],+ [pro, { duration: 6 }, /This video model does not take duration/],+ [pro, { aspect_ratio: '16:9' }, /This video model does not take aspect_ratio/],+ [pro, { scenes: scenes(2, 5), estimated_duration: 10 }, /does not list the clip lengths it supports/],+ // fal.ai publishes no schema for some models, and knows none for an unknown ID.+ [{ id: 'fal-ai/kling-video/v2.1/standard/text-to-video', schema: null }, { scenes: scenes(2, 5), estimated_duration: 10 }, /does not list the clip lengths/],+ [{ id: 'fal-ai/unknown', schema: 404 }, { scenes: scenes(2, 5), estimated_duration: 10 }, /does not list the clip lengths/],+ [{ id: 'fal-ai/busy', schema: 503 }, { duration: 5 }, /fal\.ai model schema failed \(HTTP 503\)/],+ ]) {+ const api = falApi({ model: model.id, schema: model.schema });+ await assert.rejects(media.runMediaGeneration({ prompt: 'Story', video, config: falConfig(model), fetchImpl: api.fetchImpl, pollIntervalMs: 1 }),+ pattern, `${model.id} ${JSON.stringify(video)}`);+ assert.equal(api.calls.submits.length, 0, 'nothing is submitted');+ }+ // Without a schema, one clip's options go to the model as given.+ const unknown = falApi({ model: 'fal-ai/kling-video/v2.1/standard/text-to-video', schema: null });+ await media.runMediaGeneration({ prompt: 'Story', video: { duration: 5, aspect_ratio: '1:1' },+ config: falConfig({ id: 'fal-ai/kling-video/v2.1/standard/text-to-video' }), fetchImpl: unknown.fetchImpl, pollIntervalMs: 1 });+ assert.deepEqual(unknown.calls.submits, [{ prompt: 'Story', duration: 5, aspect_ratio: '1:1' }]);+ // Comfy Router: no schema for the model, or a read it refuses.+ const comfyConfig = { provider: 'comfyrouter', apiKey: 'comfy-key', model: 'bfl/flux-2-pro', videoModel: 'minimax/hailuo-02' };+ for (const [schema, pattern] of [[404, /does not list the clip lengths/], [401, /Comfy Router model schema failed \(HTTP 401\)/]]) {+ const api = comfyRouterApi({ model: 'minimax/hailuo-02', schema });+ await assert.rejects(media.runMediaGeneration({ prompt: 'Story', video: { scenes: scenes(2, 5), estimated_duration: 10 }, config: comfyConfig,+ fetchImpl: api.fetchImpl, pollIntervalMs: 1 }), pattern);+ assert.deepEqual([api.calls.schemas, api.calls.submits.length], [1, 0]);+ }+ });++ test(`${build}: Comfy Router scenes take each clip's length over the saved model input`, async () => {+ const api = comfyRouterApi({ model: rayTwo.id, schema: rayTwo.schema, clip: index => fixtures[index ? 'h264-aac-2' : 'h264-aac-1'] });+ const config = { provider: 'comfyrouter', apiKey: 'comfy-key', model: 'bfl/flux-2-pro', videoModel: rayTwo.id,+ videoParameters: JSON.stringify({ model: 'ray-2', duration: '9s', loop: false }) };+ const result = await generate({ prompt: 'Paper cut-out style', scenes: [{ prompt: 'A fox wakes', duration: 4 }, { prompt: 'The fox runs', duration: 9 }],+ estimated_duration: 13, resolution: '720P' }, { config, fetchImpl: api.fetchImpl, pollIntervalMs: 1 });+ assert.equal(result.success, true, result.error);+ assert.deepEqual(api.calls.submits, [+ { model: 'ray-2', duration: '5s', loop: false, prompt: 'Paper cut-out style\n\nA fox wakes', resolution: '720p' },+ { model: 'ray-2', duration: '9s', loop: false, prompt: 'Paper cut-out style\n\nThe fox runs', resolution: '720p' },+ ]);+ assert.deepEqual([result.provider, result.model, result.clipCount, result.durationSeconds], ['comfyrouter', rayTwo.id, 2, 2]);+ const joined = parse(await storedBytes(result.mediaId));+ const parts = [parse(fixtures['h264-aac-1']), parse(fixtures['h264-aac-2'])];+ assert.deepEqual(sampleBytes(await storedBytes(result.mediaId), videoOf(joined)),+ [...sampleBytes(fixtures['h264-aac-1'], videoOf(parts[0])), ...sampleBytes(fixtures['h264-aac-2'], videoOf(parts[1]))], 'the scenes in order');+ assert.deepEqual(api.calls.cancels, [], 'completed requests are not cancelled');+ });++ test(`${build}: fal.ai and Comfy Router clips are retried once, unless refused or in another format`, async () => {+ const video = { scenes: scenes(3, 10), estimated_duration: 30 };+ const fal = options => falApi({ model: h3Max.id, schema: h3Max.schema, ...options });+ const runFal = api => media.runMediaGeneration({ prompt: 'Story', video, config: falConfig(h3Max), fetchImpl: api.fetchImpl, pollIntervalMs: 1 });+ const flaky = fal({ fail: index => index === 1 });+ assert.equal((await runFal(flaky)).clips.length, 3);+ assert.equal(flaky.calls.submits.length, 4, 'three clips and one retry');+ const broke = fal({ fail: (_index, body) => body.prompt.endsWith('Scene idea 2') && 402 });+ await assert.rejects(runFal(broke), /^Error: Clip 2 of 3 failed: fal\.ai submit failed \(HTTP 402\): the account is out of credit; add credit and try again: .*Exhausted balance/);+ assert.equal(broke.calls.submits.filter(body => body.prompt.endsWith('Scene idea 2')).length, 1, 'tried once');+ const webm = fal({ clip: index => (index === 0 ? new TextEncoder().encode('\x1aE\xdf\xa3 webm') : fixtures['h264-aac-1']) });+ await assert.rejects(runFal(webm), /^Error: Clip 1 of 3 failed: fal\.ai did not return an MP4 video\./);+ assert.equal(webm.calls.submits.filter(body => body.prompt.endsWith('Scene idea 1')).length, 1, 'another format is not tried again');+ const comfy = comfyRouterApi({ model: rayTwo.id, schema: rayTwo.schema, fail: index => index === 0 && 402 });+ await assert.rejects(media.runMediaGeneration({ prompt: 'Story', video: { scenes: scenes(2, 5), estimated_duration: 10 },+ config: { provider: 'comfyrouter', apiKey: 'comfy-key', videoModel: rayTwo.id }, fetchImpl: comfy.fetchImpl, pollIntervalMs: 1 }),+ /^Error: Clip 1 of 2 failed: Comfy Router submit failed \(HTTP 402\): the account is out of credit; add credit and try again: .*Insufficient credits/);+ assert.equal(comfy.calls.submits.filter(body => body.prompt.endsWith('Scene idea 1')).length, 1);+ });++ test(`${build}: a refused fal.ai or Comfy Router clip cancels the clips still being made`, async () => {+ // Clip 2 is refused at once; clips 1 and 3 stay in progress until cancelled.+ const pending = api => async (url, init = {}) => {+ if (init.method !== 'POST' && /\/requests\/req-[02]\/status$/.test(url)) return json({ status: 'IN_PROGRESS' });+ return api.fetchImpl(url, init);+ };+ // The run's error and the cancellations sent before the test stops it; a+ // run that is still waiting fails the assertion.+ const outcome = async (api, options) => {+ const stop = new AbortController();+ try {+ const error = await settled(media.runMediaGeneration({ prompt: 'Story', fetchImpl: pending(api), pollIntervalMs: 1, signal: stop.signal, ...options })+ .catch(failure => failure.message));+ await new Promise(done => setTimeout(done, 20));+ return { error, cancels: api.calls.cancels.slice().sort() };+ } finally { stop.abort(new Error('Test over.')); }+ };+ const fal = falApi({ model: h3Max.id, schema: h3Max.schema, fail: index => index === 1 && 402 });+ const falRun = await outcome(fal, { video: { scenes: scenes(3, 10), estimated_duration: 30 }, config: falConfig(h3Max) });+ assert.match(falRun.error, /^Clip 2 of 3 failed: fal\.ai submit failed \(HTTP 402\)/);+ assert.deepEqual(falRun.cancels, [0, 2].map(index => `https://queue.fal.run/${h3Max.id}/requests/req-${index}/cancel`));+ const comfy = comfyRouterApi({ model: rayTwo.id, schema: rayTwo.schema, fail: index => index === 1 && 402 });+ const comfyRun = await outcome(comfy, { video: { scenes: scenes(3, 5), estimated_duration: 15 }, config: { provider: 'comfyrouter', apiKey: 'comfy-key', videoModel: rayTwo.id } });+ assert.match(comfyRun.error, /^Clip 2 of 3 failed: Comfy Router submit failed \(HTTP 402\)/);+ assert.deepEqual(comfyRun.cancels, [0, 2].map(index => `https://api.comfy.org/v2/models/${rayTwo.id}/requests/req-${index}/cancel`));+ });++ test(`${build}: Stop ends every clip of a fal.ai video`, async () => {+ const controller = new AbortController();+ const api = falApi({ model: h3Max.id, schema: h3Max.schema });+ const stored = assets.size;+ const result = await generate({ prompt: 'Story', scenes: scenes(5, 10), estimated_duration: 50 }, {+ config: falConfig(h3Max), pollIntervalMs: 1, signal: controller.signal,+ // The user presses Stop while the first clip downloads.+ fetchImpl: async (url, init) => {+ if (url.startsWith('https://v3b.fal.media/')) {+ controller.abort(new Error('Stopped by user.'));+ init.signal.throwIfAborted();+ }+ return api.fetchImpl(url, init);+ },+ });+ assert.equal(result.success, false);+ assert.equal(result.error, 'Stopped by user.');+ assert.equal(api.calls.submits.length, 3, 'clips 4 and 5 never start');+ assert.equal(assets.size, stored);+ });+test(`${build}: clips that cannot be joined are kept and shown in order`, async () => {const api = videoApi({ entry: grok, clip: job => (job.index === 1 ? fixtures['h264-aac-48px'] : fixtures['h264-aac-1']) });const result = await generate({ prompt: 'Story', scenes: scenes(2, 10), estimated_duration: 20 }, { fetchImpl: api.fetchImpl });diff --git a/test/generative-media.mjs b/test/generative-media.mjsindex a3dd41968..6ea9709cf 100644--- a/test/generative-media.mjs+++ b/test/generative-media.mjs@@ -125,6 +125,94 @@ for (const build of ['chrome', 'firefox']) {await assert.rejects(media.runMediaGeneration({ prompt: 'x', media: 'audio', config: openrouter, fetchImpl: async url => assert.fail(url) }), /media must be "image" or "video"/);});+ test(`${build}: a video model's lengths, ratios and resolutions are read from its published schema`, () => {+ // Shaped like the OpenAPI documents fal.ai and Comfy Router publish per model.+ const fal = (input, extra = {}) => ({+ openapi: '3.0.4',+ paths: {+ '/fal-ai/model/requests/{request_id}/status': { get: {} },+ '/fal-ai/model': { post: { requestBody: { content: { 'application/json': { schema: { $ref: '#/components/schemas/ModelInput' } } } } } },+ },+ components: { schemas: { ModelInput: { type: 'object', required: ['prompt'], properties: { prompt: { type: 'string' }, ...input } }, ...extra } },+ });+ const caps = openapi => {+ const entry = media.videoInputCapabilities(openapi);+ return entry && { ...entry, encodeDuration: undefined, encoded: entry.supported_durations?.map(seconds => entry.encodeDuration(seconds)) };+ };+ const lengths = duration => {+ const entry = caps(fal({ duration }));+ return [entry.supported_durations, entry.encoded];+ };+ const seconds = (from, to) => Array.from({ length: to - from + 1 }, (_, i) => from + i);+ // A range of whole seconds, sent as numbers (fal.ai's Grok Imagine).+ assert.deepEqual(caps(fal({+ duration: { type: 'integer', minimum: 1, maximum: 15, default: 6 },+ aspect_ratio: { type: 'string', enum: ['16:9', '4:3', '1:1', '9:16'] },+ resolution: { type: 'string', enum: ['480p', '720p', '1080p'] },+ })), {+ inputs: ['prompt', 'duration', 'aspect_ratio', 'resolution'], encodeDuration: undefined,+ supported_durations: seconds(1, 15), encoded: seconds(1, 15),+ supported_aspect_ratios: ['16:9', '4:3', '1:1', '9:16'], supported_resolutions: ['480p', '720p', '1080p'],+ });+ // A fractional range covers its whole seconds (MiniMax H3 Max).+ assert.deepEqual(lengths({ type: 'number', minimum: 0.92, maximum: 6 }), [seconds(1, 6), seconds(1, 6)]);+ // Listed lengths are sent as listed: "8s" (Veo), "10" (Hailuo 02), 10 (LTX).+ assert.deepEqual(lengths({ type: 'string', enum: ['4s', '6s', '8s'] }), [[4, 6, 8], ['4s', '6s', '8s']]);+ assert.deepEqual(lengths({ type: 'string', enum: ['6', '10'], default: '6' }), [[6, 10], ['6', '10']]);+ assert.deepEqual(lengths({ type: 'integer', enum: [6, 8, 10] }), [[6, 8, 10], [6, 8, 10]]);+ // "auto" is not a length; the range or the other listed values are (FLUX.3, Seedance).+ assert.deepEqual(lengths({ enum: ['auto', 5, 6, 7] }), [[5, 6, 7], [5, 6, 7]]);+ assert.deepEqual(lengths({ oneOf: [{ type: 'integer', minimum: 4, maximum: 6 }, { type: 'string', enum: ['auto'] }] }), [[4, 5, 6], [4, 5, 6]]);+ assert.deepEqual(lengths({ anyOf: [{ enum: [-1] }, { minimum: 2, maximum: 4 }] }), [[2, 3, 4], [2, 3, 4]]);+ assert.deepEqual(lengths({ anyOf: [{ type: 'integer', minimum: 2, maximum: 3 }, { type: 'null' }] }), [[2, 3], [2, 3]]);+ // Exclusive bounds and steps.+ assert.deepEqual(lengths({ type: 'integer', exclusiveMinimum: 2, exclusiveMaximum: 6 }), [[3, 4, 5], [3, 4, 5]]);+ assert.deepEqual(lengths({ type: 'number', minimum: 2, exclusiveMinimum: true, maximum: 6, exclusiveMaximum: true }), [[3, 4, 5], [3, 4, 5]]);+ assert.deepEqual(lengths({ type: 'integer', minimum: 5, maximum: 15, multipleOf: 5 }), [[5, 10, 15], [5, 10, 15]]);+ // Lengths start at 1 s.+ assert.deepEqual(lengths({ type: 'integer', maximum: 3 }), [[1, 2, 3], [1, 2, 3]]);+ assert.deepEqual(lengths({ type: 'number', minimum: 0, maximum: 2 }), [[1, 2], [1, 2]]);+ // A listed set is the set, whatever bounds it also states; a range is read up to an hour.+ assert.deepEqual(lengths({ type: 'integer', enum: [6, 8, 10], minimum: 6, maximum: 10 }), [[6, 8, 10], [6, 8, 10]]);+ assert.deepEqual(lengths({ type: 'integer', minimum: 1, maximum: 1e9 })[0].slice(-2), [3599, 3600]);+ // Without an upper bound, or as text, a length is unknown.+ assert.deepEqual(lengths({ type: 'integer', minimum: 1 }), [undefined, undefined]);+ assert.deepEqual(lengths({ type: 'string', maximum: 10 }), [undefined, undefined]);+ // An enum lists the values in their own case and type ("768P").+ assert.deepEqual(caps(fal({ resolution: { type: 'string', enum: ['480P', '768P', null] } })).supported_resolutions, ['480P', '768P']);+ // Comfy Router: the body schema inline, choices behind $ref and anyOf (Luma).+ const comfy = (schema, schemas = {}) => ({+ openapi: '3.0.2',+ paths: { '/v2/models/provider/model': { post: { requestBody: { content: { 'application/json': { schema } } } } } },+ components: { schemas },+ });+ const luma = caps(comfy({+ required: ['prompt', 'duration'],+ properties: { prompt: { type: 'string' }, duration: { $ref: '#/components/schemas/Duration' }, aspect_ratio: { allOf: [{ $ref: '#/components/schemas/Ratio' }], nullable: true }, resolution: { type: 'string' } },+ }, { Duration: { anyOf: [{ type: 'string', enum: ['5s', '9s'] }, { type: 'string' }] }, Ratio: { type: 'string', enum: ['16:9', '9:16'] } }));+ assert.deepEqual([luma.supported_durations, luma.encoded, luma.supported_aspect_ratios, luma.supported_resolutions], [[5, 9], ['5s', '9s'], ['16:9', '9:16'], undefined]);+ // The generation's body is the one that takes the prompt.+ const upload = caps({+ ...comfy({ properties: { prompt: { type: 'string' }, duration: { enum: ['5s'] } } }),+ paths: {+ '/storage/upload': { post: { requestBody: { content: { 'application/json': { schema: { properties: { file_name: { type: 'string' }, duration: { enum: [30] } } } } } } } },+ '/v2/models/provider/model': { post: { requestBody: { content: { 'application/json': { schema: { properties: { prompt: { type: 'string' }, duration: { enum: ['5s'] } } } } } } } },+ },+ });+ assert.deepEqual([upload.inputs, upload.encoded], [['prompt', 'duration'], ['5s']]);+ // A body that takes its prompt inside another field, or is made of parts.+ const content = caps(comfy({ properties: { content: { type: 'array' }, duration: { type: 'integer', minimum: 4, maximum: 6 }, ratio: { enum: ['16:9'] } } }));+ assert.deepEqual([content.inputs, content.supported_durations], [['content', 'duration', 'ratio'], [4, 5, 6]]);+ const parts = caps(comfy({ allOf: [{ $ref: '#/components/schemas/Base' }, { properties: { duration: { enum: [5] } } }] },+ { Base: { properties: { prompt: { type: 'string' }, duration: { enum: [10] } } } }));+ assert.deepEqual([parts.inputs, parts.supported_durations, parts.encoded], [['prompt', 'duration'], [5, 10], [5, 10]]);+ // Nothing to read: an open body, no paths, an error instead of a document.+ assert.equal(media.videoInputCapabilities(comfy({ type: 'object', additionalProperties: true })), null);+ assert.equal(media.videoInputCapabilities({ openapi: '3.0.4', paths: {} }), null);+ assert.equal(media.videoInputCapabilities({ error: { code: 'expansion_failed' } }), null);+ assert.equal(media.videoInputCapabilities(null), null);+ });+test(`${build}: OpenRouter Image API returns inline media without a chat request`, async () => {const calls = [];const result = await media.runMediaGeneration({ prompt: 'red apple', config: { provider: 'openrouter', apiKey: 'or-key', model: 'google/gemini-2.5-flash-image' }, fetchImpl: async (url, init) => {