Skip to content

Open source CapCut alternative.

TypeScript

feat(site): Omni 1.1 Flash with resolution and length [rebuild]

David Lef13c0af6f942
diff --git a/site/scripts/generate-stock-images.ts b/site/scripts/generate-stock-images.ts
index 11f71f1f..4ad18e3f 100644
--- a/site/scripts/generate-stock-images.ts
+++ b/site/scripts/generate-stock-images.ts
@@ -1,5 +1,5 @@
// Generates the bundled Cut stock-image catalog with the same Vertex image model
-// the app uses (gemini-2.5-flash-image), then writes the typed manifest the
+// the app uses (gemini-3.1-flash-image), then writes the typed manifest the
// editor imports. Idempotent: items whose file already exists are skipped, so
// re-running fills gaps or picks up new catalog entries only.
//
diff --git a/site/scripts/generate-stock-videos.ts b/site/scripts/generate-stock-videos.ts
index b792acce..cc7aca16 100644
--- a/site/scripts/generate-stock-videos.ts
+++ b/site/scripts/generate-stock-videos.ts
@@ -164,7 +164,7 @@ async function generateOne(client: GoogleGenAI, authClient: JWT, item: CatalogIt
model: MODEL,
input: `${item.prompt} Avoid: ${NEGATIVE_PROMPT}.`,
generation_config: { video_config: { task: "text_to_video" } } as never,
- response_format: { type: "video", aspect_ratio: item.aspect } as never,
+ response_format: { type: "video", aspect_ratio: item.aspect, resolution: "720p" } as never,
background: true,
});
// requires_action is a non-terminal state too (same treatment the
diff --git a/site/scripts/generate-voices.ts b/site/scripts/generate-voices.ts
index 2ba5b766..0a245fb5 100644
--- a/site/scripts/generate-voices.ts
+++ b/site/scripts/generate-voices.ts
@@ -1,6 +1,6 @@
// Renders the hosted voice-persona assets the Cut voice picker shows: one
// square portrait and one spoken sample clip per prebuilt speech voice. Both are
-// produced with the same Vertex models the app uses — gemini-2.5-flash-image for
+// produced with the same Vertex models the app uses — gemini-3.1-flash-image for
// the portrait, the Gemini TTS model for the sample — and written into
// public/cut/voices as <id>.webp and <id>.mp3 (lowercased ids, matching
// voicePortraitUrl / voiceSampleUrl). Idempotent: a voice whose portrait and
diff --git a/site/src/cut/components/GeneratePanel.tools.ts b/site/src/cut/components/GeneratePanel.tools.ts
index 96d1a1d7..ca8ba976 100644
--- a/site/src/cut/components/GeneratePanel.tools.ts
+++ b/site/src/cut/components/GeneratePanel.tools.ts
@@ -6,15 +6,24 @@
*/
import { bool, num, obj, str, type AiToolDef } from "@/cut/lib/aiToolDef";
+import {
+ geminiOmniDefaultResolution,
+ geminiOmniDurationSeconds,
+ geminiOmniResolutions,
+} from "@/lib/inference/gemini-models";
+
+const DURATION = geminiOmniDurationSeconds;
export const VIDEO_GEN_TOOLS = [
{
name: "generate_video",
description:
- "Generate an AI video clip from a text prompt (Donkey's hosted video model: one pass, audio included, 720p, up to ~10s — the model picks the length) — the default for any video ask: \"generate/make a video of/about X\" means ONE clip from this tool, even though it sounds whole. \"Make me a better version / another take\" of an existing clip is also this tool — render the fresh take and leave their clip in place; a grade or effect on the original delivers a tweak, and they asked for a version. Reach for generate_scene only when they name a narrated multi-shot production (a story, episode, or narrated short). When the user asks you to write or improve the prompt itself, put it in chat and wait for them to ask for the video. reference_asset_id anchors the render on one project asset: for an image or video, by default the tool first designs the opening frame from it with the image model (that still previews in this chat), then animates that exact frame — so tell the user it designs the frame, then renders; for audio, what it carries (speech transcribed, sound described) folds into the prompt instead. When the user wants the referenced image itself brought to life (\"animate this image/picture\"), pass animate_reference:true — the image becomes the literal first frame, unchanged, and the prompt describes only the motion. A safety-blocked or rejected anchor degrades the render on its own (identity reference, then text-only — the same ladder scene shots walk), so never resubmit a failed render with weaker inputs yourself. The render takes a minute or two and the tool RETURNS as soon as it's started — the clip previews as a live card in this chat when it finishes. Pass add_to_timeline:true (or an index) only when they asked for it in the cut; otherwise the user drags it in from the card. Needs the user signed in to Donkey (spends their credits).",
+ `Generate an AI video clip from a text prompt (Donkey's hosted video model: one pass, audio included, ${geminiOmniResolutions.join("/")} at ${geminiOmniDefaultResolution} by default, ${DURATION.min}–${DURATION.max}s when duration_seconds is set and the model's own length otherwise) — the default for any video ask: \"generate/make a video of/about X\" means ONE clip from this tool, even though it sounds whole. \"Make me a better version / another take\" of an existing clip is also this tool — render the fresh take and leave their clip in place; a grade or effect on the original delivers a tweak, and they asked for a version. Reach for generate_scene only when they name a narrated multi-shot production (a story, episode, or narrated short). When the user asks you to write or improve the prompt itself, put it in chat and wait for them to ask for the video. reference_asset_id anchors the render on one project asset: for an image or video, by default the tool first designs the opening frame from it with the image model (that still previews in this chat), then animates that exact frame — so tell the user it designs the frame, then renders; for audio, what it carries (speech transcribed, sound described) folds into the prompt instead. When the user wants the referenced image itself brought to life (\"animate this image/picture\"), pass animate_reference:true — the image becomes the literal first frame, unchanged, and the prompt describes only the motion. A safety-blocked or rejected anchor degrades the render on its own (identity reference, then text-only — the same ladder scene shots walk), so never resubmit a failed render with weaker inputs yourself. The render takes a minute or two and the tool RETURNS as soon as it's started — the clip previews as a live card in this chat when it finishes. Pass add_to_timeline:true (or an index) only when they asked for it in the cut; otherwise the user drags it in from the card. Needs the user signed in to Donkey (spends their credits).`,
inputSchema: obj({
prompt: str("The shot to generate — describe motion, subject, and mood"),
aspect: { type: "string", enum: ["16:9", "9:16"], description: "Clip shape (default: the supported shape closest to the project aspect)" },
+ resolution: { type: "string", enum: [...geminiOmniResolutions], description: `Output size (default ${geminiOmniDefaultResolution}; 1080p and 4k are upscaled and cost more per second)` },
+ duration_seconds: num(`Clip length in whole seconds, ${DURATION.min}–${DURATION.max}; omit to let the model pick`),
reference_asset_id: str("One project asset id to reference — an image or video anchors the clip's opening frame; audio folds what it carries into the prompt. Append @seconds to read a video/audio reference at a pinned moment (\"<id>@16.3\")"),
animate_reference: bool("The referenced image IS the opening frame: animate it directly, unchanged, skipping the frame-design step (use when the user says to animate that image)"),
add_to_timeline: bool("Insert the clip on the video track when it lands (default false — it stays on its chat card until the user asks)"),
diff --git a/site/src/cut/components/GeneratePanel.tsx b/site/src/cut/components/GeneratePanel.tsx
index dd35b285..cfc5e322 100644
--- a/site/src/cut/components/GeneratePanel.tsx
+++ b/site/src/cut/components/GeneratePanel.tsx
@@ -28,8 +28,11 @@ import { useVideoGen } from "@/cut/lib/videoGen";
import {
VIDEO_ASPECT_LABEL,
VIDEO_MODELS,
+ VIDEO_RESOLUTION_LABEL,
type VideoAspect,
type VideoModelOption,
+ type VideoResolution,
+ DEFAULT_VIDEO_RESOLUTION,
} from "@/cut/lib/videoModels";
import { cn } from "@/lib/utils";
import { CopyHandlePill, MentionTextarea, RefChips } from "./AssetRefs";
@@ -58,7 +61,7 @@ export function GenerateVideoPanel({ projectId }: { projectId: string }) {
const allJobs = useGenerate((s) => s.jobs);
// Panel renders only — a chat- or scene-owned job lives on its chat card.
const jobs = allJobs.filter((j) => j.projectId === projectId && j.kind === "video" && !j.chatId);
- const { prompt, refs, character, aspect } = useVideoGen();
+ const { prompt, refs, character, aspect, resolution, durationSeconds } = useVideoGen();
const candidates = useRefCandidates();
// OS files handed to the panel — dropped on it or pasted into the prompt —
// attach as references (media files import into the project on the way;
@@ -81,6 +84,21 @@ export function GenerateVideoPanel({ projectId }: { projectId: string }) {
// supports, so a stored pick from another model can never reach the API.
const model = VIDEO_MODELS.find((m) => m.tier === tier) ?? VIDEO_MODELS[0];
const effAspect = nearestAspect(aspect, model.aspects);
+ const effResolution: VideoResolution = model.resolutions.includes(resolution)
+ ? resolution
+ : (model.resolutions.find((r) => r === DEFAULT_VIDEO_RESOLUTION) ?? model.resolutions[0]);
+ const { min: minSeconds, max: maxSeconds } = model.durationSeconds;
+ const effDuration =
+ durationSeconds !== null && durationSeconds >= minSeconds && durationSeconds <= maxSeconds
+ ? durationSeconds
+ : null;
+ const durationOptions = [
+ { value: "auto", label: "Auto length" },
+ ...Array.from({ length: maxSeconds - minSeconds + 1 }, (_, i) => {
+ const s = minSeconds + i;
+ return { value: String(s), label: `${s} seconds` };
+ }),
+ ];
// Default the shape to the project's own orientation when the panel opens,
// same as the image panel (the user can still pick the other one). A custom
@@ -102,6 +120,8 @@ export function GenerateVideoPanel({ projectId }: { projectId: string }) {
const seedRefs = character ? [refFromStockVideo(character)] : all;
void useGenerate.getState().generateVideo(projectId, composed, {
aspect: effAspect,
+ resolution: effResolution,
+ ...(effDuration !== null ? { durationSeconds: effDuration } : {}),
refs: seedRefs,
// The character's poster seed is the point — the same person must
// deliver the line — so free-form prompts alone get the ref rewrite.
@@ -188,8 +208,8 @@ export function GenerateVideoPanel({ projectId }: { projectId: string }) {
</div>
{/* Which model renders the clip — a dropdown, since the catalog grows.
- The model renders the whole clip with audio in one pass and picks
- its own length, so aspect below is the only other knob. */}
+ The model renders the whole clip with audio in one pass; the knobs
+ below are its shape, its output size, and its length. */}
<PillSelect
className="h-7 shrink-0"
title="Model"
@@ -217,6 +237,32 @@ export function GenerateVideoPanel({ projectId }: { projectId: string }) {
onChange={(v) => useVideoGen.getState().setAspect(v)}
/>
+ {/* Output size: 720p is native, the larger sizes are upscaled and
+ cost more per second. */}
+ <PillSelect
+ className="h-7 shrink-0"
+ title="Resolution"
+ value={effResolution}
+ display={VIDEO_RESOLUTION_LABEL[effResolution]}
+ options={model.resolutions.map((r) => ({
+ value: r,
+ label: VIDEO_RESOLUTION_LABEL[r],
+ }))}
+ onChange={(v) => useVideoGen.getState().setResolution(v)}
+ />
+
+ {/* Length: a fixed number of seconds, or the model's own pick. */}
+ <PillSelect
+ className="h-7 shrink-0"
+ title="Length"
+ value={effDuration === null ? "auto" : String(effDuration)}
+ display={effDuration === null ? "Auto" : `${effDuration}s`}
+ options={durationOptions}
+ onChange={(v) =>
+ useVideoGen.getState().setDurationSeconds(v === "auto" ? null : Number(v))
+ }
+ />
+
<Button
className="gen-go w-full shrink-0"
disabled={!prompt.trim() || signedIn === false}
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index db705dff..2e7b2602 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -159,7 +159,7 @@ import { synthesizeMusic } from "./audioGen";
import { composeMusicPrompt } from "./composeGen";
import { stockAssetInDoc } from "./genvideo/docWriter";
import { resolveVoice, synthesizeSpeech, SPEECH_VOICES } from "./tts";
-import { defaultVideoAspects } from "./videoModels";
+import { clampVideoDuration, defaultVideoAspects, videoResolutionOf } from "./videoModels";
import { DUCK_DEFAULT, generateSubtitlesReadout } from "./voiceover";
import {
allFonts,
@@ -2449,10 +2449,17 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
// Audio references carry no picture: they ride in `refs` for the compose
// pass to hear, while anchor rungs take the first ref that has a frame.
const anchors = visualRefs(refs);
- // The model renders the whole clip with audio in one pass and picks its
- // own length (up to ~10s) — aspect is the only knob.
+ // The model renders the whole clip with audio in one pass; the knobs
+ // are the shape, the output size, and an optional length (unset lets
+ // the model pick).
+ const resolution = videoResolutionOf(input.resolution);
+ const durationSeconds = clampVideoDuration(
+ typeof input.duration_seconds === "number" ? input.duration_seconds : undefined
+ );
const baseOpts: Omit<VideoGenOptions, "onDone"> = {
...(input.aspect === "16:9" || input.aspect === "9:16" ? { aspect: input.aspect } : {}),
+ ...(resolution ? { resolution } : {}),
+ ...(durationSeconds !== undefined ? { durationSeconds } : {}),
};
// A one-off render walks the same identity ladder as a scene shot:
// seed image first (the strongest anchor), the picture as an identity
diff --git a/site/src/cut/lib/generate.ts b/site/src/cut/lib/generate.ts
index 161af5eb..e14cd919 100644
--- a/site/src/cut/lib/generate.ts
+++ b/site/src/cut/lib/generate.ts
@@ -25,7 +25,13 @@ import { useGenNotify } from "./genNotify";
import { IMAGE_ASPECTS, useImageGen } from "./imageGen";
import { useEditor } from "./store";
import { mediaSlug, nearestAspect, type MediaAsset, type RenderRecord } from "./types";
-import { aspectFramingNote, defaultVideoAspects, videoModel } from "./videoModels";
+import {
+ DEFAULT_VIDEO_RESOLUTION,
+ aspectFramingNote,
+ defaultVideoAspects,
+ videoModel,
+ type VideoResolution,
+} from "./videoModels";
import { walkLadder, type VideoAttempt } from "./videoLadder";
// AI generation jobs, held outside the panels so a tab switch (which unmounts
@@ -161,6 +167,11 @@ interface GenerateState {
export interface VideoGenOptions {
/** Composition shape; defaults to the project's orientation. */
aspect?: "16:9" | "9:16";
+ /** Output size; defaults to the registry's default (720p). */
+ resolution?: VideoResolution;
+ /** Clip length in whole seconds, within the registry's range; unset lets
+ * the model pick. */
+ durationSeconds?: number;
/** What the render must avoid — the wrong medium's tells, letterbox bars,
* on-screen text. The model has no negative-prompt parameter, so the
* adapter folds this into the prompt as an avoid clause. */
@@ -657,6 +668,8 @@ export const useGenerate = create<GenerateState>((set, get) => {
: {}),
parameters: {
aspectRatio,
+ resolution: opts?.resolution ?? DEFAULT_VIDEO_RESOLUTION,
+ ...(opts?.durationSeconds !== undefined ? { durationSeconds: opts.durationSeconds } : {}),
...(opts?.negativePrompt ? { negativePrompt: opts.negativePrompt } : {}),
},
});
diff --git a/site/src/cut/lib/genvideo/adapters/media.ts b/site/src/cut/lib/genvideo/adapters/media.ts
index 6fde064c..92b0fe63 100644
--- a/site/src/cut/lib/genvideo/adapters/media.ts
+++ b/site/src/cut/lib/genvideo/adapters/media.ts
@@ -37,7 +37,7 @@ import { useGenerate, videoJobSettlement } from "../../generate";
import { enrichAsset } from "../../media";
import { useEditor } from "../../store";
import { DEFAULT_VOICE, synthesizeSpeech } from "../../tts";
-import { videoModel } from "../../videoModels";
+import { clampVideoDuration, videoModel } from "../../videoModels";
import { reportActivity } from "../activity";
import { findRunAsset, stockAssetInDoc } from "../docWriter";
import { buildShotAttempts } from "../shotAttempts";
@@ -59,6 +59,10 @@ async function refsToAssetRefs(projectId: string, refs: RefAsset[]): Promise<Ass
return out;
}
+// Seconds a scene take renders beyond its slot, so the dailies review has a
+// window to choose inside and rounding never leaves the slot short.
+const SLOT_HEADROOM_SECONDS = 1;
+
export function makeImageRole(projectId: string, chatId?: string): ImageRole {
return {
async generate(input) {
@@ -98,8 +102,20 @@ export function makeVideoRole(projectId: string, chatId?: string): VideoRole {
// The shot's stable identity, stamped on every job this render starts so
// a resumed run re-adopts exactly its own in-flight take.
const genKey = input.shotId ? `${projectId}:${input.shotId}` : undefined;
+ // The shot renders at the length its slot needs, rounded up and given a
+ // second of headroom, clamped onto what the model renders. Up covers the
+ // slot: a take shorter than its slot gets speed-stretched to fill it
+ // (fillSlot), which drags the burned-in narration down with it. The
+ // headroom is what the dailies review picks its window inside — the
+ // placement clamps the chosen start to what the take has spare, so a
+ // take cut exactly to the slot leaves the review nothing to choose.
+ const durationSeconds =
+ input.durationSec === undefined
+ ? undefined
+ : clampVideoDuration(Math.ceil(input.durationSec) + SLOT_HEADROOM_SECONDS);
const base = {
aspect: input.aspect,
+ ...(durationSeconds !== undefined ? { durationSeconds } : {}),
...(input.negativePrompt ? { negativePrompt: input.negativePrompt } : {}),
};
// The seed keyframe resolves wherever the run's project lives — the
diff --git a/site/src/cut/lib/videoGen.ts b/site/src/cut/lib/videoGen.ts
index 9b489eef..3f97eada 100644
--- a/site/src/cut/lib/videoGen.ts
+++ b/site/src/cut/lib/videoGen.ts
@@ -3,7 +3,7 @@
import { create } from "zustand";
import { addRefOnce, sameRef, upsertRef, type AssetRef } from "./assetRef";
import type { StockVideo } from "./stock";
-import type { VideoAspect } from "./videoModels";
+import { DEFAULT_VIDEO_RESOLUTION, type VideoAspect, type VideoResolution } from "./videoModels";
// The generate-video panel's state, shared between the stock-video browser
// (which loads a stock clip's saved prompt into it on click) and the always-on
@@ -13,6 +13,11 @@ interface VideoGenState {
prompt: string;
/** The shape the next generation is composed in. */
aspect: VideoAspect;
+ /** The output size of the next generation. */
+ resolution: VideoResolution;
+ /** The length of the next generation in whole seconds; null lets the model
+ * pick. */
+ durationSeconds: number | null;
/** Visual references attached to the next generation (dragged in or picked
* via @name mentions resolved on send). */
refs: AssetRef[];
@@ -28,6 +33,8 @@ interface VideoGenState {
clearCharacter: () => void;
setPrompt: (prompt: string) => void;
setAspect: (aspect: VideoAspect) => void;
+ setResolution: (resolution: VideoResolution) => void;
+ setDurationSeconds: (durationSeconds: number | null) => void;
addRef: (ref: AssetRef) => void;
/** Land a ref's new shape — replaced in place when attached, attached when
* not — how the moment picker lands a pinned timestamp from a chip or a
@@ -39,6 +46,8 @@ interface VideoGenState {
export const useVideoGen = create<VideoGenState>((set) => ({
prompt: "",
aspect: "16:9",
+ resolution: DEFAULT_VIDEO_RESOLUTION,
+ durationSeconds: null,
refs: [],
character: null,
openWith: (prompt) => set({ prompt, refs: [], character: null }),
@@ -46,6 +55,8 @@ export const useVideoGen = create<VideoGenState>((set) => ({
clearCharacter: () => set({ character: null }),
setPrompt: (prompt) => set({ prompt }),
setAspect: (aspect) => set({ aspect }),
+ setResolution: (resolution) => set({ resolution }),
+ setDurationSeconds: (durationSeconds) => set({ durationSeconds }),
addRef: (ref) => set((s) => ({ refs: addRefOnce(s.refs, ref) })),
updateRef: (ref) => set((s) => ({ refs: upsertRef(s.refs, ref) })),
removeRef: (ref) => set((s) => ({ refs: s.refs.filter((r) => !sameRef(r, ref)) })),
diff --git a/site/src/cut/lib/videoModels.ts b/site/src/cut/lib/videoModels.ts
index a7e42e2c..4775ae77 100644
--- a/site/src/cut/lib/videoModels.ts
+++ b/site/src/cut/lib/videoModels.ts
@@ -2,7 +2,13 @@
// Pure data + lookups (no store, no browser), so the genvideo self-test can
// exercise everything that reads constraints from here.
-import { geminiOmniMaxReferenceImages } from "@/lib/inference/gemini-models";
+import {
+ geminiOmniDefaultResolution,
+ geminiOmniDurationSeconds,
+ geminiOmniMaxReferenceImages,
+ geminiOmniResolutions,
+ type GeminiOmniResolution,
+} from "@/lib/inference/gemini-models";
/** The shape the next generated clip is composed in — landscape or portrait. */
export type VideoAspect = "16:9" | "9:16";
@@ -12,11 +18,23 @@ export const VIDEO_ASPECT_LABEL: Record<VideoAspect, string> = {
"9:16": "Portrait (9:16)",
};
+/** The output size a generated clip is rendered at. */
+export type VideoResolution = GeminiOmniResolution;
+
+export const VIDEO_RESOLUTION_LABEL: Record<VideoResolution, string> = {
+ "360p": "360p draft",
+ "720p": "720p",
+ "1080p": "1080p",
+ "4k": "4K",
+};
+
+export const DEFAULT_VIDEO_RESOLUTION: VideoResolution = geminiOmniDefaultResolution;
+
/** A selectable video model. Every render runs on the unified Omni renderer:
* one pass takes text plus optional seed/reference images and returns the
- * whole clip with audio — the model picks the length (up to ~10s of 720p), so
- * there is no duration or resolution knob. Adding a model here is the whole
- * client-side change. */
+ * whole clip with audio at the resolution asked for, at the length asked for
+ * or the model's own pick. Adding a model here is the whole client-side
+ * change. */
export type VideoTier = "omni";
export interface VideoModelOption {
@@ -28,6 +46,10 @@ export interface VideoModelOption {
/** Identity reference images a render accepts alongside the prompt. */
maxReferenceImages: number;
aspects: VideoAspect[];
+ resolutions: readonly VideoResolution[];
+ /** Whole seconds a render can be asked for; a render with no length lets
+ * the model pick. */
+ durationSeconds: { min: number; max: number };
}
export const VIDEO_MODELS: VideoModelOption[] = [
@@ -37,9 +59,26 @@ export const VIDEO_MODELS: VideoModelOption[] = [
model: "Omni Flash",
maxReferenceImages: geminiOmniMaxReferenceImages,
aspects: ["16:9", "9:16"],
+ resolutions: geminiOmniResolutions,
+ durationSeconds: geminiOmniDurationSeconds,
},
];
+/** A requested length clamped onto what the default model renders, in whole
+ * seconds; undefined stays undefined (the model's own pick). */
+export function clampVideoDuration(seconds: number | null | undefined): number | undefined {
+ if (seconds === null || seconds === undefined || !Number.isFinite(seconds)) return undefined;
+ const { min, max } = VIDEO_MODELS[0].durationSeconds;
+ return Math.max(min, Math.min(max, Math.round(seconds)));
+}
+
+/** A requested resolution if the default model renders it. */
+export function videoResolutionOf(value: unknown): VideoResolution | undefined {
+ return (VIDEO_MODELS[0].resolutions as readonly string[]).includes(String(value))
+ ? (value as VideoResolution)
+ : undefined;
+}
+
/** The registry entry for a tier — the single source of truth for what that
* model supports. Any code that generates video (the panel, the scene pipeline)
* reads its constraints from here, so swapping models is one edit in this file. */
diff --git a/site/src/cut/server/ai/catalog.ts b/site/src/cut/server/ai/catalog.ts
index 901ba37b..dea42a15 100644
--- a/site/src/cut/server/ai/catalog.ts
+++ b/site/src/cut/server/ai/catalog.ts
@@ -10,6 +10,11 @@
*/
import { AI_PANEL_TOOLS } from "@/cut/components/AiPanel.tools";
+import {
+ geminiOmniDefaultResolution,
+ geminiOmniDurationSeconds,
+ geminiOmniResolutions,
+} from "@/lib/inference/gemini-models";
import { TEXT_LAYOUT_IDS, TEXT_LAYOUT_NOTES } from "@/cut/lib/textCompose";
import { textLookCatalog } from "@/cut/lib/textLooks";
import {
@@ -346,7 +351,7 @@ Three ways to get footage the user doesn't have: bundled stock (local, free), a
Stock: stock_search browses the bundled catalogs — footage clips and images in 8 categories plus ~20 UGC talking characters — matching prompts, categories, and tags. stock_add imports an item into the project as a chat card; add_to_timeline:true (or a start) also drops it on the timeline when the user asked. In the UI these live in the Video and Image tabs beside the generators; clicking a stock tile seeds the generate panel with its prompt.
Generation:
- generate_image(prompt, aspect?, resolution?, reference_asset_ids?, add_to_timeline?, index?): the hosted image model renders the prompt at 16:9, 9:16, or 1:1 (default: the supported shape closest to the project aspect) and 1K/2K/4K. The still previews in the chat; placed (add_to_timeline:true or an index) it rides video track 0 as a still clip (8s default, stretchable). Great for a cover/hook frame (index 0), a background, or a b-roll still.
-- generate_video(prompt, aspect?, reference_asset_id?, add_to_timeline?, index?): the hosted video model renders a short clip with audio in one pass — 720p, up to ~10s, the model picks the length (no duration knob; the user trims the clip on the timeline). With reference_asset_id it stages: the image model designs the opening frame from the reference first (that still previews in the chat), then the video model animates that exact frame. The render takes a minute or two, so the tool returns once it's started and the clip previews in the chat when it finishes (on the timeline only when asked) — tell the user it's rendering. Don't call it again for the same shot while one is in flight.
+- generate_video(prompt, aspect?, resolution?, duration_seconds?, reference_asset_id?, add_to_timeline?, index?): the hosted video model renders a short clip with audio in one pass — ${geminiOmniResolutions.join("/")} (${geminiOmniDefaultResolution} default; the upscaled sizes cost more per second), ${geminiOmniDurationSeconds.min}–${geminiOmniDurationSeconds.max}s when duration_seconds is set and the model's own length otherwise (the user trims the clip on the timeline). With reference_asset_id it stages: the image model designs the opening frame from the reference first (that still previews in the chat), then the video model animates that exact frame. The render takes a minute or two, so the tool returns once it's started and the clip previews in the chat when it finishes (on the timeline only when asked) — tell the user it's rendering. Don't call it again for the same shot while one is in flight.
- Every render's live status is \`renders\` in editor_state (running with elapsed seconds, done with its asset id, failed with the error) — report render progress from there, never from memory. When the user's ask depends on a running render ("assemble the clips", "add it when it's done"), call wait_for_renders and finish the job in the same turn — never hand the waiting to the user.
- generate_character_video(character_id, line, …): a stock talking character delivers a line to camera, same async render as generate_video. Characters come from stock_search kind:"character" — each has a persona; you write the line (chat deliverable rules apply: asked for "a script", write it in chat first).
References: users attach media to their message or the generate panels; project asset ids (see \`media\` in editor_state, including attachments — OS drops become project assets) pass through reference_asset_ids / reference_asset_id. Images draw from any number of references; video stages one reference into a designed opening frame and animates it; generate_music matches the sound of an audio reference or the mood of a video/image one. When the user says "use this clip/image/song" or "match this", pass the reference — don't just describe it in the prompt.
diff --git a/site/src/lib/inference/adapters/gemini-omni-video.test.ts b/site/src/lib/inference/adapters/gemini-omni-video.test.ts
new file mode 100644
index 00000000..c8cf41aa
--- /dev/null
+++ b/site/src/lib/inference/adapters/gemini-omni-video.test.ts
@@ -0,0 +1,65 @@
+import { describe, expect, test } from "bun:test";
+import { createGeminiOmniVideoAssetProvider } from "./gemini-omni-video";
+import type { AssetGenerationRequest } from "@/lib/inference/providers";
+
+// The unit math the credit preflight holds a render to: seconds of
+// 360p-equivalent output from the requested length and resolution.
+const provider = createGeminiOmniVideoAssetProvider(
+ {
+ GOOGLE_APPLICATION_CREDENTIALS_JSON: JSON.stringify({
+ project_id: "p",
+ client_email: "[email protected]",
+ private_key: "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n",
+ }),
+ },
+ () => ({ interactions: {} as never }),
+);
+
+const request = (parameters: Record<string, unknown>): AssetGenerationRequest =>
+ ({ kind: "video", prompt: "a clip", parameters }) as AssetGenerationRequest;
+
+// The route hands the counter a resolver that reads seed frames and reference
+// pictures out of storage. Counting seconds never needs them, so a call to it
+// is a bug: it would pull whole objects before the balance clears.
+let resolverCalls = 0;
+const resolveInputs = async (): Promise<AssetGenerationRequest> => {
+ resolverCalls++;
+ return request({});
+};
+const countFor = (parameters: Record<string, unknown>) =>
+ provider.assetGenerationCountFor!(request(parameters), resolveInputs);
+
+describe("Omni generation units", () => {
+ test("scales with length and resolution", async () => {
+ expect(await countFor({ durationSeconds: 10, resolution: "720p" })).toBe(30);
+ expect(await countFor({ durationSeconds: 4, resolution: "360p" })).toBe(4);
+ expect(await countFor({ durationSeconds: 10, resolution: "4k" })).toBe(90);
+ expect(await countFor({ durationSeconds: 6, resolution: "1080p" })).toBe(27);
+ });
+
+ test("an unset length bills the model's ceiling at the default size", async () => {
+ expect(await countFor({})).toBe(30);
+ });
+
+ test("refuses a size or length the model does not render", async () => {
+ const failure = async (parameters: Record<string, unknown>) => {
+ try {
+ await countFor(parameters);
+ } catch (error) {
+ return String(error);
+ }
+ return "";
+ };
+ expect((await failure({ resolution: "8k" })).includes("resolution")).toBe(true);
+ expect((await failure({ resolution: 1080 })).includes("resolution")).toBe(true);
+ expect((await failure({ resolution: "" })).includes("resolution")).toBe(true);
+ expect((await failure({ durationSeconds: 2 })).includes("length")).toBe(true);
+ expect((await failure({ durationSeconds: 5.5 })).includes("length")).toBe(true);
+ });
+
+ test("counts without reading the request's media", async () => {
+ resolverCalls = 0;
+ await countFor({ durationSeconds: 8, resolution: "720p" });
+ expect(resolverCalls).toBe(0);
+ });
+});
diff --git a/site/src/lib/inference/adapters/gemini-omni-video.ts b/site/src/lib/inference/adapters/gemini-omni-video.ts
index b2097467..fcbf7491 100644
--- a/site/src/lib/inference/adapters/gemini-omni-video.ts
+++ b/site/src/lib/inference/adapters/gemini-omni-video.ts
@@ -11,7 +11,16 @@ import {
type AdapterEnvironment,
} from "@/lib/inference/adapters/gemini-client";
import { providerCreditPricing } from "@/lib/credits/provider-pricing";
-import { geminiOmniMaxReferenceImages, geminiOmniModels } from "@/lib/inference/gemini-models";
+import {
+ geminiOmniDefaultResolution,
+ geminiOmniDurationSeconds,
+ geminiOmniMaxReferenceImages,
+ geminiOmniModels,
+ geminiOmniResolutions,
+ geminiOmniTokensPerSecond,
+ geminiOmniUnitTokens,
+ type GeminiOmniResolution,
+} from "@/lib/inference/gemini-models";
import { ensureConfigured } from "@/lib/inference/http";
import { isJsonObject, toJsonObject } from "@/lib/inference/json";
import {
@@ -28,10 +37,10 @@ import {
// The video generation provider (Gemini Omni Flash) on the Interactions API:
// one call takes text plus optional seed/reference images and renders a clip
-// with audio in a single pass. It submits a background interaction and the
+// with audio in a single pass at the resolution asked for, at the length asked
+// for or the model's own pick. It submits a background interaction and the
// caller polls refresh until it lands; the provider id keeps that refresh
-// routing separate from the synchronous image adapter. The model decides the
-// clip length (up to ~10s of 720p) — there is no duration knob to pass.
+// routing separate from the synchronous image adapter.
const providerID = "gemini-omni";
// Verified live against Vertex: `response_modalities` is rejected outright
@@ -91,8 +100,62 @@ export function createGeminiOmniVideoAssetProvider(
return model;
}
+ // The output size and length a request asks for, validated against the
+ // registry: an unknown resolution or an out-of-range length is a caller bug,
+ // refused before anything bills.
+ function renderShape(parameters: JsonObject): {
+ resolution: GeminiOmniResolution;
+ durationSeconds: number | undefined;
+ } {
+ const asked = parameters.resolution;
+ const resolutionRaw =
+ asked === undefined || asked === null ? geminiOmniDefaultResolution : asked;
+ if (
+ typeof resolutionRaw !== "string" ||
+ !(geminiOmniResolutions as readonly string[]).includes(resolutionRaw)
+ ) {
+ throw new InferenceProviderError("Unsupported video resolution.", {
+ statusCode: 400,
+ code: "unsupported_video_resolution",
+ details: { resolution: resolutionRaw, supported: [...geminiOmniResolutions] },
+ });
+ }
+ const durationRaw = parameters.durationSeconds;
+ let durationSeconds: number | undefined;
+ if (durationRaw !== undefined && durationRaw !== null) {
+ const { min, max } = geminiOmniDurationSeconds;
+ if (
+ typeof durationRaw !== "number" ||
+ !Number.isInteger(durationRaw) ||
+ durationRaw < min ||
+ durationRaw > max
+ ) {
+ throw new InferenceProviderError(
+ `Video length must be a whole number of seconds from ${min} to ${max}.`,
+ { statusCode: 400, code: "unsupported_video_duration", details: { durationSeconds: durationRaw } },
+ );
+ }
+ durationSeconds = durationRaw;
+ }
+ return { resolution: resolutionRaw as GeminiOmniResolution, durationSeconds };
+ }
+
+ // Billing units for a request: seconds of 360p-equivalent output, from the
+ // length (the model's ceiling when unset) times the resolution's tokens per
+ // second. The route holds the balance to this count before the submit, and
+ // the submit bills the same count.
+ function generationUnits(parameters: JsonObject): number {
+ const { resolution, durationSeconds } = renderShape(parameters);
+ const seconds = durationSeconds ?? geminiOmniDurationSeconds.max;
+ return Math.max(
+ 1,
+ Math.ceil((seconds * geminiOmniTokensPerSecond[resolution]) / geminiOmniUnitTokens),
+ );
+ }
+
async function generateAsset({
request,
+ generationCount,
}: AssetGenerationProviderRequest): Promise<AssetGenerationProviderResult> {
ensureConfigured(configured);
@@ -152,6 +215,17 @@ export function createGeminiOmniVideoAssetProvider(
? "reference_to_video"
: "text_to_video";
const aspectRatio = stringValue(parameters.aspectRatio);
+ const { resolution, durationSeconds } = renderShape(parameters);
+ // The route settles the count through assetGenerationCountFor and the
+ // preflight holds the balance to it; a submit that recounted here would
+ // bill a charge no preflight covered.
+ const units = Math.floor(generationCount ?? 0);
+ if (units < 1) {
+ throw new InferenceProviderError("The video charge was not counted before submit.", {
+ statusCode: 500,
+ code: "video_charge_uncounted",
+ });
+ }
const client = clientFactory(clientConfig.options);
let interaction: Interaction;
@@ -164,6 +238,8 @@ export function createGeminiOmniVideoAssetProvider(
generation_config: { video_config: { task } } as never,
response_format: {
type: "video",
+ resolution,
+ ...(durationSeconds !== undefined ? { duration: `${durationSeconds}s` } : {}),
...(aspectRatio === "16:9" || aspectRatio === "9:16" ? { aspect_ratio: aspectRatio } : {}),
} as never,
background: true,
@@ -178,11 +254,12 @@ export function createGeminiOmniVideoAssetProvider(
return settled;
}
- // In progress: the clip is committed, so the flat clip price bills now —
- // generationCount is the billing unit (provider-pricing.ts), since the
- // async submit carries no token counts and the polls that follow are free.
- // Only the submit carries it; a refresh poll's in-progress result must not.
- return { ...inProgressResult(interaction.id, model), usage: { generationCount: 1 } };
+ // In progress: the clip is committed, so its price bills now — the unit
+ // count the requested length and resolution add up to (provider-pricing.ts),
+ // since the async submit carries no token counts and the polls that follow
+ // are free. Only the submit carries it; a refresh poll's in-progress result
+ // must not.
+ return { ...inProgressResult(interaction.id, model), usage: { generationCount: units } };
}
async function refreshAsset(
@@ -287,6 +364,10 @@ export function createGeminiOmniVideoAssetProvider(
capabilities: ["video"],
listModels,
assetModelFor: (request) => resolveModel(request.model),
+ // Seconds of output, read off the parameters alone: the resolver goes
+ // untouched so a seed frame never leaves storage before the balance clears.
+ assetGenerationCountFor: async (request) =>
+ generationUnits(toJsonObject(request.parameters ?? {})),
generateAsset,
refreshAsset,
};