The VS Code of video editing, built for agents.
TypeScript
Enhance model configuration with resolution support
konstantin-paulus8616127c500a
diff --git a/apps/web/src/components/genai/config.ts b/apps/web/src/components/genai/config.tsindex 216461df..3f239004 100644--- a/apps/web/src/components/genai/config.ts+++ b/apps/web/src/components/genai/config.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import type { ModelId } from "@diffusionstudio/api-contract";+import type { ModelId, Resolution } from "@diffusionstudio/api-contract";/*** What each model of the API makes. Keyed by the contract's model ids, so a@@ -11,18 +11,23 @@ import type { ModelId } from "@diffusionstudio/api-contract";* the text models (transcripts, analysis) never land on the canvas.*/export const MODEL_MODES: Record<ModelId, PromptMode | "TEXT"> = {- "gpt-image-2": "IMAGE",+ "gpt-image-2.5-sunburst": "IMAGE","nano-banana-2": "IMAGE","nano-banana-pro": "IMAGE",- "seedream-4.5": "IMAGE",+ "seedream-5.0-pro": "IMAGE",+ "grok-imagine-image-2.0": "IMAGE",+ "krea-2-large": "IMAGE",+ "flux-2-pro": "IMAGE","flux-2-klein": "IMAGE","remove-background": "IMAGE","upscale-image": "IMAGE","kling-3-pro": "VIDEO","kling-o3-pro": "VIDEO",- "wan-2.6": "VIDEO",- "hailuo-3-max": "VIDEO",- "seedance-2.0": "VIDEO",+ "wan-3.0": "VIDEO",+ "minimax-h3": "VIDEO",+ "seedance-2.5": "VIDEO",+ "flux-3-video": "VIDEO",+ "grok-imagine-video-1.5": "VIDEO","veo-3.1": "VIDEO","veo-3.1-fast": "VIDEO","upscale-video": "VIDEO",@@ -63,13 +68,19 @@ export const ASPECT_RATIO_OPTIONS: { value: AspectRatio; label: string; triggerL{ value: "3:4", label: "3:4 \u00B7 Tall", triggerLabel: "3:4", icon: "aspect-ratio-3-4" },];-export const PROMPT_INPUT_RESOLUTION_OPTIONS = [+/** Output resolutions, highest first; a model offers some of them. */+export const RESOLUTION_OPTIONS: { value: Resolution; label: string }[] = [{ value: "4K", label: "4K" },- { value: "1440p", label: "1440p" },+ { value: "2K", label: "2K" },+ { value: "1K", label: "1K" },{ value: "1080p", label: "1080p" },{ value: "720p", label: "720p" },+ { value: "480p", label: "480p" },];+/** The resolution a mode's models start at, when they offer it. */+export const DEFAULT_RESOLUTION: Partial<Record<PromptMode, Resolution>> = { IMAGE: "1K", VIDEO: "720p" };+export const PROMPT_INPUT_VOICE_OPTIONS = [{value: "CwhRBWXzGAHq8TQ4Fs17",@@ -255,6 +266,10 @@ export interface ModelOption {counts?: number[];/** Seconds. */durations?: number[];+ /** Output resolutions; a single one when the model offers no choice. */+ resolutions?: Resolution[];+ /** Resolutions only some durations allow, with those durations. */+ resolutionDurations?: Partial<Record<Resolution, number[]>>;voices?: VoiceOption[];/** How many reference images the model takes. */references?: number;@@ -276,14 +291,25 @@ export const MODEL_OPTIONS: ModelOption[] = [icon: "large-bfl",description: "Low budget, high quality, fast.",references: 4,+ resolutions: ["1K"],},{...image,- id: "gpt-image-2",- name: "GPT Image 2",+ id: "flux-2-pro",+ name: "FLUX.2 Pro",+ icon: "large-bfl",+ description: "Frontier quality, consistent references.",+ references: 8,+ resolutions: ["1K"],+ },+ {+ ...image,+ id: "gpt-image-2.5-sunburst",+ name: "GPT Image 2.5 Sunburst",icon: "large-openai",- description: "Flexible sizes up to 4K, true aspect ratios.",+ description: "Precise detail, strong text, true edits.",references: 16,+ resolutions: ["1K"],},{...image,@@ -292,6 +318,7 @@ export const MODEL_OPTIONS: ModelOption[] = [icon: "large-google",description: "Fast, high quality, flexible edits.",references: 14,+ resolutions: ["1K", "2K", "4K"],},{...image,@@ -300,14 +327,35 @@ export const MODEL_OPTIONS: ModelOption[] = [icon: "large-google",description: "Pro control, readable text, ultra consistent.",references: 14,+ resolutions: ["1K", "2K", "4K"],},{...image,- id: "seedream-4.5",- name: "Seedream 4.5",+ id: "seedream-5.0-pro",+ name: "Seedream 5.0 Pro",icon: "large-bytedance",- description: "Up to 4K, multi image edits, strong text.",+ description: "Lifelike scenes, precise multi image edits.",references: 14,+ resolutions: ["1K", "2K"],+ },+ {+ ...image,+ id: "grok-imagine-image-2.0",+ name: "Grok Imagine 2.0",+ icon: "ai-generate",+ description: "Fast, vivid, photoreal images.",+ references: 3,+ resolutions: ["1K", "2K"],+ },+ {+ ...image,+ id: "krea-2-large",+ name: "Krea 2 Large",+ icon: "ai-generate",+ description: "Raw, textured, artistic looks.",+ references: 1,+ aspectRatios: ["16:9", "9:16", "1:1", "4:3"],+ resolutions: ["1K"],},{...video,@@ -318,6 +366,7 @@ export const MODEL_OPTIONS: ModelOption[] = [frames: ["start", "end"],durations: seconds(3, 15),aspectRatios: ["16:9", "9:16", "1:1"],+ resolutions: ["720p"],},{...video,@@ -327,16 +376,18 @@ export const MODEL_OPTIONS: ModelOption[] = [description: "Multi-modal reasoning, strong scenes.",frames: ["start"],durations: seconds(3, 15),+ resolutions: ["1080p"],},{...video,- id: "seedance-2.0",- name: "Seedance 2.0",+ id: "seedance-2.5",+ name: "Seedance 2.5",icon: "large-bytedance",- description: "Rich motion, lip-synced audio and SFX.",+ description: "Rich motion, lip-synced audio, up to 30s.",frames: ["start", "end"],- durations: seconds(4, 15),+ durations: seconds(4, 30),aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ resolutions: ["480p", "720p"],},{...video,@@ -347,6 +398,8 @@ export const MODEL_OPTIONS: ModelOption[] = [frames: ["start", "end"],durations: [4, 6, 8],aspectRatios: ["16:9", "9:16"],+ resolutions: ["720p", "1080p"],+ resolutionDurations: { "1080p": [8] },},{...video,@@ -357,26 +410,52 @@ export const MODEL_OPTIONS: ModelOption[] = [frames: ["start", "end"],durations: [4, 6, 8],aspectRatios: ["16:9", "9:16"],+ resolutions: ["720p", "1080p"],+ resolutionDurations: { "1080p": [8] },},{...video,- id: "wan-2.6",- name: "Wan 2.6",+ id: "flux-3-video",+ name: "FLUX.3 Video",+ icon: "large-bfl",+ description: "Sharp detail with native audio.",+ frames: ["start", "end"],+ durations: seconds(5, 20),+ aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ resolutions: ["720p", "1080p"],+ },+ {+ ...video,+ id: "wan-3.0",+ name: "Wan 3.0",icon: "large-wan",- description: "Expressive motion, stylized aesthetics.",+ description: "Expressive motion, clips up to 30s.",frames: ["start"],- durations: [5, 10],- aspectRatios: ["16:9", "9:16"],+ durations: seconds(2, 30),+ aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ resolutions: ["480p", "720p", "1080p"],},{...video,- id: "hailuo-3-max",- name: "Hailuo 3 Max",+ id: "minimax-h3",+ name: "MiniMax H3",icon: "large-hailuo",- description: "Smooth motion, cinematic lighting.",+ description: "Native 2K with built-in audio.",frames: ["start", "end"],durations: seconds(5, 15),aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ resolutions: ["2K"],+ },+ {+ ...video,+ id: "grok-imagine-video-1.5",+ name: "Grok Imagine Video 1.5",+ icon: "ai-generate",+ description: "Quick clips from text or a frame.",+ frames: ["start"],+ durations: seconds(1, 15),+ aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ resolutions: ["480p", "720p", "1080p"],},{id: "elevenlabs-v3",@@ -407,3 +486,7 @@ export const modelOptions = (mode: PromptMode) => MODEL_OPTIONS.filter((option)/** The prompt box's option for `id`; undefined for a model it doesn't make requests with (tools, text). */export const modelOption = (id: ModelId) => MODEL_OPTIONS.find((option) => option.id === id);++/** The durations `option` takes at `resolution`: fewer for a resolution only some durations allow. */+export const durationsAt = (option: ModelOption, resolution: Resolution | undefined) =>+ (resolution && option.resolutionDurations?.[resolution]) || option.durations;diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsxindex f4a04849..53f43562 100644--- a/apps/web/src/components/genai/prompt-input.tsx+++ b/apps/web/src/components/genai/prompt-input.tsx@@ -33,9 +33,10 @@ import {import {ASPECT_RATIO_OPTIONS,PROMPT_INPUT_MODE_OPTIONS,+ RESOLUTION_OPTIONS,+ durationsAt,modelOption,modelOptions,- // PROMPT_INPUT_RESOLUTION_OPTIONStype AspectRatio,type PromptMode,} from "./config";@@ -52,7 +53,7 @@ import { droppedFiles, importFiles, pickFiles } from "@/engine/asset-actions";import { useMediaSelection } from "./selection";import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";-import type { ModelId } from "@diffusionstudio/api-contract";+import type { ModelId, Resolution } from "@diffusionstudio/api-contract";import type { AssetCache } from "@diffusionstudio/assets";import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";@@ -98,8 +99,11 @@ export function PromptInput(props: PromptInputProps) {ASPECT_RATIO_OPTIONS.filter((o) => option().aspectRatios?.includes(o.value)),);const countOptions = createMemo(() => (option().counts ?? []).map((n) => ({ value: String(n), label: String(n) })));+ const resolutionOptions = createMemo(() =>+ RESOLUTION_OPTIONS.filter((o) => option().resolutions?.includes(o.value)),+ );const durationOptions = createMemo(() =>- (option().durations ?? []).map((seconds) => ({ value: String(seconds), label: `${seconds}s` })),+ (durationsAt(option(), config().resolution) ?? []).map((seconds) => ({ value: String(seconds), label: `${seconds}s` })),);// Pictures on the canvas are offered as references and as video frames:@@ -357,6 +361,7 @@ export function PromptInput(props: PromptInputProps) {const aspectRatioAccessor: Accessor<string> = () => config().aspectRatio ?? "";const countAccessor: Accessor<string> = () => String(config().count ?? "");const durationAccessor: Accessor<string> = () => String(config().duration ?? "");+ const resolutionAccessor: Accessor<string> = () => config().resolution ?? "";const modelAccessor: Accessor<string> = () => config().model;const voiceAccessor: Accessor<string> = () => config().voice ?? "";@@ -541,14 +546,16 @@ export function PromptInput(props: PromptInputProps) {triggerIcon="aspect-ratio-16-9"/></Show>- {/* <PromptInputCompactMenu- aria-label="Select resolution"- menuLabel="Resolution"- value={resolution}- options={PROMPT_INPUT_RESOLUTION_OPTIONS}- onChange={setResolution}- triggerIcon="resolution"- /> */}+ <Show when={resolutionOptions().length > 0}>+ <PromptInputCompactMenu+ aria-label="Select resolution"+ menuLabel="Resolution"+ value={resolutionAccessor}+ options={resolutionOptions()}+ onChange={(v) => setConfig(fitToModel({ ...config(), resolution: v as Resolution }))}+ triggerIcon="resolution"+ />+ </Show><Show when={countOptions().length > 0}><PromptInputCompactMenuaria-label="Select amount of variants"diff --git a/apps/web/src/components/genai/requests.ts b/apps/web/src/components/genai/requests.tsindex 9fbf2c19..44a0beb3 100644--- a/apps/web/src/components/genai/requests.ts+++ b/apps/web/src/components/genai/requests.ts@@ -10,10 +10,10 @@import { uploadAsset } from "@/engine/generate";import { AUDIO_SIZE } from "@/engine/insert-asset";-import { ASPECT_RATIO_DIMENSIONS, MODEL_MODES, modelOption, modelOptions } from "./config";+import { ASPECT_RATIO_DIMENSIONS, DEFAULT_RESOLUTION, MODEL_MODES, durationsAt, modelOption, modelOptions } from "./config";import type { AssetLibrary } from "@diffusionstudio/assets";-import type { AssetRef, GenerateRequest, GenerateRequestInput, ModelId } from "@diffusionstudio/api-contract";+import type { AssetRef, GenerateRequest, GenerateRequestInput, ModelId, Resolution } from "@diffusionstudio/api-contract";import type { GenerationOutput } from "@/engine/generate";import type { AspectRatio, PromptMode } from "./config";import type { GenerationConfig } from "./types";@@ -31,11 +31,12 @@ const DEFAULT_SIZE = ASPECT_RATIO_DIMENSIONS["16:9"]!;/*** What a request puts on the canvas, read off the request itself — so a past* job (`job.request`) is placed the way the prompt that made it was. Text- * models have nothing to place.+ * models have nothing to place, and neither have models the API has retired+ * (a past job's `seedream-4.5`): they can't be run again.*/export function outputOf(request: { model: ModelId; count?: number; aspectRatio?: string }): GenerationOutput | undefined {- const mode = MODEL_MODES[request.model];- if (mode === "TEXT") return undefined;+ const mode = MODEL_MODES[request.model] as PromptMode | "TEXT" | undefined;+ if (!mode || mode === "TEXT") return undefined;return {kind: mode === "IMAGE" ? "image" : mode === "VIDEO" ? "video" : "audio",@@ -68,17 +69,20 @@ function pick<T>(value: T | undefined, allowed: readonly T[] | undefined, fallba* model allows is kept, one it allows other values of falls back to a* default, and one it doesn't take is dropped. Every config the prompt box* holds has been through here, so its requests are within the model's bounds.+ * The resolution is fitted first: it may narrow the durations (Veo's 1080p).*/export function fitToModel(config: GenerationConfig): GenerationConfig {const option = modelOption(config.model) ?? modelOptions(config.mode)[0]!;const frame = (frame: "start" | "end", id: string | undefined) => (option.frames?.includes(frame) ? id : undefined);+ const resolution = pick(config.resolution, option.resolutions, DEFAULT_RESOLUTION[option.mode]);return {mode: option.mode,model: option.id,prompt: config.prompt,aspectRatio: pick(config.aspectRatio, option.aspectRatios, "16:9"),count: pick(config.count, option.counts, 1),- duration: pick(config.duration, option.durations, 6),+ duration: pick(config.duration, durationsAt(option, resolution), 6),+ resolution,voice: pick(config.voice, option.voices?.map((voice) => voice.value)),imageRefIds: option.references ? config.imageRefIds?.slice(0, option.references) : undefined,startFrameImageId: frame("start", config.startFrameImageId),@@ -93,14 +97,14 @@ export function createDefaultConfig(mode: PromptMode, prompt = ""): GenerationCo/** The request `config` makes, its library inputs uploaded. Settings the config leaves unset are left out. */export async function toRequest(library: AssetLibrary, config: GenerationConfig): Promise<GenerateRequestInput> {- const { model, prompt, aspectRatio, count, duration, voice } = config;+ const { model, prompt, aspectRatio, count, duration, resolution, voice } = config;const [startFrame, endFrame, ...images] = await refs(library, [config.startFrameImageId,config.endFrameImageId,...(config.imageRefIds ?? []),]);const uploaded = images.filter((ref) => ref !== undefined);- const request = { model, prompt, aspectRatio, count, duration, voice, startFrame, endFrame, images: uploaded.length ? uploaded : undefined };+ const request = { model, prompt, aspectRatio, count, duration, resolution, voice, startFrame, endFrame, images: uploaded.length ? uploaded : undefined };return Object.fromEntries(Object.entries(request).filter(([, value]) => value !== undefined)) as GenerateRequestInput;}@@ -113,7 +117,7 @@ export function toConfig(request: GenerateRequest): GenerationConfig | undefinedconst option = modelOption(request.model);if (!option || !("prompt" in request) || request.prompt === undefined) return undefined;// The settings a request of the prompt box's models may carry; `fitToModel` checks them.- const settings = request as { aspectRatio?: string; count?: number; duration?: number; voice?: string };+ const settings = request as { aspectRatio?: string; count?: number; duration?: number; resolution?: Resolution; voice?: string };return fitToModel({mode: option.mode,model: option.id,@@ -121,6 +125,7 @@ export function toConfig(request: GenerateRequest): GenerationConfig | undefinedaspectRatio: settings.aspectRatio as AspectRatio | undefined,count: settings.count,duration: settings.duration,+ resolution: settings.resolution,voice: settings.voice,});}diff --git a/apps/web/src/components/genai/types.ts b/apps/web/src/components/genai/types.tsindex df7c05ef..d1608321 100644--- a/apps/web/src/components/genai/types.ts+++ b/apps/web/src/components/genai/types.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import type { ModelId } from "@diffusionstudio/api-contract";+import type { ModelId, Resolution } from "@diffusionstudio/api-contract";import type { AspectRatio, PromptMode } from "./config";/**@@ -18,6 +18,7 @@ export interface GenerationConfig {count?: number;/** Seconds. */duration?: number;+ resolution?: Resolution;voice?: string;imageRefIds?: string[];startFrameImageId?: string;