The VS Code of video editing, built for agents.
TypeScript
Refactor generative AI components and consolidate generation logic
konstantin-paulus42753381fff9
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsxindex edb70d2b..79f21405 100644--- a/apps/web/src/components/genai/action-bar.tsx+++ b/apps/web/src/components/genai/action-bar.tsx@@ -15,10 +15,7 @@ import {} from "@/components/ui/dropdown-menu";import { createMemo, Show } from "solid-js";import { useGenerationRecords } from "./use-generation-records";-import { useGenerateImage } from "./use-generate-image";-import { useGenerateVideo } from "./use-generate-video";-import { useGenerateVoice } from "./use-generate-voice";-import { useGenerateAudio } from "./use-generate-audio";+import { useGenerate } from "./use-generate";import { useAutoCaptions } from "./use-auto-captions";import { useMediaSelection } from "./selection";import { useTransforms } from "./use-transforms";@@ -32,48 +29,28 @@ interface ActionBarProps {}export function ActionBar(props: ActionBarProps) {- const { imageNodes, videoNodes } = useMediaSelection();- const { isOn, toggle } = useTransforms();+ const { imageMedia, videoMedia } = useMediaSelection();+ const { run: runTransform } = useTransforms();- const { generate: generateImage } = useGenerateImage();- const { generate: generateVideo } = useGenerateVideo();- const { generate: generateVoice } = useGenerateVoice();- const { generate: generateAudio } = useGenerateAudio();+ const { rerun } = useGenerate();const { generate: autoCaptions, hasScene } = useAutoCaptions();- const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();+ const { isGenerated, totalCredits, firstJob, firstConfig } = useGenerationRecords();- const isImage = createMemo(() => imageNodes().length > 0);- const isVideo = createMemo(() => videoNodes().length > 0);+ const isImage = createMemo(() => imageMedia().length > 0);+ const isVideo = createMemo(() => videoMedia().length > 0);const visible = createMemo(() => {return isImage() || isVideo() || hasScene();})const handleRerun = () => {- const config = firstConfig();- if (!config) {- toast("No generation config found", { description: "This asset wasn't generated with a prompt." });+ const job = firstJob();+ if (!job) {+ toast("No generation found", { description: "This asset wasn't generated with a prompt." });return;}-- const promise = (() => {- switch (config.mode) {- case "IMAGE":- return generateImage(config);- case "VIDEO":- return generateVideo(config);- case "VOICE":- return generateVoice(config);- case "AUDIO":- return generateAudio(config);- }- })();-- promise.catch((err) => {- toast("Rerun failed", {- description: err instanceof Error ? err.message : String(err),- });- });++ rerun(job.request);};const handleEditWithPrompt = () => {@@ -108,8 +85,7 @@ export function ActionBar(props: ActionBarProps) {<Buttonvariant="ghost"class="gap-0 pl-0.5 text-muted-foreground"- classList={{ "text-foreground": isOn("removeBackground") }}- onClick={() => toggle("removeBackground")}+ onClick={() => runTransform("removeBackground")}><Icon name="ai-generate" />Remove background@@ -117,8 +93,7 @@ export function ActionBar(props: ActionBarProps) {<Buttonvariant="ghost"class="gap-0 pl-0.5 text-muted-foreground"- classList={{ "text-foreground": isOn("upscale") }}- onClick={() => toggle("upscale")}+ onClick={() => runTransform("upscale")}><Icon name="arrow-scale" />Upscale@@ -172,8 +147,7 @@ export function ActionBar(props: ActionBarProps) {<Buttonvariant="ghost"class="gap-0 pl-0.5 text-muted-foreground"- classList={{ "text-foreground": isOn("addAudio") }}- onClick={() => toggle("addAudio")}+ onClick={() => runTransform("addAudio")}><Icon name="generate-audio" />Add audio@@ -181,8 +155,7 @@ export function ActionBar(props: ActionBarProps) {<Buttonvariant="ghost"class="gap-0 pl-0.5 text-muted-foreground"- classList={{ "text-foreground": isOn("upscale") }}- onClick={() => toggle("upscale")}+ onClick={() => runTransform("upscale")}><Icon name="arrow-scale" />Upscalediff --git a/apps/web/src/components/genai/config.ts b/apps/web/src/components/genai/config.tsindex c9092c43..acc00c83 100644--- a/apps/web/src/components/genai/config.ts+++ b/apps/web/src/components/genai/config.ts@@ -2,8 +2,42 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+import type { ModelId } from "@diffusionstudio/api-contract";++/**+ * What each model of the API makes. Keyed by the contract's model ids, so a+ * model added to (or dropped from) the API fails the type check here first.+ * The modes are the prompt box's; the tools are run from the action bar, and+ * the text models (transcripts, analysis) never land on the canvas.+ */+export const MODEL_MODES: Record<ModelId, PromptMode | "TEXT"> = {+ "gpt-image-2": "IMAGE",+ "nano-banana-2": "IMAGE",+ "nano-banana-pro": "IMAGE",+ "seedream-4.5": "IMAGE",+ "flux-2-klein": "IMAGE",+ "remove-background": "IMAGE",+ "upscale-image": "IMAGE",+ "kling-3-pro": "VIDEO",+ "kling-o3-pro": "VIDEO",+ "wan-2.6": "VIDEO",+ "hailuo-3-max": "VIDEO",+ "seedance-2.0": "VIDEO",+ "veo-3.1": "VIDEO",+ "veo-3.1-fast": "VIDEO",+ "upscale-video": "VIDEO",+ "add-audio": "VIDEO",+ "elevenlabs-music": "AUDIO",+ "elevenlabs-sfx": "AUDIO",+ "elevenlabs-v3": "VOICE",+ "transcribe": "TEXT",+ "analyze": "TEXT",+};++export type PromptMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";+/** The one model the voice mode speaks with; the UI picks the voice, not the model. */-export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3";+export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3" satisfies ModelId;/** What an aspect ratio is worth in pixels, at 1080p. */export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {@@ -67,51 +101,72 @@ export const PROMPT_INPUT_RESOLUTION_OPTIONS = [{ value: "720p", label: "720p" },];-export const PROMPT_INPUT_IMAGE_MODEL_OPTIONS = [+export type ImageModelOption = {+ name: string;+ id: ModelId;+ icon: string;+ description: string;+ maxReferences: number;+};++export const PROMPT_INPUT_IMAGE_MODEL_OPTIONS: ImageModelOption[] = [{- name: "FLUX.2 [DEV] Turbo",- id: "flux-2-turbo",+ name: "FLUX.2 Klein",+ id: "flux-2-klein",icon: "large-bfl",- description: "Low budget, high quality, fast turbo mode.",+ description: "Low budget, high quality, fast.",+ maxReferences: 4,},{name: "GPT Image 2",id: "gpt-image-2",icon: "large-openai",description: "Flexible sizes up to 4K, true aspect ratios.",+ maxReferences: 16,},{name: "Nano Banana 2",id: "nano-banana-2",icon: "large-google",description: "Fast, high quality, flexible edits.",+ maxReferences: 14,},{name: "Nano Banana Pro",id: "nano-banana-pro",icon: "large-google",description: "Pro control, readable text, ultra consistent.",+ maxReferences: 14,},{name: "Seedream 4.5",id: "seedream-4.5",icon: "large-bytedance",description: "Up to 4K, multi image edits, strong text.",+ maxReferences: 14,},];+/**+ * What a video model takes besides the prompt: a first and a last frame, and+ * a switch for the sound it makes (models without one score their video+ * themselves, or not at all).+ */export type VideoModelFeature = "start-frame" | "end-frame" | "audio";export type VideoModelOption = {name: string;- id: string;+ id: ModelId;icon: string;description: string;features: VideoModelFeature[];durations: string[];+ /** Empty for a model that takes its proportions from the start frame. */aspectRatios: string[];};+const seconds = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => `${from + i}s`);+export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [{name: "Kling 3.0",@@ -119,7 +174,7 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [icon: "large-kling",description: "Cinematic motion with built-in audio.",features: ["start-frame", "end-frame", "audio"],- durations: ["3s", "4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],+ durations: seconds(3, 15),aspectRatios: ["16:9", "9:16", "1:1"],},{@@ -127,26 +182,17 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [id: "kling-o3-pro",icon: "large-kling",description: "Multi-modal reasoning, strong scenes.",- features: ["start-frame", "end-frame", "audio"],- durations: ["3s", "4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],- aspectRatios: ["16:9", "9:16", "1:1"],- },- {- name: "Kling 2.5 Turbo",- id: "kling-2.5-turbo",- icon: "large-kling",- description: "Fast and affordable for quick iterations.",- features: ["start-frame", "end-frame", "audio"],- durations: ["5s", "10s"],- aspectRatios: ["16:9", "9:16", "1:1"],+ features: ["start-frame", "audio"],+ durations: seconds(3, 15),+ aspectRatios: [],},{name: "Seedance 2.0",id: "seedance-2.0",icon: "large-bytedance",description: "Rich motion, lip-synced audio and SFX.",- features: ["start-frame", "end-frame", "audio"],- durations: ["4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],+ features: ["start-frame", "end-frame"],+ durations: seconds(4, 15),aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],},{@@ -173,17 +219,17 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [icon: "large-wan",description: "Expressive motion, stylized aesthetics.",features: ["start-frame"],- durations: ["5s", "10s", "15s"],- aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],+ durations: ["5s", "10s"],+ aspectRatios: ["16:9", "9:16"],},{- name: "Hailuo 2.3",- id: "hailuo-2.3",+ name: "Hailuo 3 Max",+ id: "hailuo-3-max",icon: "large-hailuo",description: "Smooth motion, cinematic lighting.",- features: ["start-frame"],- durations: ["5s"],- aspectRatios: ["16:9"],+ features: ["start-frame", "end-frame"],+ durations: seconds(5, 15),+ aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],},];@@ -351,7 +397,7 @@ export const PROMPT_INPUT_VOICE_OPTIONS = [},];-export const PROMPT_INPUT_AUDIO_MODEL_OPTIONS = [+export const PROMPT_INPUT_AUDIO_MODEL_OPTIONS: { name: string; id: ModelId; icon: string; description: string }[] = [{name: "ElevenLabs Music",id: "elevenlabs-music",diff --git a/apps/web/src/components/genai/placeholder.ts b/apps/web/src/components/genai/placeholder.tsdeleted file mode 100644index 201f6189..00000000--- a/apps/web/src/components/genai/placeholder.ts+++ /dev/null@@ -1,16 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// Placeholders for the generative AI features, which are being rebuilt. The-// prompt box and the action bar stay in place; every action that would have reached a model lands here instead.--import { toast } from "somoto";--/** Tells the user `feature` is not wired up yet. */-export function notAvailable(feature: string): void {- toast(`${feature} is not available yet`, {- id: `genai-placeholder:${feature}`,- description: "Generative AI features are being rebuilt.",- });-}diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsxindex b3cacf90..eb2ddd2a 100644--- a/apps/web/src/components/genai/prompt-input-actions.tsx+++ b/apps/web/src/components/genai/prompt-input-actions.tsx@@ -18,55 +18,31 @@ import { createMemo, Show } from "solid-js";import { toast } from "somoto";import { useGenerationRecords } from "./use-generation-records";-import { useGenerateImage } from "./use-generate-image";-import { useGenerateVideo } from "./use-generate-video";-import { useGenerateVoice } from "./use-generate-voice";-import { useGenerateAudio } from "./use-generate-audio";+import { useGenerate } from "./use-generate";import { useMediaSelection } from "./selection";import { useTransforms } from "./use-transforms";import type { TransformType } from "./schemas";export function PromptInputActions() {- const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();- const { imageNodes, videoNodes } = useMediaSelection();- const { isOn, toggle } = useTransforms();+ const { isGenerated, totalCredits, firstJob } = useGenerationRecords();+ const { imageMedia, videoMedia } = useMediaSelection();+ const { run } = useTransforms();+ const { rerun } = useGenerate();- const { generate: generateImage } = useGenerateImage();- const { generate: generateVideo } = useGenerateVideo();- const { generate: generateVoice } = useGenerateVoice();- const { generate: generateAudio } = useGenerateAudio();-- const hasImageSelection = createMemo(() => imageNodes().length > 0);- const hasVideoSelection = createMemo(() => videoNodes().length > 0);+ const hasImageSelection = createMemo(() => imageMedia().length > 0);+ const hasVideoSelection = createMemo(() => videoMedia().length > 0);const handleRerun = () => {- const config = firstConfig();- if (!config) {- toast("No generation config found", {+ const job = firstJob();+ if (!job) {+ toast("No generation found", {description: "This asset wasn't generated with a prompt.",});return;}-- const promise = (() => {- switch (config.mode) {- case "IMAGE":- return generateImage(config);- case "VIDEO":- return generateVideo(config);- case "VOICE":- return generateVoice(config);- case "AUDIO":- return generateAudio(config);- }- })();-- promise.catch((err) => {- toast("Rerun failed", {- description: err instanceof Error ? err.message : String(err),- });- });++ rerun(job.request);};return (@@ -121,13 +97,13 @@ export function PromptInputActions() {</Show><DropdownMenuGroup><Show when={hasImageSelection() || hasVideoSelection()}>- <TransformItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />+ <TransformItem name="upscale" icon="arrow-scale" label="Upscale" run={run} /></Show><Show when={hasImageSelection()}>- <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />+ <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" run={run} /></Show><Show when={hasVideoSelection()}>- <TransformItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />+ <TransformItem name="addAudio" icon="audio-on" label="Add audio" run={run} /></Show></DropdownMenuGroup></DropdownMenuContent>@@ -142,19 +118,15 @@ type TransformItemProps = {name: TransformType;icon: string;label: string;- isOn(name: TransformType): boolean;- toggle(name: TransformType): void;+ run(name: TransformType): void;};-/** A transform as a menu row: a check when the selection is asking for it. */+/** A transform as a menu row. */function TransformItem(props: TransformItemProps) {return (- <DropdownMenuItem onSelect={() => props.toggle(props.name)}>+ <DropdownMenuItem onSelect={() => props.run(props.name)}><Icon name={props.icon} class="mr-2 text-foreground" /><span class="flex-1">{props.label}</span>- <Show when={props.isOn(props.name)}>- <Icon name="confirm-check" class="ml-2 text-foreground" />- </Show></DropdownMenuItem>);}diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsxindex 5fd46d02..a90e63c5 100644--- a/apps/web/src/components/genai/prompt-input.tsx+++ b/apps/web/src/components/genai/prompt-input.tsx@@ -41,14 +41,11 @@ import {ALL_DURATION_OPTIONS,PROMPT_INPUT_AUDIO_MODEL_OPTIONS,// PROMPT_INPUT_RESOLUTION_OPTIONS+ type PromptMode,} from "./config";import { createStoredSignal } from "@/lib/store";import { store } from "@/init";-import { toast } from "somoto";-import { useGenerateImage } from "./use-generate-image";-import { useGenerateVideo } from "./use-generate-video";-import { useGenerateVoice } from "./use-generate-voice";-import { useGenerateAudio } from "./use-generate-audio";+import { useGenerate } from "./use-generate";import { PromptInputActions } from "./prompt-input-actions";import type {AspectRatio,@@ -68,9 +65,10 @@ import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";import type { AssetCache } from "@diffusionstudio/assets";import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";-export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";+export type PromptInputMode = PromptMode;const DEFAULT_BUTTON_CLASS = "text-muted-foreground gap-0 pr-2 pl-0";+/** As many references as the prompt box has room for; a model may take fewer. */const MAX_IMAGE_REFERENCES = 5;type GenerationConfgs = {@@ -123,10 +121,7 @@ export function PromptInput(props: PromptInputProps) {const library = useLibrary();const editor = useEditor();const { images: selectedImages } = useMediaSelection();- const { generate: generateImage } = useGenerateImage();- const { generate: generateVideo } = useGenerateVideo();- const { generate: generateVoice } = useGenerateVoice();- const { generate: generateAudio } = useGenerateAudio();+ const { generate } = useGenerate();let textareaRef!: HTMLTextAreaElement;let dragCounter = 0;@@ -159,6 +154,11 @@ export function PromptInput(props: PromptInputProps) {return c.mode === "VOICE" ? c : undefined;};+ const maxImageReferences = createMemo(() => Math.min(+ MAX_IMAGE_REFERENCES,+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS.find((m) => m.id === config().model)?.maxReferences ?? MAX_IMAGE_REFERENCES,+ ));+const currentVideoModel = createMemo(() =>PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((m) => m.id === config().model),);@@ -184,7 +184,7 @@ export function PromptInput(props: PromptInputProps) {.filter((id) => !local.includes(id));return [...local, ...selected].filter((id) => lib.get(id))- .slice(0, MAX_IMAGE_REFERENCES);+ .slice(0, maxImageReferences());});/** The library id a frame is set to, explicitly or by the selection. */@@ -239,11 +239,11 @@ export function PromptInput(props: PromptInputProps) {// The reference is shown from a selected canvas entity, so removing it// means deselecting that entity.const entry = selectedImages().find((e) => e.asset.id === assetId);- if (entry) editor.deselect(entry.entity);+ if (entry) editor.deselect(entry.node);};const openImageReferencesPicker = async () => {- if (effectiveImageRefIds().length >= MAX_IMAGE_REFERENCES) return;+ if (effectiveImageRefIds().length >= maxImageReferences()) return;const lib = library();if (!lib) return;@@ -255,7 +255,7 @@ export function PromptInput(props: PromptInputProps) {const imageAssets = imported.filter((a) => a.type === "IMAGE");const current = imageConfig()?.imageRefIds ?? [];patch({- imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),+ imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, maxImageReferences()),});};@@ -291,7 +291,7 @@ export function PromptInput(props: PromptInputProps) {// The frame is being shown from a selected canvas entity, so clearing it// means deselecting that entity (mirrors removeImageReference).const entry = selectedImages()[frame === "start" ? 0 : 1];- if (entry) editor.deselect(entry.entity);+ if (entry) editor.deselect(entry.node);};const resizeTextarea = () => {@@ -321,31 +321,24 @@ export function PromptInput(props: PromptInputProps) {patch({ prompt: "" });resizeTextarea();- const promise = (() => {- switch (currentConfig.mode) {- case "IMAGE":- return generateImage({- ...currentConfig,- imageRefIds: effectiveImageRefIds(),- });- case "VIDEO":- return generateVideo({- ...currentConfig,- startFrameImageId: effectiveStartFrameId() || undefined,- endFrameImageId: effectiveEndFrameId() || undefined,- });- case "VOICE":- return generateVoice(currentConfig);- case "AUDIO":- return generateAudio(currentConfig);- }- })();+ let submitted: GenerationConfig;++ if (currentConfig.mode === "IMAGE") {+ submitted = {+ ...currentConfig,+ imageRefIds: effectiveImageRefIds(),+ };+ } else if (currentConfig.mode === "VIDEO") {+ submitted = {+ ...currentConfig,+ startFrameImageId: effectiveStartFrameId() || undefined,+ endFrameImageId: effectiveEndFrameId() || undefined,+ };+ } else {+ submitted = currentConfig;+ }- promise.catch((err) => {- toast("Generation failed", {- description: err instanceof Error ? err.message : String(err),- });- });+ generate(submitted);};const handleKeyDown = (e: KeyboardEvent) => {@@ -433,9 +426,7 @@ export function PromptInput(props: PromptInputProps) {}const current = imageConfig()?.imageRefIds ?? [];- patch({- imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),- });+ patch({ imageRefIds: [...current, ...imageIds].slice(0, maxImageReferences()) });};const handleVideoModelChange = (id: string) => {@@ -451,7 +442,7 @@ export function PromptInput(props: PromptInputProps) {endFrameImageId: model.features.includes("end-frame") ? c.endFrameImageId : undefined,generateAudio: model.features.includes("audio") ? (c.generateAudio ?? true) : false,duration: model.durations.includes(durationStr) ? c.duration : parseInt(model.durations[0], 10),- aspectRatio: model.aspectRatios.includes(c.aspectRatio)+ aspectRatio: model.aspectRatios.length === 0 || model.aspectRatios.includes(c.aspectRatio)? c.aspectRatio: (model.aspectRatios[0] as AspectRatio),});@@ -489,7 +480,7 @@ export function PromptInput(props: PromptInputProps) {/>)}</For>- <Show when={effectiveImageRefIds().length < MAX_IMAGE_REFERENCES}>+ <Show when={effectiveImageRefIds().length < maxImageReferences()}><PromptInputAttachButtonicon="attachment"label="Image references"@@ -664,14 +655,16 @@ export function PromptInput(props: PromptInputProps) {value={modelAccessor}onChange={handleVideoModelChange}/>- <PromptInputCompactMenu- aria-label="Select aspect ratio"- menuLabel="Aspect ratio"- value={aspectRatioAccessor}- options={videoAspectRatioOptions()}- onChange={(v) => patch({ aspectRatio: v as AspectRatio })}- triggerIcon="aspect-ratio-16-9"- />+ <Show when={videoAspectRatioOptions().length > 0}>+ <PromptInputCompactMenu+ aria-label="Select aspect ratio"+ menuLabel="Aspect ratio"+ value={aspectRatioAccessor}+ options={videoAspectRatioOptions()}+ onChange={(v) => patch({ aspectRatio: v as AspectRatio })}+ triggerIcon="aspect-ratio-16-9"+ />+ </Show>{/* <PromptInputCompactMenuaria-label="Select resolution"menuLabel="Resolution"diff --git a/apps/web/src/components/genai/requests.ts b/apps/web/src/components/genai/requests.tsnew file mode 100644index 00000000..57f583e1--- /dev/null+++ b/apps/web/src/components/genai/requests.ts@@ -0,0 +1,147 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * The prompt box's settings in the API's vocabulary and back: what a config+ * asks the API for (`toRequest`), what a request puts on the canvas+ * (`outputOf`), and what settings a past request was made with (`toConfig`).+ */++import { uploadAsset } from "@/engine/generate";+import { AUDIO_SIZE } from "@/engine/insert-asset";+import { ASPECT_RATIO_DIMENSIONS, MODEL_MODES, PROMPT_INPUT_VIDEO_MODEL_OPTIONS, PROMPT_INPUT_VOICE_OPTIONS } from "./config";+import { aspectRatioSchema } from "./schemas";++import type { AssetLibrary } from "@diffusionstudio/assets";+import type { AssetRef, GenerateRequest, GenerateRequestInput, ModelId } from "@diffusionstudio/api-contract";+import type { GenerationOutput } from "@/engine/generate";+import type { GenerationConfig } from "./schemas";++const TITLES = {+ IMAGE: "Image generation failed",+ VIDEO: "Video generation failed",+ VOICE: "Voice generation failed",+ AUDIO: "Audio generation failed",+} as const;++/** The default box of a picture or a clip: its aspect ratio at 1080p. */+const DEFAULT_SIZE = ASPECT_RATIO_DIMENSIONS["16:9"]!;++/**+ * What a request puts on the canvas, read off the request itself — so a past+ * job (`job.request`) is placed the way the prompt that made it was. Text+ * models have nothing to place.+ */+export function outputOf(request: { model: ModelId; count?: number; aspectRatio?: string }): GenerationOutput | undefined {+ const mode = MODEL_MODES[request.model];+ if (mode === "TEXT") return undefined;++ return {+ kind: mode === "IMAGE" ? "image" : mode === "VIDEO" ? "video" : "audio",+ count: request.count ?? 1,+ size: mode === "VOICE" || mode === "AUDIO"+ ? { ...AUDIO_SIZE }+ : ASPECT_RATIO_DIMENSIONS[request.aspectRatio ?? ""] ?? DEFAULT_SIZE,+ title: TITLES[mode],+ };+}++/** The library assets `ids` name, uploaded for a model to read; ids no longer in the library are dropped. */+function refs(library: AssetLibrary, ids: (string | undefined)[]): Promise<(AssetRef | undefined)[]> {+ return Promise.all(ids.map((id) => {+ const asset = id ? library.get(id) : undefined;+ return asset ? uploadAsset(library, asset) : undefined;+ }));+}++/**+ * The request `config` makes, its library inputs uploaded. The API validates+ * the rest: a model that does not take a setting says so.+ */+export async function toRequest(library: AssetLibrary, config: GenerationConfig): Promise<GenerateRequestInput> {+ const model = config.model as ModelId;++ switch (config.mode) {+ case "IMAGE": {+ const images = (await refs(library, config.imageRefIds ?? [])).filter((ref) => ref !== undefined);+ return {+ model,+ prompt: config.prompt,+ aspectRatio: config.aspectRatio,+ count: config.count,+ ...(images.length ? { images } : {}),+ } as GenerateRequestInput;+ }+ case "VIDEO": {+ const option = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((candidate) => candidate.id === model);+ const [startFrame, endFrame] = await refs(library, [config.startFrameImageId, config.endFrameImageId]);+ return {+ model,+ prompt: config.prompt,+ duration: config.duration,+ ...(option?.aspectRatios.length ? { aspectRatio: config.aspectRatio } : {}),+ ...(option?.features.includes("audio") ? { generateAudio: config.generateAudio ?? false } : {}),+ ...(startFrame ? { startFrame } : {}),+ ...(endFrame ? { endFrame } : {}),+ } as GenerateRequestInput;+ }+ case "VOICE":+ return { model: "elevenlabs-v3", prompt: config.prompt, voice: config.voice };+ case "AUDIO":+ return { model, prompt: config.prompt } as GenerateRequestInput;+ }+}++/**+ * The prompt box set the way `request` was made — what "Reuse" opens it with.+ * Inputs are left out: the request names them by upload, not by library+ * asset. Undefined for a request the prompt box does not make (a tool).+ */+export function toConfig(request: GenerateRequest): GenerationConfig | undefined {+ const aspectRatio = (value: unknown) => aspectRatioSchema.safeParse(value).data ?? "16:9";++ switch (request.model) {+ case "gpt-image-2":+ case "nano-banana-2":+ case "nano-banana-pro":+ case "seedream-4.5":+ case "flux-2-klein":+ return {+ mode: "IMAGE",+ model: request.model,+ prompt: request.prompt,+ aspectRatio: aspectRatio(request.aspectRatio),+ count: Math.min(request.count, 4),+ };+ case "kling-3-pro":+ case "kling-o3-pro":+ case "wan-2.6":+ case "hailuo-3-max":+ case "seedance-2.0":+ case "veo-3.1":+ case "veo-3.1-fast":+ return {+ mode: "VIDEO",+ model: request.model,+ prompt: request.prompt,+ aspectRatio: aspectRatio("aspectRatio" in request ? request.aspectRatio : undefined),+ duration: request.duration,+ generateAudio: "generateAudio" in request ? request.generateAudio : undefined,+ };+ case "elevenlabs-v3":+ return {+ mode: "VOICE",+ model: request.model,+ prompt: request.prompt,+ voice: PROMPT_INPUT_VOICE_OPTIONS.some((option) => option.value === request.voice)+ ? request.voice+ : PROMPT_INPUT_VOICE_OPTIONS[0]!.value,+ };+ case "elevenlabs-music":+ case "elevenlabs-sfx":+ return { mode: "AUDIO", model: request.model, prompt: request.prompt };+ default:+ return undefined;+ }+}diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.tsindex 54f37e73..feba5d7f 100644--- a/apps/web/src/components/genai/selection.ts+++ b/apps/web/src/components/genai/selection.ts@@ -3,75 +3,58 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createMemo } from "solid-js";-import { AssetId, Cache, Paint, PaintType, getIntrinsicPaint } from "@diffusionstudio/runtime";-import { useSelection } from "@/engine/hooks";+import { AssetId, PaintType } from "@diffusionstudio/runtime";+import { useDerived, useSelection } from "@/engine/hooks";+import { topMedia } from "@/engine/generate";import { useLibrary } from "@/engine/library";import type { Asset } from "@diffusionstudio/assets";-import type { Entity } from "koota";+import type { NodeMedia } from "@/engine/generate";-/** The kinds of paint the prompt box has anything to say about. */-type MediaPaint = PaintType.IMAGE | PaintType.VIDEO;--/** A selected node and the element its source props live on. */-export interface SelectedNode {- entity: Entity;- source: Entity;- paint?: MediaPaint;+/** A selected node's top media, and the library asset it shows once that has loaded. */+export interface SelectedMedia extends NodeMedia {+ asset?: Asset;}-/** A selected node and the library asset its source element is bound to. */-export interface BoundNode extends SelectedNode {- asset: Asset;-}+/** A node's top media with the id of the asset bound to it so far. */+type Shown = NodeMedia & { assetId?: string };-function isMediaPaint(paint: PaintType | undefined): paint is MediaPaint {- return paint === PaintType.IMAGE || paint === PaintType.VIDEO;-}+const sameMedia = (a: Shown[], b: Shown[]) =>+ a.length === b.length && a.every((media, index) => {+ const other = b[index]!;+ return media.node === other.node && media.paint === other.paint && media.assetId === other.assetId;+ });/**- * Where `entity` keeps its source: itself when its own paint is the media,- * else its first media fill. A node that paints no media at all is still its- * own source element — an `<audio>` carries its `src` like any other.+ * The media the selected nodes show on top (see `topMedia`) — what a prompt+ * references and a transform is run over. Sampled every frame: which paint+ * is on top is derived state, and changes without the selection changing (a+ * transform lands, a fill is hidden).*/-function resolve(entity: Entity): SelectedNode {- const intrinsic = getIntrinsicPaint(entity);- if (isMediaPaint(intrinsic)) return { entity, source: entity, paint: intrinsic };-- for (const fill of entity.get(Cache)?.fills ?? []) {- const paint = fill.get(Paint)?.value;- if (isMediaPaint(paint)) return { entity, source: fill, paint };- }-- return { entity, source: entity };-}-export function useMediaSelection() {const library = useLibrary();const { nodes } = useSelection();- const selected = createMemo(() => nodes().map(resolve));-- const painted = (paint: MediaPaint) =>- createMemo(() => selected().filter((node) => node.paint === paint).map((node) => node.source));+ const shown = useDerived<Shown[]>(+ () => nodes()+ .map(topMedia)+ .filter((media) => media !== undefined)+ .map((media) => ({ ...media, assetId: media.paint.get(AssetId)?.value })),+ sameMedia,+ );- const imageNodes = painted(PaintType.IMAGE);- const videoNodes = painted(PaintType.VIDEO);-- const bound = createMemo(() => {+ const media = createMemo<SelectedMedia[]>(() => {const lib = library();- if (!lib) return [];-- const entries: BoundNode[] = [];- for (const node of selected()) {- const id = node.source.get(AssetId)?.value;- const asset = id ? lib.get(id) : undefined;- if (asset) entries.push({ ...node, asset });- }- return entries;+ return shown().map(({ assetId, ...entry }) => ({ ...entry, asset: assetId ? lib?.get(assetId) : undefined }));});- const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));+ const imageMedia = createMemo(() => media().filter((entry) => entry.type === PaintType.IMAGE));+ const videoMedia = createMemo(() => media().filter((entry) => entry.type === PaintType.VIDEO));++ /** Selected pictures whose asset is in the library: what a prompt can reference. */+ const images = createMemo(() =>+ imageMedia().filter((entry): entry is SelectedMedia & { asset: Asset } => entry.asset?.type === "IMAGE"),+ );- return { bound, images, imageNodes, videoNodes };+ return { media, imageMedia, videoMedia, images };}diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsxindex f874b43a..025eaf25 100644--- a/apps/web/src/components/genai/use-auto-captions.tsx+++ b/apps/web/src/components/genai/use-auto-captions.tsx@@ -3,15 +3,25 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createMemo } from "solid-js";+import { useWorld } from "@diffusionstudio/koota-solid";import { isScene } from "@diffusionstudio/runtime";+import { useProject } from "@/context/project";+import { generateCaptions } from "@/engine/generate";import { useSelection } from "@/engine/hooks";-import { notAvailable } from "./placeholder";export function useAutoCaptions() {+ const world = useWorld();+ const project = useProject();const { nodes } = useSelection();- const hasScene = createMemo(() => nodes().some(isScene));- const generate = () => notAvailable("Auto captions");+ const scenes = createMemo(() => nodes().filter(isScene));+ const hasScene = createMemo(() => scenes().length > 0);++ const generate = () => {+ for (const scene of scenes()) {+ generateCaptions(world, scene, project.dir());+ }+ };return { hasScene, generate };}diff --git a/apps/web/src/components/genai/use-generate-audio.tsx b/apps/web/src/components/genai/use-generate-audio.tsxdeleted file mode 100644index 77ac9429..00000000--- a/apps/web/src/components/genai/use-generate-audio.tsx+++ /dev/null@@ -1,12 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { notAvailable } from "./placeholder";--import type { AudioGenerationConfig } from "./schemas";--export function useGenerateAudio() {- const run = async (_config: AudioGenerationConfig) => notAvailable("Audio generation");- return { generate: run } as const;-}diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsxdeleted file mode 100644index 882cca78..00000000--- a/apps/web/src/components/genai/use-generate-image.tsx+++ /dev/null@@ -1,12 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { notAvailable } from "./placeholder";--import type { ImageGenerationConfig } from "./schemas";--export function useGenerateImage() {- const run = async (_config: ImageGenerationConfig) => notAvailable("Image generation");- return { generate: run } as const;-}diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsxdeleted file mode 100644index 48b5111c..00000000--- a/apps/web/src/components/genai/use-generate-video.tsx+++ /dev/null@@ -1,12 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { notAvailable } from "./placeholder";--import type { VideoGenerationConfig } from "./schemas";--export function useGenerateVideo() {- const run = async (_config: VideoGenerationConfig) => notAvailable("Video generation");- return { generate: run } as const;-}diff --git a/apps/web/src/components/genai/use-generate-voice.tsx b/apps/web/src/components/genai/use-generate-voice.tsxdeleted file mode 100644index bb999a55..00000000--- a/apps/web/src/components/genai/use-generate-voice.tsx+++ /dev/null@@ -1,12 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { notAvailable } from "./placeholder";--import type { VoiceGenerationConfig } from "./schemas";--export function useGenerateVoice() {- const run = async (_config: VoiceGenerationConfig) => notAvailable("Voice generation");- return { generate: run } as const;-}diff --git a/apps/web/src/components/genai/use-generate.ts b/apps/web/src/components/genai/use-generate.tsnew file mode 100644index 00000000..03f4f69f--- /dev/null+++ b/apps/web/src/components/genai/use-generate.ts@@ -0,0 +1,38 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { useWorld } from "@diffusionstudio/koota-solid";+import { Library } from "@diffusionstudio/runtime";+import { generate } from "@/engine/generate";+import { outputOf, toRequest } from "./requests";++import type { GenerateRequest } from "@diffusionstudio/api-contract";+import type { GenerationConfig } from "./schemas";++/**+ * Generating onto the canvas: from the prompt box's settings (`generate`), or+ * again from a past job's request (`rerun`), which is resubmitted as it was —+ * same model, settings and inputs, a new take.+ */+export function useGenerate() {+ const world = useWorld();++ const run = (config: GenerationConfig): Promise<void> => {+ const library = world.get(Library);+ const output = outputOf({+ model: config.model as GenerateRequest["model"],+ count: config.mode === "IMAGE" ? config.count : 1,+ aspectRatio: "aspectRatio" in config ? config.aspectRatio : undefined,+ });+ if (!library || !output) return Promise.resolve();+ return generate(world, output, () => toRequest(library, config));+ };++ const rerun = (request: GenerateRequest): Promise<void> => {+ const output = outputOf(request);+ return output ? generate(world, output, request) : Promise.resolve();+ };++ return { generate: run, rerun };+}diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.tsindex 381ef80f..eb50f04d 100644--- a/apps/web/src/components/genai/use-generation-records.ts+++ b/apps/web/src/components/genai/use-generation-records.ts@@ -2,15 +2,44 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// How the selection was generated and what it cost: a placeholder until the-// generative features are back (see ./placeholder). Nothing reads as-// generated, so "Rerun", "Reuse" and the credits line stay hidden.+// How the selection was generated and what it cost. A generated asset keeps+// the id of the job it came from (`Asset.job`); the job is the API's account+// of the run — what it was asked, which "Rerun" and "Reuse" work from, and+// what it was charged.+import { createMemo, createResource } from "solid-js";+import { api } from "@/lib/api";+import { useMediaSelection } from "./selection";+import { toConfig } from "./requests";++import type { Job } from "@diffusionstudio/api-contract";import type { GenerationConfig } from "./schemas";export function useGenerationRecords() {- const isGenerated = (): boolean => false;- const totalCredits = (): number => 0;- const firstConfig = (): GenerationConfig | undefined => undefined;- return { isGenerated, totalCredits, firstConfig };+ const { media } = useMediaSelection();++ const jobIds = createMemo(+ () => [...new Set(media().map((entry) => entry.asset?.job).filter((id) => id !== undefined))],+ undefined,+ { equals: (a, b) => a.length === b.length && a.every((id, index) => id === b[index]) },+ );++ const [jobs] = createResource(jobIds, async (ids): Promise<Job[]> => {+ const settled = await Promise.allSettled(ids.map((id) => api.jobs.get.query({ id })));+ return settled.flatMap((result) => (result.status === "fulfilled" ? [result.value] : []));+ }, { initialValue: [] });++ const isGenerated = createMemo(() => jobIds().length > 0);+ const totalCredits = createMemo(() => jobs().reduce((sum, job) => sum + job.credits, 0));++ /** The job behind the first generated asset of the selection, once it is fetched. */+ const firstJob = (): Job | undefined => jobs()[0];++ /** The prompt box's settings for that job, when the prompt box makes such requests. */+ const firstConfig = (): GenerationConfig | undefined => {+ const job = firstJob();+ return job ? toConfig(job.request) : undefined;+ };++ return { isGenerated, totalCredits, firstJob, firstConfig };}diff --git a/apps/web/src/components/genai/use-transforms.ts b/apps/web/src/components/genai/use-transforms.tsindex 8f6f991f..9dca04a5 100644--- a/apps/web/src/components/genai/use-transforms.ts+++ b/apps/web/src/components/genai/use-transforms.ts@@ -2,21 +2,59 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Upscaling, taking a background out, scoring footage: placeholders until the-// generative features are back (see ./placeholder).+// Upscaling, taking a background out, scoring footage: tools run over the+// media the selected nodes show on top. The result is laid over that paint,+// which is hidden rather than removed (see `transform`).-import { notAvailable } from "./placeholder";+import { useWorld } from "@diffusionstudio/koota-solid";+import { Generating, PaintType } from "@diffusionstudio/runtime";+import { transform } from "@/engine/generate";+import { useMediaSelection } from "./selection";+import type { AssetRef, GenerateRequestInput } from "@diffusionstudio/api-contract";+import type { NodeMedia } from "@/engine/generate";import type { TransformType } from "./schemas";-const LABELS: Record<TransformType, string> = {- upscale: "Upscaling",- removeBackground: "Background removal",- addAudio: "Adding audio",+interface Tool {+ /** What a failure is reported under. */+ title: string;+ /** Whether the tool can be run over media of `type`. */+ takes(type: NodeMedia["type"]): boolean;+ /** The request over `input`, an upload of media of `type`. */+ request(type: NodeMedia["type"], input: AssetRef): GenerateRequestInput;+}++const TOOLS: Record<TransformType, Tool> = {+ upscale: {+ title: "Upscale failed",+ takes: () => true,+ request: (type, input) =>+ type === PaintType.VIDEO ? { model: "upscale-video", video: input } : { model: "upscale-image", image: input },+ },+ removeBackground: {+ title: "Background removal failed",+ takes: (type) => type === PaintType.IMAGE,+ request: (_type, image) => ({ model: "remove-background", image }),+ },+ addAudio: {+ title: "Adding audio failed",+ takes: (type) => type === PaintType.VIDEO,+ request: (_type, video) => ({ model: "add-audio", video }),+ },};export function useTransforms() {- const isOn = (_type: TransformType): boolean => false;- const toggle = (type: TransformType): void => notAvailable(LABELS[type]);- return { isOn, toggle };+ const world = useWorld();+ const { media } = useMediaSelection();++ /** Runs `type` over every selected node it applies to that is not generating already. */+ const run = (type: TransformType): void => {+ const tool = TOOLS[type];+ for (const entry of media()) {+ if (!tool.takes(entry.type) || entry.node.has(Generating)) continue;+ transform(world, entry, tool.title, (input) => tool.request(entry.type, input));+ }+ };++ return { run };}diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsxnew file mode 100644index 00000000..0206c237--- /dev/null+++ b/apps/web/src/engine/generate/captions.tsx@@ -0,0 +1,122 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Auto-captions: the scene's audible mix, encoded and transcribed, mounted as+ * the scene's `<captions>`. A scene has no geometry to stand in with — it is+ * the frame everything else sits in — so the wait is told in a toast that+ * follows the job instead of a pulse.+ */++import { Captions } from '@diffusionstudio/reconciler';+import { AssetId, Audio, Caption, getEntityTree, Hidden, Library, Muted, Paint, PaintType } from '@diffusionstudio/runtime';+import { assetName } from '@diffusionstudio/assets';+import { createEncoder } from '@diffusionstudio/encoder';+import { toast } from 'somoto';++import { jobFailure, runJob } from '@/lib/jobs';+import { uploadFile } from '@/lib/uploads';++import { createCapture } from '../capture';+import { getDocumentEditor } from '../editor';++import type { AssetLibrary } from '@diffusionstudio/assets';+import type { Job } from '@diffusionstudio/api-contract';+import type { Entity, World } from 'koota';++const TITLE = 'Caption generation failed';++/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */+function hasAudio(world: World, scene: Entity): boolean {+ return getEntityTree(world, scene).some((entity) =>+ !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)+ && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));+}++/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */+async function encodeAudio(world: World, scene: Entity, dir?: string): Promise<File> {+ const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });+ try {+ const encoder = await createEncoder(capture.world, {+ format: 'ogg',+ video: { enabled: false },+ audio: { enabled: true, codec: 'opus', sampleRate: 24000 },+ });+ const result = await encoder.render();+ if (result.type !== 'success' || !result.data) {+ throw new Error('Could not encode the scene audio.');+ }+ return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });+ } finally {+ capture.dispose();+ }+}++/** The next free `Captions N.json`. */+function captionsName(library: AssetLibrary): string {+ let max = 0;+ for (const asset of library.list()) {+ const match = assetName(asset).match(/^Captions (\d+)\.json$/);+ if (match) max = Math.max(max, Number(match[1]));+ }+ return `Captions ${max + 1}.json`;+}++/** The toast line for a transcription in flight. */+function status(job: Job | undefined): string {+ if (!job) return 'Preparing the audio…';+ if (job.status === 'queued') return 'Queued';+ if (job.phase === 'finalizing' || job.status === 'succeeded') return 'Almost done…';+ return 'Transcribing…';+}++/**+ * Transcribes `scene` and mounts the transcript as its captions: the scene's+ * `<captions>` is pointed at the new transcript, or one is added.+ */+export async function generateCaptions(world: World, scene: Entity, dir?: string): Promise<void> {+ const library = world.get(Library);+ if (!library) return;+ if (!hasAudio(world, scene)) {+ toast(TITLE, { description: 'No audio found. Add an audio or video clip to the scene to generate captions.' });+ return;+ }++ const id = `captions:${scene.id()}`;+ toast.loading('Generating captions', { id, description: status(undefined) });++ try {+ const audio = await uploadFile(await encodeAudio(world, scene, dir));+ const job = await runJob({ model: 'transcribe', audio }, (update) => {+ toast.loading('Generating captions', { id, description: status(update) });+ });+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));++ const [file] = job.assets;+ if (!file) throw new Error('The transcription returned nothing.');+ const response = await fetch(file.url);+ if (!response.ok) throw new Error(`Could not download the transcript (${response.status})`);+ const transcript = await response.json() as { words: unknown[] }[];+ if (!transcript.some((segment) => segment.words.length > 0)) {+ throw new Error('No speech detected. The audio does not appear to contain recognizable speech.');+ }++ const blob = new Blob([JSON.stringify(transcript)], { type: 'application/json' });+ const asset = library.update(await library.store(blob, { name: captionsName(library) }), { job: job.id });++ const editor = getDocumentEditor(world);+ const existing = getEntityTree(world, scene).find((entity) => entity !== scene && entity.has(Caption));+ if (existing) {+ editor.editProperty(existing, 'src', asset.path);+ editor.select(existing);+ } else {+ const [captions] = editor.insertElement(scene, () => <Captions src={asset.path} />);+ if (captions) editor.select(captions);+ }+ toast.dismiss(id);+ } catch (error) {+ console.error(`[generate] ${TITLE}:`, error);+ toast.error(TITLE, { id, description: error instanceof Error ? error.message : String(error) });+ }+}diff --git a/apps/web/src/engine/generate/index.ts b/apps/web/src/engine/generate/index.tsnew file mode 100644index 00000000..21a80f4a--- /dev/null+++ b/apps/web/src/engine/generate/index.ts@@ -0,0 +1,13 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++export { generateCaptions } from './captions';+export { topMedia } from './media';+export { findPlacement, PLACEMENT_GAP } from './placement';+export { generate, transform, uploadAsset } from './run';+export { trackPlaceholder } from './placeholder';++export type { MediaPaintType, NodeMedia } from './media';+export type { GenerationOutput } from './run';+export type { GenerationState, Placeholder } from './placeholder';diff --git a/apps/web/src/engine/generate/media.ts b/apps/web/src/engine/generate/media.tsnew file mode 100644index 00000000..0a1a8b88--- /dev/null+++ b/apps/web/src/engine/generate/media.ts@@ -0,0 +1,42 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Cache, Hidden, Paint, PaintType, getIntrinsicPaint } from '@diffusionstudio/runtime';++import type { Entity } from 'koota';++/** The kinds of paint a model reads or writes. */+export type MediaPaintType = PaintType.IMAGE | PaintType.VIDEO;++/** The media a node shows, and the element that carries its `src`. */+export interface NodeMedia {+ node: Entity;+ paint: Entity;+ type: MediaPaintType;+}++const isMedia = (type: PaintType | undefined): type is MediaPaintType =>+ type === PaintType.IMAGE || type === PaintType.VIDEO;++/**+ * The media `node` shows on top: its topmost visible media fill (fills stack+ * in document order, the last on top), else its intrinsic media, which sits+ * beneath every fill. What a reference or a transform is taken from, so a+ * node that was transformed before — the old paint hidden, the result laid+ * over it — is read as the result.+ */+export function topMedia(node: Entity): NodeMedia | undefined {+ const fills = node.get(Cache)?.fills ?? [];+ for (let index = fills.length - 1; index >= 0; index--) {+ const fill = fills[index]!;+ const type = fill.get(Paint)?.value;++ if (!fill.has(Hidden) && isMedia(type)) {+ return { node, paint: fill, type };+ }+ }++ const intrinsic = getIntrinsicPaint(node);+ return isMedia(intrinsic) ? { node, paint: node, type: intrinsic } : undefined;+}diff --git a/apps/web/src/engine/generate/placeholder.ts b/apps/web/src/engine/generate/placeholder.tsnew file mode 100644index 00000000..2f7713a9--- /dev/null+++ b/apps/web/src/engine/generate/placeholder.ts@@ -0,0 +1,73 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * An element standing in for a generation in flight. The app puts the state+ * onto the runtime — `Generating` on the element's entity, which the stage+ * and the timeline draw as the pulse, the label and the progress — and takes+ * it off again on release; the runtime only draws what it is given.+ *+ * The element is held by its `Source` stamp, never by entity: a remount or an+ * undo/redo makes new entities, a stamp survives (and a pending stamp is+ * followed to the real one the write answers with). The state is put on+ * again every `REFRESH` — the progress moves with the clock, and an entity+ * that replaced the old one picks it up — not once a frame.+ */++import { Generating, Source } from '@diffusionstudio/runtime';++import { getDocumentEditor } from '../editor';++import type { Entity, World } from 'koota';++/** How often a placeholder's state is put onto its entity. */+const REFRESH = 250; // ms++/** What a placeholder shows: what the generation is doing, and how far it has come (0–1) when that is known. */+export interface GenerationState {+ label: string;+ progress?: number;+}++export interface Placeholder {+ /** The entity carrying the element right now, or undefined when it left the document. */+ entity(): Entity | undefined;+ /** Lets go of the element: it stops pulsing and reads its own name again. */+ release(): void;+}++/**+ * Makes `entity` a placeholder showing `state`, read on every refresh.+ * Returns undefined for an entity the file cannot name.+ */+export function trackPlaceholder(world: World, entity: Entity, state: () => GenerationState): Placeholder | undefined {+ let source = entity.get(Source)?.value;+ if (!source) return undefined;++ const find = () => world.query(Source).find((candidate) => candidate.isAlive() && candidate.get(Source)!.value === source);++ const apply = () => {+ const target = find();+ if (!target) return;+ const { label, progress } = state();+ if (!target.has(Generating)) target.add(Generating);+ const current = target.get(Generating)!;+ if (current.label !== label || current.progress !== progress) target.set(Generating, { label, progress });+ };++ const unsubscribe = getDocumentEditor(world).onRename((ids) => {+ source = ids[source!] ?? source;+ });+ const timer = setInterval(apply, REFRESH);+ apply();++ return {+ entity: find,+ release: () => {+ clearInterval(timer);+ unsubscribe();+ find()?.remove(Generating);+ },+ };+}diff --git a/apps/web/src/engine/generate/placement.ts b/apps/web/src/engine/generate/placement.tsnew file mode 100644index 00000000..d2f36229--- /dev/null+++ b/apps/web/src/engine/generate/placement.ts@@ -0,0 +1,136 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Where generations go on the infinite canvas: beside the work rather than+ * over it. Of the nodes the user is looking at, the one with the most free+ * room to its left or right — inside the view — takes the block beside it,+ * level with it. When no side has room in view, the free side nearest the+ * middle of the view does, wherever it is; the camera follows (`revealRect`).+ */++import { getEntityBounds, getEntityChildren, getViewport, Hidden, Root, screenToWorld } from '@diffusionstudio/runtime';++import type { AABB, Rect } from '@diffusionstudio/runtime';+import type { World } from 'koota';++/** Between the items of a block, and between the block and its neighbour. */+export const PLACEMENT_GAP = 40;++/** The visible part of the canvas in document space, or null before a surface is mounted. */+function viewBounds(world: World): AABB | null {+ const viewport = getViewport(world);+ if (!viewport) return null;++ // All four corners, so the box still holds if the camera is ever rotated.+ const corners = [+ screenToWorld(world, 0, 0),+ screenToWorld(world, viewport.width, 0),+ screenToWorld(world, viewport.width, viewport.height),+ screenToWorld(world, 0, viewport.height),+ ];++ return {+ minX: Math.min(...corners.map((corner) => corner.x)),+ minY: Math.min(...corners.map((corner) => corner.y)),+ maxX: Math.max(...corners.map((corner) => corner.x)),+ maxY: Math.max(...corners.map((corner) => corner.y)),+ };+}++/**+ * What stands on the canvas: the bounds of every visible top-level node. A+ * node the transform system has not measured yet has no place to avoid.+ */+function occupied(world: World): AABB[] {+ const boxes: AABB[] = [];+ for (const entity of getEntityChildren(world, world.get(Root)!)) {+ if (entity.has(Hidden)) continue;+ const rect = getEntityBounds(world, [entity]);+ if (rect) boxes.push({ minX: rect.x, minY: rect.y, maxX: rect.x + rect.width, maxY: rect.y + rect.height });+ }+ return boxes;+}++/** Overlap with area: boxes that only touch along an edge leave each other alone. */+const overlaps = (a: AABB, b: AABB): boolean =>+ a.minX < b.maxX && b.minX < a.maxX && a.minY < b.maxY && b.minY < a.maxY;++const contains = (outer: AABB, inner: AABB): boolean =>+ inner.minX >= outer.minX && inner.maxX <= outer.maxX && inner.minY >= outer.minY && inner.maxY <= outer.maxY;++const center = (box: AABB) => ({ x: (box.minX + box.maxX) / 2, y: (box.minY + box.maxY) / 2 });++const toRect = (box: AABB): Rect => ({ x: box.minX, y: box.minY, width: box.maxX - box.minX, height: box.maxY - box.minY });++/** A spot level with `box`, `gap` off its side (1 right, -1 left), and the room that side has. */+interface Beside {+ spot: AABB;+ /** Free span off that side of `box` within the band the spot covers: to the nearest node, or the edge of the view. */+ room: number;+}++function beside(box: AABB, side: 1 | -1, width: number, height: number, gap: number, boxes: AABB[], view: AABB): Beside {+ const minY = (box.minY + box.maxY) / 2 - height / 2;+ const minX = side > 0 ? box.maxX + gap : box.minX - gap - width;+ const spot = { minX, minY, maxX: minX + width, maxY: minY + height };++ let room = side > 0 ? view.maxX - box.maxX : box.minX - view.minX;+ for (const other of boxes) {+ if (other === box || other.maxY <= spot.minY || other.minY >= spot.maxY) continue;+ if (side > 0 && other.minX >= box.maxX) room = Math.min(room, other.minX - box.maxX);+ if (side < 0 && other.maxX <= box.minX) room = Math.min(room, box.minX - other.maxX);+ }++ return { spot, room };+}++/**+ * Where a `width`×`height` block goes (see the module comment), in document+ * space. Falls back to the middle of the view on an empty canvas, and to the+ * right of everything when even that is taken.+ */+export function findPlacement(world: World, width: number, height: number, gap = PLACEMENT_GAP): Rect {+ return placeBlock(occupied(world), viewBounds(world) ?? { minX: 0, minY: 0, maxX: 0, maxY: 0 }, width, height, gap);+}++/** `findPlacement` over what stands on the canvas (`boxes`) and the part of it in view. */+export function placeBlock(boxes: AABB[], view: AABB, width: number, height: number, gap: number): Rect {+ const middle = center(view);+ const free = (spot: AABB) => !boxes.some((box) => overlaps(spot, box));++ let roomiest: Beside | undefined;+ let nearest: { spot: AABB; distance: number } | undefined;++ for (const box of boxes) {+ const inView = overlaps(box, view);+ for (const side of [1, -1] as const) {+ const candidate = beside(box, side, width, height, gap, boxes, view);+ if (!free(candidate.spot)) continue;++ if (inView && contains(view, candidate.spot) && (!roomiest || candidate.room > roomiest.room)) {+ roomiest = candidate;+ }++ const { x, y } = center(candidate.spot);+ const distance = Math.hypot(x - middle.x, y - middle.y);+ if (!nearest || distance < nearest.distance) nearest = { spot: candidate.spot, distance };+ }+ }++ if (roomiest) return toRect(roomiest.spot);++ const centered = {+ minX: middle.x - width / 2,+ minY: middle.y - height / 2,+ maxX: middle.x + width / 2,+ maxY: middle.y + height / 2,+ };+ if (free(centered)) return toRect(centered);+ if (nearest) return toRect(nearest.spot);++ // Every side of every node is taken: past the right edge of all of them.+ const right = Math.max(...boxes.map((box) => box.maxX)) + gap;+ return { x: right, y: middle.y - height / 2, width, height };+}diff --git a/apps/web/src/engine/generate/run.tsx b/apps/web/src/engine/generate/run.tsxnew file mode 100644index 00000000..e9b5d137--- /dev/null+++ b/apps/web/src/engine/generate/run.tsx@@ -0,0 +1,278 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Generations on the canvas, from the request to the file in the library.+ *+ * - A prompt (`generate`) gets one empty geometry per output, placed beside+ * the work (see ./placement) and brought into view. Each stands in for its+ * output — the pulse, headed by the job's progress (see ./placeholder) — until+ * the result lands as the geometry's paint.+ * - A transform (`transform`) is run over the media a node shows on top (see+ * ./media); the node pulses meanwhile, and the result is laid over the paint+ * it was made from, which is hidden rather than removed.+ *+ * What a model returns is downloaded into the project library, which only+ * ever holds files that exist: nothing is reserved there for a job in flight,+ * and a job that fails leaves no trace but the toast saying why.+ */++import { Audio, ImagePaint, Rect, VideoPaint, authoredElement, renderAuthored } from '@diffusionstudio/reconciler';+import { AssetId, Cache, getCameraMatrix, getNextName, getParentEntity, Library, Paint, revealRect, Root, Selected, Size, Source } from '@diffusionstudio/runtime';+import { assetName } from '@diffusionstudio/assets';+import { TRPCClientError } from '@trpc/client';+import { toast } from 'somoto';++import { jobFailure, runJob } from '@/lib/jobs';+import { uploadFile } from '@/lib/uploads';++import { getDocumentEditor } from '../editor';+import { findPlacement, PLACEMENT_GAP } from './placement';+import { trackPlaceholder } from './placeholder';+import { topMedia } from './media';++import type { Asset, AssetLibrary } from '@diffusionstudio/assets';+import type { AssetRef, GenerateRequestInput, Job } from '@diffusionstudio/api-contract';+import type { Entity, World } from 'koota';+import type { NodeMedia } from './media';+import type { GenerationState, Placeholder } from './placeholder';++/** What a prompt asks for, as far as the canvas is concerned. */+export interface GenerationOutput {+ /** What each output lands as: a paint of its placeholder, or an `<audio>` in its place. */+ kind: 'image' | 'video' | 'audio';+ /** How many outputs the request makes, each with a placeholder of its own. */+ count: number;+ /** The box of each placeholder; a picture's is corrected to the result's proportions. */+ size: { width: number; height: number };+ /** What a failure is reported under, e.g. "Image generation failed". */+ title: string;+}++/**+ * The progress of one job, as its placeholders show it: "Queued" until the+ * job runs, then "Generating" with the share of its estimate that has passed+ * — counted on to the end, so a job that lands in time goes from a progress+ * straight to its name. Once the estimate is spent there is no progress left+ * to show: it reads "Taking a bit longer" while the model works, and "Almost+ * done" once the model is done and the result is on its way into the project.+ */+class Progress {+ private job: Job | undefined;+ /** When the job was first seen running, on this clock. */+ private since = 0;++ public readonly update = (job: Job): void => {+ if (job.status !== 'queued' && !this.since) this.since = Date.now();+ this.job = job;+ };++ public readonly state = (): GenerationState => {+ const job = this.job;+ if (!job || job.status === 'queued') return { label: 'Queued' };+ const progress = (Date.now() - this.since) / (job.etaSeconds * 1000);+ if (progress < 1) return { label: 'Generating', progress };+ return { label: job.status === 'succeeded' || job.phase === 'finalizing' ? 'Almost done' : 'Taking a bit longer' };+ };+}++/**+ * Generates what `request` asks for into empty geometries on the canvas (see+ * the module comment). `request` may take making — inputs to upload — so it+ * is made once the placeholders are there, from the moment of the prompt.+ */+export async function generate(+ world: World,+ output: GenerationOutput,+ request: GenerateRequestInput | (() => Promise<GenerateRequestInput>),+): Promise<void> {+ const library = world.get(Library);+ const root = world.get(Root);+ if (!library || !root?.get(Source)?.value) {+ toast('Nothing to generate into', { description: 'Open a project first.' });+ return;+ }++ const editor = getDocumentEditor(world);+ const { width, height } = output.size;+ const block = findPlacement(world, output.count * width + (output.count - 1) * PLACEMENT_GAP, height);++ revealRect(world, block);+ editor.reportEdit(root, 'camera', getCameraMatrix(world));++ const progress = new Progress();+ const placeholders: Placeholder[] = [];+ const inserted: Entity[] = [];++ // A picture or a clip keeps its proportions; a sound's box is only where it sits.+ const proportions = output.kind === 'audio' ? {} : { keepAspectRatio: true };++ for (let index = 0; index < output.count; index++) {+ const x = Math.round(block.x + index * (width + PLACEMENT_GAP));+ const y = Math.round(block.y);+ const [entity] = editor.insertElement(root, () => (+ <Rect {...proportions} x={x} y={y} width={width} height={height} />+ ));+ const placeholder = entity && trackPlaceholder(world, entity, progress.state);+ if (!placeholder) continue;+ inserted.push(entity);+ placeholders.push(placeholder);+ }+ if (inserted.length) editor.select(inserted);++ try {+ const job = await runJob(typeof request === 'function' ? await request() : request, progress.update);+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));++ const results = await Promise.all(job.assets.map((file) => storeResult(library, job, file)));+ let orphaned = 0;+ placeholders.forEach((placeholder, index) => {+ const entity = placeholder.entity();+ const result = results[index];+ if (!entity) orphaned += result ? 1 : 0;+ else if (result) land(world, entity, result);+ else editor.remove(entity);+ });+ if (orphaned) {+ toast('Generation finished', { description: 'Its placeholder was removed, so the result is in the library only.' });+ }+ } catch (error) {+ editor.remove(placeholders.map((placeholder) => placeholder.entity()).filter((entity) => entity !== undefined));+ report(output.title, error);+ } finally {+ for (const placeholder of placeholders) placeholder.release();+ }+}++/**+ * Runs `request` over the media `media` shows (see the module comment);+ * `request` is handed the uploaded input.+ */+export async function transform(+ world: World,+ media: NodeMedia,+ title: string,+ request: (input: AssetRef) => GenerateRequestInput,+): Promise<void> {+ const library = world.get(Library);+ const input = library?.get(media.paint.get(AssetId)?.value ?? '');+ if (!library || !input) {+ toast(title, { description: 'The media has not loaded yet.' });+ return;+ }++ const progress = new Progress();+ const placeholder = trackPlaceholder(world, media.node, progress.state);+ const paint = media.paint === media.node ? undefined : media.paint.get(Source)?.value;++ try {+ const job = await runJob(request(await uploadAsset(library, input)), progress.update);+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));++ const [file] = job.assets;+ if (!file) throw new Error('The model returned nothing.');+ const result = await storeResult(library, job, file);++ const node = placeholder?.entity();+ if (node) overlay(world, node, paint, result);+ else toast('Generation finished', { description: 'Its element was removed, so the result is in the library only.' });+ } catch (error) {+ report(title, error);+ } finally {+ placeholder?.release();+ }+}++/** Uploads made this session, by library asset id: an input is sent once. */+const uploads = new Map<string, Promise<AssetRef>>();++/** The ref a request names a library asset by, uploading its bytes when the API lacks them. */+export function uploadAsset(library: AssetLibrary, asset: Asset): Promise<AssetRef> {+ let ref = uploads.get(asset.id);+ if (!ref) {+ ref = library.file(asset).then(uploadFile);+ ref.catch(() => uploads.delete(asset.id));+ uploads.set(asset.id, ref);+ }+ return ref;+}++/** Downloads one output of `job` into the library, noting the job it came from. */+async function storeResult(library: AssetLibrary, job: Job, file: Job['assets'][number]): Promise<Asset> {+ const response = await fetch(file.url);+ if (!response.ok) throw new Error(`Could not download the result (${response.status})`);+ const asset = await library.store(await response.blob(), { name: file.filename });+ return library.update(asset, { job: job.id });+}++const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, '');++/** Fills a prompt's placeholder with what it stood in for, named after it. */+function land(world: World, placeholder: Entity, asset: Asset): void {+ const editor = getDocumentEditor(world);+ const name = getNextName(world, stem(asset));++ if (asset.type === 'AUDIO') {+ // A sound is an element of its own, not a paint: it takes the+ // placeholder's place, and its selection.+ const parent = getParentEntity(placeholder);+ const props = authoredElement(placeholder)?.props ?? {};+ const selected = placeholder.has(Selected);+ if (!parent) return;+ const [audio] = editor.insertElement(parent, () => (+ <Audio name={name} src={asset.path} x={props.x as number} y={props.y as number} width={props.width as number} height={props.height as number} />+ ), placeholder);+ editor.remove(placeholder);+ if (audio && selected) editor.select(audio, { extend: true });+ return;+ }++ editor.insertElement(placeholder, () => (+ asset.type === 'VIDEO' ? <VideoPaint src={asset.path} /> : <ImagePaint src={asset.path} />+ ));+ editor.editProperty(placeholder, 'name', name);++ // The box was a guess at the result's proportions (a model may not take+ // the aspect ratio asked for); the width stays, the height follows.+ const size = placeholder.get(Size);+ if ('width' in asset && asset.width > 0 && size) {+ const height = Math.round((size.width * asset.height) / asset.width);+ if (Math.abs(height - size.height) > 1) editor.editProperty(placeholder, 'height', height);+ }+}++/**+ * Lays a transform's result over the paint it was made from, which is hidden+ * — not removed — so the original is a toggle away. Media that is the node's+ * own (`<image src>`) has no paint to hide; its `src` is pointed at the+ * result instead.+ */+function overlay(world: World, node: Entity, paintSource: string | undefined, asset: Asset): void {+ const editor = getDocumentEditor(world);+ const fills = node.get(Cache)?.fills ?? [];+ const paint = (paintSource && fills.find((fill) => fill.get(Source)?.value === paintSource)) || topMedia(node)?.paint;++ if (!paint || paint === node || !paint.has(Paint)) {+ editor.editProperty(node, 'src', asset.path);+ return;+ }++ // The new paint is drawn the way the old one was: fit, opacity, blending.+ const props = { ...authoredElement(paint)?.props };+ delete props.src;+ delete props.hidden;+ editor.editProperty(paint, 'hidden', true);+ editor.insertElement(node, () => renderAuthored({+ tag: asset.type === 'VIDEO' ? 'videoPaint' : 'imagePaint',+ props: { ...props, src: asset.path },+ children: [],+ }));+}++/** Tells the user why a generation came to nothing — unless the upgrade dialog already does. */+function report(title: string, error: unknown): void {+ if (error instanceof TRPCClientError && error.data?.code === 'PAYMENT_REQUIRED') return;+ console.error(`[generate] ${title}:`, error);+ toast.error(title, { description: error instanceof Error ? error.message : String(error) });+}diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.tsindex 600ac39f..7d035571 100644--- a/apps/web/src/engine/hud/hud-system.ts+++ b/apps/web/src/engine/hud/hud-system.ts@@ -7,7 +7,7 @@ import {Active, Computed, Culled, FrameRate, Geometry, Group, Hidden,HitRegions, Hovering, Name, Playback, RenderSurface, Root, Selected,Sequential, ChildOf,- entityQuad, entityWorldMat, getMaskSelection, getSelectionMask, getSourceFailure,+ entityQuad, entityWorldMat, getGeneratingLabel, getGeneratingProgress, getMaskSelection, getSelectionMask, getSourceFailure,invert2D,multiply2D, rectToQuad, rotate2D, scale2D, store, transformPoint,translate2D,@@ -106,11 +106,14 @@ function drawHoverOutlines(world: World, ctx: Ctx2D, resolution: number): void {/*** The bar above a top-level node: play button, name, active badge, duration.- * It sits in the node's own rotation, one line above its top edge.+ * It sits in the node's own rotation, one line above its top edge. A node in+ * generation is headed by what the generation is doing instead of its name,+ * and by its progress where the duration would be.*/function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number): void {const failure = getSourceFailure(entity);- const label = failure || entity.get(Name)?.value;+ const generating = getGeneratingLabel(world, entity);+ const label = failure || generating || entity.get(Name)?.value;if (!label) return;const header = getHeaderLayout(world, entity, resolution);@@ -195,17 +198,19 @@ function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: numberctx.fillText('Active', 0, 0);}- if (playback && (fitted?.width ?? 0) + badgeWidth + 60 < header.width) {+ const trailing = generating+ ? getGeneratingProgress(entity)+ : playback+ ? formatDuration(world, entity)+ : undefined;++ if (trailing && (fitted?.width ?? 0) + badgeWidth + 60 < header.width) {ctx.setTransform(header.mat.a, header.mat.b, header.mat.c, header.mat.d, header.mat.e, header.mat.f);ctx.translate(header.width, 3);ctx.textAlign = 'right';ctx.textBaseline = 'top';ctx.font = '400 11px JetBrains Mono';- const seconds = (store(world, Computed).duration[entity.id()] ?? 0) / (world.get(FrameRate)?.value ?? 30);- const minutes = Math.floor(seconds / 60).toString().padStart(2, '0');- const rest = Math.floor(seconds % 60).toString().padStart(2, '0');-if (selected) {ctx.globalAlpha = 0.7;ctx.fillStyle = '#cce8ff';@@ -213,11 +218,19 @@ function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: numberctx.fillStyle = 'rgba(121, 121, 121, 1)';}- ctx.fillText(`${minutes}:${rest}`, 0, 0);+ ctx.fillText(trailing, 0, 0);ctx.globalAlpha = 1;}}+/** A node's duration as `mm:ss`. */+function formatDuration(world: World, entity: Entity): string {+ const seconds = (store(world, Computed).duration[entity.id()] ?? 0) / (world.get(FrameRate)?.value ?? 30);+ const minutes = Math.floor(seconds / 60).toString().padStart(2, '0');+ const rest = Math.floor(seconds % 60).toString().padStart(2, '0');+ return `${minutes}:${rest}`;+}+/*** The box around the selection, its resize/rotate regions, and the handles* that show where they are. Rotation regions sit outside the corners, resizediff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.tsindex 20b929b2..7b8e7452 100644--- a/apps/web/src/engine/timeline/render/clip.ts+++ b/apps/web/src/engine/timeline/render/clip.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getSourceFailure, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';+import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getGeneratingColor, getGeneratingLabel, getSourceFailure, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';import { getDocumentEditor } from '../../editor';import {@@ -57,6 +57,7 @@ export function renderClip(const asset = getClipAsset(world, entity);const error = getSourceFailure(entity);+ const generating = getGeneratingLabel(world, entity);const style = getClipStyle(entity, asset, !!error?.length);handleBody(world, surface, entity, left, width, row, resolution);@@ -70,11 +71,11 @@ export function renderClip(ctx.save();ctx.beginPath();ctx.roundRect(left, 0, width, row.height, CLIP_CORNER_RADIUS);- ctx.fillStyle = style.background;+ ctx.fillStyle = generating ? getGeneratingColor(world) : style.background;ctx.fill();ctx.restore();- if (!error) {+ if (!error && !generating) {renderContent(world, scene, surface, entity, asset, style, row);}@@ -86,6 +87,7 @@ export function renderClip(if (isCaption(entity)) label = label || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;else if (isText(entity)) label = entity.get(Chars)?.value ?? label ?? '';if (!label) label = getClipFallbackName(world, entity);+ if (generating) label = generating;if (error) label = error;const [viewportLeft] = getViewport(world, scene, surface.layout.width);diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.tsindex 0423a789..1875d10f 100644--- a/apps/web/src/lib/api.ts+++ b/apps/web/src/lib/api.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { createTRPCClient, httpBatchLink, TRPCClientError, type TRPCLink } from "@trpc/client";+import { createTRPCClient, httpBatchLink, httpSubscriptionLink, splitLink, TRPCClientError, type TRPCLink } from "@trpc/client";import { observable } from "@trpc/server/observable";import { supabase } from "./supabase";import { showUpgradeDialog } from "@/components/upgrade-dialog";@@ -25,17 +25,39 @@ const paymentRequiredLink: TRPCLink<AppRouter> = () => ({ next, op }) =>return () => sub.unsubscribe();});-/** The Diffusion Studio API (v2), authenticated with the Supabase session. */+const url = `${import.meta.env.VITE_API_URL ?? ""}/api/v2/trpc`;++/** The Supabase session as an Authorization header value, or undefined when signed out. */+async function authorization(): Promise<string | undefined> {+ const session = await supabase?.auth.getSession();+ const token = session?.data.session?.access_token;+ return token ? `Bearer ${token}` : undefined;+}++/**+ * The Diffusion Studio API (v2), authenticated with the Supabase session.+ * Subscriptions (`jobs.watch`) are server-sent events, which cannot carry+ * headers, so the token travels as a connection param there.+ */export const api = createTRPCClient<AppRouter>({links: [paymentRequiredLink,- httpBatchLink({- url: `${import.meta.env.VITE_API_URL ?? ""}/api/v2/trpc`,- async headers() {- const session = await supabase?.auth.getSession();- const token = session?.data.session?.access_token;- return token ? { Authorization: `Bearer ${token}` } : {};- },+ splitLink({+ condition: (op) => op.type === "subscription",+ true: httpSubscriptionLink({+ url,+ connectionParams: async () => {+ const value = await authorization();+ return value ? { authorization: value } : {};+ },+ }),+ false: httpBatchLink({+ url,+ async headers() {+ const value = await authorization();+ return value ? { Authorization: value } : {};+ },+ }),}),],});diff --git a/apps/web/src/lib/jobs.ts b/apps/web/src/lib/jobs.tsnew file mode 100644index 00000000..8567ba73--- /dev/null+++ b/apps/web/src/lib/jobs.ts@@ -0,0 +1,71 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Generation jobs on the API: everything that produces a file is one+ * `generate` call that answers with a job right away, followed to its end+ * through `jobs.watch`. The stream closes when the job ends, and every few+ * minutes on the server's side regardless; reopening it hands back the+ * current state first, so a reconnect misses nothing.+ */++import { api } from "./api";++import type { GenerateRequestInput, Job } from "@diffusionstudio/api-contract";++/** How often a dropped watch is reopened before the job is given up on. */+const MAX_RETRIES = 5;+const RETRY_DELAY = 2000;++export const isTerminal = (job: Job): boolean =>+ job.status === "succeeded" || job.status === "failed" || job.status === "canceled";++/**+ * Starts a job for `request` and follows it to its end. `onUpdate` hears+ * every state the job passes through, the first one included. Resolves with+ * the job as it ended — failed and canceled jobs included, which say why in+ * `error`; rejects only when the job could not be started or followed.+ */+export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void): Promise<Job> {+ const job = await api.generate.mutate({ request, idempotencyKey: crypto.randomUUID() });+ onUpdate(job);+ return isTerminal(job) ? job : watchJob(job.id, onUpdate);+}++/** Follows job `id` until it ends. */+export function watchJob(id: string, onUpdate: (job: Job) => void): Promise<Job> {+ return new Promise((resolve, reject) => {+ let failures = 0;+ let ended = false;++ const subscribe = () => {+ const subscription = api.jobs.watch.subscribe({ id }, {+ onData: ({ data }) => {+ failures = 0;+ onUpdate(data);+ if (!isTerminal(data)) return;+ ended = true;+ subscription.unsubscribe();+ resolve(data);+ },+ onError: (error) => {+ if (++failures > MAX_RETRIES) reject(error);+ else setTimeout(subscribe, RETRY_DELAY * failures);+ },+ // The server closes the stream after a while; the job goes on.+ onComplete: () => {+ if (!ended) subscribe();+ },+ });+ };++ subscribe();+ });+}++/** Why a job did not succeed, in a sentence for the user. */+export function jobFailure(job: Job): string {+ if (job.status === "canceled") return "The generation was canceled.";+ return job.error?.message || "The generation failed.";+}diff --git a/apps/web/src/lib/uploads.ts b/apps/web/src/lib/uploads.tsnew file mode 100644index 00000000..378a9083--- /dev/null+++ b/apps/web/src/lib/uploads.ts@@ -0,0 +1,45 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Getting a file to where a model can read it. The API stores inputs by the+ * sha256 of their bytes, per user, so a file is uploaded once however many+ * generations name it: `uploads.create` says whether the bytes are already+ * there, and hands out a signed resumable session when they are not.+ */++import { api } from "./api";++import type { AssetRef } from "@diffusionstudio/api-contract";++/** Lowercase hex sha256 of `blob`'s bytes: what the API names an upload by. */+async function sha256(blob: Blob): Promise<string> {+ const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");+}++/** Uploads `blob` unless the API already has its bytes; returns the ref a request names it by. */+export async function uploadFile(blob: Blob): Promise<AssetRef> {+ const contentType = blob.type || "application/octet-stream";+ const { asset, uploadUrl, uploadHeaders } = await api.uploads.create.mutate({+ sha256: await sha256(blob),+ contentType,+ size: blob.size,+ });+ if (!uploadUrl) return asset;++ // A signed resumable session (valid 15 minutes), opened with exactly the+ // headers it was signed for, then the bytes in one PUT.+ const start = await fetch(uploadUrl, {+ method: "POST",+ headers: uploadHeaders ?? { "x-goog-resumable": "start", "content-type": contentType },+ });+ const session = start.headers.get("location");+ if (!start.ok || !session) throw new Error(`Could not start the upload (${start.status})`);++ const put = await fetch(session, { method: "PUT", body: blob });+ if (!put.ok) throw new Error(`The upload failed (${put.status})`);++ return asset;+}diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.tsindex 65da731a..fa6ab8c3 100644--- a/packages/assets/src/types.ts+++ b/packages/assets/src/types.ts@@ -47,6 +47,11 @@ interface AssetBase {createdAt: string;mimeType: string;stat?: AssetStat;+ /**+ * The API job a model produced the file in, for a generated asset: what+ * it was asked (to run it again, or reuse its settings) and what it cost.+ */+ job?: string;/*** Resolved on the fly for a `src` that names a path or URL outside the* library; lives in memory only and is never written to the manifest.diff --git a/packages/runtime/src/actions/camera.ts b/packages/runtime/src/actions/camera.tsindex 03f46358..5f9bef7e 100644--- a/packages/runtime/src/actions/camera.ts+++ b/packages/runtime/src/actions/camera.ts@@ -105,14 +105,36 @@ export function resetCameraZoom(world: World): void {* rect should bring it into view, not magnify it.*/export function focusRect(world: World, rect: Rect, padding = FOCUS_PADDING): void {+ const fit = fitScale(world, rect, padding);+ if (fit !== null) centerOn(world, rect, clamp(fit, MIN_CAMERA_ZOOM, 1));+}++/**+ * Bring a document-space rect into view: centered, and fully visible with+ * `padding` CSS pixels to spare. Unlike `focusRect` the zoom is kept where it+ * fits — the view only zooms out, and only as far as the rect needs.+ */+export function revealRect(world: World, rect: Rect, padding = FOCUS_PADDING): void {+ const fit = fitScale(world, rect, padding);+ if (fit !== null) centerOn(world, rect, clamp(Math.min(fit, getCameraScale(world)), MIN_CAMERA_ZOOM, MAX_CAMERA_ZOOM));+}++/** The zoom at which `rect` just fits the viewport inside `padding`, or null when nothing can. */+function fitScale(world: World, rect: Rect, padding: number): number | null {const viewport = getViewport(world);- if (!viewport || rect.width <= 0 || rect.height <= 0) return;+ if (!viewport || rect.width <= 0 || rect.height <= 0) return null;const width = viewport.width - padding * 2;const height = viewport.height - padding * 2;- if (width <= 0 || height <= 0) return;+ if (width <= 0 || height <= 0) return null;++ return Math.min(width / rect.width, height / rect.height);+}- const scale = clamp(Math.min(width / rect.width, height / rect.height), MIN_CAMERA_ZOOM, 1);+/** Put the middle of `rect` in the middle of the viewport, at `scale`. */+function centerOn(world: World, rect: Rect, scale: number): void {+ const viewport = getViewport(world);+ if (!viewport) return;setCamera(world, {a: scale,diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex b21a8c55..fd2e8f99 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -30,6 +30,7 @@ import { getParentNode } from '../queries/hierarchy';import { getViewMatrix } from '../queries/camera';import { colorToHex } from '../utils/color';import { FAILED_COLOR, getSourceFailure } from '../utils/source-failure';+import { getGeneratingColor, isGenerating } from '../utils/generating';import { applyStrokeStyle } from '../utils/stroke';import { renderText } from '../utils/text';import { getTransitionWindow } from '../utils/transition';@@ -589,13 +590,15 @@ function renderStrokes(world: World, entity: Entity): void {}/**- * The still fill a node whose source failed is left with+ * What covers a node's fills while they cannot be shown: the pulse while a+ * generation is filling it in, the still fill when its source failed.*/-function renderSourceFailure(world: World, entity: Entity): void {- if (getSourceFailure(entity) === undefined) return;+function renderPendingFill(world: World, entity: Entity): void {+ const generating = isGenerating(entity);+ if (!generating && !getSourceFailure(entity)) return;const ctx = getCtx(world);- ctx.fillStyle = FAILED_COLOR;+ ctx.fillStyle = generating ? getGeneratingColor(world) : FAILED_COLOR;ctx.fill();}@@ -666,7 +669,7 @@ function renderShapeNode(world: World, entity: Entity): void {renderShadows(world, entity);renderIntrinsicFill(world, entity);renderFills(world, entity);- renderSourceFailure(world, entity);+ renderPendingFill(world, entity);renderStrokes(world, entity);}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex d7455338..f199f8e1 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -47,6 +47,8 @@ export const Hidden = trait();export const ClipsContent = trait();+export const Generating = trait({ label: '', progress: undefined as number | undefined });+export const Name = trait({ value: '' });// Stable identity for entities.diff --git a/packages/runtime/src/utils/generating.ts b/packages/runtime/src/utils/generating.tsnew file mode 100644index 00000000..d7fddbf6--- /dev/null+++ b/packages/runtime/src/utils/generating.ts@@ -0,0 +1,74 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * The pulse a node waiting on a generation is filled with (see `Generating`).+ * Everything that draws one reads it from here — the stage and the timeline+ * both — so a node in generation pulses in step wherever it is shown.+ *+ * Easing: cubic-bezier(0.52, 0.18, 0.56, 0.88)+ * Duration: 0.6s per half-cycle (alternating)+ * Delay: 0.2s before the first transition+ * Full cycle: 0.2 delay + 0.6 forward + 0.6 reverse = 1.4s+ *+ * Pulses between --background (#1c1c1c) and --secondary (#292929).+ */++import { cubicBezier } from 'animejs';++import { Generating, Time } from '../traits';++import type { Entity, World } from 'koota';++const ease = cubicBezier(0.52, 0.18, 0.56, 0.88);++const DELAY = 200; // ms+const HALF_CYCLE = 600; // ms+const CYCLE = DELAY + HALF_CYCLE * 2;++const FROM = [28, 28, 28] as const; // #1c1c1c+const TO = [41, 41, 41] as const; // #292929++/** Whether the node stands in for a generation in flight. */+export function isGenerating(entity: Entity): boolean {+ return entity.has(Generating);+}++/** How long each step of the trailing dots lasts. */+const DOT_STEP = 400; // ms++/**+ * What the header of a node in generation reads this frame — its label with+ * dots that count up on the world's clock (".", "..", "...") — or undefined+ * for any other node.+ */+export function getGeneratingLabel(world: World, entity: Entity): string | undefined {+ const label = entity.get(Generating)?.label;+ if (label === undefined) return undefined;+ const dots = 1 + (Math.floor((world.get(Time)?.now ?? 0) / DOT_STEP) % 3);+ return label + '.'.repeat(dots);+}++/** How far a node's generation has come, as a whole percentage, when there is a measure of it. */+export function getGeneratingProgress(entity: Entity): string | undefined {+ const progress = entity.get(Generating)?.progress;+ return progress === undefined ? undefined : `${Math.floor(progress * 100)}%`;+}++/** Where the pulse stands on the world's clock: 0 at rest, 1 fully lit. */+function pulse(world: World): number {+ const elapsed = (world.get(Time)?.now ?? 0) % CYCLE;+ if (elapsed < DELAY) return 0;++ const t = elapsed - DELAY;+ const eased = ease((t % HALF_CYCLE) / HALF_CYCLE);+ return t >= HALF_CYCLE ? 1 - eased : eased;+}++/** The fill a node in generation is painted with this frame. */+export function getGeneratingColor(world: World): string {+ const factor = pulse(world);+ const [r, g, b] = FROM.map((from, i) => Math.round(from + (TO[i]! - from) * factor));+ return `rgb(${r},${g},${b})`;+}