Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor generative AI components and consolidate generation logic

konstantin-paulus42753381fff9
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsx
index edb70d2b..79f21405 100644
--- a/apps/web/src/components/genai/action-bar.tsx
+++ b/apps/web/src/components/genai/action-bar.tsx
@@ -15,10 +15,7 @@ import {
} from "@/components/ui/dropdown-menu";
import { createMemo, Show } from "solid-js";
import { useGenerationRecords } from "./use-generation-records";
-import { useGenerateImage } from "./use-generate-image";
-import { useGenerateVideo } from "./use-generate-video";
-import { useGenerateVoice } from "./use-generate-voice";
-import { useGenerateAudio } from "./use-generate-audio";
+import { useGenerate } from "./use-generate";
import { useAutoCaptions } from "./use-auto-captions";
import { useMediaSelection } from "./selection";
import { useTransforms } from "./use-transforms";
@@ -32,48 +29,28 @@ interface ActionBarProps {
}
export function ActionBar(props: ActionBarProps) {
- const { imageNodes, videoNodes } = useMediaSelection();
- const { isOn, toggle } = useTransforms();
+ const { imageMedia, videoMedia } = useMediaSelection();
+ const { run: runTransform } = useTransforms();
- const { generate: generateImage } = useGenerateImage();
- const { generate: generateVideo } = useGenerateVideo();
- const { generate: generateVoice } = useGenerateVoice();
- const { generate: generateAudio } = useGenerateAudio();
+ const { rerun } = useGenerate();
const { generate: autoCaptions, hasScene } = useAutoCaptions();
- const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
+ const { isGenerated, totalCredits, firstJob, firstConfig } = useGenerationRecords();
- const isImage = createMemo(() => imageNodes().length > 0);
- const isVideo = createMemo(() => videoNodes().length > 0);
+ const isImage = createMemo(() => imageMedia().length > 0);
+ const isVideo = createMemo(() => videoMedia().length > 0);
const visible = createMemo(() => {
return isImage() || isVideo() || hasScene();
})
const handleRerun = () => {
- const config = firstConfig();
- if (!config) {
- toast("No generation config found", { description: "This asset wasn't generated with a prompt." });
+ const job = firstJob();
+ if (!job) {
+ toast("No generation found", { description: "This asset wasn't generated with a prompt." });
return;
}
-
- const promise = (() => {
- switch (config.mode) {
- case "IMAGE":
- return generateImage(config);
- case "VIDEO":
- return generateVideo(config);
- case "VOICE":
- return generateVoice(config);
- case "AUDIO":
- return generateAudio(config);
- }
- })();
-
- promise.catch((err) => {
- toast("Rerun failed", {
- description: err instanceof Error ? err.message : String(err),
- });
- });
+
+ rerun(job.request);
};
const handleEditWithPrompt = () => {
@@ -108,8 +85,7 @@ export function ActionBar(props: ActionBarProps) {
<Button
variant="ghost"
class="gap-0 pl-0.5 text-muted-foreground"
- classList={{ "text-foreground": isOn("removeBackground") }}
- onClick={() => toggle("removeBackground")}
+ onClick={() => runTransform("removeBackground")}
>
<Icon name="ai-generate" />
Remove background
@@ -117,8 +93,7 @@ export function ActionBar(props: ActionBarProps) {
<Button
variant="ghost"
class="gap-0 pl-0.5 text-muted-foreground"
- classList={{ "text-foreground": isOn("upscale") }}
- onClick={() => toggle("upscale")}
+ onClick={() => runTransform("upscale")}
>
<Icon name="arrow-scale" />
Upscale
@@ -172,8 +147,7 @@ export function ActionBar(props: ActionBarProps) {
<Button
variant="ghost"
class="gap-0 pl-0.5 text-muted-foreground"
- classList={{ "text-foreground": isOn("addAudio") }}
- onClick={() => toggle("addAudio")}
+ onClick={() => runTransform("addAudio")}
>
<Icon name="generate-audio" />
Add audio
@@ -181,8 +155,7 @@ export function ActionBar(props: ActionBarProps) {
<Button
variant="ghost"
class="gap-0 pl-0.5 text-muted-foreground"
- classList={{ "text-foreground": isOn("upscale") }}
- onClick={() => toggle("upscale")}
+ onClick={() => runTransform("upscale")}
>
<Icon name="arrow-scale" />
Upscale
diff --git a/apps/web/src/components/genai/config.ts b/apps/web/src/components/genai/config.ts
index c9092c43..acc00c83 100644
--- a/apps/web/src/components/genai/config.ts
+++ b/apps/web/src/components/genai/config.ts
@@ -2,8 +2,42 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import type { ModelId } from "@diffusionstudio/api-contract";
+
+/**
+ * What each model of the API makes. Keyed by the contract's model ids, so a
+ * model added to (or dropped from) the API fails the type check here first.
+ * The modes are the prompt box's; the tools are run from the action bar, and
+ * the text models (transcripts, analysis) never land on the canvas.
+ */
+export const MODEL_MODES: Record<ModelId, PromptMode | "TEXT"> = {
+ "gpt-image-2": "IMAGE",
+ "nano-banana-2": "IMAGE",
+ "nano-banana-pro": "IMAGE",
+ "seedream-4.5": "IMAGE",
+ "flux-2-klein": "IMAGE",
+ "remove-background": "IMAGE",
+ "upscale-image": "IMAGE",
+ "kling-3-pro": "VIDEO",
+ "kling-o3-pro": "VIDEO",
+ "wan-2.6": "VIDEO",
+ "hailuo-3-max": "VIDEO",
+ "seedance-2.0": "VIDEO",
+ "veo-3.1": "VIDEO",
+ "veo-3.1-fast": "VIDEO",
+ "upscale-video": "VIDEO",
+ "add-audio": "VIDEO",
+ "elevenlabs-music": "AUDIO",
+ "elevenlabs-sfx": "AUDIO",
+ "elevenlabs-v3": "VOICE",
+ "transcribe": "TEXT",
+ "analyze": "TEXT",
+};
+
+export type PromptMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";
+
/** The one model the voice mode speaks with; the UI picks the voice, not the model. */
-export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3";
+export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3" satisfies ModelId;
/** What an aspect ratio is worth in pixels, at 1080p. */
export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {
@@ -67,51 +101,72 @@ export const PROMPT_INPUT_RESOLUTION_OPTIONS = [
{ value: "720p", label: "720p" },
];
-export const PROMPT_INPUT_IMAGE_MODEL_OPTIONS = [
+export type ImageModelOption = {
+ name: string;
+ id: ModelId;
+ icon: string;
+ description: string;
+ maxReferences: number;
+};
+
+export const PROMPT_INPUT_IMAGE_MODEL_OPTIONS: ImageModelOption[] = [
{
- name: "FLUX.2 [DEV] Turbo",
- id: "flux-2-turbo",
+ name: "FLUX.2 Klein",
+ id: "flux-2-klein",
icon: "large-bfl",
- description: "Low budget, high quality, fast turbo mode.",
+ description: "Low budget, high quality, fast.",
+ maxReferences: 4,
},
{
name: "GPT Image 2",
id: "gpt-image-2",
icon: "large-openai",
description: "Flexible sizes up to 4K, true aspect ratios.",
+ maxReferences: 16,
},
{
name: "Nano Banana 2",
id: "nano-banana-2",
icon: "large-google",
description: "Fast, high quality, flexible edits.",
+ maxReferences: 14,
},
{
name: "Nano Banana Pro",
id: "nano-banana-pro",
icon: "large-google",
description: "Pro control, readable text, ultra consistent.",
+ maxReferences: 14,
},
{
name: "Seedream 4.5",
id: "seedream-4.5",
icon: "large-bytedance",
description: "Up to 4K, multi image edits, strong text.",
+ maxReferences: 14,
},
];
+/**
+ * What a video model takes besides the prompt: a first and a last frame, and
+ * a switch for the sound it makes (models without one score their video
+ * themselves, or not at all).
+ */
export type VideoModelFeature = "start-frame" | "end-frame" | "audio";
export type VideoModelOption = {
name: string;
- id: string;
+ id: ModelId;
icon: string;
description: string;
features: VideoModelFeature[];
durations: string[];
+ /** Empty for a model that takes its proportions from the start frame. */
aspectRatios: string[];
};
+const seconds = (from: number, to: number) => Array.from({ length: to - from + 1 }, (_, i) => `${from + i}s`);
+
export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [
{
name: "Kling 3.0",
@@ -119,7 +174,7 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [
icon: "large-kling",
description: "Cinematic motion with built-in audio.",
features: ["start-frame", "end-frame", "audio"],
- durations: ["3s", "4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],
+ durations: seconds(3, 15),
aspectRatios: ["16:9", "9:16", "1:1"],
},
{
@@ -127,26 +182,17 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [
id: "kling-o3-pro",
icon: "large-kling",
description: "Multi-modal reasoning, strong scenes.",
- features: ["start-frame", "end-frame", "audio"],
- durations: ["3s", "4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],
- aspectRatios: ["16:9", "9:16", "1:1"],
- },
- {
- name: "Kling 2.5 Turbo",
- id: "kling-2.5-turbo",
- icon: "large-kling",
- description: "Fast and affordable for quick iterations.",
- features: ["start-frame", "end-frame", "audio"],
- durations: ["5s", "10s"],
- aspectRatios: ["16:9", "9:16", "1:1"],
+ features: ["start-frame", "audio"],
+ durations: seconds(3, 15),
+ aspectRatios: [],
},
{
name: "Seedance 2.0",
id: "seedance-2.0",
icon: "large-bytedance",
description: "Rich motion, lip-synced audio and SFX.",
- features: ["start-frame", "end-frame", "audio"],
- durations: ["4s", "5s", "6s", "7s", "8s", "9s", "10s", "11s", "12s", "13s", "14s", "15s"],
+ features: ["start-frame", "end-frame"],
+ durations: seconds(4, 15),
aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],
},
{
@@ -173,17 +219,17 @@ export const PROMPT_INPUT_VIDEO_MODEL_OPTIONS: VideoModelOption[] = [
icon: "large-wan",
description: "Expressive motion, stylized aesthetics.",
features: ["start-frame"],
- durations: ["5s", "10s", "15s"],
- aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],
+ durations: ["5s", "10s"],
+ aspectRatios: ["16:9", "9:16"],
},
{
- name: "Hailuo 2.3",
- id: "hailuo-2.3",
+ name: "Hailuo 3 Max",
+ id: "hailuo-3-max",
icon: "large-hailuo",
description: "Smooth motion, cinematic lighting.",
- features: ["start-frame"],
- durations: ["5s"],
- aspectRatios: ["16:9"],
+ features: ["start-frame", "end-frame"],
+ durations: seconds(5, 15),
+ aspectRatios: ["16:9", "9:16", "1:1", "4:3", "3:4"],
},
];
@@ -351,7 +397,7 @@ export const PROMPT_INPUT_VOICE_OPTIONS = [
},
];
-export const PROMPT_INPUT_AUDIO_MODEL_OPTIONS = [
+export const PROMPT_INPUT_AUDIO_MODEL_OPTIONS: { name: string; id: ModelId; icon: string; description: string }[] = [
{
name: "ElevenLabs Music",
id: "elevenlabs-music",
diff --git a/apps/web/src/components/genai/placeholder.ts b/apps/web/src/components/genai/placeholder.ts
deleted file mode 100644
index 201f6189..00000000
--- a/apps/web/src/components/genai/placeholder.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Placeholders for the generative AI features, which are being rebuilt. The
-// prompt box and the action bar stay in place; every action that would have reached a model lands here instead.
-
-import { toast } from "somoto";
-
-/** Tells the user `feature` is not wired up yet. */
-export function notAvailable(feature: string): void {
- toast(`${feature} is not available yet`, {
- id: `genai-placeholder:${feature}`,
- description: "Generative AI features are being rebuilt.",
- });
-}
diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsx
index b3cacf90..eb2ddd2a 100644
--- a/apps/web/src/components/genai/prompt-input-actions.tsx
+++ b/apps/web/src/components/genai/prompt-input-actions.tsx
@@ -18,55 +18,31 @@ import { createMemo, Show } from "solid-js";
import { toast } from "somoto";
import { useGenerationRecords } from "./use-generation-records";
-import { useGenerateImage } from "./use-generate-image";
-import { useGenerateVideo } from "./use-generate-video";
-import { useGenerateVoice } from "./use-generate-voice";
-import { useGenerateAudio } from "./use-generate-audio";
+import { useGenerate } from "./use-generate";
import { useMediaSelection } from "./selection";
import { useTransforms } from "./use-transforms";
import type { TransformType } from "./schemas";
export function PromptInputActions() {
- const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
- const { imageNodes, videoNodes } = useMediaSelection();
- const { isOn, toggle } = useTransforms();
+ const { isGenerated, totalCredits, firstJob } = useGenerationRecords();
+ const { imageMedia, videoMedia } = useMediaSelection();
+ const { run } = useTransforms();
+ const { rerun } = useGenerate();
- const { generate: generateImage } = useGenerateImage();
- const { generate: generateVideo } = useGenerateVideo();
- const { generate: generateVoice } = useGenerateVoice();
- const { generate: generateAudio } = useGenerateAudio();
-
- const hasImageSelection = createMemo(() => imageNodes().length > 0);
- const hasVideoSelection = createMemo(() => videoNodes().length > 0);
+ const hasImageSelection = createMemo(() => imageMedia().length > 0);
+ const hasVideoSelection = createMemo(() => videoMedia().length > 0);
const handleRerun = () => {
- const config = firstConfig();
- if (!config) {
- toast("No generation config found", {
+ const job = firstJob();
+ if (!job) {
+ toast("No generation found", {
description: "This asset wasn't generated with a prompt.",
});
return;
}
-
- const promise = (() => {
- switch (config.mode) {
- case "IMAGE":
- return generateImage(config);
- case "VIDEO":
- return generateVideo(config);
- case "VOICE":
- return generateVoice(config);
- case "AUDIO":
- return generateAudio(config);
- }
- })();
-
- promise.catch((err) => {
- toast("Rerun failed", {
- description: err instanceof Error ? err.message : String(err),
- });
- });
+
+ rerun(job.request);
};
return (
@@ -121,13 +97,13 @@ export function PromptInputActions() {
</Show>
<DropdownMenuGroup>
<Show when={hasImageSelection() || hasVideoSelection()}>
- <TransformItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />
+ <TransformItem name="upscale" icon="arrow-scale" label="Upscale" run={run} />
</Show>
<Show when={hasImageSelection()}>
- <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />
+ <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" run={run} />
</Show>
<Show when={hasVideoSelection()}>
- <TransformItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />
+ <TransformItem name="addAudio" icon="audio-on" label="Add audio" run={run} />
</Show>
</DropdownMenuGroup>
</DropdownMenuContent>
@@ -142,19 +118,15 @@ type TransformItemProps = {
name: TransformType;
icon: string;
label: string;
- isOn(name: TransformType): boolean;
- toggle(name: TransformType): void;
+ run(name: TransformType): void;
};
-/** A transform as a menu row: a check when the selection is asking for it. */
+/** A transform as a menu row. */
function TransformItem(props: TransformItemProps) {
return (
- <DropdownMenuItem onSelect={() => props.toggle(props.name)}>
+ <DropdownMenuItem onSelect={() => props.run(props.name)}>
<Icon name={props.icon} class="mr-2 text-foreground" />
<span class="flex-1">{props.label}</span>
- <Show when={props.isOn(props.name)}>
- <Icon name="confirm-check" class="ml-2 text-foreground" />
- </Show>
</DropdownMenuItem>
);
}
diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsx
index 5fd46d02..a90e63c5 100644
--- a/apps/web/src/components/genai/prompt-input.tsx
+++ b/apps/web/src/components/genai/prompt-input.tsx
@@ -41,14 +41,11 @@ import {
ALL_DURATION_OPTIONS,
PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
// PROMPT_INPUT_RESOLUTION_OPTIONS
+ type PromptMode,
} from "./config";
import { createStoredSignal } from "@/lib/store";
import { store } from "@/init";
-import { toast } from "somoto";
-import { useGenerateImage } from "./use-generate-image";
-import { useGenerateVideo } from "./use-generate-video";
-import { useGenerateVoice } from "./use-generate-voice";
-import { useGenerateAudio } from "./use-generate-audio";
+import { useGenerate } from "./use-generate";
import { PromptInputActions } from "./prompt-input-actions";
import type {
AspectRatio,
@@ -68,9 +65,10 @@ import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";
import type { AssetCache } from "@diffusionstudio/assets";
import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";
-export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";
+export type PromptInputMode = PromptMode;
const DEFAULT_BUTTON_CLASS = "text-muted-foreground gap-0 pr-2 pl-0";
+/** As many references as the prompt box has room for; a model may take fewer. */
const MAX_IMAGE_REFERENCES = 5;
type GenerationConfgs = {
@@ -123,10 +121,7 @@ export function PromptInput(props: PromptInputProps) {
const library = useLibrary();
const editor = useEditor();
const { images: selectedImages } = useMediaSelection();
- const { generate: generateImage } = useGenerateImage();
- const { generate: generateVideo } = useGenerateVideo();
- const { generate: generateVoice } = useGenerateVoice();
- const { generate: generateAudio } = useGenerateAudio();
+ const { generate } = useGenerate();
let textareaRef!: HTMLTextAreaElement;
let dragCounter = 0;
@@ -159,6 +154,11 @@ export function PromptInput(props: PromptInputProps) {
return c.mode === "VOICE" ? c : undefined;
};
+ const maxImageReferences = createMemo(() => Math.min(
+ MAX_IMAGE_REFERENCES,
+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS.find((m) => m.id === config().model)?.maxReferences ?? MAX_IMAGE_REFERENCES,
+ ));
+
const currentVideoModel = createMemo(() =>
PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((m) => m.id === config().model),
);
@@ -184,7 +184,7 @@ export function PromptInput(props: PromptInputProps) {
.filter((id) => !local.includes(id));
return [...local, ...selected]
.filter((id) => lib.get(id))
- .slice(0, MAX_IMAGE_REFERENCES);
+ .slice(0, maxImageReferences());
});
/** The library id a frame is set to, explicitly or by the selection. */
@@ -239,11 +239,11 @@ export function PromptInput(props: PromptInputProps) {
// The reference is shown from a selected canvas entity, so removing it
// means deselecting that entity.
const entry = selectedImages().find((e) => e.asset.id === assetId);
- if (entry) editor.deselect(entry.entity);
+ if (entry) editor.deselect(entry.node);
};
const openImageReferencesPicker = async () => {
- if (effectiveImageRefIds().length >= MAX_IMAGE_REFERENCES) return;
+ if (effectiveImageRefIds().length >= maxImageReferences()) return;
const lib = library();
if (!lib) return;
@@ -255,7 +255,7 @@ export function PromptInput(props: PromptInputProps) {
const imageAssets = imported.filter((a) => a.type === "IMAGE");
const current = imageConfig()?.imageRefIds ?? [];
patch({
- imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),
+ imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, maxImageReferences()),
});
};
@@ -291,7 +291,7 @@ export function PromptInput(props: PromptInputProps) {
// The frame is being shown from a selected canvas entity, so clearing it
// means deselecting that entity (mirrors removeImageReference).
const entry = selectedImages()[frame === "start" ? 0 : 1];
- if (entry) editor.deselect(entry.entity);
+ if (entry) editor.deselect(entry.node);
};
const resizeTextarea = () => {
@@ -321,31 +321,24 @@ export function PromptInput(props: PromptInputProps) {
patch({ prompt: "" });
resizeTextarea();
- const promise = (() => {
- switch (currentConfig.mode) {
- case "IMAGE":
- return generateImage({
- ...currentConfig,
- imageRefIds: effectiveImageRefIds(),
- });
- case "VIDEO":
- return generateVideo({
- ...currentConfig,
- startFrameImageId: effectiveStartFrameId() || undefined,
- endFrameImageId: effectiveEndFrameId() || undefined,
- });
- case "VOICE":
- return generateVoice(currentConfig);
- case "AUDIO":
- return generateAudio(currentConfig);
- }
- })();
+ let submitted: GenerationConfig;
+
+ if (currentConfig.mode === "IMAGE") {
+ submitted = {
+ ...currentConfig,
+ imageRefIds: effectiveImageRefIds(),
+ };
+ } else if (currentConfig.mode === "VIDEO") {
+ submitted = {
+ ...currentConfig,
+ startFrameImageId: effectiveStartFrameId() || undefined,
+ endFrameImageId: effectiveEndFrameId() || undefined,
+ };
+ } else {
+ submitted = currentConfig;
+ }
- promise.catch((err) => {
- toast("Generation failed", {
- description: err instanceof Error ? err.message : String(err),
- });
- });
+ generate(submitted);
};
const handleKeyDown = (e: KeyboardEvent) => {
@@ -433,9 +426,7 @@ export function PromptInput(props: PromptInputProps) {
}
const current = imageConfig()?.imageRefIds ?? [];
- patch({
- imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),
- });
+ patch({ imageRefIds: [...current, ...imageIds].slice(0, maxImageReferences()) });
};
const handleVideoModelChange = (id: string) => {
@@ -451,7 +442,7 @@ export function PromptInput(props: PromptInputProps) {
endFrameImageId: model.features.includes("end-frame") ? c.endFrameImageId : undefined,
generateAudio: model.features.includes("audio") ? (c.generateAudio ?? true) : false,
duration: model.durations.includes(durationStr) ? c.duration : parseInt(model.durations[0], 10),
- aspectRatio: model.aspectRatios.includes(c.aspectRatio)
+ aspectRatio: model.aspectRatios.length === 0 || model.aspectRatios.includes(c.aspectRatio)
? c.aspectRatio
: (model.aspectRatios[0] as AspectRatio),
});
@@ -489,7 +480,7 @@ export function PromptInput(props: PromptInputProps) {
/>
)}
</For>
- <Show when={effectiveImageRefIds().length < MAX_IMAGE_REFERENCES}>
+ <Show when={effectiveImageRefIds().length < maxImageReferences()}>
<PromptInputAttachButton
icon="attachment"
label="Image references"
@@ -664,14 +655,16 @@ export function PromptInput(props: PromptInputProps) {
value={modelAccessor}
onChange={handleVideoModelChange}
/>
- <PromptInputCompactMenu
- aria-label="Select aspect ratio"
- menuLabel="Aspect ratio"
- value={aspectRatioAccessor}
- options={videoAspectRatioOptions()}
- onChange={(v) => patch({ aspectRatio: v as AspectRatio })}
- triggerIcon="aspect-ratio-16-9"
- />
+ <Show when={videoAspectRatioOptions().length > 0}>
+ <PromptInputCompactMenu
+ aria-label="Select aspect ratio"
+ menuLabel="Aspect ratio"
+ value={aspectRatioAccessor}
+ options={videoAspectRatioOptions()}
+ onChange={(v) => patch({ aspectRatio: v as AspectRatio })}
+ triggerIcon="aspect-ratio-16-9"
+ />
+ </Show>
{/* <PromptInputCompactMenu
aria-label="Select resolution"
menuLabel="Resolution"
diff --git a/apps/web/src/components/genai/requests.ts b/apps/web/src/components/genai/requests.ts
new file mode 100644
index 00000000..57f583e1
--- /dev/null
+++ b/apps/web/src/components/genai/requests.ts
@@ -0,0 +1,147 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The prompt box's settings in the API's vocabulary and back: what a config
+ * asks the API for (`toRequest`), what a request puts on the canvas
+ * (`outputOf`), and what settings a past request was made with (`toConfig`).
+ */
+
+import { uploadAsset } from "@/engine/generate";
+import { AUDIO_SIZE } from "@/engine/insert-asset";
+import { ASPECT_RATIO_DIMENSIONS, MODEL_MODES, PROMPT_INPUT_VIDEO_MODEL_OPTIONS, PROMPT_INPUT_VOICE_OPTIONS } from "./config";
+import { aspectRatioSchema } from "./schemas";
+
+import type { AssetLibrary } from "@diffusionstudio/assets";
+import type { AssetRef, GenerateRequest, GenerateRequestInput, ModelId } from "@diffusionstudio/api-contract";
+import type { GenerationOutput } from "@/engine/generate";
+import type { GenerationConfig } from "./schemas";
+
+const TITLES = {
+ IMAGE: "Image generation failed",
+ VIDEO: "Video generation failed",
+ VOICE: "Voice generation failed",
+ AUDIO: "Audio generation failed",
+} as const;
+
+/** The default box of a picture or a clip: its aspect ratio at 1080p. */
+const DEFAULT_SIZE = ASPECT_RATIO_DIMENSIONS["16:9"]!;
+
+/**
+ * What a request puts on the canvas, read off the request itself — so a past
+ * job (`job.request`) is placed the way the prompt that made it was. Text
+ * models have nothing to place.
+ */
+export function outputOf(request: { model: ModelId; count?: number; aspectRatio?: string }): GenerationOutput | undefined {
+ const mode = MODEL_MODES[request.model];
+ if (mode === "TEXT") return undefined;
+
+ return {
+ kind: mode === "IMAGE" ? "image" : mode === "VIDEO" ? "video" : "audio",
+ count: request.count ?? 1,
+ size: mode === "VOICE" || mode === "AUDIO"
+ ? { ...AUDIO_SIZE }
+ : ASPECT_RATIO_DIMENSIONS[request.aspectRatio ?? ""] ?? DEFAULT_SIZE,
+ title: TITLES[mode],
+ };
+}
+
+/** The library assets `ids` name, uploaded for a model to read; ids no longer in the library are dropped. */
+function refs(library: AssetLibrary, ids: (string | undefined)[]): Promise<(AssetRef | undefined)[]> {
+ return Promise.all(ids.map((id) => {
+ const asset = id ? library.get(id) : undefined;
+ return asset ? uploadAsset(library, asset) : undefined;
+ }));
+}
+
+/**
+ * The request `config` makes, its library inputs uploaded. The API validates
+ * the rest: a model that does not take a setting says so.
+ */
+export async function toRequest(library: AssetLibrary, config: GenerationConfig): Promise<GenerateRequestInput> {
+ const model = config.model as ModelId;
+
+ switch (config.mode) {
+ case "IMAGE": {
+ const images = (await refs(library, config.imageRefIds ?? [])).filter((ref) => ref !== undefined);
+ return {
+ model,
+ prompt: config.prompt,
+ aspectRatio: config.aspectRatio,
+ count: config.count,
+ ...(images.length ? { images } : {}),
+ } as GenerateRequestInput;
+ }
+ case "VIDEO": {
+ const option = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((candidate) => candidate.id === model);
+ const [startFrame, endFrame] = await refs(library, [config.startFrameImageId, config.endFrameImageId]);
+ return {
+ model,
+ prompt: config.prompt,
+ duration: config.duration,
+ ...(option?.aspectRatios.length ? { aspectRatio: config.aspectRatio } : {}),
+ ...(option?.features.includes("audio") ? { generateAudio: config.generateAudio ?? false } : {}),
+ ...(startFrame ? { startFrame } : {}),
+ ...(endFrame ? { endFrame } : {}),
+ } as GenerateRequestInput;
+ }
+ case "VOICE":
+ return { model: "elevenlabs-v3", prompt: config.prompt, voice: config.voice };
+ case "AUDIO":
+ return { model, prompt: config.prompt } as GenerateRequestInput;
+ }
+}
+
+/**
+ * The prompt box set the way `request` was made — what "Reuse" opens it with.
+ * Inputs are left out: the request names them by upload, not by library
+ * asset. Undefined for a request the prompt box does not make (a tool).
+ */
+export function toConfig(request: GenerateRequest): GenerationConfig | undefined {
+ const aspectRatio = (value: unknown) => aspectRatioSchema.safeParse(value).data ?? "16:9";
+
+ switch (request.model) {
+ case "gpt-image-2":
+ case "nano-banana-2":
+ case "nano-banana-pro":
+ case "seedream-4.5":
+ case "flux-2-klein":
+ return {
+ mode: "IMAGE",
+ model: request.model,
+ prompt: request.prompt,
+ aspectRatio: aspectRatio(request.aspectRatio),
+ count: Math.min(request.count, 4),
+ };
+ case "kling-3-pro":
+ case "kling-o3-pro":
+ case "wan-2.6":
+ case "hailuo-3-max":
+ case "seedance-2.0":
+ case "veo-3.1":
+ case "veo-3.1-fast":
+ return {
+ mode: "VIDEO",
+ model: request.model,
+ prompt: request.prompt,
+ aspectRatio: aspectRatio("aspectRatio" in request ? request.aspectRatio : undefined),
+ duration: request.duration,
+ generateAudio: "generateAudio" in request ? request.generateAudio : undefined,
+ };
+ case "elevenlabs-v3":
+ return {
+ mode: "VOICE",
+ model: request.model,
+ prompt: request.prompt,
+ voice: PROMPT_INPUT_VOICE_OPTIONS.some((option) => option.value === request.voice)
+ ? request.voice
+ : PROMPT_INPUT_VOICE_OPTIONS[0]!.value,
+ };
+ case "elevenlabs-music":
+ case "elevenlabs-sfx":
+ return { mode: "AUDIO", model: request.model, prompt: request.prompt };
+ default:
+ return undefined;
+ }
+}
diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.ts
index 54f37e73..feba5d7f 100644
--- a/apps/web/src/components/genai/selection.ts
+++ b/apps/web/src/components/genai/selection.ts
@@ -3,75 +3,58 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createMemo } from "solid-js";
-import { AssetId, Cache, Paint, PaintType, getIntrinsicPaint } from "@diffusionstudio/runtime";
-import { useSelection } from "@/engine/hooks";
+import { AssetId, PaintType } from "@diffusionstudio/runtime";
+import { useDerived, useSelection } from "@/engine/hooks";
+import { topMedia } from "@/engine/generate";
import { useLibrary } from "@/engine/library";
import type { Asset } from "@diffusionstudio/assets";
-import type { Entity } from "koota";
+import type { NodeMedia } from "@/engine/generate";
-/** The kinds of paint the prompt box has anything to say about. */
-type MediaPaint = PaintType.IMAGE | PaintType.VIDEO;
-
-/** A selected node and the element its source props live on. */
-export interface SelectedNode {
- entity: Entity;
- source: Entity;
- paint?: MediaPaint;
+/** A selected node's top media, and the library asset it shows once that has loaded. */
+export interface SelectedMedia extends NodeMedia {
+ asset?: Asset;
}
-/** A selected node and the library asset its source element is bound to. */
-export interface BoundNode extends SelectedNode {
- asset: Asset;
-}
+/** A node's top media with the id of the asset bound to it so far. */
+type Shown = NodeMedia & { assetId?: string };
-function isMediaPaint(paint: PaintType | undefined): paint is MediaPaint {
- return paint === PaintType.IMAGE || paint === PaintType.VIDEO;
-}
+const sameMedia = (a: Shown[], b: Shown[]) =>
+ a.length === b.length && a.every((media, index) => {
+ const other = b[index]!;
+ return media.node === other.node && media.paint === other.paint && media.assetId === other.assetId;
+ });
/**
- * Where `entity` keeps its source: itself when its own paint is the media,
- * else its first media fill. A node that paints no media at all is still its
- * own source element — an `<audio>` carries its `src` like any other.
+ * The media the selected nodes show on top (see `topMedia`) — what a prompt
+ * references and a transform is run over. Sampled every frame: which paint
+ * is on top is derived state, and changes without the selection changing (a
+ * transform lands, a fill is hidden).
*/
-function resolve(entity: Entity): SelectedNode {
- const intrinsic = getIntrinsicPaint(entity);
- if (isMediaPaint(intrinsic)) return { entity, source: entity, paint: intrinsic };
-
- for (const fill of entity.get(Cache)?.fills ?? []) {
- const paint = fill.get(Paint)?.value;
- if (isMediaPaint(paint)) return { entity, source: fill, paint };
- }
-
- return { entity, source: entity };
-}
-
export function useMediaSelection() {
const library = useLibrary();
const { nodes } = useSelection();
- const selected = createMemo(() => nodes().map(resolve));
-
- const painted = (paint: MediaPaint) =>
- createMemo(() => selected().filter((node) => node.paint === paint).map((node) => node.source));
+ const shown = useDerived<Shown[]>(
+ () => nodes()
+ .map(topMedia)
+ .filter((media) => media !== undefined)
+ .map((media) => ({ ...media, assetId: media.paint.get(AssetId)?.value })),
+ sameMedia,
+ );
- const imageNodes = painted(PaintType.IMAGE);
- const videoNodes = painted(PaintType.VIDEO);
-
- const bound = createMemo(() => {
+ const media = createMemo<SelectedMedia[]>(() => {
const lib = library();
- if (!lib) return [];
-
- const entries: BoundNode[] = [];
- for (const node of selected()) {
- const id = node.source.get(AssetId)?.value;
- const asset = id ? lib.get(id) : undefined;
- if (asset) entries.push({ ...node, asset });
- }
- return entries;
+ return shown().map(({ assetId, ...entry }) => ({ ...entry, asset: assetId ? lib?.get(assetId) : undefined }));
});
- const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));
+ const imageMedia = createMemo(() => media().filter((entry) => entry.type === PaintType.IMAGE));
+ const videoMedia = createMemo(() => media().filter((entry) => entry.type === PaintType.VIDEO));
+
+ /** Selected pictures whose asset is in the library: what a prompt can reference. */
+ const images = createMemo(() =>
+ imageMedia().filter((entry): entry is SelectedMedia & { asset: Asset } => entry.asset?.type === "IMAGE"),
+ );
- return { bound, images, imageNodes, videoNodes };
+ return { media, imageMedia, videoMedia, images };
}
diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsx
index f874b43a..025eaf25 100644
--- a/apps/web/src/components/genai/use-auto-captions.tsx
+++ b/apps/web/src/components/genai/use-auto-captions.tsx
@@ -3,15 +3,25 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createMemo } from "solid-js";
+import { useWorld } from "@diffusionstudio/koota-solid";
import { isScene } from "@diffusionstudio/runtime";
+import { useProject } from "@/context/project";
+import { generateCaptions } from "@/engine/generate";
import { useSelection } from "@/engine/hooks";
-import { notAvailable } from "./placeholder";
export function useAutoCaptions() {
+ const world = useWorld();
+ const project = useProject();
const { nodes } = useSelection();
- const hasScene = createMemo(() => nodes().some(isScene));
- const generate = () => notAvailable("Auto captions");
+ const scenes = createMemo(() => nodes().filter(isScene));
+ const hasScene = createMemo(() => scenes().length > 0);
+
+ const generate = () => {
+ for (const scene of scenes()) {
+ generateCaptions(world, scene, project.dir());
+ }
+ };
return { hasScene, generate };
}
diff --git a/apps/web/src/components/genai/use-generate-audio.tsx b/apps/web/src/components/genai/use-generate-audio.tsx
deleted file mode 100644
index 77ac9429..00000000
--- a/apps/web/src/components/genai/use-generate-audio.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { notAvailable } from "./placeholder";
-
-import type { AudioGenerationConfig } from "./schemas";
-
-export function useGenerateAudio() {
- const run = async (_config: AudioGenerationConfig) => notAvailable("Audio generation");
- return { generate: run } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsx
deleted file mode 100644
index 882cca78..00000000
--- a/apps/web/src/components/genai/use-generate-image.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { notAvailable } from "./placeholder";
-
-import type { ImageGenerationConfig } from "./schemas";
-
-export function useGenerateImage() {
- const run = async (_config: ImageGenerationConfig) => notAvailable("Image generation");
- return { generate: run } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsx
deleted file mode 100644
index 48b5111c..00000000
--- a/apps/web/src/components/genai/use-generate-video.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { notAvailable } from "./placeholder";
-
-import type { VideoGenerationConfig } from "./schemas";
-
-export function useGenerateVideo() {
- const run = async (_config: VideoGenerationConfig) => notAvailable("Video generation");
- return { generate: run } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-voice.tsx b/apps/web/src/components/genai/use-generate-voice.tsx
deleted file mode 100644
index bb999a55..00000000
--- a/apps/web/src/components/genai/use-generate-voice.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { notAvailable } from "./placeholder";
-
-import type { VoiceGenerationConfig } from "./schemas";
-
-export function useGenerateVoice() {
- const run = async (_config: VoiceGenerationConfig) => notAvailable("Voice generation");
- return { generate: run } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate.ts b/apps/web/src/components/genai/use-generate.ts
new file mode 100644
index 00000000..03f4f69f
--- /dev/null
+++ b/apps/web/src/components/genai/use-generate.ts
@@ -0,0 +1,38 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { Library } from "@diffusionstudio/runtime";
+import { generate } from "@/engine/generate";
+import { outputOf, toRequest } from "./requests";
+
+import type { GenerateRequest } from "@diffusionstudio/api-contract";
+import type { GenerationConfig } from "./schemas";
+
+/**
+ * Generating onto the canvas: from the prompt box's settings (`generate`), or
+ * again from a past job's request (`rerun`), which is resubmitted as it was —
+ * same model, settings and inputs, a new take.
+ */
+export function useGenerate() {
+ const world = useWorld();
+
+ const run = (config: GenerationConfig): Promise<void> => {
+ const library = world.get(Library);
+ const output = outputOf({
+ model: config.model as GenerateRequest["model"],
+ count: config.mode === "IMAGE" ? config.count : 1,
+ aspectRatio: "aspectRatio" in config ? config.aspectRatio : undefined,
+ });
+ if (!library || !output) return Promise.resolve();
+ return generate(world, output, () => toRequest(library, config));
+ };
+
+ const rerun = (request: GenerateRequest): Promise<void> => {
+ const output = outputOf(request);
+ return output ? generate(world, output, request) : Promise.resolve();
+ };
+
+ return { generate: run, rerun };
+}
diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.ts
index 381ef80f..eb50f04d 100644
--- a/apps/web/src/components/genai/use-generation-records.ts
+++ b/apps/web/src/components/genai/use-generation-records.ts
@@ -2,15 +2,44 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// How the selection was generated and what it cost: a placeholder until the
-// generative features are back (see ./placeholder). Nothing reads as
-// generated, so "Rerun", "Reuse" and the credits line stay hidden.
+// How the selection was generated and what it cost. A generated asset keeps
+// the id of the job it came from (`Asset.job`); the job is the API's account
+// of the run — what it was asked, which "Rerun" and "Reuse" work from, and
+// what it was charged.
+import { createMemo, createResource } from "solid-js";
+import { api } from "@/lib/api";
+import { useMediaSelection } from "./selection";
+import { toConfig } from "./requests";
+
+import type { Job } from "@diffusionstudio/api-contract";
import type { GenerationConfig } from "./schemas";
export function useGenerationRecords() {
- const isGenerated = (): boolean => false;
- const totalCredits = (): number => 0;
- const firstConfig = (): GenerationConfig | undefined => undefined;
- return { isGenerated, totalCredits, firstConfig };
+ const { media } = useMediaSelection();
+
+ const jobIds = createMemo(
+ () => [...new Set(media().map((entry) => entry.asset?.job).filter((id) => id !== undefined))],
+ undefined,
+ { equals: (a, b) => a.length === b.length && a.every((id, index) => id === b[index]) },
+ );
+
+ const [jobs] = createResource(jobIds, async (ids): Promise<Job[]> => {
+ const settled = await Promise.allSettled(ids.map((id) => api.jobs.get.query({ id })));
+ return settled.flatMap((result) => (result.status === "fulfilled" ? [result.value] : []));
+ }, { initialValue: [] });
+
+ const isGenerated = createMemo(() => jobIds().length > 0);
+ const totalCredits = createMemo(() => jobs().reduce((sum, job) => sum + job.credits, 0));
+
+ /** The job behind the first generated asset of the selection, once it is fetched. */
+ const firstJob = (): Job | undefined => jobs()[0];
+
+ /** The prompt box's settings for that job, when the prompt box makes such requests. */
+ const firstConfig = (): GenerationConfig | undefined => {
+ const job = firstJob();
+ return job ? toConfig(job.request) : undefined;
+ };
+
+ return { isGenerated, totalCredits, firstJob, firstConfig };
}
diff --git a/apps/web/src/components/genai/use-transforms.ts b/apps/web/src/components/genai/use-transforms.ts
index 8f6f991f..9dca04a5 100644
--- a/apps/web/src/components/genai/use-transforms.ts
+++ b/apps/web/src/components/genai/use-transforms.ts
@@ -2,21 +2,59 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Upscaling, taking a background out, scoring footage: placeholders until the
-// generative features are back (see ./placeholder).
+// Upscaling, taking a background out, scoring footage: tools run over the
+// media the selected nodes show on top. The result is laid over that paint,
+// which is hidden rather than removed (see `transform`).
-import { notAvailable } from "./placeholder";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { Generating, PaintType } from "@diffusionstudio/runtime";
+import { transform } from "@/engine/generate";
+import { useMediaSelection } from "./selection";
+import type { AssetRef, GenerateRequestInput } from "@diffusionstudio/api-contract";
+import type { NodeMedia } from "@/engine/generate";
import type { TransformType } from "./schemas";
-const LABELS: Record<TransformType, string> = {
- upscale: "Upscaling",
- removeBackground: "Background removal",
- addAudio: "Adding audio",
+interface Tool {
+ /** What a failure is reported under. */
+ title: string;
+ /** Whether the tool can be run over media of `type`. */
+ takes(type: NodeMedia["type"]): boolean;
+ /** The request over `input`, an upload of media of `type`. */
+ request(type: NodeMedia["type"], input: AssetRef): GenerateRequestInput;
+}
+
+const TOOLS: Record<TransformType, Tool> = {
+ upscale: {
+ title: "Upscale failed",
+ takes: () => true,
+ request: (type, input) =>
+ type === PaintType.VIDEO ? { model: "upscale-video", video: input } : { model: "upscale-image", image: input },
+ },
+ removeBackground: {
+ title: "Background removal failed",
+ takes: (type) => type === PaintType.IMAGE,
+ request: (_type, image) => ({ model: "remove-background", image }),
+ },
+ addAudio: {
+ title: "Adding audio failed",
+ takes: (type) => type === PaintType.VIDEO,
+ request: (_type, video) => ({ model: "add-audio", video }),
+ },
};
export function useTransforms() {
- const isOn = (_type: TransformType): boolean => false;
- const toggle = (type: TransformType): void => notAvailable(LABELS[type]);
- return { isOn, toggle };
+ const world = useWorld();
+ const { media } = useMediaSelection();
+
+ /** Runs `type` over every selected node it applies to that is not generating already. */
+ const run = (type: TransformType): void => {
+ const tool = TOOLS[type];
+ for (const entry of media()) {
+ if (!tool.takes(entry.type) || entry.node.has(Generating)) continue;
+ transform(world, entry, tool.title, (input) => tool.request(entry.type, input));
+ }
+ };
+
+ return { run };
}
diff --git a/apps/web/src/engine/generate/captions.tsx b/apps/web/src/engine/generate/captions.tsx
new file mode 100644
index 00000000..0206c237
--- /dev/null
+++ b/apps/web/src/engine/generate/captions.tsx
@@ -0,0 +1,122 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Auto-captions: the scene's audible mix, encoded and transcribed, mounted as
+ * the scene's `<captions>`. A scene has no geometry to stand in with — it is
+ * the frame everything else sits in — so the wait is told in a toast that
+ * follows the job instead of a pulse.
+ */
+
+import { Captions } from '@diffusionstudio/reconciler';
+import { AssetId, Audio, Caption, getEntityTree, Hidden, Library, Muted, Paint, PaintType } from '@diffusionstudio/runtime';
+import { assetName } from '@diffusionstudio/assets';
+import { createEncoder } from '@diffusionstudio/encoder';
+import { toast } from 'somoto';
+
+import { jobFailure, runJob } from '@/lib/jobs';
+import { uploadFile } from '@/lib/uploads';
+
+import { createCapture } from '../capture';
+import { getDocumentEditor } from '../editor';
+
+import type { AssetLibrary } from '@diffusionstudio/assets';
+import type { Job } from '@diffusionstudio/api-contract';
+import type { Entity, World } from 'koota';
+
+const TITLE = 'Caption generation failed';
+
+/** Whether anything in the scene is heard: an unmuted, visible clip or video paint with its asset bound. */
+function hasAudio(world: World, scene: Entity): boolean {
+ return getEntityTree(world, scene).some((entity) =>
+ !entity.has(Hidden) && !entity.has(Muted) && entity.has(AssetId)
+ && (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO));
+}
+
+/** The scene's audible mix as an Ogg/Opus file; `dir` is the project's folder, compiled fresh. */
+async function encodeAudio(world: World, scene: Entity, dir?: string): Promise<File> {
+ const capture = await createCapture(world, scene, { mode: 'offline-audio', dir });
+ try {
+ const encoder = await createEncoder(capture.world, {
+ format: 'ogg',
+ video: { enabled: false },
+ audio: { enabled: true, codec: 'opus', sampleRate: 24000 },
+ });
+ const result = await encoder.render();
+ if (result.type !== 'success' || !result.data) {
+ throw new Error('Could not encode the scene audio.');
+ }
+ return new File([result.data], 'scene.ogg', { type: 'audio/ogg' });
+ } finally {
+ capture.dispose();
+ }
+}
+
+/** The next free `Captions N.json`. */
+function captionsName(library: AssetLibrary): string {
+ let max = 0;
+ for (const asset of library.list()) {
+ const match = assetName(asset).match(/^Captions (\d+)\.json$/);
+ if (match) max = Math.max(max, Number(match[1]));
+ }
+ return `Captions ${max + 1}.json`;
+}
+
+/** The toast line for a transcription in flight. */
+function status(job: Job | undefined): string {
+ if (!job) return 'Preparing the audio…';
+ if (job.status === 'queued') return 'Queued';
+ if (job.phase === 'finalizing' || job.status === 'succeeded') return 'Almost done…';
+ return 'Transcribing…';
+}
+
+/**
+ * Transcribes `scene` and mounts the transcript as its captions: the scene's
+ * `<captions>` is pointed at the new transcript, or one is added.
+ */
+export async function generateCaptions(world: World, scene: Entity, dir?: string): Promise<void> {
+ const library = world.get(Library);
+ if (!library) return;
+ if (!hasAudio(world, scene)) {
+ toast(TITLE, { description: 'No audio found. Add an audio or video clip to the scene to generate captions.' });
+ return;
+ }
+
+ const id = `captions:${scene.id()}`;
+ toast.loading('Generating captions', { id, description: status(undefined) });
+
+ try {
+ const audio = await uploadFile(await encodeAudio(world, scene, dir));
+ const job = await runJob({ model: 'transcribe', audio }, (update) => {
+ toast.loading('Generating captions', { id, description: status(update) });
+ });
+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));
+
+ const [file] = job.assets;
+ if (!file) throw new Error('The transcription returned nothing.');
+ const response = await fetch(file.url);
+ if (!response.ok) throw new Error(`Could not download the transcript (${response.status})`);
+ const transcript = await response.json() as { words: unknown[] }[];
+ if (!transcript.some((segment) => segment.words.length > 0)) {
+ throw new Error('No speech detected. The audio does not appear to contain recognizable speech.');
+ }
+
+ const blob = new Blob([JSON.stringify(transcript)], { type: 'application/json' });
+ const asset = library.update(await library.store(blob, { name: captionsName(library) }), { job: job.id });
+
+ const editor = getDocumentEditor(world);
+ const existing = getEntityTree(world, scene).find((entity) => entity !== scene && entity.has(Caption));
+ if (existing) {
+ editor.editProperty(existing, 'src', asset.path);
+ editor.select(existing);
+ } else {
+ const [captions] = editor.insertElement(scene, () => <Captions src={asset.path} />);
+ if (captions) editor.select(captions);
+ }
+ toast.dismiss(id);
+ } catch (error) {
+ console.error(`[generate] ${TITLE}:`, error);
+ toast.error(TITLE, { id, description: error instanceof Error ? error.message : String(error) });
+ }
+}
diff --git a/apps/web/src/engine/generate/index.ts b/apps/web/src/engine/generate/index.ts
new file mode 100644
index 00000000..21a80f4a
--- /dev/null
+++ b/apps/web/src/engine/generate/index.ts
@@ -0,0 +1,13 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export { generateCaptions } from './captions';
+export { topMedia } from './media';
+export { findPlacement, PLACEMENT_GAP } from './placement';
+export { generate, transform, uploadAsset } from './run';
+export { trackPlaceholder } from './placeholder';
+
+export type { MediaPaintType, NodeMedia } from './media';
+export type { GenerationOutput } from './run';
+export type { GenerationState, Placeholder } from './placeholder';
diff --git a/apps/web/src/engine/generate/media.ts b/apps/web/src/engine/generate/media.ts
new file mode 100644
index 00000000..0a1a8b88
--- /dev/null
+++ b/apps/web/src/engine/generate/media.ts
@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Cache, Hidden, Paint, PaintType, getIntrinsicPaint } from '@diffusionstudio/runtime';
+
+import type { Entity } from 'koota';
+
+/** The kinds of paint a model reads or writes. */
+export type MediaPaintType = PaintType.IMAGE | PaintType.VIDEO;
+
+/** The media a node shows, and the element that carries its `src`. */
+export interface NodeMedia {
+ node: Entity;
+ paint: Entity;
+ type: MediaPaintType;
+}
+
+const isMedia = (type: PaintType | undefined): type is MediaPaintType =>
+ type === PaintType.IMAGE || type === PaintType.VIDEO;
+
+/**
+ * The media `node` shows on top: its topmost visible media fill (fills stack
+ * in document order, the last on top), else its intrinsic media, which sits
+ * beneath every fill. What a reference or a transform is taken from, so a
+ * node that was transformed before — the old paint hidden, the result laid
+ * over it — is read as the result.
+ */
+export function topMedia(node: Entity): NodeMedia | undefined {
+ const fills = node.get(Cache)?.fills ?? [];
+ for (let index = fills.length - 1; index >= 0; index--) {
+ const fill = fills[index]!;
+ const type = fill.get(Paint)?.value;
+
+ if (!fill.has(Hidden) && isMedia(type)) {
+ return { node, paint: fill, type };
+ }
+ }
+
+ const intrinsic = getIntrinsicPaint(node);
+ return isMedia(intrinsic) ? { node, paint: node, type: intrinsic } : undefined;
+}
diff --git a/apps/web/src/engine/generate/placeholder.ts b/apps/web/src/engine/generate/placeholder.ts
new file mode 100644
index 00000000..2f7713a9
--- /dev/null
+++ b/apps/web/src/engine/generate/placeholder.ts
@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * An element standing in for a generation in flight. The app puts the state
+ * onto the runtime — `Generating` on the element's entity, which the stage
+ * and the timeline draw as the pulse, the label and the progress — and takes
+ * it off again on release; the runtime only draws what it is given.
+ *
+ * The element is held by its `Source` stamp, never by entity: a remount or an
+ * undo/redo makes new entities, a stamp survives (and a pending stamp is
+ * followed to the real one the write answers with). The state is put on
+ * again every `REFRESH` — the progress moves with the clock, and an entity
+ * that replaced the old one picks it up — not once a frame.
+ */
+
+import { Generating, Source } from '@diffusionstudio/runtime';
+
+import { getDocumentEditor } from '../editor';
+
+import type { Entity, World } from 'koota';
+
+/** How often a placeholder's state is put onto its entity. */
+const REFRESH = 250; // ms
+
+/** What a placeholder shows: what the generation is doing, and how far it has come (0–1) when that is known. */
+export interface GenerationState {
+ label: string;
+ progress?: number;
+}
+
+export interface Placeholder {
+ /** The entity carrying the element right now, or undefined when it left the document. */
+ entity(): Entity | undefined;
+ /** Lets go of the element: it stops pulsing and reads its own name again. */
+ release(): void;
+}
+
+/**
+ * Makes `entity` a placeholder showing `state`, read on every refresh.
+ * Returns undefined for an entity the file cannot name.
+ */
+export function trackPlaceholder(world: World, entity: Entity, state: () => GenerationState): Placeholder | undefined {
+ let source = entity.get(Source)?.value;
+ if (!source) return undefined;
+
+ const find = () => world.query(Source).find((candidate) => candidate.isAlive() && candidate.get(Source)!.value === source);
+
+ const apply = () => {
+ const target = find();
+ if (!target) return;
+ const { label, progress } = state();
+ if (!target.has(Generating)) target.add(Generating);
+ const current = target.get(Generating)!;
+ if (current.label !== label || current.progress !== progress) target.set(Generating, { label, progress });
+ };
+
+ const unsubscribe = getDocumentEditor(world).onRename((ids) => {
+ source = ids[source!] ?? source;
+ });
+ const timer = setInterval(apply, REFRESH);
+ apply();
+
+ return {
+ entity: find,
+ release: () => {
+ clearInterval(timer);
+ unsubscribe();
+ find()?.remove(Generating);
+ },
+ };
+}
diff --git a/apps/web/src/engine/generate/placement.ts b/apps/web/src/engine/generate/placement.ts
new file mode 100644
index 00000000..d2f36229
--- /dev/null
+++ b/apps/web/src/engine/generate/placement.ts
@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Where generations go on the infinite canvas: beside the work rather than
+ * over it. Of the nodes the user is looking at, the one with the most free
+ * room to its left or right — inside the view — takes the block beside it,
+ * level with it. When no side has room in view, the free side nearest the
+ * middle of the view does, wherever it is; the camera follows (`revealRect`).
+ */
+
+import { getEntityBounds, getEntityChildren, getViewport, Hidden, Root, screenToWorld } from '@diffusionstudio/runtime';
+
+import type { AABB, Rect } from '@diffusionstudio/runtime';
+import type { World } from 'koota';
+
+/** Between the items of a block, and between the block and its neighbour. */
+export const PLACEMENT_GAP = 40;
+
+/** The visible part of the canvas in document space, or null before a surface is mounted. */
+function viewBounds(world: World): AABB | null {
+ const viewport = getViewport(world);
+ if (!viewport) return null;
+
+ // All four corners, so the box still holds if the camera is ever rotated.
+ const corners = [
+ screenToWorld(world, 0, 0),
+ screenToWorld(world, viewport.width, 0),
+ screenToWorld(world, viewport.width, viewport.height),
+ screenToWorld(world, 0, viewport.height),
+ ];
+
+ return {
+ minX: Math.min(...corners.map((corner) => corner.x)),
+ minY: Math.min(...corners.map((corner) => corner.y)),
+ maxX: Math.max(...corners.map((corner) => corner.x)),
+ maxY: Math.max(...corners.map((corner) => corner.y)),
+ };
+}
+
+/**
+ * What stands on the canvas: the bounds of every visible top-level node. A
+ * node the transform system has not measured yet has no place to avoid.
+ */
+function occupied(world: World): AABB[] {
+ const boxes: AABB[] = [];
+ for (const entity of getEntityChildren(world, world.get(Root)!)) {
+ if (entity.has(Hidden)) continue;
+ const rect = getEntityBounds(world, [entity]);
+ if (rect) boxes.push({ minX: rect.x, minY: rect.y, maxX: rect.x + rect.width, maxY: rect.y + rect.height });
+ }
+ return boxes;
+}
+
+/** Overlap with area: boxes that only touch along an edge leave each other alone. */
+const overlaps = (a: AABB, b: AABB): boolean =>
+ a.minX < b.maxX && b.minX < a.maxX && a.minY < b.maxY && b.minY < a.maxY;
+
+const contains = (outer: AABB, inner: AABB): boolean =>
+ inner.minX >= outer.minX && inner.maxX <= outer.maxX && inner.minY >= outer.minY && inner.maxY <= outer.maxY;
+
+const center = (box: AABB) => ({ x: (box.minX + box.maxX) / 2, y: (box.minY + box.maxY) / 2 });
+
+const toRect = (box: AABB): Rect => ({ x: box.minX, y: box.minY, width: box.maxX - box.minX, height: box.maxY - box.minY });
+
+/** A spot level with `box`, `gap` off its side (1 right, -1 left), and the room that side has. */
+interface Beside {
+ spot: AABB;
+ /** Free span off that side of `box` within the band the spot covers: to the nearest node, or the edge of the view. */
+ room: number;
+}
+
+function beside(box: AABB, side: 1 | -1, width: number, height: number, gap: number, boxes: AABB[], view: AABB): Beside {
+ const minY = (box.minY + box.maxY) / 2 - height / 2;
+ const minX = side > 0 ? box.maxX + gap : box.minX - gap - width;
+ const spot = { minX, minY, maxX: minX + width, maxY: minY + height };
+
+ let room = side > 0 ? view.maxX - box.maxX : box.minX - view.minX;
+ for (const other of boxes) {
+ if (other === box || other.maxY <= spot.minY || other.minY >= spot.maxY) continue;
+ if (side > 0 && other.minX >= box.maxX) room = Math.min(room, other.minX - box.maxX);
+ if (side < 0 && other.maxX <= box.minX) room = Math.min(room, box.minX - other.maxX);
+ }
+
+ return { spot, room };
+}
+
+/**
+ * Where a `width`×`height` block goes (see the module comment), in document
+ * space. Falls back to the middle of the view on an empty canvas, and to the
+ * right of everything when even that is taken.
+ */
+export function findPlacement(world: World, width: number, height: number, gap = PLACEMENT_GAP): Rect {
+ return placeBlock(occupied(world), viewBounds(world) ?? { minX: 0, minY: 0, maxX: 0, maxY: 0 }, width, height, gap);
+}
+
+/** `findPlacement` over what stands on the canvas (`boxes`) and the part of it in view. */
+export function placeBlock(boxes: AABB[], view: AABB, width: number, height: number, gap: number): Rect {
+ const middle = center(view);
+ const free = (spot: AABB) => !boxes.some((box) => overlaps(spot, box));
+
+ let roomiest: Beside | undefined;
+ let nearest: { spot: AABB; distance: number } | undefined;
+
+ for (const box of boxes) {
+ const inView = overlaps(box, view);
+ for (const side of [1, -1] as const) {
+ const candidate = beside(box, side, width, height, gap, boxes, view);
+ if (!free(candidate.spot)) continue;
+
+ if (inView && contains(view, candidate.spot) && (!roomiest || candidate.room > roomiest.room)) {
+ roomiest = candidate;
+ }
+
+ const { x, y } = center(candidate.spot);
+ const distance = Math.hypot(x - middle.x, y - middle.y);
+ if (!nearest || distance < nearest.distance) nearest = { spot: candidate.spot, distance };
+ }
+ }
+
+ if (roomiest) return toRect(roomiest.spot);
+
+ const centered = {
+ minX: middle.x - width / 2,
+ minY: middle.y - height / 2,
+ maxX: middle.x + width / 2,
+ maxY: middle.y + height / 2,
+ };
+ if (free(centered)) return toRect(centered);
+ if (nearest) return toRect(nearest.spot);
+
+ // Every side of every node is taken: past the right edge of all of them.
+ const right = Math.max(...boxes.map((box) => box.maxX)) + gap;
+ return { x: right, y: middle.y - height / 2, width, height };
+}
diff --git a/apps/web/src/engine/generate/run.tsx b/apps/web/src/engine/generate/run.tsx
new file mode 100644
index 00000000..e9b5d137
--- /dev/null
+++ b/apps/web/src/engine/generate/run.tsx
@@ -0,0 +1,278 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Generations on the canvas, from the request to the file in the library.
+ *
+ * - A prompt (`generate`) gets one empty geometry per output, placed beside
+ * the work (see ./placement) and brought into view. Each stands in for its
+ * output — the pulse, headed by the job's progress (see ./placeholder) — until
+ * the result lands as the geometry's paint.
+ * - A transform (`transform`) is run over the media a node shows on top (see
+ * ./media); the node pulses meanwhile, and the result is laid over the paint
+ * it was made from, which is hidden rather than removed.
+ *
+ * What a model returns is downloaded into the project library, which only
+ * ever holds files that exist: nothing is reserved there for a job in flight,
+ * and a job that fails leaves no trace but the toast saying why.
+ */
+
+import { Audio, ImagePaint, Rect, VideoPaint, authoredElement, renderAuthored } from '@diffusionstudio/reconciler';
+import { AssetId, Cache, getCameraMatrix, getNextName, getParentEntity, Library, Paint, revealRect, Root, Selected, Size, Source } from '@diffusionstudio/runtime';
+import { assetName } from '@diffusionstudio/assets';
+import { TRPCClientError } from '@trpc/client';
+import { toast } from 'somoto';
+
+import { jobFailure, runJob } from '@/lib/jobs';
+import { uploadFile } from '@/lib/uploads';
+
+import { getDocumentEditor } from '../editor';
+import { findPlacement, PLACEMENT_GAP } from './placement';
+import { trackPlaceholder } from './placeholder';
+import { topMedia } from './media';
+
+import type { Asset, AssetLibrary } from '@diffusionstudio/assets';
+import type { AssetRef, GenerateRequestInput, Job } from '@diffusionstudio/api-contract';
+import type { Entity, World } from 'koota';
+import type { NodeMedia } from './media';
+import type { GenerationState, Placeholder } from './placeholder';
+
+/** What a prompt asks for, as far as the canvas is concerned. */
+export interface GenerationOutput {
+ /** What each output lands as: a paint of its placeholder, or an `<audio>` in its place. */
+ kind: 'image' | 'video' | 'audio';
+ /** How many outputs the request makes, each with a placeholder of its own. */
+ count: number;
+ /** The box of each placeholder; a picture's is corrected to the result's proportions. */
+ size: { width: number; height: number };
+ /** What a failure is reported under, e.g. "Image generation failed". */
+ title: string;
+}
+
+/**
+ * The progress of one job, as its placeholders show it: "Queued" until the
+ * job runs, then "Generating" with the share of its estimate that has passed
+ * — counted on to the end, so a job that lands in time goes from a progress
+ * straight to its name. Once the estimate is spent there is no progress left
+ * to show: it reads "Taking a bit longer" while the model works, and "Almost
+ * done" once the model is done and the result is on its way into the project.
+ */
+class Progress {
+ private job: Job | undefined;
+ /** When the job was first seen running, on this clock. */
+ private since = 0;
+
+ public readonly update = (job: Job): void => {
+ if (job.status !== 'queued' && !this.since) this.since = Date.now();
+ this.job = job;
+ };
+
+ public readonly state = (): GenerationState => {
+ const job = this.job;
+ if (!job || job.status === 'queued') return { label: 'Queued' };
+ const progress = (Date.now() - this.since) / (job.etaSeconds * 1000);
+ if (progress < 1) return { label: 'Generating', progress };
+ return { label: job.status === 'succeeded' || job.phase === 'finalizing' ? 'Almost done' : 'Taking a bit longer' };
+ };
+}
+
+/**
+ * Generates what `request` asks for into empty geometries on the canvas (see
+ * the module comment). `request` may take making — inputs to upload — so it
+ * is made once the placeholders are there, from the moment of the prompt.
+ */
+export async function generate(
+ world: World,
+ output: GenerationOutput,
+ request: GenerateRequestInput | (() => Promise<GenerateRequestInput>),
+): Promise<void> {
+ const library = world.get(Library);
+ const root = world.get(Root);
+ if (!library || !root?.get(Source)?.value) {
+ toast('Nothing to generate into', { description: 'Open a project first.' });
+ return;
+ }
+
+ const editor = getDocumentEditor(world);
+ const { width, height } = output.size;
+ const block = findPlacement(world, output.count * width + (output.count - 1) * PLACEMENT_GAP, height);
+
+ revealRect(world, block);
+ editor.reportEdit(root, 'camera', getCameraMatrix(world));
+
+ const progress = new Progress();
+ const placeholders: Placeholder[] = [];
+ const inserted: Entity[] = [];
+
+ // A picture or a clip keeps its proportions; a sound's box is only where it sits.
+ const proportions = output.kind === 'audio' ? {} : { keepAspectRatio: true };
+
+ for (let index = 0; index < output.count; index++) {
+ const x = Math.round(block.x + index * (width + PLACEMENT_GAP));
+ const y = Math.round(block.y);
+ const [entity] = editor.insertElement(root, () => (
+ <Rect {...proportions} x={x} y={y} width={width} height={height} />
+ ));
+ const placeholder = entity && trackPlaceholder(world, entity, progress.state);
+ if (!placeholder) continue;
+ inserted.push(entity);
+ placeholders.push(placeholder);
+ }
+ if (inserted.length) editor.select(inserted);
+
+ try {
+ const job = await runJob(typeof request === 'function' ? await request() : request, progress.update);
+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));
+
+ const results = await Promise.all(job.assets.map((file) => storeResult(library, job, file)));
+ let orphaned = 0;
+ placeholders.forEach((placeholder, index) => {
+ const entity = placeholder.entity();
+ const result = results[index];
+ if (!entity) orphaned += result ? 1 : 0;
+ else if (result) land(world, entity, result);
+ else editor.remove(entity);
+ });
+ if (orphaned) {
+ toast('Generation finished', { description: 'Its placeholder was removed, so the result is in the library only.' });
+ }
+ } catch (error) {
+ editor.remove(placeholders.map((placeholder) => placeholder.entity()).filter((entity) => entity !== undefined));
+ report(output.title, error);
+ } finally {
+ for (const placeholder of placeholders) placeholder.release();
+ }
+}
+
+/**
+ * Runs `request` over the media `media` shows (see the module comment);
+ * `request` is handed the uploaded input.
+ */
+export async function transform(
+ world: World,
+ media: NodeMedia,
+ title: string,
+ request: (input: AssetRef) => GenerateRequestInput,
+): Promise<void> {
+ const library = world.get(Library);
+ const input = library?.get(media.paint.get(AssetId)?.value ?? '');
+ if (!library || !input) {
+ toast(title, { description: 'The media has not loaded yet.' });
+ return;
+ }
+
+ const progress = new Progress();
+ const placeholder = trackPlaceholder(world, media.node, progress.state);
+ const paint = media.paint === media.node ? undefined : media.paint.get(Source)?.value;
+
+ try {
+ const job = await runJob(request(await uploadAsset(library, input)), progress.update);
+ if (job.status !== 'succeeded') throw new Error(jobFailure(job));
+
+ const [file] = job.assets;
+ if (!file) throw new Error('The model returned nothing.');
+ const result = await storeResult(library, job, file);
+
+ const node = placeholder?.entity();
+ if (node) overlay(world, node, paint, result);
+ else toast('Generation finished', { description: 'Its element was removed, so the result is in the library only.' });
+ } catch (error) {
+ report(title, error);
+ } finally {
+ placeholder?.release();
+ }
+}
+
+/** Uploads made this session, by library asset id: an input is sent once. */
+const uploads = new Map<string, Promise<AssetRef>>();
+
+/** The ref a request names a library asset by, uploading its bytes when the API lacks them. */
+export function uploadAsset(library: AssetLibrary, asset: Asset): Promise<AssetRef> {
+ let ref = uploads.get(asset.id);
+ if (!ref) {
+ ref = library.file(asset).then(uploadFile);
+ ref.catch(() => uploads.delete(asset.id));
+ uploads.set(asset.id, ref);
+ }
+ return ref;
+}
+
+/** Downloads one output of `job` into the library, noting the job it came from. */
+async function storeResult(library: AssetLibrary, job: Job, file: Job['assets'][number]): Promise<Asset> {
+ const response = await fetch(file.url);
+ if (!response.ok) throw new Error(`Could not download the result (${response.status})`);
+ const asset = await library.store(await response.blob(), { name: file.filename });
+ return library.update(asset, { job: job.id });
+}
+
+const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, '');
+
+/** Fills a prompt's placeholder with what it stood in for, named after it. */
+function land(world: World, placeholder: Entity, asset: Asset): void {
+ const editor = getDocumentEditor(world);
+ const name = getNextName(world, stem(asset));
+
+ if (asset.type === 'AUDIO') {
+ // A sound is an element of its own, not a paint: it takes the
+ // placeholder's place, and its selection.
+ const parent = getParentEntity(placeholder);
+ const props = authoredElement(placeholder)?.props ?? {};
+ const selected = placeholder.has(Selected);
+ if (!parent) return;
+ const [audio] = editor.insertElement(parent, () => (
+ <Audio name={name} src={asset.path} x={props.x as number} y={props.y as number} width={props.width as number} height={props.height as number} />
+ ), placeholder);
+ editor.remove(placeholder);
+ if (audio && selected) editor.select(audio, { extend: true });
+ return;
+ }
+
+ editor.insertElement(placeholder, () => (
+ asset.type === 'VIDEO' ? <VideoPaint src={asset.path} /> : <ImagePaint src={asset.path} />
+ ));
+ editor.editProperty(placeholder, 'name', name);
+
+ // The box was a guess at the result's proportions (a model may not take
+ // the aspect ratio asked for); the width stays, the height follows.
+ const size = placeholder.get(Size);
+ if ('width' in asset && asset.width > 0 && size) {
+ const height = Math.round((size.width * asset.height) / asset.width);
+ if (Math.abs(height - size.height) > 1) editor.editProperty(placeholder, 'height', height);
+ }
+}
+
+/**
+ * Lays a transform's result over the paint it was made from, which is hidden
+ * — not removed — so the original is a toggle away. Media that is the node's
+ * own (`<image src>`) has no paint to hide; its `src` is pointed at the
+ * result instead.
+ */
+function overlay(world: World, node: Entity, paintSource: string | undefined, asset: Asset): void {
+ const editor = getDocumentEditor(world);
+ const fills = node.get(Cache)?.fills ?? [];
+ const paint = (paintSource && fills.find((fill) => fill.get(Source)?.value === paintSource)) || topMedia(node)?.paint;
+
+ if (!paint || paint === node || !paint.has(Paint)) {
+ editor.editProperty(node, 'src', asset.path);
+ return;
+ }
+
+ // The new paint is drawn the way the old one was: fit, opacity, blending.
+ const props = { ...authoredElement(paint)?.props };
+ delete props.src;
+ delete props.hidden;
+ editor.editProperty(paint, 'hidden', true);
+ editor.insertElement(node, () => renderAuthored({
+ tag: asset.type === 'VIDEO' ? 'videoPaint' : 'imagePaint',
+ props: { ...props, src: asset.path },
+ children: [],
+ }));
+}
+
+/** Tells the user why a generation came to nothing — unless the upgrade dialog already does. */
+function report(title: string, error: unknown): void {
+ if (error instanceof TRPCClientError && error.data?.code === 'PAYMENT_REQUIRED') return;
+ console.error(`[generate] ${title}:`, error);
+ toast.error(title, { description: error instanceof Error ? error.message : String(error) });
+}
diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.ts
index 600ac39f..7d035571 100644
--- a/apps/web/src/engine/hud/hud-system.ts
+++ b/apps/web/src/engine/hud/hud-system.ts
@@ -7,7 +7,7 @@ import {
Active, Computed, Culled, FrameRate, Geometry, Group, Hidden,
HitRegions, Hovering, Name, Playback, RenderSurface, Root, Selected,
Sequential, ChildOf,
- entityQuad, entityWorldMat, getMaskSelection, getSelectionMask, getSourceFailure,
+ entityQuad, entityWorldMat, getGeneratingLabel, getGeneratingProgress, getMaskSelection, getSelectionMask, getSourceFailure,
invert2D,
multiply2D, rectToQuad, rotate2D, scale2D, store, transformPoint,
translate2D,
@@ -106,11 +106,14 @@ function drawHoverOutlines(world: World, ctx: Ctx2D, resolution: number): void {
/**
* The bar above a top-level node: play button, name, active badge, duration.
- * It sits in the node's own rotation, one line above its top edge.
+ * It sits in the node's own rotation, one line above its top edge. A node in
+ * generation is headed by what the generation is doing instead of its name,
+ * and by its progress where the duration would be.
*/
function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number): void {
const failure = getSourceFailure(entity);
- const label = failure || entity.get(Name)?.value;
+ const generating = getGeneratingLabel(world, entity);
+ const label = failure || generating || entity.get(Name)?.value;
if (!label) return;
const header = getHeaderLayout(world, entity, resolution);
@@ -195,17 +198,19 @@ function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number
ctx.fillText('Active', 0, 0);
}
- if (playback && (fitted?.width ?? 0) + badgeWidth + 60 < header.width) {
+ const trailing = generating
+ ? getGeneratingProgress(entity)
+ : playback
+ ? formatDuration(world, entity)
+ : undefined;
+
+ if (trailing && (fitted?.width ?? 0) + badgeWidth + 60 < header.width) {
ctx.setTransform(header.mat.a, header.mat.b, header.mat.c, header.mat.d, header.mat.e, header.mat.f);
ctx.translate(header.width, 3);
ctx.textAlign = 'right';
ctx.textBaseline = 'top';
ctx.font = '400 11px JetBrains Mono';
- const seconds = (store(world, Computed).duration[entity.id()] ?? 0) / (world.get(FrameRate)?.value ?? 30);
- const minutes = Math.floor(seconds / 60).toString().padStart(2, '0');
- const rest = Math.floor(seconds % 60).toString().padStart(2, '0');
-
if (selected) {
ctx.globalAlpha = 0.7;
ctx.fillStyle = '#cce8ff';
@@ -213,11 +218,19 @@ function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number
ctx.fillStyle = 'rgba(121, 121, 121, 1)';
}
- ctx.fillText(`${minutes}:${rest}`, 0, 0);
+ ctx.fillText(trailing, 0, 0);
ctx.globalAlpha = 1;
}
}
+/** A node's duration as `mm:ss`. */
+function formatDuration(world: World, entity: Entity): string {
+ const seconds = (store(world, Computed).duration[entity.id()] ?? 0) / (world.get(FrameRate)?.value ?? 30);
+ const minutes = Math.floor(seconds / 60).toString().padStart(2, '0');
+ const rest = Math.floor(seconds % 60).toString().padStart(2, '0');
+ return `${minutes}:${rest}`;
+}
+
/**
* The box around the selection, its resize/rotate regions, and the handles
* that show where they are. Rotation regions sit outside the corners, resize
diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.ts
index 20b929b2..7b8e7452 100644
--- a/apps/web/src/engine/timeline/render/clip.ts
+++ b/apps/web/src/engine/timeline/render/clip.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getSourceFailure, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';
+import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getGeneratingColor, getGeneratingLabel, getSourceFailure, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';
import { getDocumentEditor } from '../../editor';
import {
@@ -57,6 +57,7 @@ export function renderClip(
const asset = getClipAsset(world, entity);
const error = getSourceFailure(entity);
+ const generating = getGeneratingLabel(world, entity);
const style = getClipStyle(entity, asset, !!error?.length);
handleBody(world, surface, entity, left, width, row, resolution);
@@ -70,11 +71,11 @@ export function renderClip(
ctx.save();
ctx.beginPath();
ctx.roundRect(left, 0, width, row.height, CLIP_CORNER_RADIUS);
- ctx.fillStyle = style.background;
+ ctx.fillStyle = generating ? getGeneratingColor(world) : style.background;
ctx.fill();
ctx.restore();
- if (!error) {
+ if (!error && !generating) {
renderContent(world, scene, surface, entity, asset, style, row);
}
@@ -86,6 +87,7 @@ export function renderClip(
if (isCaption(entity)) label = label || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;
else if (isText(entity)) label = entity.get(Chars)?.value ?? label ?? '';
if (!label) label = getClipFallbackName(world, entity);
+ if (generating) label = generating;
if (error) label = error;
const [viewportLeft] = getViewport(world, scene, surface.layout.width);
diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts
index 0423a789..1875d10f 100644
--- a/apps/web/src/lib/api.ts
+++ b/apps/web/src/lib/api.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { createTRPCClient, httpBatchLink, TRPCClientError, type TRPCLink } from "@trpc/client";
+import { createTRPCClient, httpBatchLink, httpSubscriptionLink, splitLink, TRPCClientError, type TRPCLink } from "@trpc/client";
import { observable } from "@trpc/server/observable";
import { supabase } from "./supabase";
import { showUpgradeDialog } from "@/components/upgrade-dialog";
@@ -25,17 +25,39 @@ const paymentRequiredLink: TRPCLink<AppRouter> = () => ({ next, op }) =>
return () => sub.unsubscribe();
});
-/** The Diffusion Studio API (v2), authenticated with the Supabase session. */
+const url = `${import.meta.env.VITE_API_URL ?? ""}/api/v2/trpc`;
+
+/** The Supabase session as an Authorization header value, or undefined when signed out. */
+async function authorization(): Promise<string | undefined> {
+ const session = await supabase?.auth.getSession();
+ const token = session?.data.session?.access_token;
+ return token ? `Bearer ${token}` : undefined;
+}
+
+/**
+ * The Diffusion Studio API (v2), authenticated with the Supabase session.
+ * Subscriptions (`jobs.watch`) are server-sent events, which cannot carry
+ * headers, so the token travels as a connection param there.
+ */
export const api = createTRPCClient<AppRouter>({
links: [
paymentRequiredLink,
- httpBatchLink({
- url: `${import.meta.env.VITE_API_URL ?? ""}/api/v2/trpc`,
- async headers() {
- const session = await supabase?.auth.getSession();
- const token = session?.data.session?.access_token;
- return token ? { Authorization: `Bearer ${token}` } : {};
- },
+ splitLink({
+ condition: (op) => op.type === "subscription",
+ true: httpSubscriptionLink({
+ url,
+ connectionParams: async () => {
+ const value = await authorization();
+ return value ? { authorization: value } : {};
+ },
+ }),
+ false: httpBatchLink({
+ url,
+ async headers() {
+ const value = await authorization();
+ return value ? { Authorization: value } : {};
+ },
+ }),
}),
],
});
diff --git a/apps/web/src/lib/jobs.ts b/apps/web/src/lib/jobs.ts
new file mode 100644
index 00000000..8567ba73
--- /dev/null
+++ b/apps/web/src/lib/jobs.ts
@@ -0,0 +1,71 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Generation jobs on the API: everything that produces a file is one
+ * `generate` call that answers with a job right away, followed to its end
+ * through `jobs.watch`. The stream closes when the job ends, and every few
+ * minutes on the server's side regardless; reopening it hands back the
+ * current state first, so a reconnect misses nothing.
+ */
+
+import { api } from "./api";
+
+import type { GenerateRequestInput, Job } from "@diffusionstudio/api-contract";
+
+/** How often a dropped watch is reopened before the job is given up on. */
+const MAX_RETRIES = 5;
+const RETRY_DELAY = 2000;
+
+export const isTerminal = (job: Job): boolean =>
+ job.status === "succeeded" || job.status === "failed" || job.status === "canceled";
+
+/**
+ * Starts a job for `request` and follows it to its end. `onUpdate` hears
+ * every state the job passes through, the first one included. Resolves with
+ * the job as it ended — failed and canceled jobs included, which say why in
+ * `error`; rejects only when the job could not be started or followed.
+ */
+export async function runJob(request: GenerateRequestInput, onUpdate: (job: Job) => void): Promise<Job> {
+ const job = await api.generate.mutate({ request, idempotencyKey: crypto.randomUUID() });
+ onUpdate(job);
+ return isTerminal(job) ? job : watchJob(job.id, onUpdate);
+}
+
+/** Follows job `id` until it ends. */
+export function watchJob(id: string, onUpdate: (job: Job) => void): Promise<Job> {
+ return new Promise((resolve, reject) => {
+ let failures = 0;
+ let ended = false;
+
+ const subscribe = () => {
+ const subscription = api.jobs.watch.subscribe({ id }, {
+ onData: ({ data }) => {
+ failures = 0;
+ onUpdate(data);
+ if (!isTerminal(data)) return;
+ ended = true;
+ subscription.unsubscribe();
+ resolve(data);
+ },
+ onError: (error) => {
+ if (++failures > MAX_RETRIES) reject(error);
+ else setTimeout(subscribe, RETRY_DELAY * failures);
+ },
+ // The server closes the stream after a while; the job goes on.
+ onComplete: () => {
+ if (!ended) subscribe();
+ },
+ });
+ };
+
+ subscribe();
+ });
+}
+
+/** Why a job did not succeed, in a sentence for the user. */
+export function jobFailure(job: Job): string {
+ if (job.status === "canceled") return "The generation was canceled.";
+ return job.error?.message || "The generation failed.";
+}
diff --git a/apps/web/src/lib/uploads.ts b/apps/web/src/lib/uploads.ts
new file mode 100644
index 00000000..378a9083
--- /dev/null
+++ b/apps/web/src/lib/uploads.ts
@@ -0,0 +1,45 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Getting a file to where a model can read it. The API stores inputs by the
+ * sha256 of their bytes, per user, so a file is uploaded once however many
+ * generations name it: `uploads.create` says whether the bytes are already
+ * there, and hands out a signed resumable session when they are not.
+ */
+
+import { api } from "./api";
+
+import type { AssetRef } from "@diffusionstudio/api-contract";
+
+/** Lowercase hex sha256 of `blob`'s bytes: what the API names an upload by. */
+async function sha256(blob: Blob): Promise<string> {
+ const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());
+ return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+}
+
+/** Uploads `blob` unless the API already has its bytes; returns the ref a request names it by. */
+export async function uploadFile(blob: Blob): Promise<AssetRef> {
+ const contentType = blob.type || "application/octet-stream";
+ const { asset, uploadUrl, uploadHeaders } = await api.uploads.create.mutate({
+ sha256: await sha256(blob),
+ contentType,
+ size: blob.size,
+ });
+ if (!uploadUrl) return asset;
+
+ // A signed resumable session (valid 15 minutes), opened with exactly the
+ // headers it was signed for, then the bytes in one PUT.
+ const start = await fetch(uploadUrl, {
+ method: "POST",
+ headers: uploadHeaders ?? { "x-goog-resumable": "start", "content-type": contentType },
+ });
+ const session = start.headers.get("location");
+ if (!start.ok || !session) throw new Error(`Could not start the upload (${start.status})`);
+
+ const put = await fetch(session, { method: "PUT", body: blob });
+ if (!put.ok) throw new Error(`The upload failed (${put.status})`);
+
+ return asset;
+}
diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.ts
index 65da731a..fa6ab8c3 100644
--- a/packages/assets/src/types.ts
+++ b/packages/assets/src/types.ts
@@ -47,6 +47,11 @@ interface AssetBase {
createdAt: string;
mimeType: string;
stat?: AssetStat;
+ /**
+ * The API job a model produced the file in, for a generated asset: what
+ * it was asked (to run it again, or reuse its settings) and what it cost.
+ */
+ job?: string;
/**
* Resolved on the fly for a `src` that names a path or URL outside the
* library; lives in memory only and is never written to the manifest.
diff --git a/packages/runtime/src/actions/camera.ts b/packages/runtime/src/actions/camera.ts
index 03f46358..5f9bef7e 100644
--- a/packages/runtime/src/actions/camera.ts
+++ b/packages/runtime/src/actions/camera.ts
@@ -105,14 +105,36 @@ export function resetCameraZoom(world: World): void {
* rect should bring it into view, not magnify it.
*/
export function focusRect(world: World, rect: Rect, padding = FOCUS_PADDING): void {
+ const fit = fitScale(world, rect, padding);
+ if (fit !== null) centerOn(world, rect, clamp(fit, MIN_CAMERA_ZOOM, 1));
+}
+
+/**
+ * Bring a document-space rect into view: centered, and fully visible with
+ * `padding` CSS pixels to spare. Unlike `focusRect` the zoom is kept where it
+ * fits — the view only zooms out, and only as far as the rect needs.
+ */
+export function revealRect(world: World, rect: Rect, padding = FOCUS_PADDING): void {
+ const fit = fitScale(world, rect, padding);
+ if (fit !== null) centerOn(world, rect, clamp(Math.min(fit, getCameraScale(world)), MIN_CAMERA_ZOOM, MAX_CAMERA_ZOOM));
+}
+
+/** The zoom at which `rect` just fits the viewport inside `padding`, or null when nothing can. */
+function fitScale(world: World, rect: Rect, padding: number): number | null {
const viewport = getViewport(world);
- if (!viewport || rect.width <= 0 || rect.height <= 0) return;
+ if (!viewport || rect.width <= 0 || rect.height <= 0) return null;
const width = viewport.width - padding * 2;
const height = viewport.height - padding * 2;
- if (width <= 0 || height <= 0) return;
+ if (width <= 0 || height <= 0) return null;
+
+ return Math.min(width / rect.width, height / rect.height);
+}
- const scale = clamp(Math.min(width / rect.width, height / rect.height), MIN_CAMERA_ZOOM, 1);
+/** Put the middle of `rect` in the middle of the viewport, at `scale`. */
+function centerOn(world: World, rect: Rect, scale: number): void {
+ const viewport = getViewport(world);
+ if (!viewport) return;
setCamera(world, {
a: scale,
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index b21a8c55..fd2e8f99 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -30,6 +30,7 @@ import { getParentNode } from '../queries/hierarchy';
import { getViewMatrix } from '../queries/camera';
import { colorToHex } from '../utils/color';
import { FAILED_COLOR, getSourceFailure } from '../utils/source-failure';
+import { getGeneratingColor, isGenerating } from '../utils/generating';
import { applyStrokeStyle } from '../utils/stroke';
import { renderText } from '../utils/text';
import { getTransitionWindow } from '../utils/transition';
@@ -589,13 +590,15 @@ function renderStrokes(world: World, entity: Entity): void {
}
/**
- * The still fill a node whose source failed is left with
+ * What covers a node's fills while they cannot be shown: the pulse while a
+ * generation is filling it in, the still fill when its source failed.
*/
-function renderSourceFailure(world: World, entity: Entity): void {
- if (getSourceFailure(entity) === undefined) return;
+function renderPendingFill(world: World, entity: Entity): void {
+ const generating = isGenerating(entity);
+ if (!generating && !getSourceFailure(entity)) return;
const ctx = getCtx(world);
- ctx.fillStyle = FAILED_COLOR;
+ ctx.fillStyle = generating ? getGeneratingColor(world) : FAILED_COLOR;
ctx.fill();
}
@@ -666,7 +669,7 @@ function renderShapeNode(world: World, entity: Entity): void {
renderShadows(world, entity);
renderIntrinsicFill(world, entity);
renderFills(world, entity);
- renderSourceFailure(world, entity);
+ renderPendingFill(world, entity);
renderStrokes(world, entity);
}
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index d7455338..f199f8e1 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -47,6 +47,8 @@ export const Hidden = trait();
export const ClipsContent = trait();
+export const Generating = trait({ label: '', progress: undefined as number | undefined });
+
export const Name = trait({ value: '' });
// Stable identity for entities.
diff --git a/packages/runtime/src/utils/generating.ts b/packages/runtime/src/utils/generating.ts
new file mode 100644
index 00000000..d7fddbf6
--- /dev/null
+++ b/packages/runtime/src/utils/generating.ts
@@ -0,0 +1,74 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * The pulse a node waiting on a generation is filled with (see `Generating`).
+ * Everything that draws one reads it from here — the stage and the timeline
+ * both — so a node in generation pulses in step wherever it is shown.
+ *
+ * Easing: cubic-bezier(0.52, 0.18, 0.56, 0.88)
+ * Duration: 0.6s per half-cycle (alternating)
+ * Delay: 0.2s before the first transition
+ * Full cycle: 0.2 delay + 0.6 forward + 0.6 reverse = 1.4s
+ *
+ * Pulses between --background (#1c1c1c) and --secondary (#292929).
+ */
+
+import { cubicBezier } from 'animejs';
+
+import { Generating, Time } from '../traits';
+
+import type { Entity, World } from 'koota';
+
+const ease = cubicBezier(0.52, 0.18, 0.56, 0.88);
+
+const DELAY = 200; // ms
+const HALF_CYCLE = 600; // ms
+const CYCLE = DELAY + HALF_CYCLE * 2;
+
+const FROM = [28, 28, 28] as const; // #1c1c1c
+const TO = [41, 41, 41] as const; // #292929
+
+/** Whether the node stands in for a generation in flight. */
+export function isGenerating(entity: Entity): boolean {
+ return entity.has(Generating);
+}
+
+/** How long each step of the trailing dots lasts. */
+const DOT_STEP = 400; // ms
+
+/**
+ * What the header of a node in generation reads this frame — its label with
+ * dots that count up on the world's clock (".", "..", "...") — or undefined
+ * for any other node.
+ */
+export function getGeneratingLabel(world: World, entity: Entity): string | undefined {
+ const label = entity.get(Generating)?.label;
+ if (label === undefined) return undefined;
+ const dots = 1 + (Math.floor((world.get(Time)?.now ?? 0) / DOT_STEP) % 3);
+ return label + '.'.repeat(dots);
+}
+
+/** How far a node's generation has come, as a whole percentage, when there is a measure of it. */
+export function getGeneratingProgress(entity: Entity): string | undefined {
+ const progress = entity.get(Generating)?.progress;
+ return progress === undefined ? undefined : `${Math.floor(progress * 100)}%`;
+}
+
+/** Where the pulse stands on the world's clock: 0 at rest, 1 fully lit. */
+function pulse(world: World): number {
+ const elapsed = (world.get(Time)?.now ?? 0) % CYCLE;
+ if (elapsed < DELAY) return 0;
+
+ const t = elapsed - DELAY;
+ const eased = ease((t % HALF_CYCLE) / HALF_CYCLE);
+ return t >= HALF_CYCLE ? 1 - eased : eased;
+}
+
+/** The fill a node in generation is painted with this frame. */
+export function getGeneratingColor(world: World): string {
+ const factor = pulse(world);
+ const [r, g, b] = FROM.map((from, i) => Math.round(from + (TO[i]! - from) * factor));
+ return `rgb(${r},${g},${b})`;
+}