The VS Code of video editing, built for agents.
TypeScript
Refactor media selection and generation handling in the ActionBar and PromptInput components
konstantin-paulusa4ab6440b0bd
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsxindex 862d41c8..59fc5a29 100644--- a/apps/web/src/components/genai/action-bar.tsx+++ b/apps/web/src/components/genai/action-bar.tsx@@ -14,54 +14,27 @@ import {DropdownMenuTrigger,} from "@/components/ui/dropdown-menu";import { createMemo, Show } from "solid-js";-import { PaintType, isAudio } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { useRemoveBackground } from "./use-remove-background";-import { useUpscaleImage } from "./use-upscale-image";-import { useUpscaleVideo } from "./use-upscale-video";-import { useAddAudioToVideo } from "./use-add-audio-to-video";import { useGenerationRecords } from "./use-generation-records";import { useGenerateImage } from "./use-generate-image";import { useGenerateVideo } from "./use-generate-video";import { useGenerateVoice } from "./use-generate-voice";import { useGenerateAudio } from "./use-generate-audio";import { useAutoCaptions } from "./use-auto-captions";+import { useMediaSelection } from "./selection";+import { useModifiers } from "./use-modifiers";import { createDefaultConfig } from "./prompt-input";import { toast } from "somoto";-import { useECS } from "@/context/ecs";--import type { EngineWorld } from "@/components/engine";import type { GenerationConfig } from "@/components/engine/db";-function getNodeMediaType(world: EngineWorld, eid: number) {- const c = world.components;- if (isAudio(world, eid)) {- return "AUDIO";- }- for (const fid of c.Cache.fills[eid]) {- if (c.Paint[fid] === PaintType.VIDEO) {- return "VIDEO";- }- if (c.Paint[fid] === PaintType.IMAGE) {- return "IMAGE";- }- }- return null;-}-interface ActionBarProps {openPromptInput?(config: GenerationConfig): void;}export function ActionBar(props: ActionBarProps) {- const { world } = useEngine();- const { selectedNodes } = useECS();+ const { imageNodes, videoNodes } = useMediaSelection();+ const { isOn, toggle } = useModifiers();- const { generate: upscaleImage } = useUpscaleImage();- const { generate: upscaleVideo } = useUpscaleVideo();- const { generate: addAudioToVideo } = useAddAudioToVideo();- const { generate: removeBackground } = useRemoveBackground();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();const { generate: generateVoice } = useGenerateVoice();@@ -69,19 +42,8 @@ export function ActionBar(props: ActionBarProps) {const { generate: autoCaptions, hasScene } = useAutoCaptions();const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();- const isImage = createMemo(() => {- return Array.from(selectedNodes()).some(eid => {- const mediaType = getNodeMediaType(world, eid);- return mediaType === "IMAGE";- });- });-- const isVideo = createMemo(() => {- return Array.from(selectedNodes()).some(eid => {- const mediaType = getNodeMediaType(world, eid);- return mediaType === "VIDEO";- });- });+ const isImage = createMemo(() => imageNodes().length > 0);+ const isVideo = createMemo(() => videoNodes().length > 0);const visible = createMemo(() => {return isImage() || isVideo() || hasScene();@@ -143,11 +105,21 @@ export function ActionBar(props: ActionBarProps) {</Show><Show when={isImage()}><div class="flex gap-1 items-center">- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={removeBackground}>+ <Button+ variant="ghost"+ class="gap-0 pl-0.5 text-muted-foreground"+ classList={{ "text-foreground": isOn("removeBackground") }}+ onClick={() => toggle("removeBackground")}+ ><Icon name="ai-generate" />Remove background</Button>- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={upscaleImage}>+ <Button+ variant="ghost"+ class="gap-0 pl-0.5 text-muted-foreground"+ classList={{ "text-foreground": isOn("upscale") }}+ onClick={() => toggle("upscale")}+ ><Icon name="arrow-scale" />Upscale</Button>@@ -197,11 +169,21 @@ export function ActionBar(props: ActionBarProps) {</Show><Show when={isVideo()}><div class="flex gap-1 items-center">- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={addAudioToVideo}>+ <Button+ variant="ghost"+ class="gap-0 pl-0.5 text-muted-foreground"+ classList={{ "text-foreground": isOn("addAudio") }}+ onClick={() => toggle("addAudio")}+ ><Icon name="generate-audio" />Add audio</Button>- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={upscaleVideo}>+ <Button+ variant="ghost"+ class="gap-0 pl-0.5 text-muted-foreground"+ classList={{ "text-foreground": isOn("upscale") }}+ onClick={() => toggle("upscale")}+ ><Icon name="arrow-scale" />Upscale</Button>diff --git a/apps/web/src/components/genai/config.ts b/apps/web/src/components/genai/config.tsindex c1deb1c6..c9092c43 100644--- a/apps/web/src/components/genai/config.ts+++ b/apps/web/src/components/genai/config.ts@@ -2,6 +2,18 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */+/** The one model the voice mode speaks with; the UI picks the voice, not the model. */+export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3";++/** What an aspect ratio is worth in pixels, at 1080p. */+export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {+ "16:9": { width: 1920, height: 1080 },+ "9:16": { width: 1080, height: 1920 },+ "1:1": { width: 1080, height: 1080 },+ "4:3": { width: 1440, height: 1080 },+ "3:4": { width: 1080, height: 1440 },+};+export const PROMPT_INPUT_MODE_OPTIONS = [{ value: "IMAGE", label: "Image", icon: "image" },{ value: "VIDEO", label: "Video", icon: "film-video-export" },diff --git a/apps/web/src/components/genai/insert.tsx b/apps/web/src/components/genai/insert.tsxnew file mode 100644index 00000000..c602baf5--- /dev/null+++ b/apps/web/src/components/genai/insert.tsx@@ -0,0 +1,67 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { Root } from "@diffusionstudio/runtime";+import { findEmptyPlacement } from "@/engine/placement";+import { toast } from "somoto";++import type { DocumentEditor } from "@/engine/editor";+import type { Entity, World } from "koota";++/** Between the variants of one batch, and between the batch and its neighbour. */+const GAP = 40;++/** A declaration is keyed by its spec, so a fresh seed makes a new take. */+export const randomSeed = () => Math.floor(Math.random() * 1_000_000);++/** The box an inserted element is given, in canvas space. */+export interface GenerationBox {+ x: number;+ y: number;+ width: number;+ height: number;+}++/**+ * Inserts `count` elements of `size` at the root of the canvas, side by side+ * as one block, and selects them. `element` is called once per variant with+ * the box it goes in; each call declares its own source, so the variants of a+ * batch are separate takes rather than one asset shown four times.+ *+ * Returns the entities, or nothing (having said so) when there is no project+ * to insert into.+ */+export function insertGenerated(+ world: World,+ editor: DocumentEditor,+ size: { width: number; height: number },+ element: (box: GenerationBox, index: number) => unknown,+ count = 1,+): Entity[] {+ const root = world.get(Root);+ const inserted: Entity[] = [];++ if (root) {+ const totalWidth = count * size.width + (count - 1) * GAP;+ const center = findEmptyPlacement(world, totalWidth, size.height, GAP);+ const left = center.x - totalWidth / 2;+ const y = Math.round(center.y - size.height / 2);++ for (let index = 0; index < count; index++) {+ const x = Math.round(left + index * (size.width + GAP));+ const [entity] = editor.insertElement(root, () => element({ x, y, ...size }, index));+ if (entity) inserted.push(entity);+ }+ }++ if (!inserted.length) {+ toast("Could not add the generation", {+ description: "Open a project to generate into.",+ });+ return inserted;+ }++ editor.select(inserted);+ return inserted;+}diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsxindex e0ce85a3..239fc8af 100644--- a/apps/web/src/components/genai/prompt-input-actions.tsx+++ b/apps/web/src/components/genai/prompt-input-actions.tsx@@ -17,54 +17,28 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltipimport { createMemo, Show } from "solid-js";import { toast } from "somoto";-import { PaintType } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { useUpscaleImage } from "./use-upscale-image";-import { useUpscaleVideo } from "./use-upscale-video";-import { useAddAudioToVideo } from "./use-add-audio-to-video";-import { useRemoveBackground } from "./use-remove-background";import { useGenerationRecords } from "./use-generation-records";import { useGenerateImage } from "./use-generate-image";import { useGenerateVideo } from "./use-generate-video";import { useGenerateVoice } from "./use-generate-voice";import { useGenerateAudio } from "./use-generate-audio";-import { useECS } from "@/context/ecs";+import { useMediaSelection } from "./selection";+import { useModifiers } from "./use-modifiers";++import type { ModifierName } from "./use-modifiers";export function PromptInputActions() {- const { world } = useEngine();- const { selectedNodes } = useECS();const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();+ const { imageNodes, videoNodes } = useMediaSelection();+ const { isOn, toggle } = useModifiers();- const { generate: upscaleImage } = useUpscaleImage();- const { generate: upscaleVideo } = useUpscaleVideo();- const { generate: addAudioToVideo } = useAddAudioToVideo();- const { generate: removeBackground } = useRemoveBackground();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();const { generate: generateVoice } = useGenerateVoice();const { generate: generateAudio } = useGenerateAudio();- const hasImageSelection = createMemo(() => {- const Fills = world.components.Cache.fills;- const Paint = world.components.Paint;- for (const eid of selectedNodes()) {- for (const fid of Fills[eid] ?? []) {- if (Paint[fid] === PaintType.IMAGE) return true;- }- }- return false;- });-- const hasVideoSelection = createMemo(() => {- const Fills = world.components.Cache.fills;- const Paint = world.components.Paint;- for (const eid of selectedNodes()) {- for (const fid of Fills[eid] ?? []) {- if (Paint[fid] === PaintType.VIDEO) return true;- }- }- return false;- });+ const hasImageSelection = createMemo(() => imageNodes().length > 0);+ const hasVideoSelection = createMemo(() => videoNodes().length > 0);const handleRerun = () => {const config = firstConfig();@@ -146,25 +120,14 @@ export function PromptInputActions() {<Separator class="my-1" /></Show><DropdownMenuGroup>+ <Show when={hasImageSelection() || hasVideoSelection()}>+ <ModifierItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />+ </Show><Show when={hasImageSelection()}>- <DropdownMenuItem onSelect={upscaleImage}>- <Icon name="arrow-scale" class="mr-2 text-foreground" />- Upscale- </DropdownMenuItem>- <DropdownMenuItem onSelect={removeBackground}>- <Icon name="ai-generate" class="mr-2 text-foreground" />- Remove background- </DropdownMenuItem>+ <ModifierItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} /></Show><Show when={hasVideoSelection()}>- <DropdownMenuItem onSelect={upscaleVideo}>- <Icon name="arrow-scale" class="mr-2 text-foreground" />- Upscale- </DropdownMenuItem>- <DropdownMenuItem onSelect={addAudioToVideo}>- <Icon name="audio-on" class="mr-2 text-foreground" />- Add audio- </DropdownMenuItem>+ <ModifierItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} /></Show></DropdownMenuGroup></DropdownMenuContent>@@ -174,3 +137,24 @@ export function PromptInputActions() {</Show>);}++type ModifierItemProps = {+ name: ModifierName;+ icon: string;+ label: string;+ isOn(name: ModifierName): boolean;+ toggle(name: ModifierName): void;+};++/** A modifier as a menu row: a check when the selection is asking for it. */+function ModifierItem(props: ModifierItemProps) {+ return (+ <DropdownMenuItem onSelect={() => props.toggle(props.name)}>+ <Icon name={props.icon} class="mr-2 text-foreground" />+ <span class="flex-1">{props.label}</span>+ <Show when={props.isOn(props.name)}>+ <Icon name="confirm-check" class="ml-2 text-foreground" />+ </Show>+ </DropdownMenuItem>+ );+}diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsxindex 058da27a..1c456df4 100644--- a/apps/web/src/components/genai/prompt-input.tsx+++ b/apps/web/src/components/genai/prompt-input.tsx@@ -31,6 +31,7 @@ import {import {PROMPT_INPUT_IMAGE_MODEL_OPTIONS,PROMPT_INPUT_VIDEO_MODEL_OPTIONS,+ PROMPT_INPUT_VOICE_MODEL,PROMPT_INPUT_VOICE_OPTIONS,PROMPT_INPUT_MODE_OPTIONS,PROMPT_INPUT_IMAGE_ASPECT_RATIO_OPTIONS,@@ -48,8 +49,6 @@ import { useGenerateVideo } from "./use-generate-video";import { useGenerateVoice } from "./use-generate-voice";import { useGenerateAudio } from "./use-generate-audio";import { PromptInputActions } from "./prompt-input-actions";-import { useEngine } from "@/context/engine";-import { showFileDialog } from "@/utils";import type {AspectRatio,GenerationConfig,@@ -59,18 +58,13 @@ import type {AudioGenerationConfig,} from "@/components/engine/db";import { AssetThumbnail } from "@/components/ui/asset-thumbnail";-import { ChildOf, PaintType, removeComponent } from "@/components/engine";-import { useAssets, loadAsset } from "@/components/engine";-import { useECS } from "@/context/ecs";-import { Not, query } from 'bitecs';-import { AssetId } from "@diffusionstudio/runtime";import { useLibrary } from "@/engine/library";-import { useEditor, useSelection } from "@/engine/hooks";+import { useEditor } from "@/engine/hooks";import { droppedFiles, importFiles, pickFiles } from "@/engine/asset-actions";+import { useMediaSelection } from "./selection";import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";-import type { Entity } from "koota";-import type { Asset as LibraryAsset, AssetCache } from "@diffusionstudio/assets";+import type { AssetCache } from "@diffusionstudio/assets";import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";@@ -107,7 +101,7 @@ export function createDefaultConfig<K extends keyof GenerationConfgs>(mode: K, pcase "VOICE":return {mode: "VOICE",- model: "elevenlabs-v3",+ model: PROMPT_INPUT_VOICE_MODEL,prompt,voice: PROMPT_INPUT_VOICE_OPTIONS[0].value,} as GenerationConfgs[K];@@ -125,15 +119,9 @@ export interface PromptInputProps {}export function PromptInput(props: PromptInputProps) {- const { world } = useEngine();- const assets = useAssets(world);- const { selectedNodes } = useECS();- const c = world.components;- // The runtime the IMAGE mode works against: references come from the- // project library and the canvas selection, and land as declarations.const library = useLibrary();const editor = useEditor();- const { nodes } = useSelection();+ const { images: selectedImages } = useMediaSelection();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();const { generate: generateVoice } = useGenerateVoice();@@ -184,46 +172,13 @@ export function PromptInput(props: PromptInputProps) {return model ? ALL_VIDEO_ASPECT_RATIO_OPTIONS.filter((o) => model.aspectRatios.includes(o.value)) : [];});- const selectedImageEntries = createMemo(() => {- const entries: { assetId: string; eid: number }[] = [];- const AssetId = world.components.AssetId;- for (const eid of selectedNodes()) {- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {- if (c.Paint[fid] === PaintType.IMAGE) {- const assetId = AssetId[fid];- if (!assetId) continue;- entries.push({ assetId, eid });- break; // we just need the first image fill- }- }- }- return entries;- });-- const selectedImageIds = createMemo(() =>- selectedImageEntries().map((e) => e.assetId),- );-- // The image assets of the current canvas selection, from the runtime's- // world: entities bound to a library image count as references.- const selectedLibraryImages = createMemo(() => {- const lib = library();- if (!lib) return [];-- const entries: { asset: LibraryAsset; entity: Entity }[] = [];- for (const entity of nodes()) {- const id = entity.get(AssetId)?.value;- const asset = id ? lib.get(id) : undefined;- if (asset?.type === "IMAGE") entries.push({ asset, entity });- }- return entries;- });-+ // Pictures on the canvas are offered as references and as video frames:+ // selecting one is another way of attaching it.const effectiveImageRefIds = createMemo(() => {const lib = library();if (!lib) return [];const local = imageConfig()?.imageRefIds ?? [];- const selected = selectedLibraryImages()+ const selected = selectedImages().map((entry) => entry.asset.id).filter((id) => !local.includes(id));return [...local, ...selected]@@ -231,15 +186,22 @@ export function PromptInput(props: PromptInputProps) {.slice(0, MAX_IMAGE_REFERENCES);});- const effectiveStartFrameId = createMemo(() => {- const id = videoConfig()?.startFrameImageId ?? selectedImageIds()[0] ?? null;- return id && assets.get(id) ? id : null;- });+ /** The library id a frame is set to, explicitly or by the selection. */+ const frameId = (frame: "start" | "end") => {+ // A frame the model cannot take is not one the prompt box offers, and+ // not one a canvas selection quietly attaches either: the declaration is+ // checked against the model before it runs.+ if (!currentVideoModel()?.features.includes(`${frame}-frame`)) return null;- const effectiveEndFrameId = createMemo(() => {- const id = videoConfig()?.endFrameImageId ?? selectedImageIds()[1] ?? null;- return id && assets.get(id) ? id : null;- });+ const lib = library();+ const config = videoConfig();+ const explicit = frame === "start" ? config?.startFrameImageId : config?.endFrameImageId;+ const id = explicit ?? selectedImages()[frame === "start" ? 0 : 1]?.asset.id ?? null;+ return id && lib?.get(id) ? id : null;+ };++ const effectiveStartFrameId = createMemo(() => frameId("start"));+ const effectiveEndFrameId = createMemo(() => frameId("end"));const [recentPrompts, setRecentPrompts] = createStoredSignal(store.define<string[]>("prompt-input.recent-prompts", []),@@ -275,7 +237,7 @@ export function PromptInput(props: PromptInputProps) {}// The reference is shown from a selected canvas entity, so removing it// means deselecting that entity.- const entry = selectedLibraryImages().find((e) => e.asset.id === assetId);+ const entry = selectedImages().find((e) => e.asset.id === assetId);if (entry) editor.deselect(entry.entity);};@@ -297,22 +259,17 @@ export function PromptInput(props: PromptInputProps) {};const openVideoFramePicker = async (frame: "start" | "end") => {- const handles = await showFileDialog("image/*", false);- if (handles.length !== 1) return;-- try {- const loadedAsset = await loadAsset(world, handles[0]!);- if (loadedAsset.type !== "IMAGE") return;- if (frame === "start") {- patch({ startFrameImageId: loadedAsset.id });- } else {- patch({ endFrameImageId: loadedAsset.id });- }- } catch (e) {- toast("Failed to import assets", {- description: (e as Error).message,- });- }+ const lib = library();+ if (!lib) return;++ const files = await pickFiles({ accept: "image/*", multiple: false });+ if (files.length === 0) return;++ // `importFiles` reports its own failures.+ const [imported] = (await importFiles(lib, files, "")).filter((a) => a.type === "IMAGE");+ if (!imported) return;++ patch(frame === "start" ? { startFrameImageId: imported.id } : { endFrameImageId: imported.id });};const swapVideoFrameImages = () => {@@ -332,10 +289,8 @@ export function PromptInput(props: PromptInputProps) {}// The frame is being shown from a selected canvas entity, so clearing it// means deselecting that entity (mirrors removeImageReference).- const entry = frame === "start" ? selectedImageEntries()[0] : selectedImageEntries()[1];- if (entry) {- removeComponent(world, entry.eid, c.Selected, false);- }+ const entry = selectedImages()[frame === "start" ? 0 : 1];+ if (entry) editor.deselect(entry.entity);};const resizeTextarea = () => {@@ -444,59 +399,42 @@ export function PromptInput(props: PromptInputProps) {dragCounter = 0;setIsDragging(false);+ if (mode() !== "IMAGE" && mode() !== "VIDEO") return;++ const lib = library();+ if (!lib) return;++ // What a generation is given to work from lives in the project library:+ // dragged library assets are taken as they are, external files imported.const assetIdData = event.dataTransfer?.getData(ASSET_DRAG_TYPE);const existingAssetIds = assetIdData?.split(",").filter(Boolean) ?? [];+ const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));+ if (existingAssetIds.length === 0 && files.length === 0) return;- // References for an image generation live in the project library:- // dragged library assets are taken as they are, external files imported.- if (mode() === "IMAGE") {- const lib = library();- if (!lib) return;-- const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));- if (existingAssetIds.length === 0 && files.length === 0) return;-- const newAssetIds =- existingAssetIds.length > 0- ? existingAssetIds- : (await importFiles(lib, files, "")).map((a) => a.id);-- const imageIds = newAssetIds.filter((id) => lib.get(id)?.type === "IMAGE");- const current = imageConfig()?.imageRefIds ?? [];- patch({- imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),- });+ // `importFiles` reports its own failures.+ const droppedIds =+ existingAssetIds.length > 0+ ? existingAssetIds+ : (await importFiles(lib, files, "")).map((a) => a.id);++ const imageIds = droppedIds.filter((id) => lib.get(id)?.type === "IMAGE");+ if (imageIds.length === 0) return;++ if (mode() === "VIDEO") {+ const c = videoConfig();+ if (!c?.startFrameImageId && imageIds[0]) {+ patch({ startFrameImageId: imageIds[0] });+ }+ if (!c?.endFrameImageId && imageIds[1]) {+ patch({ endFrameImageId: imageIds[1] });+ }return;}- const fileItems = Array.from(event.dataTransfer?.items ?? []).filter(- (item) => item.kind === "file" && item.type.startsWith("image/"),- );-- if (existingAssetIds.length === 0 && fileItems.length === 0) return;-- try {- let newAssetIds =- existingAssetIds.length > 0- ? existingAssetIds- : (await Promise.all(fileItems.map((item) => loadAsset(world, item)))).map((a) => a.id);-- newAssetIds = newAssetIds.filter((id) => world.assets.get(id)?.type === "IMAGE");-- if (mode() === "VIDEO") {- const c = videoConfig();- if (!c?.startFrameImageId && newAssetIds[0]) {- patch({ startFrameImageId: newAssetIds[0] });- }- if (!c?.endFrameImageId && newAssetIds[1]) {- patch({ endFrameImageId: newAssetIds[1] });- }- }- } catch (e) {- toast("Failed to import assets", {- description: (e as Error).message,- });- }+ const current = imageConfig()?.imageRefIds ?? [];+ patch({+ imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),+ });};const handleVideoModelChange = (id: string) => {@@ -510,7 +448,7 @@ export function PromptInput(props: PromptInputProps) {...c,model: id,endFrameImageId: model.features.includes("end-frame") ? c.endFrameImageId : undefined,- generateAudio: model.features.includes("audio") ? c.generateAudio : true,+ generateAudio: model.features.includes("audio") ? (c.generateAudio ?? true) : false,duration: model.durations.includes(durationStr) ? c.duration : parseInt(model.durations[0], 10),aspectRatio: model.aspectRatios.includes(c.aspectRatio)? c.aspectRatio@@ -577,7 +515,8 @@ export function PromptInput(props: PromptInputProps) {>{(src) => (<PromptInputReferenceImageButton- asset={assets.get(src())!}+ asset={library()!.get(src())!}+ cache={library()!.cache}class="h-16 w-28"size={{ width: 112, height: 64 }}onRemove={() => clearVideoFrame("start")}@@ -611,7 +550,8 @@ export function PromptInput(props: PromptInputProps) {>{(src) => (<PromptInputReferenceImageButton- asset={assets.get(src())!}+ asset={library()!.get(src())!}+ cache={library()!.cache}class="h-16 w-28"size={{ width: 112, height: 64 }}onRemove={() => clearVideoFrame("end")}diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.tsnew file mode 100644index 00000000..23a28d77--- /dev/null+++ b/apps/web/src/components/genai/selection.ts@@ -0,0 +1,56 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// What the prompt box has to work with. Two readings of the selection, for+// two different questions:+//+// `imageNodes`/`videoNodes` are what the element *is*, which is what a+// source modifier applies to — a picture that is still generating is still a+// picture, and `removeBackground` can be asked of it before it exists.+//+// `images`/`videos` are what the element currently *shows*, which is what a+// reference needs: a library asset with a path to name it by.++import { createMemo } from "solid-js";+import { AssetId } from "@diffusionstudio/runtime";+import { authoredElement } from "@diffusionstudio/reconciler";+import { useSelection } from "@/engine/hooks";+import { useLibrary } from "@/engine/library";++import type { Asset } from "@diffusionstudio/assets";+import type { Entity } from "koota";++/** A selected node and the library asset it is bound to. */+export interface BoundNode {+ entity: Entity;+ asset: Asset;+}++export function useMediaSelection() {+ const library = useLibrary();+ const { nodes } = useSelection();++ const tagged = (tag: string) =>+ createMemo(() => nodes().filter((entity) => authoredElement(entity)?.tag === tag));++ const imageNodes = tagged("image");+ const videoNodes = tagged("video");++ const bound = createMemo(() => {+ const lib = library();+ if (!lib) return [];++ const entries: BoundNode[] = [];+ for (const entity of nodes()) {+ const id = entity.get(AssetId)?.value;+ const asset = id ? lib.get(id) : undefined;+ if (asset) entries.push({ entity, asset });+ }+ return entries;+ });++ const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));++ return { bound, images, imageNodes, videoNodes };+}diff --git a/apps/web/src/components/genai/use-add-audio-to-video.ts b/apps/web/src/components/genai/use-add-audio-to-video.tsdeleted file mode 100644index a413dbf8..00000000--- a/apps/web/src/components/genai/use-add-audio-to-video.ts+++ /dev/null@@ -1,75 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { Not, Or, query } from 'bitecs';-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent } from "@/components/engine";-import { loadAsset, uploadAsset } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { assert } from "@/utils";-import { toast } from "somoto";--import type { EngineWorld } from "@/components/engine";--function findVideoFill(world: EngineWorld, eid: number): number | null {- const c = world.components;-- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {- if (c.Paint[fid] === PaintType.VIDEO) {- return fid;- }- }- return null;-}--export function useAddAudioToVideo() {- const { world } = useEngine();- const c = world.components;-- const generate = async () => {- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];-- for (const nid of nodes) {- const fid = findVideoFill(world, nid);- if (!fid) continue;-- const assetId = c.AssetId[fid];- if (!assetId) continue;-- const asset = world.assets.get(assetId);- if (!asset) continue;-- try {- addComponent(world, nid, c.Generating);-- const fileRef = await uploadAsset(world, assetId);- if (!fileRef) throw new Error("Failed to upload asset");-- const { url, generationId } = await trpc.addAudioToVideo.mutate({ video: fileRef });-- const dot = asset.name.lastIndexOf(".");- const name = dot > 0- ? asset.name.slice(0, dot) + " (With audio)" + asset.name.slice(dot)- : asset.name + " (With audio)";- const newAsset = await loadAsset(world, url, { name, generationId });-- assert(newAsset.type === "VIDEO", "New asset is not a video");-- addComponent(world, fid, c.Hidden);-- const newFid = createEntity(world);- setComponent(world, newFid, c.Paint, PaintType.VIDEO);- setComponent(world, newFid, c.AssetId, newAsset.id);- appendChild(world, newFid, nid);- } catch (err) {- console.error("Add audio failed:", err);- toast.error("Failed to add audio to video");- } finally {- removeComponent(world, nid, c.Generating);- }- }- };-- return { generate };-}diff --git a/apps/web/src/components/genai/use-generate-audio.ts b/apps/web/src/components/genai/use-generate-audio.tsdeleted file mode 100644index e20c3172..00000000--- a/apps/web/src/components/genai/use-generate-audio.ts+++ /dev/null@@ -1,90 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { track } from "@/lib/analytics";-import { findEmptyPlacement, recordGeneration, addWaveformFill, ensureGenerationFolder } from "@/utils/genai";-import { assert } from "@/utils";--import { GeometryType, createEntity, deleteEntity, addComponent, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset } from "@/components/engine";--import type { AudioGenerationConfig } from "@/components/engine/db";---export function useGenerateAudio() {- const { world } = useEngine();- const c = world.components;-- const generate = async (config: AudioGenerationConfig) => {- let eid: number | undefined;- const startedAt = performance.now();- try {- track('generation_started', {- mode: 'audio',- model: config.model,- prompt_length: config.prompt.length,- });-- const placement = findEmptyPlacement(world, 400, 100, 40);-- // In-flight placeholder is transient UI — build it untracked so it never- // reaches the undo stack. The whole generation is recorded as one step below.- world.history.untrack(() => {- eid = createEntity(world);- setComponent(world, eid, c.Geometry, GeometryType.RECT);- addComponent(world, eid, c.Audio);- setComponent(world, eid, c.Playback, {});- setComponent(world, eid, c.Position, {- x: Math.round(placement.x - 200),- y: Math.round(placement.y - 50),- });- resizeEntity(world, eid, {- width: 400,- height: 100,- });- addComponent(world, eid, c.Generating);- });-- const { results, name, generationId } = await trpc.generateSound.mutate({- model: config.model,- prompt: config.prompt,- });-- assert(results.length > 0, "No results returned from the model");- const folderId = await ensureGenerationFolder(world, name, results.length);- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });-- assert(asset.type === "AUDIO", "Asset is not an audio");-- world.history.untrack(() => {- setComponent(world, eid!, c.Name, name);- setComponent(world, eid!, c.AssetId, asset.id);- addWaveformFill(world, eid!, asset.id);- removeComponent(world, eid!, c.Generating);- });-- recordGeneration(world, 'Generate audio', [eid!]);-- track('generation_completed', {- mode: 'audio',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- });- } catch (err) {- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));- console.error(err);- track('generation_failed', {- mode: 'audio',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',- });- throw err;- }- };-- return { generate } as const;-}diff --git a/apps/web/src/components/genai/use-generate-audio.tsx b/apps/web/src/components/genai/use-generate-audio.tsxnew file mode 100644index 00000000..b5cc4cce--- /dev/null+++ b/apps/web/src/components/genai/use-generate-audio.tsx@@ -0,0 +1,34 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Music and sound effects, declared: the prompt box writes a `generate.audio`+// source and inserts an <Audio> consuming it (see ./insert).++import { generate } from "@diffusionstudio/jsx";+import { Audio } from "@diffusionstudio/reconciler";+import { useWorld } from "@diffusionstudio/koota-solid";+import { useEditor } from "@/engine/hooks";+import { AUDIO_SIZE } from "@/engine/insert-asset";+import { insertGenerated, randomSeed } from "./insert";++import type { AudioGenerationConfig } from "@/components/engine/db";++export function useGenerateAudio() {+ const world = useWorld();+ const editor = useEditor();++ const run = async (config: AudioGenerationConfig) => {+ const src = generate.audio({+ prompt: config.prompt,+ model: config.model,+ seed: randomSeed(),+ });++ insertGenerated(world, editor, AUDIO_SIZE, (box) => (+ <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />+ ));+ };++ return { generate: run } as const;+}diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsxindex 92606c80..2ce4f64f 100644--- a/apps/web/src/components/genai/use-generate-image.tsx+++ b/apps/web/src/components/genai/use-generate-image.tsx@@ -8,27 +8,16 @@// declaration through the world's Ai (see `EditorGenAi`), and the element// lands in the project file with the `generate.image({...})` call that// reproduces it — same spec, same asset, this session and the next.-//-// Generations go on the canvas rather than into a scene: they are material to-// work with, not yet part of a composition, and a scene owns a timeline they-// have no place on until someone puts them there.import { generate } from "@diffusionstudio/jsx";import { Image } from "@diffusionstudio/reconciler";-import { Library, Root } from "@diffusionstudio/runtime";+import { Library } from "@diffusionstudio/runtime";import { useWorld } from "@diffusionstudio/koota-solid";import { useEditor } from "@/engine/hooks";-import { findEmptyPlacement } from "@/engine/placement";-import { ASPECT_RATIO_DIMENSIONS } from "@/utils/genai";-import { toast } from "somoto";+import { ASPECT_RATIO_DIMENSIONS } from "./config";+import { insertGenerated, randomSeed } from "./insert";import type { ImageGenerationConfig } from "@/components/engine/db";-import type { Entity } from "koota";--const IMAGE_GAP = 40;--/** A declaration is keyed by its spec, so a fresh seed makes a new take. */-const randomSeed = () => Math.floor(Math.random() * 1_000_000);export function useGenerateImage() {const world = useWorld();@@ -36,8 +25,6 @@ export function useGenerateImage() {const run = async (config: ImageGenerationConfig) => {const library = world.get(Library);- const root = world.get(Root);- if (!root) return;// References travel by library path: durable in the file, and readable// there. Ids that no longer name a library asset are dropped.@@ -46,35 +33,20 @@ export function useGenerateImage() {.filter((path): path is string => path !== undefined);const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };- const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;- const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);- const left = center.x - totalWidth / 2;- const top = Math.round(center.y - dims.height / 2);-- const inserted: Entity[] = [];- for (let index = 0; index < config.count; index++) {- const src = generate.image({- prompt: config.prompt,- model: config.model,- aspectRatio: config.aspectRatio,- ...(refs.length ? { refs } : {}),- seed: randomSeed(),- });-- const x = Math.round(left + index * (dims.width + IMAGE_GAP));- const [entity] = editor.insertElement(root, () => (- <Image src={src} x={x} y={top} width={dims.width} height={dims.height} />- ));- if (entity) inserted.push(entity);- }- if (!inserted.length) {- toast("Could not add the generation", {- description: "Open a project to generate into.",- });- return;- }- editor.select(inserted);+ // One declaration per variant, each with its own seed: a batch is four+ // takes of the prompt rather than one asset shown four times.+ const sources = Array.from({ length: config.count }, () => generate.image({+ prompt: config.prompt,+ model: config.model,+ aspectRatio: config.aspectRatio,+ ...(refs.length ? { refs } : {}),+ seed: randomSeed(),+ }));++ insertGenerated(world, editor, dims, (box, index) => (+ <Image src={sources[index]} x={box.x} y={box.y} width={box.width} height={box.height} />+ ), config.count);};return { generate: run } as const;diff --git a/apps/web/src/components/genai/use-generate-video.ts b/apps/web/src/components/genai/use-generate-video.tsdeleted file mode 100644index aef56326..00000000--- a/apps/web/src/components/genai/use-generate-video.ts+++ /dev/null@@ -1,111 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { track } from "@/lib/analytics";-import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement, recordGeneration, ensureGenerationFolder } from "@/utils/genai";--import { GeometryType, PaintType, createEntity, deleteEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset, uploadAsset } from "@/components/engine";-import { assert } from "@/utils";--import type { VideoGenerationConfig } from "@/components/engine/db";--export function useGenerateVideo() {- const { world } = useEngine();- const c = world.components;-- const generate = async (config: VideoGenerationConfig) => {- let eid: number | undefined;- const startedAt = performance.now();- try {- track('generation_started', {- mode: 'video',- model: config.model,- aspect_ratio: config.aspectRatio,- duration: config.duration,- generate_audio: config.generateAudio,- has_start_frame: !!config.startFrameImageId,- has_end_frame: !!config.endFrameImageId,- prompt_length: config.prompt.length,- });-- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };- const placement = findEmptyPlacement(world, dims.width, dims.height, 40);-- // In-flight placeholder is transient UI — build it untracked so it never- // reaches the undo stack. The whole generation is recorded as one step below.- world.history.untrack(() => {- eid = createEntity(world);- setComponent(world, eid, c.Geometry, GeometryType.RECT);- setComponent(world, eid, c.Position, {- x: Math.round(placement.x - dims.width / 2),- y: Math.round(placement.y - dims.height / 2),- });- resizeEntity(world, eid, {- width: dims.width,- height: dims.height,- });- setComponent(world, eid, c.KeepAspectRatio, {- width: dims.width,- height: dims.height,- });- addComponent(world, eid, c.Generating);- });-- const [startFrame, endFrame] = await Promise.all([- config.startFrameImageId ? uploadAsset(world, config.startFrameImageId) : undefined,- config.endFrameImageId ? uploadAsset(world, config.endFrameImageId) : undefined,- ]);-- const { results, name, generationId } = await trpc.generateVideo.mutate({- model: config.model,- prompt: config.prompt,- aspectRatio: config.aspectRatio,- duration: config.duration,- generateAudio: config.generateAudio,- startFrame: startFrame ?? undefined,- endFrame: endFrame ?? undefined,- });-- assert(results.length > 0, "No results returned from the model");- const folderId = await ensureGenerationFolder(world, name, results.length);- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });-- world.history.untrack(() => {- setComponent(world, eid!, c.Name, name);- removeComponent(world, eid!, c.Generating);-- const fid = createEntity(world);- setComponent(world, fid, c.Paint, PaintType.VIDEO);- setComponent(world, fid, c.AssetId, asset.id);- appendChild(world, fid, eid!);-- setComponent(world, eid!, c.Playback, {});- });-- recordGeneration(world, 'Generate video', [eid!]);-- track('generation_completed', {- mode: 'video',- model: config.model,- aspect_ratio: config.aspectRatio,- duration_ms: Math.round(performance.now() - startedAt),- });- } catch (err) {- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));- console.error(err);- track('generation_failed', {- mode: 'video',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',- });- throw err;- }- };-- return { generate } as const;-}diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsxnew file mode 100644index 00000000..5d1b2c88--- /dev/null+++ b/apps/web/src/components/genai/use-generate-video.tsx@@ -0,0 +1,49 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Video generation, declared rather than performed: the prompt box writes a+// `generate.video` source and inserts a <Video> consuming it (see ./insert).+// The frames a model animates between are library assets, named by path, so+// the call in the file says what it was made from as well as what to make.++import { generate } from "@diffusionstudio/jsx";+import { Video } from "@diffusionstudio/reconciler";+import { Library } from "@diffusionstudio/runtime";+import { useWorld } from "@diffusionstudio/koota-solid";+import { useEditor } from "@/engine/hooks";+import { ASPECT_RATIO_DIMENSIONS } from "./config";+import { insertGenerated, randomSeed } from "./insert";++import type { VideoGenerationConfig } from "@/components/engine/db";++export function useGenerateVideo() {+ const world = useWorld();+ const editor = useEditor();++ const run = async (config: VideoGenerationConfig) => {+ const library = world.get(Library);+ const frame = (id: string | undefined) => (id ? library?.get(id)?.path : undefined);++ const startFrame = frame(config.startFrameImageId);+ const endFrame = frame(config.endFrameImageId);+ const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };++ const src = generate.video({+ prompt: config.prompt,+ model: config.model,+ aspectRatio: config.aspectRatio,+ duration: config.duration,+ audio: config.generateAudio ?? false,+ ...(startFrame ? { startFrame } : {}),+ ...(endFrame ? { endFrame } : {}),+ seed: randomSeed(),+ });++ insertGenerated(world, editor, dims, (box) => (+ <Video src={src} x={box.x} y={box.y} width={box.width} height={box.height} />+ ));+ };++ return { generate: run } as const;+}diff --git a/apps/web/src/components/genai/use-generate-voice.ts b/apps/web/src/components/genai/use-generate-voice.tsdeleted file mode 100644index 7236938b..00000000--- a/apps/web/src/components/genai/use-generate-voice.ts+++ /dev/null@@ -1,89 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */---import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { track } from "@/lib/analytics";-import { findEmptyPlacement, recordGeneration, addWaveformFill, ensureGenerationFolder } from "@/utils/genai";-import { GeometryType, createEntity, deleteEntity, addComponent, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset } from "@/components/engine";--import type { VoiceGenerationConfig } from "@/components/engine/db";--export function useGenerateVoice() {- const { world } = useEngine();- const c = world.components;--- const generate = async (config: VoiceGenerationConfig) => {- let eid: number | undefined;- const startedAt = performance.now();- try {- track('generation_started', {- mode: 'voice',- model: config.model,- voice: config.voice,- prompt_length: config.prompt.length,- });-- const placement = findEmptyPlacement(world, 400, 100, 40);- // In-flight placeholder is transient UI — build it untracked so it never- // reaches the undo stack. The whole generation is recorded as one step below.- world.history.untrack(() => {- eid = createEntity(world);- setComponent(world, eid, c.Geometry, GeometryType.RECT);- addComponent(world, eid, c.Audio);- setComponent(world, eid, c.Playback, {});- setComponent(world, eid, c.Position, {- x: Math.round(placement.x - 200),- y: Math.round(placement.y - 50),- });- resizeEntity(world, eid, {- width: 400,- height: 100,- });- addComponent(world, eid, c.Generating);- });-- const { results, name, generationId } = await trpc.textToSpeech.mutate({- model: config.model,- prompt: config.prompt,- voice: config.voice,- });-- if (results.length > 0) {- const folderId = await ensureGenerationFolder(world, name, results.length);- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });-- world.history.untrack(() => {- setComponent(world, eid!, c.Name, name);- setComponent(world, eid!, c.AssetId, asset.id);- addWaveformFill(world, eid!, asset.id);- removeComponent(world, eid!, c.Generating);- });-- recordGeneration(world, 'Generate voice', [eid!]);- }-- track('generation_completed', {- mode: 'voice',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- });- } catch (err) {- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));- console.error(err);- track('generation_failed', {- mode: 'voice',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',- });- throw err;- }- };-- return { generate } as const;-}diff --git a/apps/web/src/components/genai/use-generate-voice.tsx b/apps/web/src/components/genai/use-generate-voice.tsxnew file mode 100644index 00000000..9850ed34--- /dev/null+++ b/apps/web/src/components/genai/use-generate-voice.tsx@@ -0,0 +1,35 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Spoken lines, declared: the prompt box writes a `generate.voice` source and+// inserts an <Audio> consuming it (see ./insert), so the file holds the line+// and the voice that reads it rather than a sound nobody can trace back.++import { generate } from "@diffusionstudio/jsx";+import { Audio } from "@diffusionstudio/reconciler";+import { useWorld } from "@diffusionstudio/koota-solid";+import { useEditor } from "@/engine/hooks";+import { AUDIO_SIZE } from "@/engine/insert-asset";+import { insertGenerated, randomSeed } from "./insert";++import type { VoiceGenerationConfig } from "@/components/engine/db";++export function useGenerateVoice() {+ const world = useWorld();+ const editor = useEditor();++ const run = async (config: VoiceGenerationConfig) => {+ const src = generate.voice({+ prompt: config.prompt,+ voice: config.voice,+ seed: randomSeed(),+ });++ insertGenerated(world, editor, AUDIO_SIZE, (box) => (+ <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />+ ));+ };++ return { generate: run } as const;+}diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.tsindex 1e013c38..0503377f 100644--- a/apps/web/src/components/genai/use-generation-records.ts+++ b/apps/web/src/components/genai/use-generation-records.ts@@ -3,14 +3,23 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createMemo, createResource } from "solid-js";-import { hasComponent } from "bitecs";-import { PaintType } from "@/components/engine";-import { useEngine } from "@/context/engine";+import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";+import { authoredElement } from "@diffusionstudio/reconciler";+import { useSelection } from "@/engine/hooks";+import { useLibrary } from "@/engine/library";import { supabase } from "@/lib/supabase";+import { useMediaSelection } from "./selection";+import {+ PROMPT_INPUT_AUDIO_MODEL_OPTIONS,+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS,+ PROMPT_INPUT_VIDEO_MODEL_OPTIONS,+ PROMPT_INPUT_VOICE_MODEL,+ PROMPT_INPUT_VOICE_OPTIONS,+} from "./config";import { generationConfigSchema, type GenerationConfig } from "@/components/engine/db";-import type { EngineWorld } from "@/components/engine";-import { useECS } from "@/context/ecs";+import type { AssetInput, AssetSpecInput } from "@diffusionstudio/jsx";+import type { AssetLibrary } from "@diffusionstudio/assets";/*** The stored `usage_records.config` IS the server-side adapter input, which@@ -48,36 +57,73 @@ export function toClientConfig(stored: unknown): GenerationConfig | undefined {return parsed.success ? parsed.data : undefined;}-function getNodeAssetId(world: EngineWorld, eid: number): string | undefined {- const c = world.components;- if (hasComponent(world, eid, c.Audio)) {- return c.AssetId[eid];- }- for (const fid of c.Cache.fills[eid]) {- if (hasComponent(world, fid, c.Hidden)) continue;- if (c.Paint[fid] === PaintType.VIDEO || c.Paint[fid] === PaintType.IMAGE) {- return c.AssetId[fid];- }+/**+ * A declaration read back as what the prompt box would have to be set to to+ * produce it — the element's own account of how it was made, which is what+ * "Rerun" and "Reuse" work from. Inputs are named by library path in the+ * file and by asset id in the prompt box, so they are looked up on the way.+ */+function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): GenerationConfig {+ const idOf = (input: AssetInput | undefined): string | undefined =>+ typeof input === "string" ? library.get(input)?.id : undefined;++ switch (spec.type) {+ case "image":+ return {+ mode: "IMAGE",+ model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio ?? "16:9",+ count: 1,+ imageRefIds: (spec.refs ?? []).map(idOf).filter((id): id is string => id !== undefined),+ };+ case "video":+ return {+ mode: "VIDEO",+ model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio ?? "16:9",+ duration: spec.duration ?? 5,+ generateAudio: spec.audio ?? false,+ startFrameImageId: idOf(spec.startFrame),+ endFrameImageId: idOf(spec.endFrame),+ };+ case "voice":+ return {+ mode: "VOICE",+ model: PROMPT_INPUT_VOICE_MODEL,+ prompt: spec.prompt,+ voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,+ };+ case "audio":+ return {+ mode: "AUDIO",+ model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ };}- return undefined;}export function useGenerationRecords() {- const { selectedNodes } = useECS();- const { world } = useEngine();+ const library = useLibrary();+ const { nodes } = useSelection();+ const { bound } = useMediaSelection();++ // What the selected elements declare their source to be. An element made by+ // the prompt box carries the whole spec, so this answers before the asset+ // exists — and without asking the server what it was asked for.+ const declarations = createMemo(() =>+ nodes()+ .map((entity) => authoredElement(entity)?.props.src)+ .filter(isAssetRef)+ .map(getAssetSpec),+ );+ // Credits are the server's to know, and it knows them per generation.const generationIds = createMemo(() => {const ids = new Set<string>();- for (const eid of selectedNodes()) {- const assetId = getNodeAssetId(world, eid);- if (!assetId) {- continue;- }-- const asset = world.assets.get(assetId);- if (asset?.generationId) {- ids.add(asset.generationId);- }+ for (const { asset } of bound()) {+ if (asset.generation?.id) ids.add(asset.generation.id);}return [...ids];});@@ -106,9 +152,15 @@ export function useGenerationRecords() {return (records() ?? []).reduce((sum, r) => sum + r.credits, 0);});- const isGenerated = createMemo(() => generationIds().length > 0);+ const isGenerated = createMemo(() => declarations().length > 0 || generationIds().length > 0);const firstConfig = (): GenerationConfig | undefined => {+ const lib = library();+ const declared = declarations()[0];+ if (declared && lib) return toPromptConfig(declared, lib);++ // Nothing declared: an asset generated before the project was written in+ // JSX, whose settings only the record of the run still has.const ids = generationIds();if (ids.length === 0) return undefined;const record = (records() ?? []).find((r) => r.id === ids[0]);diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.tsnew file mode 100644index 00000000..17fc24d8--- /dev/null+++ b/apps/web/src/components/genai/use-modifiers.ts@@ -0,0 +1,72 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Upscaling, taking a background out, scoring footage: the editor's side of+// the source modifiers (see the runtime's `SourceModifiers` and+// reference/jsx/media.md). The action bar sets a prop and stops there — the+// asset system is what notices, calls the model and binds the result — so+// these are toggles rather than commands, and turning one off gives the+// original back rather than running anything.++import { authoredElement } from "@diffusionstudio/reconciler";+import { useEditor } from "@/engine/hooks";+import { useMediaSelection } from "./selection";++import type { Entity } from "koota";++/** The props an element carries to say what its source is put through. */+export type ModifierName = "removeBackground" | "upscale" | "addAudio";++/**+ * What `upscale` is set to when it is switched on. The endpoint takes no+ * factor today, so this is the one the language says and the number every+ * upscale is; see the cache key in `EditorGenAi.transform`.+ */+const UPSCALE_FACTOR = 2;++/** Whether `entity` currently asks for `name`, as its element says it. */+function isSet(entity: Entity, name: ModifierName): boolean {+ const value = authoredElement(entity)?.props[name];+ return name === "upscale" ? typeof value === "number" && value > 1 : value === true;+}++export function useModifiers() {+ const editor = useEditor();+ const { imageNodes, videoNodes } = useMediaSelection();++ /** The selected elements a modifier can be asked of. */+ const targets = (name: ModifierName): Entity[] => {+ if (name === "removeBackground") return imageNodes();+ if (name === "addAudio") return videoNodes();+ return [...imageNodes(), ...videoNodes()];+ };++ /** On when every element it applies to asks for it, so the toggle turns the odd one on. */+ const isOn = (name: ModifierName): boolean => {+ const selected = targets(name);+ return selected.length > 0 && selected.every((entity) => isSet(entity, name));+ };++ /**+ * Turns `name` on for the selection, or off when they all have it. Off is+ * written as `false`, which the source writer spells as the prop's absence+ * — an element says what it asks for, and nothing about what it does not.+ */+ const toggle = (name: ModifierName): void => {+ const next = !isOn(name);+ const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;++ for (const entity of targets(name)) {+ // A failure is the answer to what was asked for (see `sourceErrorSystem`),+ // and asking for a different set of modifiers is a different question.+ // Taking the prop off is what asks again, and this is the user doing it.+ if (authoredElement(entity)?.props.error !== undefined) {+ editor.editProperty(entity, "error", false);+ }+ editor.editProperty(entity, name, value);+ }+ };++ return { isOn, toggle };+}diff --git a/apps/web/src/components/genai/use-remove-background.ts b/apps/web/src/components/genai/use-remove-background.tsdeleted file mode 100644index f98877c5..00000000--- a/apps/web/src/components/genai/use-remove-background.ts+++ /dev/null@@ -1,77 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { Not, Or, query } from 'bitecs';-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent } from "@/components/engine";-import { loadAsset, uploadAsset } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { toast } from "somoto";--import type { EngineWorld } from "@/components/engine";--/**- * For a given shape entity, find the first image fill and its index.- */-function findImageFill(world: EngineWorld, eid: number): number | null {- const c = world.components;-- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {- if (c.Paint[fid] === PaintType.IMAGE) {- return fid;- }- }- return null;-}--export function useRemoveBackground() {- const { world } = useEngine();- const c = world.components;-- const generate = async () => {- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];-- for (const nid of nodes) {- const fid = findImageFill(world, nid);- if (!fid) continue;-- const assetId = world.components.AssetId[fid];- if (!assetId) continue;-- const asset = world.assets.get(assetId);- if (!asset) continue;-- try {- // Show shimmer- addComponent(world, nid, world.components.Generating);-- // Upload asset and call API- const fileRef = await uploadAsset(world, assetId);- if (!fileRef) throw new Error("Failed to upload asset");-- const { url, generationId } = await trpc.removeBackground.mutate({ image: fileRef });-- // Load result as new asset- const newAsset = await loadAsset(world, url, {- name: asset.name + " (Background removed)",- generationId,- });-- addComponent(world, fid, c.Hidden);-- const newFid = createEntity(world);- setComponent(world, newFid, c.Paint, PaintType.IMAGE);- setComponent(world, newFid, c.AssetId, newAsset.id);- appendChild(world, newFid, nid);- } catch (err) {- console.error("Background removal failed:", err);- toast.error("Failed to remove background");- } finally {- removeComponent(world, nid, c.Generating);- }- }- };-- return { generate };-}diff --git a/apps/web/src/components/genai/use-upscale-image.ts b/apps/web/src/components/genai/use-upscale-image.tsdeleted file mode 100644index 3db6e767..00000000--- a/apps/web/src/components/genai/use-upscale-image.ts+++ /dev/null@@ -1,78 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { Not, Or, query } from 'bitecs';-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset, uploadAsset } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { assert } from "@/utils";-import { toast } from "somoto";--import type { EngineWorld } from "@/components/engine";--function findImageFill(world: EngineWorld, eid: number): number | null {- const c = world.components;-- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {- if (c.Paint[fid] === PaintType.IMAGE) {- return fid;- }- }- return null;-}--export function useUpscaleImage() {- const { world } = useEngine();- const c = world.components;-- const generate = async () => {- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];-- for (const nid of nodes) {- const fid = findImageFill(world, nid);- if (!fid) continue;-- const assetId = world.components.AssetId[fid];- if (!assetId) continue;-- const asset = world.assets.get(assetId);- if (!asset) continue;-- try {- // Show shimmer- addComponent(world, nid, c.Generating);-- const fileRef = await uploadAsset(world, assetId);- if (!fileRef) throw new Error("Failed to upload asset");-- const { url, generationId } = await trpc.upscaleImage.mutate({ image: fileRef });-- const dot = asset.name.lastIndexOf(".");- const name = dot > 0- ? asset.name.slice(0, dot) + " (Upscaled)" + asset.name.slice(dot)- : asset.name + " (Upscaled)";- const newAsset = await loadAsset(world, url, { name, generationId });-- assert(newAsset.type === "IMAGE", "New asset is not an image");-- addComponent(world, fid, c.Hidden);- setComponent(world, nid, c.KeepAspectRatio, { width: newAsset.width, height: newAsset.height });- resizeEntity(world, nid, { width: newAsset.width, height: newAsset.height });-- const newFid = createEntity(world);- setComponent(world, newFid, c.Paint, PaintType.IMAGE);- setComponent(world, newFid, c.AssetId, newAsset.id);- appendChild(world, newFid, nid);- } catch (err) {- console.error("Upscale failed:", err);- toast.error("Failed to upscale image");- } finally {- removeComponent(world, nid, c.Generating);- }- }- };-- return { generate };-}diff --git a/apps/web/src/components/genai/use-upscale-video.ts b/apps/web/src/components/genai/use-upscale-video.tsdeleted file mode 100644index 27530b40..00000000--- a/apps/web/src/components/genai/use-upscale-video.ts+++ /dev/null@@ -1,77 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { Not, Or, query } from 'bitecs';-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset, uploadAsset } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { assert } from "@/utils";-import { toast } from "somoto";--import type { EngineWorld } from "@/components/engine";--function findVideoFill(world: EngineWorld, eid: number): number | null {- const c = world.components;-- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {- if (c.Paint[fid] === PaintType.VIDEO) {- return fid;- }- }- return null;-}--export function useUpscaleVideo() {- const { world } = useEngine();- const c = world.components;-- const generate = async () => {- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];-- for (const nid of nodes) {- const fid = findVideoFill(world, nid);- if (!fid) continue;-- const assetId = world.components.AssetId[fid];- if (!assetId) continue;-- const asset = world.assets.get(assetId);- if (!asset) continue;-- try {- addComponent(world, nid, c.Generating);-- const fileRef = await uploadAsset(world, assetId);- if (!fileRef) throw new Error("Failed to upload asset");-- const { url, generationId } = await trpc.upscaleVideo.mutate({ video: fileRef });-- const dot = asset.name.lastIndexOf(".");- const name = dot > 0- ? asset.name.slice(0, dot) + " (Upscaled)" + asset.name.slice(dot)- : asset.name + " (Upscaled)";- const newAsset = await loadAsset(world, url, { name, generationId });-- assert(newAsset.type === "VIDEO", "New asset is not a video");-- addComponent(world, fid, c.Hidden);- setComponent(world, nid, c.KeepAspectRatio, { width: newAsset.width, height: newAsset.height });- resizeEntity(world, nid, { width: newAsset.width, height: newAsset.height });-- const newFid = createEntity(world);- setComponent(world, newFid, c.Paint, PaintType.VIDEO);- setComponent(world, newFid, c.AssetId, newAsset.id);- appendChild(world, newFid, nid);- } catch (err) {- console.error("Upscale failed:", err);- toast.error("Failed to upscale video");- } finally {- removeComponent(world, nid, c.Generating);- }- }- };-- return { generate };-}diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsxindex ebe42be1..3ebeca91 100644--- a/apps/web/src/engine/insert-asset.tsx+++ b/apps/web/src/engine/insert-asset.tsx@@ -25,8 +25,8 @@ export interface InsertAssetOptions {start?: number;}-const AUDIO_WIDTH = 500;-const AUDIO_HEIGHT = 150;+/** The box an audio clip gets on the canvas: it has no size of its own. */+export const AUDIO_SIZE = { width: 500, height: 150 } as const;/*** Inserts `asset` into the project as the element of its type and returns@@ -73,7 +73,7 @@ function sizeOf(asset: Asset): { width: number; height: number } | undefined {case 'SEQUENCE':return { width: Math.round(asset.width), height: Math.round(asset.height) };case 'AUDIO':- return { width: AUDIO_WIDTH, height: AUDIO_HEIGHT };+ return { ...AUDIO_SIZE };default:return undefined;}diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.tsindex e6ab122c..58139f16 100644--- a/apps/web/src/engine/library.ts+++ b/apps/web/src/engine/library.ts@@ -40,11 +40,16 @@ export function isLibraryFile(path: string): boolean {return path === MANIFEST_FILE || path === ASSETS_DIR || path.startsWith(`${ASSETS_DIR}/`);}-/** Rewrites `src` on every element that named an asset by its old path. */+/**+ * Rewrites `src` on every element that named an asset by its old path. Matched+ * on the path alone, not on what the element is bound to: an element showing a+ * modified source (`removeBackground`, `upscale`) is bound to what the+ * modifiers made of the asset rather than to the asset itself, and its `src`+ * still has to follow the rename.+ */function followRename(world: World, asset: Asset, from: string): void {const editor = getDocumentEditor(world);for (const entity of world.query(AssetId)) {- if (entity.get(AssetId)?.value !== asset.id) continue;const src = authoredElement(entity)?.props.src;if (src !== from) continue;editor.editProperty(entity, 'src', asset.path);diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 8e234d41..d855c1e5 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -7,12 +7,14 @@ import {Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,Paint, PaintType, Project, Source,} from "@diffusionstudio/runtime";+import type { SourceModifierValues } from "@diffusionstudio/runtime";import { createEncoder } from "@diffusionstudio/encoder";import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";import {PROMPT_INPUT_AUDIO_MODEL_OPTIONS,PROMPT_INPUT_IMAGE_MODEL_OPTIONS,PROMPT_INPUT_VIDEO_MODEL_OPTIONS,+ PROMPT_INPUT_VOICE_MODEL,PROMPT_INPUT_VOICE_OPTIONS,} from "@/components/genai/config";import { assert } from "@/utils";@@ -22,12 +24,10 @@ import { trpc } from "@/lib/trpc";import { toast } from "somoto";import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";-import type { Asset, AssetLibrary } from "@diffusionstudio/assets";+import type { Asset, AssetLibrary, AssetType } from "@diffusionstudio/assets";+import type { FileRef } from "@diffusionstudio/api-contract";import type { Entity, World } from "koota";-// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).-const VOICE_MODEL = "elevenlabs-v3";-/** What a failure is called where the user reads about it. */const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {image: "Image generation failed",@@ -37,6 +37,31 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {transcript: "Caption generation failed",};+/**+ * One step of a source modifier (see the runtime's `SourceModifiers`): a+ * model call that makes a new asset out of one the library already holds.+ * An element asks for these by prop and goes on naming what it was made+ * from, so each step is stored and cached on its own, and a set of modifiers+ * is these run in order (see `derive`).+ */+export type TransformKind = "remove-background" | "upscale" | "add-audio";++/** What each step is called where the user reads about it, and names its result. */+const TRANSFORMS: Record<TransformKind, { title: string; suffix: string }> = {+ "remove-background": { title: "Background removal failed", suffix: "Background removed" },+ "upscale": { title: "Upscale failed", suffix: "Upscaled" },+ "add-audio": { title: "Adding audio failed", suffix: "With audio" },+};++/** The steps a set of modifiers comes to, in the order they are applied. */+function steps(modifiers: SourceModifierValues): TransformKind[] {+ const kinds: TransformKind[] = [];+ if (modifiers.removeBackground) kinds.push("remove-background");+ if (modifiers.upscale > 1) kinds.push("upscale");+ if (modifiers.addAudio) kinds.push("add-audio");+ return kinds;+}+/*** A spec with defaults applied and every `AssetInput` reduced to an asset id.* Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.@@ -44,8 +69,8 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {type ResolvedSpec =| { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }| { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }- | { type: "voice"; model: string; prompt: string; voice: string }- | { type: "audio"; model: string; prompt: string; duration?: number };+ | { type: "voice"; model: string; prompt: string; voice: string; seed?: number }+ | { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */export function attachAi(world: World, library: AssetLibrary): EditorGenAi {@@ -222,9 +247,10 @@ export class EditorGenAi extends GenAi {case "voice": {return {type: "voice",- model: VOICE_MODEL,+ model: PROMPT_INPUT_VOICE_MODEL,prompt: spec.prompt,voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,+ seed: spec.seed,};}case "audio": {@@ -233,6 +259,7 @@ export class EditorGenAi extends GenAi {model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,prompt: spec.prompt,duration: spec.duration,+ seed: spec.seed,};}}@@ -254,14 +281,7 @@ export class EditorGenAi extends GenAi {const { name, results, generationId } = await this.requestGeneration(spec);assert(results.length > 0, "No results returned from the model");- const url = results[0].url;- const response = await fetch(url);- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);- const blob = await response.blob();- const extension = url.split(/[?#]/)[0].split(".").pop();- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;-- const asset = await this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });+ const asset = await this.store(results[0].url, name, { key: generationKey, id: generationId });track("generation_completed", {mode: spec.type,model: spec.model,@@ -279,6 +299,108 @@ export class EditorGenAi extends GenAi {}}+ /**+ * `asset` through every modifier the element asked for, one step at a time+ * and in a fixed order. Each step is cached in its own right, so turning+ * one on leaves what the others already made alone — adding `upscale` to a+ * cut-out picture pays for the enlarging, not for the matte again.+ */+ public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {+ let derived = asset;+ for (const kind of steps(modifiers)) {+ derived = await this.transform(kind, derived);+ }+ return derived;+ }++ /**+ * Runs one step over `asset` and returns what it produced, stored under+ * `generated/` like any other model output. Keyed by step and input, so the+ * same call on the same asset is the same result in this session and the+ * next: upscaling a picture twice costs what upscaling it once did.+ */+ public async transform(kind: TransformKind, asset: Asset): Promise<Asset> {+ // No upscale factor in the key: the endpoint takes none, so every factor+ // is the same call and would otherwise be billed once per number asked+ // for. It belongs here the moment the API can be told one.+ const key = `transform:v1:${kind}:${asset.id}`;++ const cached = this.library.list().find((entry) => entry.generation?.key === key);+ if (cached) return cached;++ const running = this.inflight.get(key);+ if (running) return await running;++ const promise = this.runTransform(kind, asset, key);+ this.inflight.set(key, promise);+ try {+ return await promise;+ } catch (error) {+ throw reportFailure(error, TRANSFORMS[kind].title);+ } finally {+ this.inflight.delete(key);+ }+ }++ /** Uploads the input, runs the call, and stores the result beside the generations. */+ private async runTransform(kind: TransformKind, asset: Asset, key: string): Promise<Asset> {+ const startedAt = performance.now();+ track("generation_started", { mode: kind });++ try {+ console.log(`[gen-ai] running ${kind} on ${asset.path}`);+ const input = await this.uploadInput(asset.id);+ const { url, generationId } = await this.requestTransform(kind, asset, input);++ const base = assetName(asset).replace(/\.[^.]+$/, "");+ const stored = await this.store(url, `${base} (${TRANSFORMS[kind].suffix})`, { key, id: generationId });++ track("generation_completed", {+ mode: kind,+ duration_ms: Math.round(performance.now() - startedAt),+ });+ return stored;+ } catch (err) {+ track("generation_failed", {+ mode: kind,+ duration_ms: Math.round(performance.now() - startedAt),+ error: err instanceof Error ? err.message.slice(0, 200) : "unknown",+ });+ throw err;+ }+ }++ private requestTransform(kind: TransformKind, asset: Asset, input: FileRef) {+ switch (kind) {+ case "remove-background":+ assert(asset.type === "IMAGE", "Only a picture has a background to remove");+ return trpc.removeBackground.mutate({ image: input });+ case "upscale":+ return isMoving(asset.type)+ ? trpc.upscaleVideo.mutate({ video: input })+ : trpc.upscaleImage.mutate({ image: input });+ case "add-audio":+ assert(isMoving(asset.type), "Only footage can be scored");+ return trpc.addAudioToVideo.mutate({ video: input });+ }+ }++ /**+ * Downloads what a model returned and files it in the library. The name is+ * the model's, the extension the URL's — what came back decides what the+ * file is called, not what was asked for.+ */+ private async store(url: string, name: string, generation: { key: string; id?: string | null }): Promise<Asset> {+ const response = await fetch(url);+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);+ const blob = await response.blob();++ const extension = url.split(/[?#]/)[0].split(".").pop();+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;++ return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation });+ }+private requestGeneration(spec: ResolvedSpec) {switch (spec.type) {case "image": {@@ -321,6 +443,7 @@ export class EditorGenAi extends GenAi {model: spec.model,prompt: spec.prompt,voice: spec.voice,+ seed: spec.seed,});}case "audio": {@@ -328,6 +451,7 @@ export class EditorGenAi extends GenAi {model: spec.model,prompt: spec.prompt,duration: spec.duration,+ seed: spec.seed,});}}@@ -343,6 +467,9 @@ export class EditorGenAi extends GenAi {}}+/** Whether an asset type is footage, for the calls that only take footage. */+const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";+/*** Says what a generation failed with, and hands the error on. The message is* what the element ends up carrying too (the runtime's `SourceError`, whichdiff --git a/apps/web/src/utils/genai.ts b/apps/web/src/utils/genai.tsdeleted file mode 100644index b9009ee8..00000000--- a/apps/web/src/utils/genai.ts+++ /dev/null@@ -1,177 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { query, Not, Or } from "bitecs";-import { ChildOf, PaintType, aabbFromTransformedRect, aabbsIntersect, invert2D, transformPoint, getEntityTree, createEntity, setComponent, appendChild, addComponent, removeComponent, createFolder, ensureFolder, uniqueFolderName, type AABB } from "@/components/engine";-import { MAIN_CANVAS_ID } from "@/components/canvas";--import type { EngineWorld } from "@/components/engine";--const GEN_AI_FOLDER_NAME = 'Gen AI';--/**- * Map aspect ratio to pixel dimensions assuming 1080p as baseline.- */-export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {- "16:9": { width: 1920, height: 1080 },- "9:16": { width: 1080, height: 1920 },- "1:1": { width: 1080, height: 1080 },- "4:3": { width: 1440, height: 1080 },- "3:4": { width: 1080, height: 1440 },-};--/**- * Get the world-space center of the current viewport.- */-export function getViewportCenter(world: EngineWorld) {- const canvasEl = document.getElementById(MAIN_CANVAS_ID) as HTMLCanvasElement | null;- if (!canvasEl) return { x: 0, y: 0 };- const rect = canvasEl.getBoundingClientRect();- const invMat = invert2D(world.camera);- const pt = transformPoint(invMat, rect.width / 2, rect.height / 2);- return pt ?? { x: 0, y: 0 };-}--/**- * Get the world-space AABB of the current viewport.- */-function getViewportBounds(world: EngineWorld): AABB | null {- const canvasEl = document.getElementById(MAIN_CANVAS_ID) as HTMLCanvasElement | null;- if (!canvasEl) return null;- const rect = canvasEl.getBoundingClientRect();- const invMat = invert2D(world.camera);- const tl = transformPoint(invMat, 0, 0);- const tr = transformPoint(invMat, rect.width, 0);- const br = transformPoint(invMat, rect.width, rect.height);- const bl = transformPoint(invMat, 0, rect.height);- return {- minX: Math.min(tl.x, tr.x, br.x, bl.x),- minY: Math.min(tl.y, tr.y, br.y, bl.y),- maxX: Math.max(tl.x, tr.x, br.x, bl.x),- maxY: Math.max(tl.y, tr.y, br.y, bl.y),- };-}--/**- * Find a placement spot for a `width`x`height` block on the canvas.- */-export function findEmptyPlacement(- world: EngineWorld,- width: number,- height: number,- gap: number,- extra: AABB[] = [],-): { x: number; y: number } {- const center = getViewportCenter(world);- const viewport = getViewportBounds(world);-- const c = world.components;- const LocalTransform = c.LocalTransform;- const Computed = c.Computed;-- // `extra` boxes are content the world can't report yet — e.g. sibling roots- // placed earlier in the same mount, whose layout has not been computed. They- // are avoided and anchored beside like any existing box.- const boxes: AABB[] = [...extra];- for (const cid of query(world, [Or(c.Geometry, c.Group), Not(ChildOf('*')), Not(c.Deleted), Not(c.Hidden), Not(c.Generating)])) {- const w = Computed.width[cid];- const h = Computed.height[cid];- if (!w || !h) continue;- if (LocalTransform.a[cid] === undefined) continue;- const aabb = aabbFromTransformedRect(- {- a: LocalTransform.a[cid], b: LocalTransform.b[cid],- c: LocalTransform.c[cid], d: LocalTransform.d[cid],- e: LocalTransform.e[cid], f: LocalTransform.f[cid],- },- w,- h,- );- boxes.push(aabb);- }-- if (boxes.length === 0) {- return center;- }-- const candidates = viewport- ? boxes.filter((b) => aabbsIntersect(b, viewport))- : boxes;- const ordered = (candidates.length > 0 ? candidates : boxes).slice().sort((a, b) => {- const acx = (a.minX + a.maxX) / 2, acy = (a.minY + a.maxY) / 2;- const bcx = (b.minX + b.maxX) / 2, bcy = (b.minY + b.maxY) / 2;- return Math.hypot(acx - center.x, acy - center.y)- - Math.hypot(bcx - center.x, bcy - center.y);- });-- const overlapsAny = (rect: AABB): boolean => {- for (const b of boxes) {- if (aabbsIntersect(rect, b)) return true;- }- return false;- };-- for (const cand of ordered) {- const cy = (cand.minY + cand.maxY) / 2;- const minY = cy - height / 2;- const maxY = cy + height / 2;-- const rightMinX = cand.maxX + gap;- const right: AABB = { minX: rightMinX, minY, maxX: rightMinX + width, maxY };- if (!overlapsAny(right)) {- return { x: rightMinX + width / 2, y: cy };- }-- const leftMaxX = cand.minX - gap;- const left: AABB = { minX: leftMaxX - width, minY, maxX: leftMaxX, maxY };- if (!overlapsAny(left)) {- return { x: leftMaxX - width / 2, y: cy };- }- }-- return center;-}--/**- * Attach the waveform fill to a generated audio entity.- */-export function addWaveformFill(world: EngineWorld, eid: number, assetId: string) {- const c = world.components;- const fillEid = createEntity(world);- setComponent(world, fillEid, c.Paint, PaintType.WAVEFORM);- setComponent(world, fillEid, c.AssetId, assetId);- appendChild(world, fillEid, eid);-}--/**- * The library folder for a finished generation: every generated asset lands in- * a root-level "Gen AI" folder, and a multi-variant batch gets its own- * subfolder inside it, named after the generation. Both are created on demand.- * Call only after the generation succeeded, so failures leave no empty folders.- */-export async function ensureGenerationFolder(- world: EngineWorld,- name: string,- count: number,-): Promise<string> {- const genAi = await ensureFolder(world, GEN_AI_FOLDER_NAME, null);- if (count <= 1) return genAi.id;-- const batch = await createFolder(world, uniqueFolderName(world, name, genAi.id), genAi.id);- return batch.id;-}--/**- * Record a finished generation as a single undoable step.- */-export function recordGeneration(world: EngineWorld, label: string, rootEids: number[]) {- const c = world.components;- const eids = rootEids.flatMap((root) => getEntityTree(world, root));-- world.history.push({- label,- execute: () => eids.forEach((e) => removeComponent(world, e, c.Deleted)),- reverse: () => eids.forEach((e) => addComponent(world, e, c.Deleted)),- });-}diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.tsindex 34b6184f..08a5be99 100644--- a/packages/jsx/src/generate.ts+++ b/packages/jsx/src/generate.ts@@ -44,12 +44,14 @@ export type GenerateVoiceOptions = {prompt: string;/** Voice id; default the first voice from `dapi voices`. */voice?: string;+ seed?: number;};export type GenerateAudioOptions = {prompt: string;model?: string;duration?: number;+ seed?: number;};export type AssetSpecInput =diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 50257f1c..12a874ba 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -295,6 +295,22 @@ type MediaProps = {error?: string;};+/**+ * Model calls the source is put through before the element shows it. The+ * `src` goes on naming what it was made from, so taking a modifier off gives+ * the original back; what they made is cached by source and modifiers, so it+ * is made once however many elements ask for it, and adding a second+ * modifier does not re-run the first. Applied in the order below.+ */+type UpscaleProps = {+ /**+ * Resolution multiplier: 2 asks for twice the pixels. Enlarges the source,+ * not the box — the element keeps the width and height it was given, and+ * renders sharper. Default 1, the source as it is.+ */+ upscale?: number;+};+type FitProps = {/** How the source maps into the box. Default "cover" on `<Video>`, "contain" on `<Image>`. */objectFit?: Fit;@@ -572,12 +588,24 @@ export type ColorStopProps = ColorProps & OpacityProps & TrackChildren & {*/export type MediaPaintProps = PaintProps & MediaProps & FitProps & TrackChildren;-export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps & {+export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps & UpscaleProps & {+ /**+ * Scores the footage: a generated soundtrack for a clip that has none. See+ * `UpscaleProps` for what a modifier is; applied last, after `upscale`, so+ * a re-encode cannot drop the track. Independent of `volume` and `muted`,+ * which mix whatever track the clip ends up with.+ */+ addAudio?: boolean;/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */children?: SolidJSX.Element;};-export type ImageProps = CommonProps & MediaProps & FitProps & {+export type ImageProps = CommonProps & MediaProps & FitProps & UpscaleProps & {+ /**+ * Cuts the subject out, leaving the rest of the picture transparent. See+ * `UpscaleProps` for what a modifier is; applied before `upscale`.+ */+ removeBackground?: boolean;/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */children?: SolidJSX.Element;};@@ -638,7 +666,13 @@ export type SurfacePaintProps = PaintProps & {/** `<Surface>` — a rectangle carrying a `<SurfacePaint>` with the given ref. */export type SurfaceProps = CommonProps & Pick<SurfacePaintProps, "ref">;-export type AudioProps = IdentityProps & TimingProps & MediaProps & AudioTrackProps & {+/**+ * `<audio>` — a clip with a sound and no picture. It draws nothing inside a+ * scene, but on the canvas it is still something to point at: the editor+ * shows its waveform in a box, and `x`/`y`/`width`/`height` are where that+ * box is. Left off inside a scene, where they mean nothing.+ */+export type AudioProps = IdentityProps & PositionProps & SizeProps & TimingProps & MediaProps & AudioTrackProps & {/** `<KeyframeTrack>` (a `volume` track) and `<Animation>` children. */children?: SolidJSX.Element;};diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 7ba1779e..3ffcf46f 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';@@ -63,6 +63,15 @@ const Authored = trait(() => ({ tag: '', props: {} as Record<string, unknown> })/** Props that address or wire an element rather than describe it. */const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);+/** What an element with no `SourceModifiers` trait is asking for: nothing. */+const NO_MODIFIERS = { removeBackground: false, upscale: 1, addAudio: false };++/** `upscale` as a factor; anything that is not one above 1 is natural size. */+function upscaleFactor(value: unknown): number {+ const factor = typeof value === 'number' ? value : Number(value);+ return Number.isFinite(factor) && factor > 1 ? factor : 1;+}+export interface AuthoredElement {/** The camelCase tag the project used. */tag: string;@@ -1168,6 +1177,47 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {});return;}+ case 'removeBackground':+ case 'addAudio':+ case 'upscale': {+ const current = entity.get(SourceModifiers) ?? NO_MODIFIERS;+ const next = {+ ...current,+ [name]: name === 'upscale' ? upscaleFactor(value) : value === true,+ };++ if (+ next.removeBackground === current.removeBackground+ && next.upscale === current.upscale+ && next.addAudio === current.addAudio+ ) return;++ if (hasModifier(next)) {+ entity.add(SourceModifiers);+ entity.set(SourceModifiers, next);+ } else {+ entity.remove(SourceModifiers);+ }++ if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;++ const src = entity.get(Authored)?.props.src;+ if (src === undefined || src === null || src === '') return;++ // A resolution running for the old modifiers must not bind late.+ entity.remove(PendingSource, Generating);++ if (typeof src === 'string') {+ entity.add(LoadRequest);+ entity.set(LoadRequest, { value: src });+ return;+ }++ entity.add(GenerationRequest);+ entity.set(GenerationRequest, { ref: src as AssetRef });++ return;+ }case 'seed': {if (!entity.has(Caption)) return;// An authored src mounts a transcript directly; there is nodiff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex 58f16e7b..785b4fa9 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -6,7 +6,7 @@// goes through the AssetLibrary (@diffusionstudio/assets); these are the// lookups the decoders and hosts share.-import { Ai, AssetId, Library, Paint } from '../traits';+import { Ai, AssetId, Library, Paint, SourceModifiers } from '../traits';import { PaintType } from '../constants';import type { Entity, World } from 'koota';@@ -27,6 +27,28 @@ export function getAi(world: World): GenAi {return ai;}+/** What a source is put through after it resolves (see `SourceModifiers`). */+export interface SourceModifierValues {+ removeBackground: boolean;+ /** Factor, 1 = natural size. */+ upscale: number;+ addAudio: boolean;+}++/** Whether a set of modifiers asks for anything at all. */+export const hasModifier = (modifiers: SourceModifierValues): boolean =>+ modifiers.removeBackground || modifiers.upscale > 1 || modifiers.addAudio;++/**+ * The modifiers `entity` asks its source to be put through, or undefined+ * when it asks for none — which is what the trait's absence means, so this+ * is also the question "is this element showing a derived source".+ */+export function getModifiers(entity: Entity): SourceModifierValues | undefined {+ const modifiers = entity.get(SourceModifiers);+ return modifiers && hasModifier(modifiers) ? modifiers : undefined;+}+/*** Binds an entity to an asset: stamps its AssetId, and follows the asset* with the paint — a frames directory on a `<video>` or `<image>` plays asdiff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsindex 18bc7910..25e6516c 100644--- a/packages/runtime/src/ai.ts+++ b/packages/runtime/src/ai.ts@@ -6,6 +6,7 @@ import { generate } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';+import type { SourceModifierValues } from './actions/assets';import type {AssetRef,GenerateAudioOptions,@@ -31,6 +32,14 @@ export abstract class GenAi {*/public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;+ /**+ * `asset` put through the modifiers an element asked of its source (see+ * `SourceModifiers`): the same asset and the same modifiers are the same+ * result, in this session and the next, and every step is cached on its+ * own, so adding one to an element does not re-run the others.+ */+ public abstract derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset>;+/*** The imperative surface: `ai.generate.image({...})` declares and resolves* in one call, returning the finished asset.diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex ea68dd7b..675a7918 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';-import { bindAsset } from '../actions/assets';+import { bindAsset, getModifiers } from '../actions/assets';import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';import type { Entity, World } from 'koota';@@ -15,7 +15,7 @@ export function assetSystem(world: World): void {for (const entity of world.query(LoadRequest)) {const source = entity.get(LoadRequest)!.value;entity.remove(LoadRequest);- resolve(world, entity, source, library.resolve(source));+ start(world, entity, library.resolve(source));}}@@ -25,7 +25,7 @@ export function assetSystem(world: World): void {const ref = entity.get(GenerationRequest)!.ref;entity.remove(GenerationRequest);if (ref === null || entity.has(SourceError)) continue;- resolve(world, entity, ref, ai.resolve(ref), true);+ start(world, entity, ai.resolve(ref), true);}for (const entity of world.query(TranscriptionRequest)) {@@ -35,11 +35,29 @@ export function assetSystem(world: World): void {const seed = entity.get(TranscriptionRequest)!.seed;entity.remove(TranscriptionRequest);if (entity.has(SourceError)) continue;- resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed), true);+ resolve(world, entity, ai.transcribe(world, scene, seed), true);}}}+/**+ * Starts a resolution for whatever the element's src named, putting it+ * through the modifiers the element asks of it (see `SourceModifiers`). The+ * two are one wait: the element binds the asset it is going to show, not+ * first the one it was made from.+ */+function start(world: World, entity: Entity, base: Promise<Asset>, generating = false): void {+ const modifiers = getModifiers(entity);+ const ai = world.get(Ai);++ if (!modifiers || !ai) {+ resolve(world, entity, base, generating);+ return;+ }++ resolve(world, entity, base.then((asset) => ai.derive(asset, modifiers)), true);+}+/*** Whether anything in `scene`'s subtree (besides `except`, the requesting* element itself) is still waiting on a source: a request this system has@@ -56,24 +74,28 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean}/**- * Tracks one started resolution: the entity remembers what it is waiting on- * (PendingSource), so of overlapping resolutions only the latest binds, and- * one that outlives its element (or its src) is dropped.+ * Tracks one started resolution: the entity remembers which one it is waiting+ * on (PendingSource), so of overlapping resolutions only the latest binds, and+ * one that outlives its element (or its src) is dropped. The wait itself is+ * the identity rather than what was asked for — the same src put through+ * different modifiers is a different answer, and the later question is always+ * the one being asked.*/-function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>, generating = false): void {+function resolve(world: World, entity: Entity, promise: Promise<Asset>, generating = false): void {+ const token = {};entity.remove(SourceError);entity.add(PendingSource);- entity.set(PendingSource, { value });+ entity.set(PendingSource, { value: token });if (generating) entity.add(Generating);const done = promise.then((asset) => {- if (!current(entity, value)) return;+ if (!current(entity, token)) return;entity.remove(PendingSource, Generating);bindAsset(entity, asset);},(error: unknown) => {- if (!current(entity, value)) return;+ if (!current(entity, token)) return;entity.remove(PendingSource, Generating);entity.add(SourceError);entity.set(SourceError, { value: errorMessage(error), generated: generating });@@ -90,6 +112,6 @@ function errorMessage(error: unknown): string {return message.trim() || 'Something went wrong';}-function current(entity: Entity, value: unknown): boolean {- return entity.isAlive() && entity.get(PendingSource)?.value === value;+function current(entity: Entity, token: object): boolean {+ return entity.isAlive() && entity.get(PendingSource)?.value === token;}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex 408263ca..ad77b4dd 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -85,6 +85,16 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))// through the library.export const LoadRequest = trait({ value: '' });+// Model calls the src is put through once it resolves: the element shows+// what they made of what it named, and the src goes on naming the original,+// so taking a modifier off gives it back. Every field's default is off —+// `upscale` 1 is natural size — and the trait is only present while one is+// not. Applied in the order declared here: the matte before the enlarging+// (cheaper, and the model wants a normally-sized picture), the audio last so+// a re-encode cannot drop it. Each step is cached on its own, so turning one+// on does not re-pay for the ones already applied.+export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addAudio: false });+// On a `<captions>` element without a src: transcribe the scene it sits// under, through the world's Ai. The seed keys the take — the transcript is// cached by scene id + seed, so re-running with a seed replays that take anddiff --git a/reference/jsx/audio.md b/reference/jsx/audio.mdindex 01fb20c1..22dac42d 100644--- a/reference/jsx/audio.md+++ b/reference/jsx/audio.md@@ -1,6 +1,6 @@# `<audio>`-An audio clip: no visual output, no spatial props; carries volume. When timing is omitted, the node fits its natural duration (see [timing.md](./timing.md)).+An audio clip: no picture, carries volume. When timing is omitted, the node fits its natural duration (see [timing.md](./timing.md)). It draws nothing inside a scene; placed directly on the canvas (see [roots.md](./roots.md)) the editor shows its waveform in a box, which `x`/`y`/`width`/`height` place.```tsx<audio src="https://my.videoarchive.com/audio/clip.wav" start={2.2} sourceOut={16} volume={-6} />@@ -12,6 +12,8 @@ An audio clip: no visual output, no spatial props; carries volume. When timing i| ---- | ---- | ------- | ------- || `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). || `name` | `string` | none | Human-readable node name. |+| `x`, `y` | `number` | `0` | Where the waveform box sits on the canvas. No meaning inside a scene. |+| `width`, `height` | `number` | `500`, `150` | Size of that box. || `start`, `end`, `sourceIn`, `sourceOut` | `Time` | see [timing.md](./timing.md) | Temporal placement. || `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. || `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |diff --git a/reference/jsx/generate.md b/reference/jsx/generate.mdindex 834e119b..19e2df8c 100644--- a/reference/jsx/generate.md+++ b/reference/jsx/generate.md@@ -61,12 +61,14 @@ generate.video(opts: {generate.voice(opts: {prompt: string; // required: the text to speakvoice?: string; // default: first voice from `dapi voices`+ seed?: number;}): AssetRef;generate.audio(opts: {prompt: string; // requiredmodel?: string; // default: first model from `dapi models audio`duration?: number; // seconds; default 30 for music, the model's own default for sfx+ seed?: number;}): AssetRef;```diff --git a/reference/jsx/image.md b/reference/jsx/image.mdindex 245e0f6e..8e37eb8d 100644--- a/reference/jsx/image.md+++ b/reference/jsx/image.md@@ -14,5 +14,7 @@ All [common props](./elements.md#common-props), plus:| ---- | ---- | ------- | ------- || `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). || `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. |+| `removeBackground` | `boolean` | `false` | Cuts the subject out, leaving the rest transparent (see [media.md](./media.md#source-modifiers)). |+| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the image renders through the shader.diff --git a/reference/jsx/media.md b/reference/jsx/media.mdindex ff64c529..2db5a1e5 100644--- a/reference/jsx/media.md+++ b/reference/jsx/media.md@@ -8,6 +8,24 @@- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is inserted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.+## Source modifiers++`<image>` and `<video>` take props that put the source through a model before the element shows it:++| Prop | On | Meaning |+| ---- | -- | ------- |+| `removeBackground` | `<image>` | Cuts the subject out, leaving the rest transparent. |+| `upscale` | both | Resolution multiplier — `upscale={2}` asks for twice the pixels. Enlarges the source, not the box. |+| `addAudio` | `<video>` | Scores footage that has no sound. |++```tsx+<image src="footage/fox.png" removeBackground upscale={2} width={800} height={450} />+```++The `src` goes on naming what the picture was made from, so taking a modifier off gives the original back — nothing is overwritten and nothing is lost. Results are cached by source and modifiers, like [generated assets](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and adding a second modifier does not re-run the first. They are applied in the order of the table above, whatever order they are written in.++Modifiers compose with declarations: `<image src={generate.image({ prompt: "a red fox" })} removeBackground />` generates, then cuts out.+## The libraryA project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/` for files the app produced), and what it was found to be. Folders are virtual: they are the prefixes of the paths. Renaming or moving an asset in the panel rewrites the `src` props that named it. Files placed under `assets/` by hand are taken into the library on the next load.diff --git a/reference/jsx/video.md b/reference/jsx/video.mdindex 4c2484f8..54a4b834 100644--- a/reference/jsx/video.md+++ b/reference/jsx/video.md@@ -17,5 +17,7 @@ All [common props](./elements.md#common-props), plus:| `volume` | `Animatable<number>` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. || `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. || `syncTo` | `string` | none | Key of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |+| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |+| `addAudio` | `boolean` | `false` | Scores footage that has no sound (see [media.md](./media.md#source-modifiers)). |A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the frame renders through the shader.