Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor media selection and generation handling in the ActionBar and PromptInput components

konstantin-paulusa4ab6440b0bd
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsx
index 862d41c8..59fc5a29 100644
--- a/apps/web/src/components/genai/action-bar.tsx
+++ b/apps/web/src/components/genai/action-bar.tsx
@@ -14,54 +14,27 @@ import {
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { createMemo, Show } from "solid-js";
-import { PaintType, isAudio } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { useRemoveBackground } from "./use-remove-background";
-import { useUpscaleImage } from "./use-upscale-image";
-import { useUpscaleVideo } from "./use-upscale-video";
-import { useAddAudioToVideo } from "./use-add-audio-to-video";
import { useGenerationRecords } from "./use-generation-records";
import { useGenerateImage } from "./use-generate-image";
import { useGenerateVideo } from "./use-generate-video";
import { useGenerateVoice } from "./use-generate-voice";
import { useGenerateAudio } from "./use-generate-audio";
import { useAutoCaptions } from "./use-auto-captions";
+import { useMediaSelection } from "./selection";
+import { useModifiers } from "./use-modifiers";
import { createDefaultConfig } from "./prompt-input";
import { toast } from "somoto";
-import { useECS } from "@/context/ecs";
-
-import type { EngineWorld } from "@/components/engine";
import type { GenerationConfig } from "@/components/engine/db";
-function getNodeMediaType(world: EngineWorld, eid: number) {
- const c = world.components;
- if (isAudio(world, eid)) {
- return "AUDIO";
- }
- for (const fid of c.Cache.fills[eid]) {
- if (c.Paint[fid] === PaintType.VIDEO) {
- return "VIDEO";
- }
- if (c.Paint[fid] === PaintType.IMAGE) {
- return "IMAGE";
- }
- }
- return null;
-}
-
interface ActionBarProps {
openPromptInput?(config: GenerationConfig): void;
}
export function ActionBar(props: ActionBarProps) {
- const { world } = useEngine();
- const { selectedNodes } = useECS();
+ const { imageNodes, videoNodes } = useMediaSelection();
+ const { isOn, toggle } = useModifiers();
- const { generate: upscaleImage } = useUpscaleImage();
- const { generate: upscaleVideo } = useUpscaleVideo();
- const { generate: addAudioToVideo } = useAddAudioToVideo();
- const { generate: removeBackground } = useRemoveBackground();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
const { generate: generateVoice } = useGenerateVoice();
@@ -69,19 +42,8 @@ export function ActionBar(props: ActionBarProps) {
const { generate: autoCaptions, hasScene } = useAutoCaptions();
const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
- const isImage = createMemo(() => {
- return Array.from(selectedNodes()).some(eid => {
- const mediaType = getNodeMediaType(world, eid);
- return mediaType === "IMAGE";
- });
- });
-
- const isVideo = createMemo(() => {
- return Array.from(selectedNodes()).some(eid => {
- const mediaType = getNodeMediaType(world, eid);
- return mediaType === "VIDEO";
- });
- });
+ const isImage = createMemo(() => imageNodes().length > 0);
+ const isVideo = createMemo(() => videoNodes().length > 0);
const visible = createMemo(() => {
return isImage() || isVideo() || hasScene();
@@ -143,11 +105,21 @@ export function ActionBar(props: ActionBarProps) {
</Show>
<Show when={isImage()}>
<div class="flex gap-1 items-center">
- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={removeBackground}>
+ <Button
+ variant="ghost"
+ class="gap-0 pl-0.5 text-muted-foreground"
+ classList={{ "text-foreground": isOn("removeBackground") }}
+ onClick={() => toggle("removeBackground")}
+ >
<Icon name="ai-generate" />
Remove background
</Button>
- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={upscaleImage}>
+ <Button
+ variant="ghost"
+ class="gap-0 pl-0.5 text-muted-foreground"
+ classList={{ "text-foreground": isOn("upscale") }}
+ onClick={() => toggle("upscale")}
+ >
<Icon name="arrow-scale" />
Upscale
</Button>
@@ -197,11 +169,21 @@ export function ActionBar(props: ActionBarProps) {
</Show>
<Show when={isVideo()}>
<div class="flex gap-1 items-center">
- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={addAudioToVideo}>
+ <Button
+ variant="ghost"
+ class="gap-0 pl-0.5 text-muted-foreground"
+ classList={{ "text-foreground": isOn("addAudio") }}
+ onClick={() => toggle("addAudio")}
+ >
<Icon name="generate-audio" />
Add audio
</Button>
- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={upscaleVideo}>
+ <Button
+ variant="ghost"
+ class="gap-0 pl-0.5 text-muted-foreground"
+ classList={{ "text-foreground": isOn("upscale") }}
+ onClick={() => toggle("upscale")}
+ >
<Icon name="arrow-scale" />
Upscale
</Button>
diff --git a/apps/web/src/components/genai/config.ts b/apps/web/src/components/genai/config.ts
index c1deb1c6..c9092c43 100644
--- a/apps/web/src/components/genai/config.ts
+++ b/apps/web/src/components/genai/config.ts
@@ -2,6 +2,18 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+/** The one model the voice mode speaks with; the UI picks the voice, not the model. */
+export const PROMPT_INPUT_VOICE_MODEL = "elevenlabs-v3";
+
+/** What an aspect ratio is worth in pixels, at 1080p. */
+export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {
+ "16:9": { width: 1920, height: 1080 },
+ "9:16": { width: 1080, height: 1920 },
+ "1:1": { width: 1080, height: 1080 },
+ "4:3": { width: 1440, height: 1080 },
+ "3:4": { width: 1080, height: 1440 },
+};
+
export const PROMPT_INPUT_MODE_OPTIONS = [
{ value: "IMAGE", label: "Image", icon: "image" },
{ value: "VIDEO", label: "Video", icon: "film-video-export" },
diff --git a/apps/web/src/components/genai/insert.tsx b/apps/web/src/components/genai/insert.tsx
new file mode 100644
index 00000000..c602baf5
--- /dev/null
+++ b/apps/web/src/components/genai/insert.tsx
@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Root } from "@diffusionstudio/runtime";
+import { findEmptyPlacement } from "@/engine/placement";
+import { toast } from "somoto";
+
+import type { DocumentEditor } from "@/engine/editor";
+import type { Entity, World } from "koota";
+
+/** Between the variants of one batch, and between the batch and its neighbour. */
+const GAP = 40;
+
+/** A declaration is keyed by its spec, so a fresh seed makes a new take. */
+export const randomSeed = () => Math.floor(Math.random() * 1_000_000);
+
+/** The box an inserted element is given, in canvas space. */
+export interface GenerationBox {
+ x: number;
+ y: number;
+ width: number;
+ height: number;
+}
+
+/**
+ * Inserts `count` elements of `size` at the root of the canvas, side by side
+ * as one block, and selects them. `element` is called once per variant with
+ * the box it goes in; each call declares its own source, so the variants of a
+ * batch are separate takes rather than one asset shown four times.
+ *
+ * Returns the entities, or nothing (having said so) when there is no project
+ * to insert into.
+ */
+export function insertGenerated(
+ world: World,
+ editor: DocumentEditor,
+ size: { width: number; height: number },
+ element: (box: GenerationBox, index: number) => unknown,
+ count = 1,
+): Entity[] {
+ const root = world.get(Root);
+ const inserted: Entity[] = [];
+
+ if (root) {
+ const totalWidth = count * size.width + (count - 1) * GAP;
+ const center = findEmptyPlacement(world, totalWidth, size.height, GAP);
+ const left = center.x - totalWidth / 2;
+ const y = Math.round(center.y - size.height / 2);
+
+ for (let index = 0; index < count; index++) {
+ const x = Math.round(left + index * (size.width + GAP));
+ const [entity] = editor.insertElement(root, () => element({ x, y, ...size }, index));
+ if (entity) inserted.push(entity);
+ }
+ }
+
+ if (!inserted.length) {
+ toast("Could not add the generation", {
+ description: "Open a project to generate into.",
+ });
+ return inserted;
+ }
+
+ editor.select(inserted);
+ return inserted;
+}
diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsx
index e0ce85a3..239fc8af 100644
--- a/apps/web/src/components/genai/prompt-input-actions.tsx
+++ b/apps/web/src/components/genai/prompt-input-actions.tsx
@@ -17,54 +17,28 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
import { createMemo, Show } from "solid-js";
import { toast } from "somoto";
-import { PaintType } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { useUpscaleImage } from "./use-upscale-image";
-import { useUpscaleVideo } from "./use-upscale-video";
-import { useAddAudioToVideo } from "./use-add-audio-to-video";
-import { useRemoveBackground } from "./use-remove-background";
import { useGenerationRecords } from "./use-generation-records";
import { useGenerateImage } from "./use-generate-image";
import { useGenerateVideo } from "./use-generate-video";
import { useGenerateVoice } from "./use-generate-voice";
import { useGenerateAudio } from "./use-generate-audio";
-import { useECS } from "@/context/ecs";
+import { useMediaSelection } from "./selection";
+import { useModifiers } from "./use-modifiers";
+
+import type { ModifierName } from "./use-modifiers";
export function PromptInputActions() {
- const { world } = useEngine();
- const { selectedNodes } = useECS();
const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
+ const { imageNodes, videoNodes } = useMediaSelection();
+ const { isOn, toggle } = useModifiers();
- const { generate: upscaleImage } = useUpscaleImage();
- const { generate: upscaleVideo } = useUpscaleVideo();
- const { generate: addAudioToVideo } = useAddAudioToVideo();
- const { generate: removeBackground } = useRemoveBackground();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
const { generate: generateVoice } = useGenerateVoice();
const { generate: generateAudio } = useGenerateAudio();
- const hasImageSelection = createMemo(() => {
- const Fills = world.components.Cache.fills;
- const Paint = world.components.Paint;
- for (const eid of selectedNodes()) {
- for (const fid of Fills[eid] ?? []) {
- if (Paint[fid] === PaintType.IMAGE) return true;
- }
- }
- return false;
- });
-
- const hasVideoSelection = createMemo(() => {
- const Fills = world.components.Cache.fills;
- const Paint = world.components.Paint;
- for (const eid of selectedNodes()) {
- for (const fid of Fills[eid] ?? []) {
- if (Paint[fid] === PaintType.VIDEO) return true;
- }
- }
- return false;
- });
+ const hasImageSelection = createMemo(() => imageNodes().length > 0);
+ const hasVideoSelection = createMemo(() => videoNodes().length > 0);
const handleRerun = () => {
const config = firstConfig();
@@ -146,25 +120,14 @@ export function PromptInputActions() {
<Separator class="my-1" />
</Show>
<DropdownMenuGroup>
+ <Show when={hasImageSelection() || hasVideoSelection()}>
+ <ModifierItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />
+ </Show>
<Show when={hasImageSelection()}>
- <DropdownMenuItem onSelect={upscaleImage}>
- <Icon name="arrow-scale" class="mr-2 text-foreground" />
- Upscale
- </DropdownMenuItem>
- <DropdownMenuItem onSelect={removeBackground}>
- <Icon name="ai-generate" class="mr-2 text-foreground" />
- Remove background
- </DropdownMenuItem>
+ <ModifierItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />
</Show>
<Show when={hasVideoSelection()}>
- <DropdownMenuItem onSelect={upscaleVideo}>
- <Icon name="arrow-scale" class="mr-2 text-foreground" />
- Upscale
- </DropdownMenuItem>
- <DropdownMenuItem onSelect={addAudioToVideo}>
- <Icon name="audio-on" class="mr-2 text-foreground" />
- Add audio
- </DropdownMenuItem>
+ <ModifierItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />
</Show>
</DropdownMenuGroup>
</DropdownMenuContent>
@@ -174,3 +137,24 @@ export function PromptInputActions() {
</Show>
);
}
+
+type ModifierItemProps = {
+ name: ModifierName;
+ icon: string;
+ label: string;
+ isOn(name: ModifierName): boolean;
+ toggle(name: ModifierName): void;
+};
+
+/** A modifier as a menu row: a check when the selection is asking for it. */
+function ModifierItem(props: ModifierItemProps) {
+ return (
+ <DropdownMenuItem onSelect={() => props.toggle(props.name)}>
+ <Icon name={props.icon} class="mr-2 text-foreground" />
+ <span class="flex-1">{props.label}</span>
+ <Show when={props.isOn(props.name)}>
+ <Icon name="confirm-check" class="ml-2 text-foreground" />
+ </Show>
+ </DropdownMenuItem>
+ );
+}
diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsx
index 058da27a..1c456df4 100644
--- a/apps/web/src/components/genai/prompt-input.tsx
+++ b/apps/web/src/components/genai/prompt-input.tsx
@@ -31,6 +31,7 @@ import {
import {
PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
+ PROMPT_INPUT_VOICE_MODEL,
PROMPT_INPUT_VOICE_OPTIONS,
PROMPT_INPUT_MODE_OPTIONS,
PROMPT_INPUT_IMAGE_ASPECT_RATIO_OPTIONS,
@@ -48,8 +49,6 @@ import { useGenerateVideo } from "./use-generate-video";
import { useGenerateVoice } from "./use-generate-voice";
import { useGenerateAudio } from "./use-generate-audio";
import { PromptInputActions } from "./prompt-input-actions";
-import { useEngine } from "@/context/engine";
-import { showFileDialog } from "@/utils";
import type {
AspectRatio,
GenerationConfig,
@@ -59,18 +58,13 @@ import type {
AudioGenerationConfig,
} from "@/components/engine/db";
import { AssetThumbnail } from "@/components/ui/asset-thumbnail";
-import { ChildOf, PaintType, removeComponent } from "@/components/engine";
-import { useAssets, loadAsset } from "@/components/engine";
-import { useECS } from "@/context/ecs";
-import { Not, query } from 'bitecs';
-import { AssetId } from "@diffusionstudio/runtime";
import { useLibrary } from "@/engine/library";
-import { useEditor, useSelection } from "@/engine/hooks";
+import { useEditor } from "@/engine/hooks";
import { droppedFiles, importFiles, pickFiles } from "@/engine/asset-actions";
+import { useMediaSelection } from "./selection";
import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";
-import type { Entity } from "koota";
-import type { Asset as LibraryAsset, AssetCache } from "@diffusionstudio/assets";
+import type { AssetCache } from "@diffusionstudio/assets";
import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";
export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";
@@ -107,7 +101,7 @@ export function createDefaultConfig<K extends keyof GenerationConfgs>(mode: K, p
case "VOICE":
return {
mode: "VOICE",
- model: "elevenlabs-v3",
+ model: PROMPT_INPUT_VOICE_MODEL,
prompt,
voice: PROMPT_INPUT_VOICE_OPTIONS[0].value,
} as GenerationConfgs[K];
@@ -125,15 +119,9 @@ export interface PromptInputProps {
}
export function PromptInput(props: PromptInputProps) {
- const { world } = useEngine();
- const assets = useAssets(world);
- const { selectedNodes } = useECS();
- const c = world.components;
- // The runtime the IMAGE mode works against: references come from the
- // project library and the canvas selection, and land as declarations.
const library = useLibrary();
const editor = useEditor();
- const { nodes } = useSelection();
+ const { images: selectedImages } = useMediaSelection();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
const { generate: generateVoice } = useGenerateVoice();
@@ -184,46 +172,13 @@ export function PromptInput(props: PromptInputProps) {
return model ? ALL_VIDEO_ASPECT_RATIO_OPTIONS.filter((o) => model.aspectRatios.includes(o.value)) : [];
});
- const selectedImageEntries = createMemo(() => {
- const entries: { assetId: string; eid: number }[] = [];
- const AssetId = world.components.AssetId;
- for (const eid of selectedNodes()) {
- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {
- if (c.Paint[fid] === PaintType.IMAGE) {
- const assetId = AssetId[fid];
- if (!assetId) continue;
- entries.push({ assetId, eid });
- break; // we just need the first image fill
- }
- }
- }
- return entries;
- });
-
- const selectedImageIds = createMemo(() =>
- selectedImageEntries().map((e) => e.assetId),
- );
-
- // The image assets of the current canvas selection, from the runtime's
- // world: entities bound to a library image count as references.
- const selectedLibraryImages = createMemo(() => {
- const lib = library();
- if (!lib) return [];
-
- const entries: { asset: LibraryAsset; entity: Entity }[] = [];
- for (const entity of nodes()) {
- const id = entity.get(AssetId)?.value;
- const asset = id ? lib.get(id) : undefined;
- if (asset?.type === "IMAGE") entries.push({ asset, entity });
- }
- return entries;
- });
-
+ // Pictures on the canvas are offered as references and as video frames:
+ // selecting one is another way of attaching it.
const effectiveImageRefIds = createMemo(() => {
const lib = library();
if (!lib) return [];
const local = imageConfig()?.imageRefIds ?? [];
- const selected = selectedLibraryImages()
+ const selected = selectedImages()
.map((entry) => entry.asset.id)
.filter((id) => !local.includes(id));
return [...local, ...selected]
@@ -231,15 +186,22 @@ export function PromptInput(props: PromptInputProps) {
.slice(0, MAX_IMAGE_REFERENCES);
});
- const effectiveStartFrameId = createMemo(() => {
- const id = videoConfig()?.startFrameImageId ?? selectedImageIds()[0] ?? null;
- return id && assets.get(id) ? id : null;
- });
+ /** The library id a frame is set to, explicitly or by the selection. */
+ const frameId = (frame: "start" | "end") => {
+ // A frame the model cannot take is not one the prompt box offers, and
+ // not one a canvas selection quietly attaches either: the declaration is
+ // checked against the model before it runs.
+ if (!currentVideoModel()?.features.includes(`${frame}-frame`)) return null;
- const effectiveEndFrameId = createMemo(() => {
- const id = videoConfig()?.endFrameImageId ?? selectedImageIds()[1] ?? null;
- return id && assets.get(id) ? id : null;
- });
+ const lib = library();
+ const config = videoConfig();
+ const explicit = frame === "start" ? config?.startFrameImageId : config?.endFrameImageId;
+ const id = explicit ?? selectedImages()[frame === "start" ? 0 : 1]?.asset.id ?? null;
+ return id && lib?.get(id) ? id : null;
+ };
+
+ const effectiveStartFrameId = createMemo(() => frameId("start"));
+ const effectiveEndFrameId = createMemo(() => frameId("end"));
const [recentPrompts, setRecentPrompts] = createStoredSignal(
store.define<string[]>("prompt-input.recent-prompts", []),
@@ -275,7 +237,7 @@ export function PromptInput(props: PromptInputProps) {
}
// The reference is shown from a selected canvas entity, so removing it
// means deselecting that entity.
- const entry = selectedLibraryImages().find((e) => e.asset.id === assetId);
+ const entry = selectedImages().find((e) => e.asset.id === assetId);
if (entry) editor.deselect(entry.entity);
};
@@ -297,22 +259,17 @@ export function PromptInput(props: PromptInputProps) {
};
const openVideoFramePicker = async (frame: "start" | "end") => {
- const handles = await showFileDialog("image/*", false);
- if (handles.length !== 1) return;
-
- try {
- const loadedAsset = await loadAsset(world, handles[0]!);
- if (loadedAsset.type !== "IMAGE") return;
- if (frame === "start") {
- patch({ startFrameImageId: loadedAsset.id });
- } else {
- patch({ endFrameImageId: loadedAsset.id });
- }
- } catch (e) {
- toast("Failed to import assets", {
- description: (e as Error).message,
- });
- }
+ const lib = library();
+ if (!lib) return;
+
+ const files = await pickFiles({ accept: "image/*", multiple: false });
+ if (files.length === 0) return;
+
+ // `importFiles` reports its own failures.
+ const [imported] = (await importFiles(lib, files, "")).filter((a) => a.type === "IMAGE");
+ if (!imported) return;
+
+ patch(frame === "start" ? { startFrameImageId: imported.id } : { endFrameImageId: imported.id });
};
const swapVideoFrameImages = () => {
@@ -332,10 +289,8 @@ export function PromptInput(props: PromptInputProps) {
}
// The frame is being shown from a selected canvas entity, so clearing it
// means deselecting that entity (mirrors removeImageReference).
- const entry = frame === "start" ? selectedImageEntries()[0] : selectedImageEntries()[1];
- if (entry) {
- removeComponent(world, entry.eid, c.Selected, false);
- }
+ const entry = selectedImages()[frame === "start" ? 0 : 1];
+ if (entry) editor.deselect(entry.entity);
};
const resizeTextarea = () => {
@@ -444,59 +399,42 @@ export function PromptInput(props: PromptInputProps) {
dragCounter = 0;
setIsDragging(false);
+ if (mode() !== "IMAGE" && mode() !== "VIDEO") return;
+
+ const lib = library();
+ if (!lib) return;
+
+ // What a generation is given to work from lives in the project library:
+ // dragged library assets are taken as they are, external files imported.
const assetIdData = event.dataTransfer?.getData(ASSET_DRAG_TYPE);
const existingAssetIds = assetIdData?.split(",").filter(Boolean) ?? [];
+ const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));
+ if (existingAssetIds.length === 0 && files.length === 0) return;
- // References for an image generation live in the project library:
- // dragged library assets are taken as they are, external files imported.
- if (mode() === "IMAGE") {
- const lib = library();
- if (!lib) return;
-
- const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));
- if (existingAssetIds.length === 0 && files.length === 0) return;
-
- const newAssetIds =
- existingAssetIds.length > 0
- ? existingAssetIds
- : (await importFiles(lib, files, "")).map((a) => a.id);
-
- const imageIds = newAssetIds.filter((id) => lib.get(id)?.type === "IMAGE");
- const current = imageConfig()?.imageRefIds ?? [];
- patch({
- imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),
- });
+ // `importFiles` reports its own failures.
+ const droppedIds =
+ existingAssetIds.length > 0
+ ? existingAssetIds
+ : (await importFiles(lib, files, "")).map((a) => a.id);
+
+ const imageIds = droppedIds.filter((id) => lib.get(id)?.type === "IMAGE");
+ if (imageIds.length === 0) return;
+
+ if (mode() === "VIDEO") {
+ const c = videoConfig();
+ if (!c?.startFrameImageId && imageIds[0]) {
+ patch({ startFrameImageId: imageIds[0] });
+ }
+ if (!c?.endFrameImageId && imageIds[1]) {
+ patch({ endFrameImageId: imageIds[1] });
+ }
return;
}
- const fileItems = Array.from(event.dataTransfer?.items ?? []).filter(
- (item) => item.kind === "file" && item.type.startsWith("image/"),
- );
-
- if (existingAssetIds.length === 0 && fileItems.length === 0) return;
-
- try {
- let newAssetIds =
- existingAssetIds.length > 0
- ? existingAssetIds
- : (await Promise.all(fileItems.map((item) => loadAsset(world, item)))).map((a) => a.id);
-
- newAssetIds = newAssetIds.filter((id) => world.assets.get(id)?.type === "IMAGE");
-
- if (mode() === "VIDEO") {
- const c = videoConfig();
- if (!c?.startFrameImageId && newAssetIds[0]) {
- patch({ startFrameImageId: newAssetIds[0] });
- }
- if (!c?.endFrameImageId && newAssetIds[1]) {
- patch({ endFrameImageId: newAssetIds[1] });
- }
- }
- } catch (e) {
- toast("Failed to import assets", {
- description: (e as Error).message,
- });
- }
+ const current = imageConfig()?.imageRefIds ?? [];
+ patch({
+ imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),
+ });
};
const handleVideoModelChange = (id: string) => {
@@ -510,7 +448,7 @@ export function PromptInput(props: PromptInputProps) {
...c,
model: id,
endFrameImageId: model.features.includes("end-frame") ? c.endFrameImageId : undefined,
- generateAudio: model.features.includes("audio") ? c.generateAudio : true,
+ generateAudio: model.features.includes("audio") ? (c.generateAudio ?? true) : false,
duration: model.durations.includes(durationStr) ? c.duration : parseInt(model.durations[0], 10),
aspectRatio: model.aspectRatios.includes(c.aspectRatio)
? c.aspectRatio
@@ -577,7 +515,8 @@ export function PromptInput(props: PromptInputProps) {
>
{(src) => (
<PromptInputReferenceImageButton
- asset={assets.get(src())!}
+ asset={library()!.get(src())!}
+ cache={library()!.cache}
class="h-16 w-28"
size={{ width: 112, height: 64 }}
onRemove={() => clearVideoFrame("start")}
@@ -611,7 +550,8 @@ export function PromptInput(props: PromptInputProps) {
>
{(src) => (
<PromptInputReferenceImageButton
- asset={assets.get(src())!}
+ asset={library()!.get(src())!}
+ cache={library()!.cache}
class="h-16 w-28"
size={{ width: 112, height: 64 }}
onRemove={() => clearVideoFrame("end")}
diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.ts
new file mode 100644
index 00000000..23a28d77
--- /dev/null
+++ b/apps/web/src/components/genai/selection.ts
@@ -0,0 +1,56 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// What the prompt box has to work with. Two readings of the selection, for
+// two different questions:
+//
+// `imageNodes`/`videoNodes` are what the element *is*, which is what a
+// source modifier applies to — a picture that is still generating is still a
+// picture, and `removeBackground` can be asked of it before it exists.
+//
+// `images`/`videos` are what the element currently *shows*, which is what a
+// reference needs: a library asset with a path to name it by.
+
+import { createMemo } from "solid-js";
+import { AssetId } from "@diffusionstudio/runtime";
+import { authoredElement } from "@diffusionstudio/reconciler";
+import { useSelection } from "@/engine/hooks";
+import { useLibrary } from "@/engine/library";
+
+import type { Asset } from "@diffusionstudio/assets";
+import type { Entity } from "koota";
+
+/** A selected node and the library asset it is bound to. */
+export interface BoundNode {
+ entity: Entity;
+ asset: Asset;
+}
+
+export function useMediaSelection() {
+ const library = useLibrary();
+ const { nodes } = useSelection();
+
+ const tagged = (tag: string) =>
+ createMemo(() => nodes().filter((entity) => authoredElement(entity)?.tag === tag));
+
+ const imageNodes = tagged("image");
+ const videoNodes = tagged("video");
+
+ const bound = createMemo(() => {
+ const lib = library();
+ if (!lib) return [];
+
+ const entries: BoundNode[] = [];
+ for (const entity of nodes()) {
+ const id = entity.get(AssetId)?.value;
+ const asset = id ? lib.get(id) : undefined;
+ if (asset) entries.push({ entity, asset });
+ }
+ return entries;
+ });
+
+ const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));
+
+ return { bound, images, imageNodes, videoNodes };
+}
diff --git a/apps/web/src/components/genai/use-add-audio-to-video.ts b/apps/web/src/components/genai/use-add-audio-to-video.ts
deleted file mode 100644
index a413dbf8..00000000
--- a/apps/web/src/components/genai/use-add-audio-to-video.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { Not, Or, query } from 'bitecs';
-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent } from "@/components/engine";
-import { loadAsset, uploadAsset } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { assert } from "@/utils";
-import { toast } from "somoto";
-
-import type { EngineWorld } from "@/components/engine";
-
-function findVideoFill(world: EngineWorld, eid: number): number | null {
- const c = world.components;
-
- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {
- if (c.Paint[fid] === PaintType.VIDEO) {
- return fid;
- }
- }
- return null;
-}
-
-export function useAddAudioToVideo() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async () => {
- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];
-
- for (const nid of nodes) {
- const fid = findVideoFill(world, nid);
- if (!fid) continue;
-
- const assetId = c.AssetId[fid];
- if (!assetId) continue;
-
- const asset = world.assets.get(assetId);
- if (!asset) continue;
-
- try {
- addComponent(world, nid, c.Generating);
-
- const fileRef = await uploadAsset(world, assetId);
- if (!fileRef) throw new Error("Failed to upload asset");
-
- const { url, generationId } = await trpc.addAudioToVideo.mutate({ video: fileRef });
-
- const dot = asset.name.lastIndexOf(".");
- const name = dot > 0
- ? asset.name.slice(0, dot) + " (With audio)" + asset.name.slice(dot)
- : asset.name + " (With audio)";
- const newAsset = await loadAsset(world, url, { name, generationId });
-
- assert(newAsset.type === "VIDEO", "New asset is not a video");
-
- addComponent(world, fid, c.Hidden);
-
- const newFid = createEntity(world);
- setComponent(world, newFid, c.Paint, PaintType.VIDEO);
- setComponent(world, newFid, c.AssetId, newAsset.id);
- appendChild(world, newFid, nid);
- } catch (err) {
- console.error("Add audio failed:", err);
- toast.error("Failed to add audio to video");
- } finally {
- removeComponent(world, nid, c.Generating);
- }
- }
- };
-
- return { generate };
-}
diff --git a/apps/web/src/components/genai/use-generate-audio.ts b/apps/web/src/components/genai/use-generate-audio.ts
deleted file mode 100644
index e20c3172..00000000
--- a/apps/web/src/components/genai/use-generate-audio.ts
+++ /dev/null
@@ -1,90 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { track } from "@/lib/analytics";
-import { findEmptyPlacement, recordGeneration, addWaveformFill, ensureGenerationFolder } from "@/utils/genai";
-import { assert } from "@/utils";
-
-import { GeometryType, createEntity, deleteEntity, addComponent, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset } from "@/components/engine";
-
-import type { AudioGenerationConfig } from "@/components/engine/db";
-
-
-export function useGenerateAudio() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async (config: AudioGenerationConfig) => {
- let eid: number | undefined;
- const startedAt = performance.now();
- try {
- track('generation_started', {
- mode: 'audio',
- model: config.model,
- prompt_length: config.prompt.length,
- });
-
- const placement = findEmptyPlacement(world, 400, 100, 40);
-
- // In-flight placeholder is transient UI — build it untracked so it never
- // reaches the undo stack. The whole generation is recorded as one step below.
- world.history.untrack(() => {
- eid = createEntity(world);
- setComponent(world, eid, c.Geometry, GeometryType.RECT);
- addComponent(world, eid, c.Audio);
- setComponent(world, eid, c.Playback, {});
- setComponent(world, eid, c.Position, {
- x: Math.round(placement.x - 200),
- y: Math.round(placement.y - 50),
- });
- resizeEntity(world, eid, {
- width: 400,
- height: 100,
- });
- addComponent(world, eid, c.Generating);
- });
-
- const { results, name, generationId } = await trpc.generateSound.mutate({
- model: config.model,
- prompt: config.prompt,
- });
-
- assert(results.length > 0, "No results returned from the model");
- const folderId = await ensureGenerationFolder(world, name, results.length);
- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });
-
- assert(asset.type === "AUDIO", "Asset is not an audio");
-
- world.history.untrack(() => {
- setComponent(world, eid!, c.Name, name);
- setComponent(world, eid!, c.AssetId, asset.id);
- addWaveformFill(world, eid!, asset.id);
- removeComponent(world, eid!, c.Generating);
- });
-
- recordGeneration(world, 'Generate audio', [eid!]);
-
- track('generation_completed', {
- mode: 'audio',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- });
- } catch (err) {
- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));
- console.error(err);
- track('generation_failed', {
- mode: 'audio',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',
- });
- throw err;
- }
- };
-
- return { generate } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-audio.tsx b/apps/web/src/components/genai/use-generate-audio.tsx
new file mode 100644
index 00000000..b5cc4cce
--- /dev/null
+++ b/apps/web/src/components/genai/use-generate-audio.tsx
@@ -0,0 +1,34 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Music and sound effects, declared: the prompt box writes a `generate.audio`
+// source and inserts an <Audio> consuming it (see ./insert).
+
+import { generate } from "@diffusionstudio/jsx";
+import { Audio } from "@diffusionstudio/reconciler";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useEditor } from "@/engine/hooks";
+import { AUDIO_SIZE } from "@/engine/insert-asset";
+import { insertGenerated, randomSeed } from "./insert";
+
+import type { AudioGenerationConfig } from "@/components/engine/db";
+
+export function useGenerateAudio() {
+ const world = useWorld();
+ const editor = useEditor();
+
+ const run = async (config: AudioGenerationConfig) => {
+ const src = generate.audio({
+ prompt: config.prompt,
+ model: config.model,
+ seed: randomSeed(),
+ });
+
+ insertGenerated(world, editor, AUDIO_SIZE, (box) => (
+ <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />
+ ));
+ };
+
+ return { generate: run } as const;
+}
diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsx
index 92606c80..2ce4f64f 100644
--- a/apps/web/src/components/genai/use-generate-image.tsx
+++ b/apps/web/src/components/genai/use-generate-image.tsx
@@ -8,27 +8,16 @@
// declaration through the world's Ai (see `EditorGenAi`), and the element
// lands in the project file with the `generate.image({...})` call that
// reproduces it — same spec, same asset, this session and the next.
-//
-// Generations go on the canvas rather than into a scene: they are material to
-// work with, not yet part of a composition, and a scene owns a timeline they
-// have no place on until someone puts them there.
import { generate } from "@diffusionstudio/jsx";
import { Image } from "@diffusionstudio/reconciler";
-import { Library, Root } from "@diffusionstudio/runtime";
+import { Library } from "@diffusionstudio/runtime";
import { useWorld } from "@diffusionstudio/koota-solid";
import { useEditor } from "@/engine/hooks";
-import { findEmptyPlacement } from "@/engine/placement";
-import { ASPECT_RATIO_DIMENSIONS } from "@/utils/genai";
-import { toast } from "somoto";
+import { ASPECT_RATIO_DIMENSIONS } from "./config";
+import { insertGenerated, randomSeed } from "./insert";
import type { ImageGenerationConfig } from "@/components/engine/db";
-import type { Entity } from "koota";
-
-const IMAGE_GAP = 40;
-
-/** A declaration is keyed by its spec, so a fresh seed makes a new take. */
-const randomSeed = () => Math.floor(Math.random() * 1_000_000);
export function useGenerateImage() {
const world = useWorld();
@@ -36,8 +25,6 @@ export function useGenerateImage() {
const run = async (config: ImageGenerationConfig) => {
const library = world.get(Library);
- const root = world.get(Root);
- if (!root) return;
// References travel by library path: durable in the file, and readable
// there. Ids that no longer name a library asset are dropped.
@@ -46,35 +33,20 @@ export function useGenerateImage() {
.filter((path): path is string => path !== undefined);
const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };
- const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;
- const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);
- const left = center.x - totalWidth / 2;
- const top = Math.round(center.y - dims.height / 2);
-
- const inserted: Entity[] = [];
- for (let index = 0; index < config.count; index++) {
- const src = generate.image({
- prompt: config.prompt,
- model: config.model,
- aspectRatio: config.aspectRatio,
- ...(refs.length ? { refs } : {}),
- seed: randomSeed(),
- });
-
- const x = Math.round(left + index * (dims.width + IMAGE_GAP));
- const [entity] = editor.insertElement(root, () => (
- <Image src={src} x={x} y={top} width={dims.width} height={dims.height} />
- ));
- if (entity) inserted.push(entity);
- }
- if (!inserted.length) {
- toast("Could not add the generation", {
- description: "Open a project to generate into.",
- });
- return;
- }
- editor.select(inserted);
+ // One declaration per variant, each with its own seed: a batch is four
+ // takes of the prompt rather than one asset shown four times.
+ const sources = Array.from({ length: config.count }, () => generate.image({
+ prompt: config.prompt,
+ model: config.model,
+ aspectRatio: config.aspectRatio,
+ ...(refs.length ? { refs } : {}),
+ seed: randomSeed(),
+ }));
+
+ insertGenerated(world, editor, dims, (box, index) => (
+ <Image src={sources[index]} x={box.x} y={box.y} width={box.width} height={box.height} />
+ ), config.count);
};
return { generate: run } as const;
diff --git a/apps/web/src/components/genai/use-generate-video.ts b/apps/web/src/components/genai/use-generate-video.ts
deleted file mode 100644
index aef56326..00000000
--- a/apps/web/src/components/genai/use-generate-video.ts
+++ /dev/null
@@ -1,111 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { track } from "@/lib/analytics";
-import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement, recordGeneration, ensureGenerationFolder } from "@/utils/genai";
-
-import { GeometryType, PaintType, createEntity, deleteEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset, uploadAsset } from "@/components/engine";
-import { assert } from "@/utils";
-
-import type { VideoGenerationConfig } from "@/components/engine/db";
-
-export function useGenerateVideo() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async (config: VideoGenerationConfig) => {
- let eid: number | undefined;
- const startedAt = performance.now();
- try {
- track('generation_started', {
- mode: 'video',
- model: config.model,
- aspect_ratio: config.aspectRatio,
- duration: config.duration,
- generate_audio: config.generateAudio,
- has_start_frame: !!config.startFrameImageId,
- has_end_frame: !!config.endFrameImageId,
- prompt_length: config.prompt.length,
- });
-
- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };
- const placement = findEmptyPlacement(world, dims.width, dims.height, 40);
-
- // In-flight placeholder is transient UI — build it untracked so it never
- // reaches the undo stack. The whole generation is recorded as one step below.
- world.history.untrack(() => {
- eid = createEntity(world);
- setComponent(world, eid, c.Geometry, GeometryType.RECT);
- setComponent(world, eid, c.Position, {
- x: Math.round(placement.x - dims.width / 2),
- y: Math.round(placement.y - dims.height / 2),
- });
- resizeEntity(world, eid, {
- width: dims.width,
- height: dims.height,
- });
- setComponent(world, eid, c.KeepAspectRatio, {
- width: dims.width,
- height: dims.height,
- });
- addComponent(world, eid, c.Generating);
- });
-
- const [startFrame, endFrame] = await Promise.all([
- config.startFrameImageId ? uploadAsset(world, config.startFrameImageId) : undefined,
- config.endFrameImageId ? uploadAsset(world, config.endFrameImageId) : undefined,
- ]);
-
- const { results, name, generationId } = await trpc.generateVideo.mutate({
- model: config.model,
- prompt: config.prompt,
- aspectRatio: config.aspectRatio,
- duration: config.duration,
- generateAudio: config.generateAudio,
- startFrame: startFrame ?? undefined,
- endFrame: endFrame ?? undefined,
- });
-
- assert(results.length > 0, "No results returned from the model");
- const folderId = await ensureGenerationFolder(world, name, results.length);
- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });
-
- world.history.untrack(() => {
- setComponent(world, eid!, c.Name, name);
- removeComponent(world, eid!, c.Generating);
-
- const fid = createEntity(world);
- setComponent(world, fid, c.Paint, PaintType.VIDEO);
- setComponent(world, fid, c.AssetId, asset.id);
- appendChild(world, fid, eid!);
-
- setComponent(world, eid!, c.Playback, {});
- });
-
- recordGeneration(world, 'Generate video', [eid!]);
-
- track('generation_completed', {
- mode: 'video',
- model: config.model,
- aspect_ratio: config.aspectRatio,
- duration_ms: Math.round(performance.now() - startedAt),
- });
- } catch (err) {
- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));
- console.error(err);
- track('generation_failed', {
- mode: 'video',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',
- });
- throw err;
- }
- };
-
- return { generate } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsx
new file mode 100644
index 00000000..5d1b2c88
--- /dev/null
+++ b/apps/web/src/components/genai/use-generate-video.tsx
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Video generation, declared rather than performed: the prompt box writes a
+// `generate.video` source and inserts a <Video> consuming it (see ./insert).
+// The frames a model animates between are library assets, named by path, so
+// the call in the file says what it was made from as well as what to make.
+
+import { generate } from "@diffusionstudio/jsx";
+import { Video } from "@diffusionstudio/reconciler";
+import { Library } from "@diffusionstudio/runtime";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useEditor } from "@/engine/hooks";
+import { ASPECT_RATIO_DIMENSIONS } from "./config";
+import { insertGenerated, randomSeed } from "./insert";
+
+import type { VideoGenerationConfig } from "@/components/engine/db";
+
+export function useGenerateVideo() {
+ const world = useWorld();
+ const editor = useEditor();
+
+ const run = async (config: VideoGenerationConfig) => {
+ const library = world.get(Library);
+ const frame = (id: string | undefined) => (id ? library?.get(id)?.path : undefined);
+
+ const startFrame = frame(config.startFrameImageId);
+ const endFrame = frame(config.endFrameImageId);
+ const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };
+
+ const src = generate.video({
+ prompt: config.prompt,
+ model: config.model,
+ aspectRatio: config.aspectRatio,
+ duration: config.duration,
+ audio: config.generateAudio ?? false,
+ ...(startFrame ? { startFrame } : {}),
+ ...(endFrame ? { endFrame } : {}),
+ seed: randomSeed(),
+ });
+
+ insertGenerated(world, editor, dims, (box) => (
+ <Video src={src} x={box.x} y={box.y} width={box.width} height={box.height} />
+ ));
+ };
+
+ return { generate: run } as const;
+}
diff --git a/apps/web/src/components/genai/use-generate-voice.ts b/apps/web/src/components/genai/use-generate-voice.ts
deleted file mode 100644
index 7236938b..00000000
--- a/apps/web/src/components/genai/use-generate-voice.ts
+++ /dev/null
@@ -1,89 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { track } from "@/lib/analytics";
-import { findEmptyPlacement, recordGeneration, addWaveformFill, ensureGenerationFolder } from "@/utils/genai";
-import { GeometryType, createEntity, deleteEntity, addComponent, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset } from "@/components/engine";
-
-import type { VoiceGenerationConfig } from "@/components/engine/db";
-
-export function useGenerateVoice() {
- const { world } = useEngine();
- const c = world.components;
-
-
- const generate = async (config: VoiceGenerationConfig) => {
- let eid: number | undefined;
- const startedAt = performance.now();
- try {
- track('generation_started', {
- mode: 'voice',
- model: config.model,
- voice: config.voice,
- prompt_length: config.prompt.length,
- });
-
- const placement = findEmptyPlacement(world, 400, 100, 40);
- // In-flight placeholder is transient UI — build it untracked so it never
- // reaches the undo stack. The whole generation is recorded as one step below.
- world.history.untrack(() => {
- eid = createEntity(world);
- setComponent(world, eid, c.Geometry, GeometryType.RECT);
- addComponent(world, eid, c.Audio);
- setComponent(world, eid, c.Playback, {});
- setComponent(world, eid, c.Position, {
- x: Math.round(placement.x - 200),
- y: Math.round(placement.y - 50),
- });
- resizeEntity(world, eid, {
- width: 400,
- height: 100,
- });
- addComponent(world, eid, c.Generating);
- });
-
- const { results, name, generationId } = await trpc.textToSpeech.mutate({
- model: config.model,
- prompt: config.prompt,
- voice: config.voice,
- });
-
- if (results.length > 0) {
- const folderId = await ensureGenerationFolder(world, name, results.length);
- const asset = await loadAsset(world, results[0].url, { name, generationId, folderId });
-
- world.history.untrack(() => {
- setComponent(world, eid!, c.Name, name);
- setComponent(world, eid!, c.AssetId, asset.id);
- addWaveformFill(world, eid!, asset.id);
- removeComponent(world, eid!, c.Generating);
- });
-
- recordGeneration(world, 'Generate voice', [eid!]);
- }
-
- track('generation_completed', {
- mode: 'voice',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- });
- } catch (err) {
- if (eid !== undefined) world.history.untrack(() => deleteEntity(world, eid!));
- console.error(err);
- track('generation_failed', {
- mode: 'voice',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',
- });
- throw err;
- }
- };
-
- return { generate } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-voice.tsx b/apps/web/src/components/genai/use-generate-voice.tsx
new file mode 100644
index 00000000..9850ed34
--- /dev/null
+++ b/apps/web/src/components/genai/use-generate-voice.tsx
@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Spoken lines, declared: the prompt box writes a `generate.voice` source and
+// inserts an <Audio> consuming it (see ./insert), so the file holds the line
+// and the voice that reads it rather than a sound nobody can trace back.
+
+import { generate } from "@diffusionstudio/jsx";
+import { Audio } from "@diffusionstudio/reconciler";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useEditor } from "@/engine/hooks";
+import { AUDIO_SIZE } from "@/engine/insert-asset";
+import { insertGenerated, randomSeed } from "./insert";
+
+import type { VoiceGenerationConfig } from "@/components/engine/db";
+
+export function useGenerateVoice() {
+ const world = useWorld();
+ const editor = useEditor();
+
+ const run = async (config: VoiceGenerationConfig) => {
+ const src = generate.voice({
+ prompt: config.prompt,
+ voice: config.voice,
+ seed: randomSeed(),
+ });
+
+ insertGenerated(world, editor, AUDIO_SIZE, (box) => (
+ <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />
+ ));
+ };
+
+ return { generate: run } as const;
+}
diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.ts
index 1e013c38..0503377f 100644
--- a/apps/web/src/components/genai/use-generation-records.ts
+++ b/apps/web/src/components/genai/use-generation-records.ts
@@ -3,14 +3,23 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createMemo, createResource } from "solid-js";
-import { hasComponent } from "bitecs";
-import { PaintType } from "@/components/engine";
-import { useEngine } from "@/context/engine";
+import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
+import { authoredElement } from "@diffusionstudio/reconciler";
+import { useSelection } from "@/engine/hooks";
+import { useLibrary } from "@/engine/library";
import { supabase } from "@/lib/supabase";
+import { useMediaSelection } from "./selection";
+import {
+ PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
+ PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
+ PROMPT_INPUT_VOICE_MODEL,
+ PROMPT_INPUT_VOICE_OPTIONS,
+} from "./config";
import { generationConfigSchema, type GenerationConfig } from "@/components/engine/db";
-import type { EngineWorld } from "@/components/engine";
-import { useECS } from "@/context/ecs";
+import type { AssetInput, AssetSpecInput } from "@diffusionstudio/jsx";
+import type { AssetLibrary } from "@diffusionstudio/assets";
/**
* The stored `usage_records.config` IS the server-side adapter input, which
@@ -48,36 +57,73 @@ export function toClientConfig(stored: unknown): GenerationConfig | undefined {
return parsed.success ? parsed.data : undefined;
}
-function getNodeAssetId(world: EngineWorld, eid: number): string | undefined {
- const c = world.components;
- if (hasComponent(world, eid, c.Audio)) {
- return c.AssetId[eid];
- }
- for (const fid of c.Cache.fills[eid]) {
- if (hasComponent(world, fid, c.Hidden)) continue;
- if (c.Paint[fid] === PaintType.VIDEO || c.Paint[fid] === PaintType.IMAGE) {
- return c.AssetId[fid];
- }
+/**
+ * A declaration read back as what the prompt box would have to be set to to
+ * produce it — the element's own account of how it was made, which is what
+ * "Rerun" and "Reuse" work from. Inputs are named by library path in the
+ * file and by asset id in the prompt box, so they are looked up on the way.
+ */
+function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): GenerationConfig {
+ const idOf = (input: AssetInput | undefined): string | undefined =>
+ typeof input === "string" ? library.get(input)?.id : undefined;
+
+ switch (spec.type) {
+ case "image":
+ return {
+ mode: "IMAGE",
+ model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio ?? "16:9",
+ count: 1,
+ imageRefIds: (spec.refs ?? []).map(idOf).filter((id): id is string => id !== undefined),
+ };
+ case "video":
+ return {
+ mode: "VIDEO",
+ model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio ?? "16:9",
+ duration: spec.duration ?? 5,
+ generateAudio: spec.audio ?? false,
+ startFrameImageId: idOf(spec.startFrame),
+ endFrameImageId: idOf(spec.endFrame),
+ };
+ case "voice":
+ return {
+ mode: "VOICE",
+ model: PROMPT_INPUT_VOICE_MODEL,
+ prompt: spec.prompt,
+ voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,
+ };
+ case "audio":
+ return {
+ mode: "AUDIO",
+ model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ };
}
- return undefined;
}
export function useGenerationRecords() {
- const { selectedNodes } = useECS();
- const { world } = useEngine();
+ const library = useLibrary();
+ const { nodes } = useSelection();
+ const { bound } = useMediaSelection();
+
+ // What the selected elements declare their source to be. An element made by
+ // the prompt box carries the whole spec, so this answers before the asset
+ // exists — and without asking the server what it was asked for.
+ const declarations = createMemo(() =>
+ nodes()
+ .map((entity) => authoredElement(entity)?.props.src)
+ .filter(isAssetRef)
+ .map(getAssetSpec),
+ );
+ // Credits are the server's to know, and it knows them per generation.
const generationIds = createMemo(() => {
const ids = new Set<string>();
- for (const eid of selectedNodes()) {
- const assetId = getNodeAssetId(world, eid);
- if (!assetId) {
- continue;
- }
-
- const asset = world.assets.get(assetId);
- if (asset?.generationId) {
- ids.add(asset.generationId);
- }
+ for (const { asset } of bound()) {
+ if (asset.generation?.id) ids.add(asset.generation.id);
}
return [...ids];
});
@@ -106,9 +152,15 @@ export function useGenerationRecords() {
return (records() ?? []).reduce((sum, r) => sum + r.credits, 0);
});
- const isGenerated = createMemo(() => generationIds().length > 0);
+ const isGenerated = createMemo(() => declarations().length > 0 || generationIds().length > 0);
const firstConfig = (): GenerationConfig | undefined => {
+ const lib = library();
+ const declared = declarations()[0];
+ if (declared && lib) return toPromptConfig(declared, lib);
+
+ // Nothing declared: an asset generated before the project was written in
+ // JSX, whose settings only the record of the run still has.
const ids = generationIds();
if (ids.length === 0) return undefined;
const record = (records() ?? []).find((r) => r.id === ids[0]);
diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.ts
new file mode 100644
index 00000000..17fc24d8
--- /dev/null
+++ b/apps/web/src/components/genai/use-modifiers.ts
@@ -0,0 +1,72 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Upscaling, taking a background out, scoring footage: the editor's side of
+// the source modifiers (see the runtime's `SourceModifiers` and
+// reference/jsx/media.md). The action bar sets a prop and stops there — the
+// asset system is what notices, calls the model and binds the result — so
+// these are toggles rather than commands, and turning one off gives the
+// original back rather than running anything.
+
+import { authoredElement } from "@diffusionstudio/reconciler";
+import { useEditor } from "@/engine/hooks";
+import { useMediaSelection } from "./selection";
+
+import type { Entity } from "koota";
+
+/** The props an element carries to say what its source is put through. */
+export type ModifierName = "removeBackground" | "upscale" | "addAudio";
+
+/**
+ * What `upscale` is set to when it is switched on. The endpoint takes no
+ * factor today, so this is the one the language says and the number every
+ * upscale is; see the cache key in `EditorGenAi.transform`.
+ */
+const UPSCALE_FACTOR = 2;
+
+/** Whether `entity` currently asks for `name`, as its element says it. */
+function isSet(entity: Entity, name: ModifierName): boolean {
+ const value = authoredElement(entity)?.props[name];
+ return name === "upscale" ? typeof value === "number" && value > 1 : value === true;
+}
+
+export function useModifiers() {
+ const editor = useEditor();
+ const { imageNodes, videoNodes } = useMediaSelection();
+
+ /** The selected elements a modifier can be asked of. */
+ const targets = (name: ModifierName): Entity[] => {
+ if (name === "removeBackground") return imageNodes();
+ if (name === "addAudio") return videoNodes();
+ return [...imageNodes(), ...videoNodes()];
+ };
+
+ /** On when every element it applies to asks for it, so the toggle turns the odd one on. */
+ const isOn = (name: ModifierName): boolean => {
+ const selected = targets(name);
+ return selected.length > 0 && selected.every((entity) => isSet(entity, name));
+ };
+
+ /**
+ * Turns `name` on for the selection, or off when they all have it. Off is
+ * written as `false`, which the source writer spells as the prop's absence
+ * — an element says what it asks for, and nothing about what it does not.
+ */
+ const toggle = (name: ModifierName): void => {
+ const next = !isOn(name);
+ const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;
+
+ for (const entity of targets(name)) {
+ // A failure is the answer to what was asked for (see `sourceErrorSystem`),
+ // and asking for a different set of modifiers is a different question.
+ // Taking the prop off is what asks again, and this is the user doing it.
+ if (authoredElement(entity)?.props.error !== undefined) {
+ editor.editProperty(entity, "error", false);
+ }
+ editor.editProperty(entity, name, value);
+ }
+ };
+
+ return { isOn, toggle };
+}
diff --git a/apps/web/src/components/genai/use-remove-background.ts b/apps/web/src/components/genai/use-remove-background.ts
deleted file mode 100644
index f98877c5..00000000
--- a/apps/web/src/components/genai/use-remove-background.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { Not, Or, query } from 'bitecs';
-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent } from "@/components/engine";
-import { loadAsset, uploadAsset } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { toast } from "somoto";
-
-import type { EngineWorld } from "@/components/engine";
-
-/**
- * For a given shape entity, find the first image fill and its index.
- */
-function findImageFill(world: EngineWorld, eid: number): number | null {
- const c = world.components;
-
- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {
- if (c.Paint[fid] === PaintType.IMAGE) {
- return fid;
- }
- }
- return null;
-}
-
-export function useRemoveBackground() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async () => {
- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];
-
- for (const nid of nodes) {
- const fid = findImageFill(world, nid);
- if (!fid) continue;
-
- const assetId = world.components.AssetId[fid];
- if (!assetId) continue;
-
- const asset = world.assets.get(assetId);
- if (!asset) continue;
-
- try {
- // Show shimmer
- addComponent(world, nid, world.components.Generating);
-
- // Upload asset and call API
- const fileRef = await uploadAsset(world, assetId);
- if (!fileRef) throw new Error("Failed to upload asset");
-
- const { url, generationId } = await trpc.removeBackground.mutate({ image: fileRef });
-
- // Load result as new asset
- const newAsset = await loadAsset(world, url, {
- name: asset.name + " (Background removed)",
- generationId,
- });
-
- addComponent(world, fid, c.Hidden);
-
- const newFid = createEntity(world);
- setComponent(world, newFid, c.Paint, PaintType.IMAGE);
- setComponent(world, newFid, c.AssetId, newAsset.id);
- appendChild(world, newFid, nid);
- } catch (err) {
- console.error("Background removal failed:", err);
- toast.error("Failed to remove background");
- } finally {
- removeComponent(world, nid, c.Generating);
- }
- }
- };
-
- return { generate };
-}
diff --git a/apps/web/src/components/genai/use-upscale-image.ts b/apps/web/src/components/genai/use-upscale-image.ts
deleted file mode 100644
index 3db6e767..00000000
--- a/apps/web/src/components/genai/use-upscale-image.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { Not, Or, query } from 'bitecs';
-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset, uploadAsset } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { assert } from "@/utils";
-import { toast } from "somoto";
-
-import type { EngineWorld } from "@/components/engine";
-
-function findImageFill(world: EngineWorld, eid: number): number | null {
- const c = world.components;
-
- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {
- if (c.Paint[fid] === PaintType.IMAGE) {
- return fid;
- }
- }
- return null;
-}
-
-export function useUpscaleImage() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async () => {
- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];
-
- for (const nid of nodes) {
- const fid = findImageFill(world, nid);
- if (!fid) continue;
-
- const assetId = world.components.AssetId[fid];
- if (!assetId) continue;
-
- const asset = world.assets.get(assetId);
- if (!asset) continue;
-
- try {
- // Show shimmer
- addComponent(world, nid, c.Generating);
-
- const fileRef = await uploadAsset(world, assetId);
- if (!fileRef) throw new Error("Failed to upload asset");
-
- const { url, generationId } = await trpc.upscaleImage.mutate({ image: fileRef });
-
- const dot = asset.name.lastIndexOf(".");
- const name = dot > 0
- ? asset.name.slice(0, dot) + " (Upscaled)" + asset.name.slice(dot)
- : asset.name + " (Upscaled)";
- const newAsset = await loadAsset(world, url, { name, generationId });
-
- assert(newAsset.type === "IMAGE", "New asset is not an image");
-
- addComponent(world, fid, c.Hidden);
- setComponent(world, nid, c.KeepAspectRatio, { width: newAsset.width, height: newAsset.height });
- resizeEntity(world, nid, { width: newAsset.width, height: newAsset.height });
-
- const newFid = createEntity(world);
- setComponent(world, newFid, c.Paint, PaintType.IMAGE);
- setComponent(world, newFid, c.AssetId, newAsset.id);
- appendChild(world, newFid, nid);
- } catch (err) {
- console.error("Upscale failed:", err);
- toast.error("Failed to upscale image");
- } finally {
- removeComponent(world, nid, c.Generating);
- }
- }
- };
-
- return { generate };
-}
diff --git a/apps/web/src/components/genai/use-upscale-video.ts b/apps/web/src/components/genai/use-upscale-video.ts
deleted file mode 100644
index 27530b40..00000000
--- a/apps/web/src/components/genai/use-upscale-video.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { Not, Or, query } from 'bitecs';
-import { PaintType, ChildOf, createEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset, uploadAsset } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { assert } from "@/utils";
-import { toast } from "somoto";
-
-import type { EngineWorld } from "@/components/engine";
-
-function findVideoFill(world: EngineWorld, eid: number): number | null {
- const c = world.components;
-
- for (const fid of query(world, [c.Paint, ChildOf(eid), Not(c.Deleted), Not(c.Hidden)])) {
- if (c.Paint[fid] === PaintType.VIDEO) {
- return fid;
- }
- }
- return null;
-}
-
-export function useUpscaleVideo() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async () => {
- const nodes = [...query(world, [c.Selected, Or(c.Geometry, c.Group)])];
-
- for (const nid of nodes) {
- const fid = findVideoFill(world, nid);
- if (!fid) continue;
-
- const assetId = world.components.AssetId[fid];
- if (!assetId) continue;
-
- const asset = world.assets.get(assetId);
- if (!asset) continue;
-
- try {
- addComponent(world, nid, c.Generating);
-
- const fileRef = await uploadAsset(world, assetId);
- if (!fileRef) throw new Error("Failed to upload asset");
-
- const { url, generationId } = await trpc.upscaleVideo.mutate({ video: fileRef });
-
- const dot = asset.name.lastIndexOf(".");
- const name = dot > 0
- ? asset.name.slice(0, dot) + " (Upscaled)" + asset.name.slice(dot)
- : asset.name + " (Upscaled)";
- const newAsset = await loadAsset(world, url, { name, generationId });
-
- assert(newAsset.type === "VIDEO", "New asset is not a video");
-
- addComponent(world, fid, c.Hidden);
- setComponent(world, nid, c.KeepAspectRatio, { width: newAsset.width, height: newAsset.height });
- resizeEntity(world, nid, { width: newAsset.width, height: newAsset.height });
-
- const newFid = createEntity(world);
- setComponent(world, newFid, c.Paint, PaintType.VIDEO);
- setComponent(world, newFid, c.AssetId, newAsset.id);
- appendChild(world, newFid, nid);
- } catch (err) {
- console.error("Upscale failed:", err);
- toast.error("Failed to upscale video");
- } finally {
- removeComponent(world, nid, c.Generating);
- }
- }
- };
-
- return { generate };
-}
diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsx
index ebe42be1..3ebeca91 100644
--- a/apps/web/src/engine/insert-asset.tsx
+++ b/apps/web/src/engine/insert-asset.tsx
@@ -25,8 +25,8 @@ export interface InsertAssetOptions {
start?: number;
}
-const AUDIO_WIDTH = 500;
-const AUDIO_HEIGHT = 150;
+/** The box an audio clip gets on the canvas: it has no size of its own. */
+export const AUDIO_SIZE = { width: 500, height: 150 } as const;
/**
* Inserts `asset` into the project as the element of its type and returns
@@ -73,7 +73,7 @@ function sizeOf(asset: Asset): { width: number; height: number } | undefined {
case 'SEQUENCE':
return { width: Math.round(asset.width), height: Math.round(asset.height) };
case 'AUDIO':
- return { width: AUDIO_WIDTH, height: AUDIO_HEIGHT };
+ return { ...AUDIO_SIZE };
default:
return undefined;
}
diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.ts
index e6ab122c..58139f16 100644
--- a/apps/web/src/engine/library.ts
+++ b/apps/web/src/engine/library.ts
@@ -40,11 +40,16 @@ export function isLibraryFile(path: string): boolean {
return path === MANIFEST_FILE || path === ASSETS_DIR || path.startsWith(`${ASSETS_DIR}/`);
}
-/** Rewrites `src` on every element that named an asset by its old path. */
+/**
+ * Rewrites `src` on every element that named an asset by its old path. Matched
+ * on the path alone, not on what the element is bound to: an element showing a
+ * modified source (`removeBackground`, `upscale`) is bound to what the
+ * modifiers made of the asset rather than to the asset itself, and its `src`
+ * still has to follow the rename.
+ */
function followRename(world: World, asset: Asset, from: string): void {
const editor = getDocumentEditor(world);
for (const entity of world.query(AssetId)) {
- if (entity.get(AssetId)?.value !== asset.id) continue;
const src = authoredElement(entity)?.props.src;
if (src !== from) continue;
editor.editProperty(entity, 'src', asset.path);
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 8e234d41..d855c1e5 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -7,12 +7,14 @@ import {
Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,
Paint, PaintType, Project, Source,
} from "@diffusionstudio/runtime";
+import type { SourceModifierValues } from "@diffusionstudio/runtime";
import { createEncoder } from "@diffusionstudio/encoder";
import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";
import {
PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
+ PROMPT_INPUT_VOICE_MODEL,
PROMPT_INPUT_VOICE_OPTIONS,
} from "@/components/genai/config";
import { assert } from "@/utils";
@@ -22,12 +24,10 @@ import { trpc } from "@/lib/trpc";
import { toast } from "somoto";
import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
-import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
+import type { Asset, AssetLibrary, AssetType } from "@diffusionstudio/assets";
+import type { FileRef } from "@diffusionstudio/api-contract";
import type { Entity, World } from "koota";
-// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).
-const VOICE_MODEL = "elevenlabs-v3";
-
/** What a failure is called where the user reads about it. */
const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
image: "Image generation failed",
@@ -37,6 +37,31 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
transcript: "Caption generation failed",
};
+/**
+ * One step of a source modifier (see the runtime's `SourceModifiers`): a
+ * model call that makes a new asset out of one the library already holds.
+ * An element asks for these by prop and goes on naming what it was made
+ * from, so each step is stored and cached on its own, and a set of modifiers
+ * is these run in order (see `derive`).
+ */
+export type TransformKind = "remove-background" | "upscale" | "add-audio";
+
+/** What each step is called where the user reads about it, and names its result. */
+const TRANSFORMS: Record<TransformKind, { title: string; suffix: string }> = {
+ "remove-background": { title: "Background removal failed", suffix: "Background removed" },
+ "upscale": { title: "Upscale failed", suffix: "Upscaled" },
+ "add-audio": { title: "Adding audio failed", suffix: "With audio" },
+};
+
+/** The steps a set of modifiers comes to, in the order they are applied. */
+function steps(modifiers: SourceModifierValues): TransformKind[] {
+ const kinds: TransformKind[] = [];
+ if (modifiers.removeBackground) kinds.push("remove-background");
+ if (modifiers.upscale > 1) kinds.push("upscale");
+ if (modifiers.addAudio) kinds.push("add-audio");
+ return kinds;
+}
+
/**
* A spec with defaults applied and every `AssetInput` reduced to an asset id.
* Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.
@@ -44,8 +69,8 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
type ResolvedSpec =
| { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }
| { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }
- | { type: "voice"; model: string; prompt: string; voice: string }
- | { type: "audio"; model: string; prompt: string; duration?: number };
+ | { type: "voice"; model: string; prompt: string; voice: string; seed?: number }
+ | { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };
/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */
export function attachAi(world: World, library: AssetLibrary): EditorGenAi {
@@ -222,9 +247,10 @@ export class EditorGenAi extends GenAi {
case "voice": {
return {
type: "voice",
- model: VOICE_MODEL,
+ model: PROMPT_INPUT_VOICE_MODEL,
prompt: spec.prompt,
voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,
+ seed: spec.seed,
};
}
case "audio": {
@@ -233,6 +259,7 @@ export class EditorGenAi extends GenAi {
model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,
prompt: spec.prompt,
duration: spec.duration,
+ seed: spec.seed,
};
}
}
@@ -254,14 +281,7 @@ export class EditorGenAi extends GenAi {
const { name, results, generationId } = await this.requestGeneration(spec);
assert(results.length > 0, "No results returned from the model");
- const url = results[0].url;
- const response = await fetch(url);
- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
- const blob = await response.blob();
- const extension = url.split(/[?#]/)[0].split(".").pop();
- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
-
- const asset = await this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
+ const asset = await this.store(results[0].url, name, { key: generationKey, id: generationId });
track("generation_completed", {
mode: spec.type,
model: spec.model,
@@ -279,6 +299,108 @@ export class EditorGenAi extends GenAi {
}
}
+ /**
+ * `asset` through every modifier the element asked for, one step at a time
+ * and in a fixed order. Each step is cached in its own right, so turning
+ * one on leaves what the others already made alone — adding `upscale` to a
+ * cut-out picture pays for the enlarging, not for the matte again.
+ */
+ public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {
+ let derived = asset;
+ for (const kind of steps(modifiers)) {
+ derived = await this.transform(kind, derived);
+ }
+ return derived;
+ }
+
+ /**
+ * Runs one step over `asset` and returns what it produced, stored under
+ * `generated/` like any other model output. Keyed by step and input, so the
+ * same call on the same asset is the same result in this session and the
+ * next: upscaling a picture twice costs what upscaling it once did.
+ */
+ public async transform(kind: TransformKind, asset: Asset): Promise<Asset> {
+ // No upscale factor in the key: the endpoint takes none, so every factor
+ // is the same call and would otherwise be billed once per number asked
+ // for. It belongs here the moment the API can be told one.
+ const key = `transform:v1:${kind}:${asset.id}`;
+
+ const cached = this.library.list().find((entry) => entry.generation?.key === key);
+ if (cached) return cached;
+
+ const running = this.inflight.get(key);
+ if (running) return await running;
+
+ const promise = this.runTransform(kind, asset, key);
+ this.inflight.set(key, promise);
+ try {
+ return await promise;
+ } catch (error) {
+ throw reportFailure(error, TRANSFORMS[kind].title);
+ } finally {
+ this.inflight.delete(key);
+ }
+ }
+
+ /** Uploads the input, runs the call, and stores the result beside the generations. */
+ private async runTransform(kind: TransformKind, asset: Asset, key: string): Promise<Asset> {
+ const startedAt = performance.now();
+ track("generation_started", { mode: kind });
+
+ try {
+ console.log(`[gen-ai] running ${kind} on ${asset.path}`);
+ const input = await this.uploadInput(asset.id);
+ const { url, generationId } = await this.requestTransform(kind, asset, input);
+
+ const base = assetName(asset).replace(/\.[^.]+$/, "");
+ const stored = await this.store(url, `${base} (${TRANSFORMS[kind].suffix})`, { key, id: generationId });
+
+ track("generation_completed", {
+ mode: kind,
+ duration_ms: Math.round(performance.now() - startedAt),
+ });
+ return stored;
+ } catch (err) {
+ track("generation_failed", {
+ mode: kind,
+ duration_ms: Math.round(performance.now() - startedAt),
+ error: err instanceof Error ? err.message.slice(0, 200) : "unknown",
+ });
+ throw err;
+ }
+ }
+
+ private requestTransform(kind: TransformKind, asset: Asset, input: FileRef) {
+ switch (kind) {
+ case "remove-background":
+ assert(asset.type === "IMAGE", "Only a picture has a background to remove");
+ return trpc.removeBackground.mutate({ image: input });
+ case "upscale":
+ return isMoving(asset.type)
+ ? trpc.upscaleVideo.mutate({ video: input })
+ : trpc.upscaleImage.mutate({ image: input });
+ case "add-audio":
+ assert(isMoving(asset.type), "Only footage can be scored");
+ return trpc.addAudioToVideo.mutate({ video: input });
+ }
+ }
+
+ /**
+ * Downloads what a model returned and files it in the library. The name is
+ * the model's, the extension the URL's — what came back decides what the
+ * file is called, not what was asked for.
+ */
+ private async store(url: string, name: string, generation: { key: string; id?: string | null }): Promise<Asset> {
+ const response = await fetch(url);
+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
+ const blob = await response.blob();
+
+ const extension = url.split(/[?#]/)[0].split(".").pop();
+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
+
+ return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation });
+ }
+
private requestGeneration(spec: ResolvedSpec) {
switch (spec.type) {
case "image": {
@@ -321,6 +443,7 @@ export class EditorGenAi extends GenAi {
model: spec.model,
prompt: spec.prompt,
voice: spec.voice,
+ seed: spec.seed,
});
}
case "audio": {
@@ -328,6 +451,7 @@ export class EditorGenAi extends GenAi {
model: spec.model,
prompt: spec.prompt,
duration: spec.duration,
+ seed: spec.seed,
});
}
}
@@ -343,6 +467,9 @@ export class EditorGenAi extends GenAi {
}
}
+/** Whether an asset type is footage, for the calls that only take footage. */
+const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";
+
/**
* Says what a generation failed with, and hands the error on. The message is
* what the element ends up carrying too (the runtime's `SourceError`, which
diff --git a/apps/web/src/utils/genai.ts b/apps/web/src/utils/genai.ts
deleted file mode 100644
index b9009ee8..00000000
--- a/apps/web/src/utils/genai.ts
+++ /dev/null
@@ -1,177 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { query, Not, Or } from "bitecs";
-import { ChildOf, PaintType, aabbFromTransformedRect, aabbsIntersect, invert2D, transformPoint, getEntityTree, createEntity, setComponent, appendChild, addComponent, removeComponent, createFolder, ensureFolder, uniqueFolderName, type AABB } from "@/components/engine";
-import { MAIN_CANVAS_ID } from "@/components/canvas";
-
-import type { EngineWorld } from "@/components/engine";
-
-const GEN_AI_FOLDER_NAME = 'Gen AI';
-
-/**
- * Map aspect ratio to pixel dimensions assuming 1080p as baseline.
- */
-export const ASPECT_RATIO_DIMENSIONS: Record<string, { width: number; height: number }> = {
- "16:9": { width: 1920, height: 1080 },
- "9:16": { width: 1080, height: 1920 },
- "1:1": { width: 1080, height: 1080 },
- "4:3": { width: 1440, height: 1080 },
- "3:4": { width: 1080, height: 1440 },
-};
-
-/**
- * Get the world-space center of the current viewport.
- */
-export function getViewportCenter(world: EngineWorld) {
- const canvasEl = document.getElementById(MAIN_CANVAS_ID) as HTMLCanvasElement | null;
- if (!canvasEl) return { x: 0, y: 0 };
- const rect = canvasEl.getBoundingClientRect();
- const invMat = invert2D(world.camera);
- const pt = transformPoint(invMat, rect.width / 2, rect.height / 2);
- return pt ?? { x: 0, y: 0 };
-}
-
-/**
- * Get the world-space AABB of the current viewport.
- */
-function getViewportBounds(world: EngineWorld): AABB | null {
- const canvasEl = document.getElementById(MAIN_CANVAS_ID) as HTMLCanvasElement | null;
- if (!canvasEl) return null;
- const rect = canvasEl.getBoundingClientRect();
- const invMat = invert2D(world.camera);
- const tl = transformPoint(invMat, 0, 0);
- const tr = transformPoint(invMat, rect.width, 0);
- const br = transformPoint(invMat, rect.width, rect.height);
- const bl = transformPoint(invMat, 0, rect.height);
- return {
- minX: Math.min(tl.x, tr.x, br.x, bl.x),
- minY: Math.min(tl.y, tr.y, br.y, bl.y),
- maxX: Math.max(tl.x, tr.x, br.x, bl.x),
- maxY: Math.max(tl.y, tr.y, br.y, bl.y),
- };
-}
-
-/**
- * Find a placement spot for a `width`x`height` block on the canvas.
- */
-export function findEmptyPlacement(
- world: EngineWorld,
- width: number,
- height: number,
- gap: number,
- extra: AABB[] = [],
-): { x: number; y: number } {
- const center = getViewportCenter(world);
- const viewport = getViewportBounds(world);
-
- const c = world.components;
- const LocalTransform = c.LocalTransform;
- const Computed = c.Computed;
-
- // `extra` boxes are content the world can't report yet — e.g. sibling roots
- // placed earlier in the same mount, whose layout has not been computed. They
- // are avoided and anchored beside like any existing box.
- const boxes: AABB[] = [...extra];
- for (const cid of query(world, [Or(c.Geometry, c.Group), Not(ChildOf('*')), Not(c.Deleted), Not(c.Hidden), Not(c.Generating)])) {
- const w = Computed.width[cid];
- const h = Computed.height[cid];
- if (!w || !h) continue;
- if (LocalTransform.a[cid] === undefined) continue;
- const aabb = aabbFromTransformedRect(
- {
- a: LocalTransform.a[cid], b: LocalTransform.b[cid],
- c: LocalTransform.c[cid], d: LocalTransform.d[cid],
- e: LocalTransform.e[cid], f: LocalTransform.f[cid],
- },
- w,
- h,
- );
- boxes.push(aabb);
- }
-
- if (boxes.length === 0) {
- return center;
- }
-
- const candidates = viewport
- ? boxes.filter((b) => aabbsIntersect(b, viewport))
- : boxes;
- const ordered = (candidates.length > 0 ? candidates : boxes).slice().sort((a, b) => {
- const acx = (a.minX + a.maxX) / 2, acy = (a.minY + a.maxY) / 2;
- const bcx = (b.minX + b.maxX) / 2, bcy = (b.minY + b.maxY) / 2;
- return Math.hypot(acx - center.x, acy - center.y)
- - Math.hypot(bcx - center.x, bcy - center.y);
- });
-
- const overlapsAny = (rect: AABB): boolean => {
- for (const b of boxes) {
- if (aabbsIntersect(rect, b)) return true;
- }
- return false;
- };
-
- for (const cand of ordered) {
- const cy = (cand.minY + cand.maxY) / 2;
- const minY = cy - height / 2;
- const maxY = cy + height / 2;
-
- const rightMinX = cand.maxX + gap;
- const right: AABB = { minX: rightMinX, minY, maxX: rightMinX + width, maxY };
- if (!overlapsAny(right)) {
- return { x: rightMinX + width / 2, y: cy };
- }
-
- const leftMaxX = cand.minX - gap;
- const left: AABB = { minX: leftMaxX - width, minY, maxX: leftMaxX, maxY };
- if (!overlapsAny(left)) {
- return { x: leftMaxX - width / 2, y: cy };
- }
- }
-
- return center;
-}
-
-/**
- * Attach the waveform fill to a generated audio entity.
- */
-export function addWaveformFill(world: EngineWorld, eid: number, assetId: string) {
- const c = world.components;
- const fillEid = createEntity(world);
- setComponent(world, fillEid, c.Paint, PaintType.WAVEFORM);
- setComponent(world, fillEid, c.AssetId, assetId);
- appendChild(world, fillEid, eid);
-}
-
-/**
- * The library folder for a finished generation: every generated asset lands in
- * a root-level "Gen AI" folder, and a multi-variant batch gets its own
- * subfolder inside it, named after the generation. Both are created on demand.
- * Call only after the generation succeeded, so failures leave no empty folders.
- */
-export async function ensureGenerationFolder(
- world: EngineWorld,
- name: string,
- count: number,
-): Promise<string> {
- const genAi = await ensureFolder(world, GEN_AI_FOLDER_NAME, null);
- if (count <= 1) return genAi.id;
-
- const batch = await createFolder(world, uniqueFolderName(world, name, genAi.id), genAi.id);
- return batch.id;
-}
-
-/**
- * Record a finished generation as a single undoable step.
- */
-export function recordGeneration(world: EngineWorld, label: string, rootEids: number[]) {
- const c = world.components;
- const eids = rootEids.flatMap((root) => getEntityTree(world, root));
-
- world.history.push({
- label,
- execute: () => eids.forEach((e) => removeComponent(world, e, c.Deleted)),
- reverse: () => eids.forEach((e) => addComponent(world, e, c.Deleted)),
- });
-}
diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.ts
index 34b6184f..08a5be99 100644
--- a/packages/jsx/src/generate.ts
+++ b/packages/jsx/src/generate.ts
@@ -44,12 +44,14 @@ export type GenerateVoiceOptions = {
prompt: string;
/** Voice id; default the first voice from `dapi voices`. */
voice?: string;
+ seed?: number;
};
export type GenerateAudioOptions = {
prompt: string;
model?: string;
duration?: number;
+ seed?: number;
};
export type AssetSpecInput =
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 50257f1c..12a874ba 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -295,6 +295,22 @@ type MediaProps = {
error?: string;
};
+/**
+ * Model calls the source is put through before the element shows it. The
+ * `src` goes on naming what it was made from, so taking a modifier off gives
+ * the original back; what they made is cached by source and modifiers, so it
+ * is made once however many elements ask for it, and adding a second
+ * modifier does not re-run the first. Applied in the order below.
+ */
+type UpscaleProps = {
+ /**
+ * Resolution multiplier: 2 asks for twice the pixels. Enlarges the source,
+ * not the box — the element keeps the width and height it was given, and
+ * renders sharper. Default 1, the source as it is.
+ */
+ upscale?: number;
+};
+
type FitProps = {
/** How the source maps into the box. Default "cover" on `<Video>`, "contain" on `<Image>`. */
objectFit?: Fit;
@@ -572,12 +588,24 @@ export type ColorStopProps = ColorProps & OpacityProps & TrackChildren & {
*/
export type MediaPaintProps = PaintProps & MediaProps & FitProps & TrackChildren;
-export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps & {
+export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps & UpscaleProps & {
+ /**
+ * Scores the footage: a generated soundtrack for a clip that has none. See
+ * `UpscaleProps` for what a modifier is; applied last, after `upscale`, so
+ * a re-encode cannot drop the track. Independent of `volume` and `muted`,
+ * which mix whatever track the clip ends up with.
+ */
+ addAudio?: boolean;
/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */
children?: SolidJSX.Element;
};
-export type ImageProps = CommonProps & MediaProps & FitProps & {
+export type ImageProps = CommonProps & MediaProps & FitProps & UpscaleProps & {
+ /**
+ * Cuts the subject out, leaving the rest of the picture transparent. See
+ * `UpscaleProps` for what a modifier is; applied before `upscale`.
+ */
+ removeBackground?: boolean;
/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */
children?: SolidJSX.Element;
};
@@ -638,7 +666,13 @@ export type SurfacePaintProps = PaintProps & {
/** `<Surface>` — a rectangle carrying a `<SurfacePaint>` with the given ref. */
export type SurfaceProps = CommonProps & Pick<SurfacePaintProps, "ref">;
-export type AudioProps = IdentityProps & TimingProps & MediaProps & AudioTrackProps & {
+/**
+ * `<audio>` — a clip with a sound and no picture. It draws nothing inside a
+ * scene, but on the canvas it is still something to point at: the editor
+ * shows its waveform in a box, and `x`/`y`/`width`/`height` are where that
+ * box is. Left off inside a scene, where they mean nothing.
+ */
+export type AudioProps = IdentityProps & PositionProps & SizeProps & TimingProps & MediaProps & AudioTrackProps & {
/** `<KeyframeTrack>` (a `volume` track) and `<Animation>` children. */
children?: SolidJSX.Element;
};
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 7ba1779e..3ffcf46f 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
@@ -63,6 +63,15 @@ const Authored = trait(() => ({ tag: '', props: {} as Record<string, unknown> })
/** Props that address or wire an element rather than describe it. */
const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);
+/** What an element with no `SourceModifiers` trait is asking for: nothing. */
+const NO_MODIFIERS = { removeBackground: false, upscale: 1, addAudio: false };
+
+/** `upscale` as a factor; anything that is not one above 1 is natural size. */
+function upscaleFactor(value: unknown): number {
+ const factor = typeof value === 'number' ? value : Number(value);
+ return Number.isFinite(factor) && factor > 1 ? factor : 1;
+}
+
export interface AuthoredElement {
/** The camelCase tag the project used. */
tag: string;
@@ -1168,6 +1177,47 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
});
return;
}
+ case 'removeBackground':
+ case 'addAudio':
+ case 'upscale': {
+ const current = entity.get(SourceModifiers) ?? NO_MODIFIERS;
+ const next = {
+ ...current,
+ [name]: name === 'upscale' ? upscaleFactor(value) : value === true,
+ };
+
+ if (
+ next.removeBackground === current.removeBackground
+ && next.upscale === current.upscale
+ && next.addAudio === current.addAudio
+ ) return;
+
+ if (hasModifier(next)) {
+ entity.add(SourceModifiers);
+ entity.set(SourceModifiers, next);
+ } else {
+ entity.remove(SourceModifiers);
+ }
+
+ if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;
+
+ const src = entity.get(Authored)?.props.src;
+ if (src === undefined || src === null || src === '') return;
+
+ // A resolution running for the old modifiers must not bind late.
+ entity.remove(PendingSource, Generating);
+
+ if (typeof src === 'string') {
+ entity.add(LoadRequest);
+ entity.set(LoadRequest, { value: src });
+ return;
+ }
+
+ entity.add(GenerationRequest);
+ entity.set(GenerationRequest, { ref: src as AssetRef });
+
+ return;
+ }
case 'seed': {
if (!entity.has(Caption)) return;
// An authored src mounts a transcript directly; there is no
diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.ts
index 58f16e7b..785b4fa9 100644
--- a/packages/runtime/src/actions/assets.ts
+++ b/packages/runtime/src/actions/assets.ts
@@ -6,7 +6,7 @@
// goes through the AssetLibrary (@diffusionstudio/assets); these are the
// lookups the decoders and hosts share.
-import { Ai, AssetId, Library, Paint } from '../traits';
+import { Ai, AssetId, Library, Paint, SourceModifiers } from '../traits';
import { PaintType } from '../constants';
import type { Entity, World } from 'koota';
@@ -27,6 +27,28 @@ export function getAi(world: World): GenAi {
return ai;
}
+/** What a source is put through after it resolves (see `SourceModifiers`). */
+export interface SourceModifierValues {
+ removeBackground: boolean;
+ /** Factor, 1 = natural size. */
+ upscale: number;
+ addAudio: boolean;
+}
+
+/** Whether a set of modifiers asks for anything at all. */
+export const hasModifier = (modifiers: SourceModifierValues): boolean =>
+ modifiers.removeBackground || modifiers.upscale > 1 || modifiers.addAudio;
+
+/**
+ * The modifiers `entity` asks its source to be put through, or undefined
+ * when it asks for none — which is what the trait's absence means, so this
+ * is also the question "is this element showing a derived source".
+ */
+export function getModifiers(entity: Entity): SourceModifierValues | undefined {
+ const modifiers = entity.get(SourceModifiers);
+ return modifiers && hasModifier(modifiers) ? modifiers : undefined;
+}
+
/**
* Binds an entity to an asset: stamps its AssetId, and follows the asset
* with the paint — a frames directory on a `<video>` or `<image>` plays as
diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.ts
index 18bc7910..25e6516c 100644
--- a/packages/runtime/src/ai.ts
+++ b/packages/runtime/src/ai.ts
@@ -6,6 +6,7 @@ import { generate } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
+import type { SourceModifierValues } from './actions/assets';
import type {
AssetRef,
GenerateAudioOptions,
@@ -31,6 +32,14 @@ export abstract class GenAi {
*/
public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;
+ /**
+ * `asset` put through the modifiers an element asked of its source (see
+ * `SourceModifiers`): the same asset and the same modifiers are the same
+ * result, in this session and the next, and every step is cached on its
+ * own, so adding one to an element does not re-run the others.
+ */
+ public abstract derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset>;
+
/**
* The imperative surface: `ai.generate.image({...})` declares and resolves
* in one call, returning the finished asset.
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index ea68dd7b..675a7918 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, SourceError, TranscriptionRequest } from '../traits';
-import { bindAsset } from '../actions/assets';
+import { bindAsset, getModifiers } from '../actions/assets';
import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';
import type { Entity, World } from 'koota';
@@ -15,7 +15,7 @@ export function assetSystem(world: World): void {
for (const entity of world.query(LoadRequest)) {
const source = entity.get(LoadRequest)!.value;
entity.remove(LoadRequest);
- resolve(world, entity, source, library.resolve(source));
+ start(world, entity, library.resolve(source));
}
}
@@ -25,7 +25,7 @@ export function assetSystem(world: World): void {
const ref = entity.get(GenerationRequest)!.ref;
entity.remove(GenerationRequest);
if (ref === null || entity.has(SourceError)) continue;
- resolve(world, entity, ref, ai.resolve(ref), true);
+ start(world, entity, ai.resolve(ref), true);
}
for (const entity of world.query(TranscriptionRequest)) {
@@ -35,11 +35,29 @@ export function assetSystem(world: World): void {
const seed = entity.get(TranscriptionRequest)!.seed;
entity.remove(TranscriptionRequest);
if (entity.has(SourceError)) continue;
- resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed), true);
+ resolve(world, entity, ai.transcribe(world, scene, seed), true);
}
}
}
+/**
+ * Starts a resolution for whatever the element's src named, putting it
+ * through the modifiers the element asks of it (see `SourceModifiers`). The
+ * two are one wait: the element binds the asset it is going to show, not
+ * first the one it was made from.
+ */
+function start(world: World, entity: Entity, base: Promise<Asset>, generating = false): void {
+ const modifiers = getModifiers(entity);
+ const ai = world.get(Ai);
+
+ if (!modifiers || !ai) {
+ resolve(world, entity, base, generating);
+ return;
+ }
+
+ resolve(world, entity, base.then((asset) => ai.derive(asset, modifiers)), true);
+}
+
/**
* Whether anything in `scene`'s subtree (besides `except`, the requesting
* element itself) is still waiting on a source: a request this system has
@@ -56,24 +74,28 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean
}
/**
- * Tracks one started resolution: the entity remembers what it is waiting on
- * (PendingSource), so of overlapping resolutions only the latest binds, and
- * one that outlives its element (or its src) is dropped.
+ * Tracks one started resolution: the entity remembers which one it is waiting
+ * on (PendingSource), so of overlapping resolutions only the latest binds, and
+ * one that outlives its element (or its src) is dropped. The wait itself is
+ * the identity rather than what was asked for — the same src put through
+ * different modifiers is a different answer, and the later question is always
+ * the one being asked.
*/
-function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>, generating = false): void {
+function resolve(world: World, entity: Entity, promise: Promise<Asset>, generating = false): void {
+ const token = {};
entity.remove(SourceError);
entity.add(PendingSource);
- entity.set(PendingSource, { value });
+ entity.set(PendingSource, { value: token });
if (generating) entity.add(Generating);
const done = promise.then(
(asset) => {
- if (!current(entity, value)) return;
+ if (!current(entity, token)) return;
entity.remove(PendingSource, Generating);
bindAsset(entity, asset);
},
(error: unknown) => {
- if (!current(entity, value)) return;
+ if (!current(entity, token)) return;
entity.remove(PendingSource, Generating);
entity.add(SourceError);
entity.set(SourceError, { value: errorMessage(error), generated: generating });
@@ -90,6 +112,6 @@ function errorMessage(error: unknown): string {
return message.trim() || 'Something went wrong';
}
-function current(entity: Entity, value: unknown): boolean {
- return entity.isAlive() && entity.get(PendingSource)?.value === value;
+function current(entity: Entity, token: object): boolean {
+ return entity.isAlive() && entity.get(PendingSource)?.value === token;
}
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index 408263ca..ad77b4dd 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -85,6 +85,16 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))
// through the library.
export const LoadRequest = trait({ value: '' });
+// Model calls the src is put through once it resolves: the element shows
+// what they made of what it named, and the src goes on naming the original,
+// so taking a modifier off gives it back. Every field's default is off —
+// `upscale` 1 is natural size — and the trait is only present while one is
+// not. Applied in the order declared here: the matte before the enlarging
+// (cheaper, and the model wants a normally-sized picture), the audio last so
+// a re-encode cannot drop it. Each step is cached on its own, so turning one
+// on does not re-pay for the ones already applied.
+export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addAudio: false });
+
// On a `<captions>` element without a src: transcribe the scene it sits
// under, through the world's Ai. The seed keys the take — the transcript is
// cached by scene id + seed, so re-running with a seed replays that take and
diff --git a/reference/jsx/audio.md b/reference/jsx/audio.md
index 01fb20c1..22dac42d 100644
--- a/reference/jsx/audio.md
+++ b/reference/jsx/audio.md
@@ -1,6 +1,6 @@
# `<audio>`
-An audio clip: no visual output, no spatial props; carries volume. When timing is omitted, the node fits its natural duration (see [timing.md](./timing.md)).
+An audio clip: no picture, carries volume. When timing is omitted, the node fits its natural duration (see [timing.md](./timing.md)). It draws nothing inside a scene; placed directly on the canvas (see [roots.md](./roots.md)) the editor shows its waveform in a box, which `x`/`y`/`width`/`height` place.
```tsx
<audio src="https://my.videoarchive.com/audio/clip.wav" start={2.2} sourceOut={16} volume={-6} />
@@ -12,6 +12,8 @@ An audio clip: no visual output, no spatial props; carries volume. When timing i
| ---- | ---- | ------- | ------- |
| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |
| `name` | `string` | none | Human-readable node name. |
+| `x`, `y` | `number` | `0` | Where the waveform box sits on the canvas. No meaning inside a scene. |
+| `width`, `height` | `number` | `500`, `150` | Size of that box. |
| `start`, `end`, `sourceIn`, `sourceOut` | `Time` | see [timing.md](./timing.md) | Temporal placement. |
| `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. |
| `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |
diff --git a/reference/jsx/generate.md b/reference/jsx/generate.md
index 834e119b..19e2df8c 100644
--- a/reference/jsx/generate.md
+++ b/reference/jsx/generate.md
@@ -61,12 +61,14 @@ generate.video(opts: {
generate.voice(opts: {
prompt: string; // required: the text to speak
voice?: string; // default: first voice from `dapi voices`
+ seed?: number;
}): AssetRef;
generate.audio(opts: {
prompt: string; // required
model?: string; // default: first model from `dapi models audio`
duration?: number; // seconds; default 30 for music, the model's own default for sfx
+ seed?: number;
}): AssetRef;
```
diff --git a/reference/jsx/image.md b/reference/jsx/image.md
index 245e0f6e..8e37eb8d 100644
--- a/reference/jsx/image.md
+++ b/reference/jsx/image.md
@@ -14,5 +14,7 @@ All [common props](./elements.md#common-props), plus:
| ---- | ---- | ------- | ------- |
| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |
| `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. |
+| `removeBackground` | `boolean` | `false` | Cuts the subject out, leaving the rest transparent (see [media.md](./media.md#source-modifiers)). |
+| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |
A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the image renders through the shader.
diff --git a/reference/jsx/media.md b/reference/jsx/media.md
index ff64c529..2db5a1e5 100644
--- a/reference/jsx/media.md
+++ b/reference/jsx/media.md
@@ -8,6 +8,24 @@
- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.
- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is inserted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.
+## Source modifiers
+
+`<image>` and `<video>` take props that put the source through a model before the element shows it:
+
+| Prop | On | Meaning |
+| ---- | -- | ------- |
+| `removeBackground` | `<image>` | Cuts the subject out, leaving the rest transparent. |
+| `upscale` | both | Resolution multiplier — `upscale={2}` asks for twice the pixels. Enlarges the source, not the box. |
+| `addAudio` | `<video>` | Scores footage that has no sound. |
+
+```tsx
+<image src="footage/fox.png" removeBackground upscale={2} width={800} height={450} />
+```
+
+The `src` goes on naming what the picture was made from, so taking a modifier off gives the original back — nothing is overwritten and nothing is lost. Results are cached by source and modifiers, like [generated assets](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and adding a second modifier does not re-run the first. They are applied in the order of the table above, whatever order they are written in.
+
+Modifiers compose with declarations: `<image src={generate.image({ prompt: "a red fox" })} removeBackground />` generates, then cuts out.
+
## The library
A project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/` for files the app produced), and what it was found to be. Folders are virtual: they are the prefixes of the paths. Renaming or moving an asset in the panel rewrites the `src` props that named it. Files placed under `assets/` by hand are taken into the library on the next load.
diff --git a/reference/jsx/video.md b/reference/jsx/video.md
index 4c2484f8..54a4b834 100644
--- a/reference/jsx/video.md
+++ b/reference/jsx/video.md
@@ -17,5 +17,7 @@ All [common props](./elements.md#common-props), plus:
| `volume` | `Animatable<number>` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. |
| `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |
| `syncTo` | `string` | none | Key of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |
+| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |
+| `addAudio` | `boolean` | `false` | Scores footage that has no sound (see [media.md](./media.md#source-modifiers)). |
A paint child draws over the media paint created by `src` (see [paints.md](./paints.md)); a [`<shaderPaint>`](./shader-paint.md) child instead post-processes it, so the frame renders through the shader.