Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor and enhance JSX editing capabilities

konstantin-paulus71a7f30a6881
diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.ts
index 72c68437..bf0adb81 100644
--- a/apps/desktop/src/edit.ts
+++ b/apps/desktop/src/edit.ts
@@ -9,9 +9,9 @@ import { join } from "node:path";
import { IndentationText, Project, SyntaxKind } from "ts-morph";
-import { ID_ATTR, formatSource, isCompositionTag, isLoopTag, parseSource } from "@diffusionstudio/jsx";
+import { ID_ATTR, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, parseSource } from "@diffusionstudio/jsx";
-import type { PropValue } from "@diffusionstudio/jsx";
+import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";
import type {
ArrowFunction,
FunctionExpression,
@@ -24,7 +24,14 @@ import type {
SourceFile,
} from "ts-morph";
-export type { PropValue };
+export type { PropValue, SerializedAssetRef };
+
+/**
+ * A value an edit can carry: what a source spells as a literal, or a
+ * `generate.*` declaration in its wire form, spelled as the call that
+ * reproduces it (see `setProp`).
+ */
+export type EditValue = PropValue | SerializedAssetRef;
export interface SourceContext {
/** Absolute path of the project folder. */
@@ -42,7 +49,7 @@ export interface SourceContext {
export interface SourceSet {
kind: "set";
source: string;
- props: Record<string, PropValue>;
+ props: Record<string, EditValue>;
text?: string;
}
@@ -59,7 +66,7 @@ export interface SourceInsert {
source: string;
parent: string;
tag: string;
- props: Record<string, PropValue>;
+ props: Record<string, EditValue>;
before?: string;
text?: string;
}
@@ -99,7 +106,7 @@ export interface SourceRemove {
* one in `ids`. The first iteration keeps the body's own names, so it carries
* none.
*/
-export type SourceIteration = Record<string, { props: Record<string, PropValue>; text?: string; pending?: string }>;
+export type SourceIteration = Record<string, { props: Record<string, EditValue>; text?: string; pending?: string }>;
/**
* Replaces the `<For>`/`<Index>` around the element named by `source` with
@@ -245,14 +252,29 @@ const initializerText = (value: PropValue): string =>
? `"${value}"`
: `{${literalText(value)}}`;
+/** A declaration's wire form as the `generate.*` call that reproduces it. */
+function generateCallText(ref: SerializedAssetRef): string {
+ const { type, ...options } = ref.$generate;
+ return `generate.${type}(${literalText(options as Record<string, PropValue>)})`;
+}
+
/**
* Writes a prop onto a tag as one would write it: `muted`, not `muted={true}`,
* and no attribute at all rather than `muted={false}`, since absence is what a
- * boolean prop's false reads as.
+ * boolean prop's false reads as. A declaration is spelled as its `generate.*`
+ * call — the caller makes sure `generate` is imported (see
+ * `ensureGenerateImport`).
*/
-function setProp(tag: JsxTag, name: string, value: PropValue): void {
+function setProp(tag: JsxTag, name: string, value: EditValue): void {
const attribute = attributeOf(tag, name);
+ if (isSerializedAssetRef(value)) {
+ const initializer = `{${generateCallText(value)}}`;
+ if (attribute) attribute.setInitializer(initializer);
+ else tag.addAttribute({ name, initializer });
+ return;
+ }
+
if (value === true) {
if (attribute) attribute.removeInitializer();
else tag.addAttribute({ name });
@@ -268,6 +290,40 @@ function setProp(tag: JsxTag, name: string, value: PropValue): void {
else tag.addAttribute({ name, initializer: initializerText(value) });
}
+/** Where `generate` comes from — the module every project authors against. */
+const GENERATE_MODULE = "@diffusionstudio/jsx";
+
+/**
+ * Makes sure `generate` is in scope once a declaration has been spelled into
+ * the file: added to the module's existing import, or as an import of its own
+ * after the last one — or above the first statement, past the file's header
+ * comments, when there are no imports at all. Idempotent.
+ */
+function ensureGenerateImport(sourceFile: SourceFile): void {
+ const declarations = sourceFile.getImportDeclarations();
+ const importable = declarations.find(
+ (declaration) =>
+ declaration.getModuleSpecifierValue() === GENERATE_MODULE &&
+ !declaration.isTypeOnly() &&
+ !declaration.getNamespaceImport(),
+ );
+
+ if (importable) {
+ const named = importable.getNamedImports();
+ if (named.some((specifier) => specifier.getName() === "generate" && !specifier.getAliasNode())) return;
+ importable.addNamedImport("generate");
+ return;
+ }
+
+ const statement = `import { generate } from "${GENERATE_MODULE}";`;
+ const last = declarations.at(-1);
+ if (last) sourceFile.insertText(last.getEnd(), `\n${statement}`);
+ else {
+ const first = sourceFile.getStatements()[0];
+ sourceFile.insertText(first ? first.getStart() : 0, `${statement}\n\n`);
+ }
+}
+
/**
* Text content as JSX spells it: bare when JSX would read it back verbatim,
* otherwise as a string expression (JSX collapses whitespace around line
@@ -506,9 +562,25 @@ function isLiteral(node: Node): boolean {
}
/**
- * A prop is only written back when the source holds a plain literal. Anything
- * else — a signal, a prop of the surrounding component, an expression — is
- * someone's reactivity, and a drag has no business overwriting it.
+ * A `generate.*` call over a literal spec: what the writer spells a
+ * declaration as, and so an initializer it may also overwrite. One whose
+ * argument computes anything is authored reactivity, like any expression.
+ */
+function isGenerateCall(node: Node): boolean {
+ const call = node.asKind(SyntaxKind.CallExpression);
+ const callee = call?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);
+ if (!call || !callee || !callee.getExpression().isKind(SyntaxKind.Identifier)) return false;
+ if (callee.getExpression().getText() !== "generate") return false;
+
+ const args = call.getArguments();
+ return args.length === 1 && isLiteral(args[0]!);
+}
+
+/**
+ * A prop is only written back when the source holds a plain literal (or a
+ * declaration the writer itself spells). Anything else — a signal, a prop of
+ * the surrounding component, an expression — is someone's reactivity, and a
+ * drag has no business overwriting it.
*/
function isWritable(attribute: JsxAttribute): boolean {
const initializer = attribute.getInitializer();
@@ -517,7 +589,7 @@ function isWritable(attribute: JsxAttribute): boolean {
if (!initializer.isKind(SyntaxKind.JsxExpression)) return false;
const expression = initializer.getExpression();
- return expression !== undefined && isLiteral(expression);
+ return expression !== undefined && (isLiteral(expression) || isGenerateCall(expression));
}
// ---------------------------------------------------------------------------
@@ -817,6 +889,7 @@ class SourceWriter {
}
setProp(tag, name, value);
+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
wrote = true;
}
@@ -888,6 +961,7 @@ class SourceWriter {
insertChild(sourceFile, parent, child, before);
for (const [name, value] of Object.entries(edit.props)) {
setProp(findTag(sourceFile, id)!, name, value);
+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
}
ids[edit.source] = formatSource(file, id);
return true;
@@ -1127,6 +1201,7 @@ class SourceWriter {
}
for (const [name, value] of Object.entries(record.props)) {
setProp(found(), name, value);
+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
}
const element = elementOf(found());
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsx
index 8ee688d3..862d41c8 100644
--- a/apps/web/src/components/genai/action-bar.tsx
+++ b/apps/web/src/components/genai/action-bar.tsx
@@ -13,15 +13,8 @@ import {
DropdownMenuPortal,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
-import {
- Dialog,
- DialogContent,
- DialogHeader,
- DialogPortal,
- DialogTitle,
-} from "@/components/ui/dialog";
import { createMemo, Show } from "solid-js";
-import { PaintType, isScene as isSceneEntity, isAudio } from "@/components/engine";
+import { PaintType, isAudio } from "@/components/engine";
import { useEngine } from "@/context/engine";
import { useRemoveBackground } from "./use-remove-background";
import { useUpscaleImage } from "./use-upscale-image";
@@ -73,18 +66,9 @@ export function ActionBar(props: ActionBarProps) {
const { generate: generateVideo } = useGenerateVideo();
const { generate: generateVoice } = useGenerateVoice();
const { generate: generateAudio } = useGenerateAudio();
- const { generate: autoCaptions, status: autoCaptionsStatus } = useAutoCaptions();
+ const { generate: autoCaptions, hasScene } = useAutoCaptions();
const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
- const autoCaptionsStatusText = createMemo(() => {
- switch (autoCaptionsStatus()) {
- case "encoding": return "Encoding audio…";
- case "uploading": return "Uploading audio…";
- case "transcribing": return "Transcribing audio…";
- default: return "";
- }
- });
-
const isImage = createMemo(() => {
return Array.from(selectedNodes()).some(eid => {
const mediaType = getNodeMediaType(world, eid);
@@ -99,12 +83,8 @@ export function ActionBar(props: ActionBarProps) {
});
});
- const isScene = createMemo(() => {
- return Array.from(selectedNodes()).some(eid => isSceneEntity(world, eid));
- });
-
const visible = createMemo(() => {
- return isImage() || isVideo() || isScene();
+ return isImage() || isVideo() || hasScene();
})
const handleRerun = () => {
@@ -151,33 +131,12 @@ export function ActionBar(props: ActionBarProps) {
props.openPromptInput?.(config);
};
- const handleAutoCaptions = async () => {
- for (const eid of selectedNodes()) {
- if (isSceneEntity(world, eid)) {
- await autoCaptions(eid);
- }
- }
- };
-
return (
<>
- <Dialog open={autoCaptionsStatus() !== null}>
- <DialogPortal>
- <DialogContent class="sm:max-w-sm">
- <DialogHeader>
- <DialogTitle>Auto-Captions</DialogTitle>
- </DialogHeader>
- <div class="flex items-center gap-2 text-sm text-muted-foreground">
- <Icon name="spinner-loader" class="size-6 animate-spin" />
- {autoCaptionsStatusText()}
- </div>
- </DialogContent>
- </DialogPortal>
- </Dialog>
<Show when={visible()}>
<div class="absolute bottom-16 left-1/2 -translate-x-1/2 z-10 rounded-xl p-1 bg-background border border-border flex gap-1 items-center">
- <Show when={isScene()}>
- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={handleAutoCaptions}>
+ <Show when={hasScene()}>
+ <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={autoCaptions}>
<Icon name="captions" />
Auto-Captions
</Button>
diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsx
index 568e1671..058da27a 100644
--- a/apps/web/src/components/genai/prompt-input.tsx
+++ b/apps/web/src/components/genai/prompt-input.tsx
@@ -51,7 +51,6 @@ import { PromptInputActions } from "./prompt-input-actions";
import { useEngine } from "@/context/engine";
import { showFileDialog } from "@/utils";
import type {
- Asset,
AspectRatio,
GenerationConfig,
ImageGenerationConfig,
@@ -64,6 +63,15 @@ import { ChildOf, PaintType, removeComponent } from "@/components/engine";
import { useAssets, loadAsset } from "@/components/engine";
import { useECS } from "@/context/ecs";
import { Not, query } from 'bitecs';
+import { AssetId } from "@diffusionstudio/runtime";
+import { useLibrary } from "@/engine/library";
+import { useEditor, useSelection } from "@/engine/hooks";
+import { droppedFiles, importFiles, pickFiles } from "@/engine/asset-actions";
+import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";
+
+import type { Entity } from "koota";
+import type { Asset as LibraryAsset, AssetCache } from "@diffusionstudio/assets";
+import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";
export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";
@@ -121,6 +129,11 @@ export function PromptInput(props: PromptInputProps) {
const assets = useAssets(world);
const { selectedNodes } = useECS();
const c = world.components;
+ // The runtime the IMAGE mode works against: references come from the
+ // project library and the canvas selection, and land as declarations.
+ const library = useLibrary();
+ const editor = useEditor();
+ const { nodes } = useSelection();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
const { generate: generateVoice } = useGenerateVoice();
@@ -191,11 +204,30 @@ export function PromptInput(props: PromptInputProps) {
selectedImageEntries().map((e) => e.assetId),
);
+ // The image assets of the current canvas selection, from the runtime's
+ // world: entities bound to a library image count as references.
+ const selectedLibraryImages = createMemo(() => {
+ const lib = library();
+ if (!lib) return [];
+
+ const entries: { asset: LibraryAsset; entity: Entity }[] = [];
+ for (const entity of nodes()) {
+ const id = entity.get(AssetId)?.value;
+ const asset = id ? lib.get(id) : undefined;
+ if (asset?.type === "IMAGE") entries.push({ asset, entity });
+ }
+ return entries;
+ });
+
const effectiveImageRefIds = createMemo(() => {
+ const lib = library();
+ if (!lib) return [];
const local = imageConfig()?.imageRefIds ?? [];
- const selected = selectedImageIds().filter((id) => !local.includes(id));
+ const selected = selectedLibraryImages()
+ .map((entry) => entry.asset.id)
+ .filter((id) => !local.includes(id));
return [...local, ...selected]
- .filter((id) => assets.get(id))
+ .filter((id) => lib.get(id))
.slice(0, MAX_IMAGE_REFERENCES);
});
@@ -241,29 +273,27 @@ export function PromptInput(props: PromptInputProps) {
patch({ imageRefIds: refs.filter((id) => id !== assetId) });
return;
}
- const entry = selectedImageEntries().find((e) => e.assetId === assetId);
- if (entry) {
- removeComponent(world, entry.eid, c.Selected, false);
- }
+ // The reference is shown from a selected canvas entity, so removing it
+ // means deselecting that entity.
+ const entry = selectedLibraryImages().find((e) => e.asset.id === assetId);
+ if (entry) editor.deselect(entry.entity);
};
const openImageReferencesPicker = async () => {
if (effectiveImageRefIds().length >= MAX_IMAGE_REFERENCES) return;
- const handles = await showFileDialog("image/*");
- if (handles.length === 0) return;
+ const lib = library();
+ if (!lib) return;
- try {
- const loadedAssets = await Promise.all(handles.map((handle) => loadAsset(world, handle)));
- const imageAssets = loadedAssets.filter((a) => a.type === "IMAGE");
- const current = imageConfig()?.imageRefIds ?? [];
- patch({
- imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),
- });
- } catch (e) {
- toast("Failed to import assets", {
- description: (e as Error).message,
- });
- }
+ const files = await pickFiles({ accept: "image/*" });
+ if (files.length === 0) return;
+
+ // `importFiles` reports its own failures.
+ const imported = await importFiles(lib, files, "");
+ const imageAssets = imported.filter((a) => a.type === "IMAGE");
+ const current = imageConfig()?.imageRefIds ?? [];
+ patch({
+ imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),
+ });
};
const openVideoFramePicker = async (frame: "start" | "end") => {
@@ -414,9 +444,31 @@ export function PromptInput(props: PromptInputProps) {
dragCounter = 0;
setIsDragging(false);
- const assetIdData = event.dataTransfer?.getData("application/x-asset-id");
+ const assetIdData = event.dataTransfer?.getData(ASSET_DRAG_TYPE);
const existingAssetIds = assetIdData?.split(",").filter(Boolean) ?? [];
+ // References for an image generation live in the project library:
+ // dragged library assets are taken as they are, external files imported.
+ if (mode() === "IMAGE") {
+ const lib = library();
+ if (!lib) return;
+
+ const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));
+ if (existingAssetIds.length === 0 && files.length === 0) return;
+
+ const newAssetIds =
+ existingAssetIds.length > 0
+ ? existingAssetIds
+ : (await importFiles(lib, files, "")).map((a) => a.id);
+
+ const imageIds = newAssetIds.filter((id) => lib.get(id)?.type === "IMAGE");
+ const current = imageConfig()?.imageRefIds ?? [];
+ patch({
+ imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),
+ });
+ return;
+ }
+
const fileItems = Array.from(event.dataTransfer?.items ?? []).filter(
(item) => item.kind === "file" && item.type.startsWith("image/"),
);
@@ -439,11 +491,6 @@ export function PromptInput(props: PromptInputProps) {
if (!c?.endFrameImageId && newAssetIds[1]) {
patch({ endFrameImageId: newAssetIds[1] });
}
- } else {
- const current = imageConfig()?.imageRefIds ?? [];
- patch({
- imageRefIds: [...current, ...newAssetIds].slice(0, MAX_IMAGE_REFERENCES),
- });
}
} catch (e) {
toast("Failed to import assets", {
@@ -495,7 +542,8 @@ export function PromptInput(props: PromptInputProps) {
<For each={effectiveImageRefIds()}>
{(assetId) => (
<PromptInputReferenceImageButton
- asset={assets.get(assetId)!}
+ asset={library()!.get(assetId)!}
+ cache={library()!.cache}
class="size-16"
size={{ width: 64, height: 64 }}
onRemove={() => removeImageReference(assetId)}
@@ -1068,7 +1116,8 @@ function VoiceMenu(props: VoiceMenuProps) {
}
type PromptInputReferenceImageButtonProps = {
- asset: Asset;
+ asset: ThumbnailAsset;
+ cache?: AssetCache;
class?: string;
size?: { width: number; height: number };
onRemove(): void;
@@ -1078,7 +1127,7 @@ function PromptInputReferenceImageButton(props: PromptInputReferenceImageButtonP
return (
<div class={cx("group relative shrink-0", props.class)}>
<div class="size-full overflow-hidden rounded-md bg-input outline-none">
- <AssetThumbnail asset={props.asset} class="size-full" size={props.size} />
+ <AssetThumbnail asset={props.asset} cache={props.cache} class="size-full" size={props.size} />
</div>
<Show when={props.onRemove}>
<button
diff --git a/apps/web/src/components/genai/use-auto-captions.ts b/apps/web/src/components/genai/use-auto-captions.ts
deleted file mode 100644
index 990f3078..00000000
--- a/apps/web/src/components/genai/use-auto-captions.ts
+++ /dev/null
@@ -1,42 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { GeometryType, createEntity, appendChild, setComponent, transcribeScene } from "@/components/engine";
-import { useEngine } from "@/context/engine";
-import { createSignal } from "solid-js";
-import { toast } from "somoto";
-
-import type { TranscribeStatus } from "@/components/engine";
-
-export type AutoCaptionsStatus = TranscribeStatus | null;
-
-export function useAutoCaptions() {
- const engine = useEngine();
- const world = engine.world;
- const c = world.components;
-
- const [status, setStatus] = createSignal<AutoCaptionsStatus>(null);
-
- const generate = async (sceneEid: number) => {
- try {
- const { asset, trim } = await transcribeScene(engine, sceneEid, setStatus);
-
- const eid = createEntity(world);
- setComponent(world, eid, c.Geometry, GeometryType.TEXT);
- setComponent(world, eid, c.Caption, {});
- setComponent(world, eid, c.AssetId, asset.id);
- setComponent(world, eid, c.Trim, trim);
- appendChild(world, eid, sceneEid);
- } catch (error) {
- console.error("Auto-captions failed:", error);
- toast.error("Failed to generate captions", {
- description: error instanceof Error ? error.message : String(error),
- });
- } finally {
- setStatus(null);
- }
- };
-
- return { generate, status };
-}
diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsx
new file mode 100644
index 00000000..66c0256e
--- /dev/null
+++ b/apps/web/src/components/genai/use-auto-captions.tsx
@@ -0,0 +1,63 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { createMemo } from "solid-js";
+import { Captions, authoredElement } from "@diffusionstudio/reconciler";
+import { Caption, getEntityTree, isScene } from "@diffusionstudio/runtime";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useEditor, useSelection } from "@/engine/hooks";
+import { toast } from "somoto";
+
+import type { Entity } from "koota";
+
+export function useAutoCaptions() {
+ const world = useWorld();
+ const editor = useEditor();
+ const { nodes } = useSelection();
+
+ const scenes = createMemo(() => nodes().filter(isScene));
+ const hasScene = () => scenes().length > 0;
+
+ const insertFresh = (scene: Entity) => {
+ const [entity] = editor.insertElement(scene, () => <Captions />);
+ if (!entity) {
+ toast.error("Could not add captions", {
+ description: "The scene has no source to write the element into.",
+ });
+ return;
+ }
+ editor.select(entity);
+ };
+
+ const generate = () => {
+ for (const scene of scenes()) {
+ const existing = getEntityTree(world, scene).find(
+ (entity) => entity !== scene && entity.has(Caption),
+ );
+
+ if (!existing) {
+ insertFresh(scene);
+ return;
+ }
+
+ const authored = authoredElement(existing);
+
+ // Captions bound to a subtitle file say what they show; auto-captions
+ // means transcribing, so the element is replaced by one that does.
+ if (authored?.props.src !== undefined) {
+ editor.remove(existing);
+ insertFresh(scene);
+ return;
+ }
+
+ // A new take: bumping the seed re-transcribes the scene.
+ const seed = authored?.props.seed;
+ const next = (typeof seed === "number" && Number.isFinite(seed) ? Math.trunc(seed) : 0) + 1;
+ editor.editProperty(existing, "seed", next);
+ editor.select(existing);
+ }
+ };
+
+ return { hasScene, generate };
+}
diff --git a/apps/web/src/components/genai/use-generate-image.ts b/apps/web/src/components/genai/use-generate-image.ts
deleted file mode 100644
index 8b743c1b..00000000
--- a/apps/web/src/components/genai/use-generate-image.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-import { useEngine } from "@/context/engine";
-import { trpc } from "@/lib/trpc";
-import { track } from "@/lib/analytics";
-import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement, recordGeneration, ensureGenerationFolder } from "@/utils/genai";
-import { GeometryType, PaintType, createEntity, deleteEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";
-import { loadAsset, uploadAssets } from "@/components/engine";
-
-import type { ImageGenerationConfig } from "@/components/engine/db";
-
-const IMAGE_GAP = 40;
-
-export function useGenerateImage() {
- const { world } = useEngine();
- const c = world.components;
-
- const generate = async (config: ImageGenerationConfig) => {
- let placeholders: number[] = [];
- const startedAt = performance.now();
- try {
- track('generation_started', {
- mode: 'image',
- model: config.model,
- aspect_ratio: config.aspectRatio,
- count: config.count,
- prompt_length: config.prompt.length,
- reference_count: config.imageRefIds?.length ?? 0,
- });
-
- // 1. Find a slot for the whole batch, then place placeholder rectangles with the
- // Generating tag. In-flight placeholders are transient UI — build them untracked so
- // they never reach the undo stack. The whole batch is recorded as one step below.
- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };
- const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;
- const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);
-
- world.history.untrack(() => {
- placeholders = Array.from({ length: config.count }).map(() => createEntity(world));
-
- for (const [index, eid] of placeholders.entries()) {
- // When generating multiple variants, offset each horizontally so they
- // don't stack on top of each other.
- const offsetX = index * (dims.width + IMAGE_GAP) - totalWidth / 2 + dims.width / 2;
-
- setComponent(world, eid, c.Geometry, GeometryType.RECT);
- setComponent(world, eid, c.Position, {
- x: Math.round(center.x + offsetX - dims.width / 2),
- y: Math.round(center.y - dims.height / 2),
- });
- resizeEntity(world, eid, {
- width: dims.width,
- height: dims.height,
- });
- setComponent(world, eid, c.KeepAspectRatio, {
- width: dims.width,
- height: dims.height,
- });
- addComponent(world, eid, c.Generating);
- }
- });
-
- const { name, results, generationId } = await trpc.generateImage.mutate({
- model: config.model,
- prompt: config.prompt,
- aspectRatio: config.aspectRatio,
- count: config.count,
- images: await uploadAssets(world, config.imageRefIds),
- });
-
- // 2. Apply image fills and remove Generating tag
- const folderId = await ensureGenerationFolder(world, name, results.length);
- await Promise.all(
- results.map(async (result, i) => {
- const eid = placeholders[i];
- const imgName = results.length > 1 ? `${name} (${i + 1})` : name;
- const asset = await loadAsset(world, result.url, { name: imgName, generationId, folderId });
-
- world.history.untrack(() => {
- setComponent(world, eid, c.Name, imgName);
-
- const fid = createEntity(world);
- setComponent(world, fid, c.Paint, PaintType.IMAGE);
- setComponent(world, fid, c.AssetId, asset.id);
- appendChild(world, fid, eid);
-
- removeComponent(world, eid, c.Generating);
- });
- }),
- );
-
- recordGeneration(world, 'Generate image', placeholders);
-
- track('generation_completed', {
- mode: 'image',
- model: config.model,
- aspect_ratio: config.aspectRatio,
- count: results.length,
- duration_ms: Math.round(performance.now() - startedAt),
- });
- } catch (err) {
- // On failure, delete the placeholders so the shimmer stops. Untracked —
- // a failed generation leaves no trace on the undo stack.
- world.history.untrack(() => {
- for (const eid of placeholders) {
- deleteEntity(world, eid);
- }
- });
- console.error(err);
- track('generation_failed', {
- mode: 'image',
- model: config.model,
- duration_ms: Math.round(performance.now() - startedAt),
- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',
- });
- throw err;
- }
- };
-
- return { generate } as const;
-}
diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsx
new file mode 100644
index 00000000..92606c80
--- /dev/null
+++ b/apps/web/src/components/genai/use-generate-image.tsx
@@ -0,0 +1,81 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Image generation as the runtime means it: the prompt box declares a
+// `generate.image` source per variant and inserts an <Image> consuming it,
+// the way any drawn element is inserted. The asset system resolves the
+// declaration through the world's Ai (see `EditorGenAi`), and the element
+// lands in the project file with the `generate.image({...})` call that
+// reproduces it — same spec, same asset, this session and the next.
+//
+// Generations go on the canvas rather than into a scene: they are material to
+// work with, not yet part of a composition, and a scene owns a timeline they
+// have no place on until someone puts them there.
+
+import { generate } from "@diffusionstudio/jsx";
+import { Image } from "@diffusionstudio/reconciler";
+import { Library, Root } from "@diffusionstudio/runtime";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { useEditor } from "@/engine/hooks";
+import { findEmptyPlacement } from "@/engine/placement";
+import { ASPECT_RATIO_DIMENSIONS } from "@/utils/genai";
+import { toast } from "somoto";
+
+import type { ImageGenerationConfig } from "@/components/engine/db";
+import type { Entity } from "koota";
+
+const IMAGE_GAP = 40;
+
+/** A declaration is keyed by its spec, so a fresh seed makes a new take. */
+const randomSeed = () => Math.floor(Math.random() * 1_000_000);
+
+export function useGenerateImage() {
+ const world = useWorld();
+ const editor = useEditor();
+
+ const run = async (config: ImageGenerationConfig) => {
+ const library = world.get(Library);
+ const root = world.get(Root);
+ if (!root) return;
+
+ // References travel by library path: durable in the file, and readable
+ // there. Ids that no longer name a library asset are dropped.
+ const refs = (config.imageRefIds ?? [])
+ .map((id) => library?.get(id)?.path)
+ .filter((path): path is string => path !== undefined);
+
+ const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };
+ const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;
+ const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);
+ const left = center.x - totalWidth / 2;
+ const top = Math.round(center.y - dims.height / 2);
+
+ const inserted: Entity[] = [];
+ for (let index = 0; index < config.count; index++) {
+ const src = generate.image({
+ prompt: config.prompt,
+ model: config.model,
+ aspectRatio: config.aspectRatio,
+ ...(refs.length ? { refs } : {}),
+ seed: randomSeed(),
+ });
+
+ const x = Math.round(left + index * (dims.width + IMAGE_GAP));
+ const [entity] = editor.insertElement(root, () => (
+ <Image src={src} x={x} y={top} width={dims.width} height={dims.height} />
+ ));
+ if (entity) inserted.push(entity);
+ }
+
+ if (!inserted.length) {
+ toast("Could not add the generation", {
+ description: "Open a project to generate into.",
+ });
+ return;
+ }
+ editor.select(inserted);
+ };
+
+ return { generate: run } as const;
+}
diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.ts
index 84dbcbe7..f80568ec 100644
--- a/apps/web/src/engine/editor.ts
+++ b/apps/web/src/engine/editor.ts
@@ -11,15 +11,22 @@
*/
import { Active, Chars, getActiveEntity, getEntityChildren, getEntityTree, getParentEntity, isText, Loop, Selected, Sequential, setActive, Source, Stage } from '@diffusionstudio/runtime';
-import { isPropValue, SOURCE_ATTR } from '@diffusionstudio/jsx';
+import { isAssetRef, isPropValue, serializeAssetRef, SOURCE_ATTR } from '@diffusionstudio/jsx';
import { createRoot } from 'solid-js';
import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode, renderAuthored, withDocument } from '@diffusionstudio/reconciler';
-import type { PropValue } from '@diffusionstudio/jsx';
+import type { PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
import type { AuthoredTree, HostNode, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';
+/**
+ * A value an edit can carry to the file: what a source spells as a literal,
+ * or a `generate.*` declaration in its wire form (see `SerializedAssetRef`),
+ * which the writer spells as the call that reproduces it.
+ */
+export type EditValue = PropValue | SerializedAssetRef;
+
/**
* A property the editor changed, in the vocabulary of the JSX rather than of
* the traits it was written to: `source` is the element it belongs to (its
@@ -31,7 +38,7 @@ export interface PropEdit {
kind: 'prop';
source: string;
name: string;
- value: PropValue;
+ value: EditValue;
}
/**
@@ -59,7 +66,7 @@ export interface InsertEdit {
source: string;
parent: string;
tag: string;
- props: Record<string, PropValue>;
+ props: Record<string, EditValue>;
before?: string;
text?: string;
}
@@ -95,7 +102,7 @@ export interface RemoveEdit {
* own names — the pending source its entity has been re-stamped with, so the
* write can answer with the real one (see `isPendingSource`).
*/
-export type LoopIteration = Record<string, { props: Record<string, PropValue>; text?: string; pending?: string }>;
+export type LoopIteration = Record<string, { props: Record<string, EditValue>; text?: string; pending?: string }>;
/**
* A `<For>`/`<Index>` the editor needs written out as its iterations, so that
@@ -132,7 +139,30 @@ export const isLooped = (entity: Entity): boolean => entity.isAlive() && entity.
/** What `insertElement` learns about each element while it renders. */
interface Recorded {
tag: string;
- props: Record<string, PropValue>;
+ props: Record<string, EditValue>;
+}
+
+/**
+ * A prop as an edit can carry it (see `EditValue`): a value a file spells as
+ * it is, a declaration in its wire form, or undefined for anything else — an
+ * element is written with whatever of it the file can say, and the rest is
+ * the project's to keep, not the writer's to guess at. The one place that
+ * decides this, so a new kind of value is taught here once.
+ */
+function wireValue(value: unknown): EditValue | undefined {
+ if (isPropValue(value)) return value;
+ if (isAssetRef(value)) return serializeAssetRef(value);
+ return undefined;
+}
+
+/** `wireValue` over a whole authored element, dropping what will not travel. */
+function wireProps(props: Record<string, unknown>): Record<string, EditValue> {
+ const wired: Record<string, EditValue> = {};
+ for (const [name, value] of Object.entries(props)) {
+ const wire = wireValue(value);
+ if (wire !== undefined) wired[name] = wire;
+ }
+ return wired;
}
/**
@@ -329,7 +359,7 @@ export class DocumentEditor {
// know, or one the body renders twice: not spellable.
if (!source || !stamped(member) || source in iteration) return;
const pending = index > 0 ? nextPendingSource() : undefined;
- iteration[source] = { props: authored.props, ...(authored.text === undefined ? {} : { text: authored.text }), ...(pending ? { pending } : {}) };
+ iteration[source] = { props: wireProps(authored.props), ...(authored.text === undefined ? {} : { text: authored.text }), ...(pending ? { pending } : {}) };
if (pending) restamp.push([member, pending]);
}
}
@@ -770,8 +800,9 @@ export class DocumentEditor {
},
setProperty: (node, name, value) => {
const recorded = isSceneNode(node) ? created.get(node.entity) : undefined;
- if (recorded && name !== SOURCE_ATTR && name !== 'children' && name !== 'ref' && isPropValue(value)) {
- recorded.props[name] = value;
+ if (recorded && name !== SOURCE_ATTR && name !== 'children' && name !== 'ref') {
+ const wire = wireValue(value);
+ if (wire !== undefined) recorded.props[name] = wire;
}
document.setProperty(node, name, value);
},
diff --git a/apps/web/src/engine/placement.ts b/apps/web/src/engine/placement.ts
new file mode 100644
index 00000000..3ab7d8fd
--- /dev/null
+++ b/apps/web/src/engine/placement.ts
@@ -0,0 +1,102 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Where something new goes on the infinite canvas: beside the work rather
+// than over it. Carried over from the engine the editor is migrating off,
+// where this is what generations have always been placed by.
+
+import {
+ aabbsIntersect, getEntityBounds, getEntityChildren, getViewport, Hidden, Root, screenToWorld,
+} from '@diffusionstudio/runtime';
+
+import type { AABB, Point } from '@diffusionstudio/runtime';
+import type { World } from 'koota';
+
+/** The visible part of the canvas in document space, or null before a surface is mounted. */
+function viewportBounds(world: World): AABB | null {
+ const viewport = getViewport(world);
+ if (!viewport) return null;
+
+ // All four corners, so the box still holds if the camera is ever rotated.
+ const corners = [
+ screenToWorld(world, 0, 0),
+ screenToWorld(world, viewport.width, 0),
+ screenToWorld(world, viewport.width, viewport.height),
+ screenToWorld(world, 0, viewport.height),
+ ];
+
+ return {
+ minX: Math.min(...corners.map((corner) => corner.x)),
+ minY: Math.min(...corners.map((corner) => corner.y)),
+ maxX: Math.max(...corners.map((corner) => corner.x)),
+ maxY: Math.max(...corners.map((corner) => corner.y)),
+ };
+}
+
+/**
+ * What stands on the canvas: the bounds of every visible root-level node.
+ * A node the transform system has not measured yet reports nothing and is
+ * left out — it has no place on the canvas to be avoided at.
+ */
+function occupied(world: World): AABB[] {
+ const boxes: AABB[] = [];
+
+ for (const entity of getEntityChildren(world, world.get(Root)!)) {
+ if (entity.has(Hidden)) continue;
+ const rect = getEntityBounds(world, [entity]);
+ if (!rect) continue;
+ boxes.push({ minX: rect.x, minY: rect.y, maxX: rect.x + rect.width, maxY: rect.y + rect.height });
+ }
+
+ return boxes;
+}
+
+const centerOf = (box: AABB): Point => ({ x: (box.minX + box.maxX) / 2, y: (box.minY + box.maxY) / 2 });
+
+const distance = (a: Point, b: Point): number => Math.hypot(a.x - b.x, a.y - b.y);
+
+/**
+ * Where the middle of a `width`×`height` block goes on the canvas: beside
+ * something already standing there, level with it — to its right, or to its
+ * left when the right is taken — leaving `gap` between the two. Neighbours are
+ * tried nearest the middle of the view first, and those in view before those
+ * out of it, so a block lands next to what the user is looking at.
+ *
+ * The middle of the view when the canvas is empty, or when nothing has a free
+ * side left; the origin when no surface is mounted yet.
+ */
+export function findEmptyPlacement(world: World, width: number, height: number, gap: number): Point {
+ const viewport = viewportBounds(world);
+ const center = viewport ? centerOf(viewport) : { x: 0, y: 0 };
+
+ const boxes = occupied(world);
+ if (boxes.length === 0) return center;
+
+ const inView = viewport ? boxes.filter((box) => aabbsIntersect(box, viewport)) : [];
+ const ordered = (inView.length > 0 ? inView : boxes)
+ .slice()
+ .sort((a, b) => distance(centerOf(a), center) - distance(centerOf(b), center));
+
+ const free = (rect: AABB): boolean => !boxes.some((box) => aabbsIntersect(rect, box));
+
+ for (const box of ordered) {
+ // Level with the neighbour: the block keeps its middle, whatever the
+ // two are sized like.
+ const y = (box.minY + box.maxY) / 2;
+ const minY = y - height / 2;
+ const maxY = y + height / 2;
+
+ const right = box.maxX + gap;
+ if (free({ minX: right, minY, maxX: right + width, maxY })) {
+ return { x: right + width / 2, y };
+ }
+
+ const left = box.minX - gap;
+ if (free({ minX: left - width, minY, maxX: left, maxY })) {
+ return { x: left - width / 2, y };
+ }
+ }
+
+ return center;
+}
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 880e95d6..cac4b800 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -25,6 +25,7 @@ import {
} from "@/components/genai/config";
import { assert } from "@/utils";
import { uploadBlob } from "@/components/engine";
+import { track } from "@/lib/analytics";
import { trpc } from "@/lib/trpc";
import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
@@ -124,9 +125,11 @@ export class EditorGenAi extends GenAi {
const uploadId = crypto.randomUUID();
const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });
+ console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);
const fileRef = await uploadBlob(audioFile, uploadId);
assert(fileRef, "Failed to upload the scene audio for transcription");
+ console.log(`[gen-ai] transcribing scene audio for ${key}`);
const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });
assert(
transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),
@@ -226,17 +229,43 @@ export class EditorGenAi extends GenAi {
/** Runs a generation and stores its first result under `generated/`. */
private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {
- const { name, results, generationId } = await this.requestGeneration(spec);
- assert(results.length > 0, "No results returned from the model");
-
- const url = results[0].url;
- const response = await fetch(url);
- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
- const blob = await response.blob();
- const extension = url.split(/[?#]/)[0].split(".").pop();
- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
+ const startedAt = performance.now();
+ track("generation_started", {
+ mode: spec.type,
+ model: spec.model,
+ prompt_length: spec.prompt.length,
+ ...("aspectRatio" in spec ? { aspect_ratio: spec.aspectRatio } : {}),
+ ...(spec.type === "image" ? { reference_count: spec.refIds.length } : {}),
+ });
- return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
+ try {
+ console.log(`[gen-ai] generating ${spec.type} with ${spec.model}:`, spec);
+ const { name, results, generationId } = await this.requestGeneration(spec);
+ assert(results.length > 0, "No results returned from the model");
+
+ const url = results[0].url;
+ const response = await fetch(url);
+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
+ const blob = await response.blob();
+ const extension = url.split(/[?#]/)[0].split(".").pop();
+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
+
+ const asset = await this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
+ track("generation_completed", {
+ mode: spec.type,
+ model: spec.model,
+ duration_ms: Math.round(performance.now() - startedAt),
+ });
+ return asset;
+ } catch (err) {
+ track("generation_failed", {
+ mode: spec.type,
+ model: spec.model,
+ duration_ms: Math.round(performance.now() - startedAt),
+ error: err instanceof Error ? err.message.slice(0, 200) : "unknown",
+ });
+ throw err;
+ }
}
private requestGeneration(spec: ResolvedSpec) {
diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.ts
index 6b295012..34b6184f 100644
--- a/packages/jsx/src/generate.ts
+++ b/packages/jsx/src/generate.ts
@@ -76,6 +76,51 @@ export function isAssetRef(value: unknown): value is AssetRef {
return value instanceof AssetRef;
}
+/** An `AssetSpecInput` whose inputs are all strings — no nested declarations. */
+export type FlatAssetSpec = AssetSpecInput & {
+ refs?: string[];
+ startFrame?: string;
+ endFrame?: string;
+};
+
+/**
+ * A declaration as data, for the edit protocol. An `AssetRef` itself cannot
+ * travel as a prop value (see `isPropValue`: an object literal would not read
+ * back as one), so an edit carries the spec in this tagged shape instead, and
+ * the source writer spells it as the `generate.*` call that reproduces it.
+ */
+export type SerializedAssetRef = { $generate: FlatAssetSpec };
+
+/**
+ * The wire form of a declaration, or undefined when the spec references
+ * other declarations: nested refs only exist in authored code, where the
+ * call spelling them already is — the editor declares over library assets.
+ */
+export function serializeAssetRef(ref: AssetRef): SerializedAssetRef | undefined {
+ const spec = ref.spec;
+ const inputs = [
+ ...(spec.type === "image" ? (spec.refs ?? []) : []),
+ ...(spec.type === "video" ? [spec.startFrame, spec.endFrame] : []),
+ ];
+ if (inputs.some((input) => isAssetRef(input))) return undefined;
+
+ // Without the undefined entries: they are not part of the declaration, and
+ // the writer spells every entry it is handed.
+ const entries = Object.entries(spec).filter(([, value]) => value !== undefined);
+ return { $generate: Object.fromEntries(entries) as FlatAssetSpec };
+}
+
+const SPEC_TYPES = ["image", "video", "voice", "audio"];
+
+export function isSerializedAssetRef(value: unknown): value is SerializedAssetRef {
+ if (typeof value !== "object" || value === null || !("$generate" in value)) return false;
+ const spec = (value as SerializedAssetRef).$generate;
+ return (
+ typeof spec === "object" && spec !== null &&
+ SPEC_TYPES.includes(spec.type) && typeof spec.prompt === "string"
+ );
+}
+
/** Host-side accessor for a declaration's spec. */
export function getAssetSpec(ref: AssetRef): AssetSpecInput {
return ref.spec;
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.ts
index 05776aa8..f1f68f20 100644
--- a/packages/jsx/src/index.ts
+++ b/packages/jsx/src/index.ts
@@ -10,11 +10,20 @@
* @diffusionstudio/reconciler), so nothing in this package touches a host.
*/
-export { generate, AssetRef, isAssetRef, getAssetSpec } from "./generate";
+export {
+ generate,
+ AssetRef,
+ isAssetRef,
+ getAssetSpec,
+ serializeAssetRef,
+ isSerializedAssetRef,
+} from "./generate";
export type {
AspectRatio,
AssetInput,
AssetSpecInput,
+ FlatAssetSpec,
+ SerializedAssetRef,
GenerateAudioOptions,
GenerateImageOptions,
GenerateVideoOptions,
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 623d5deb..649e88a3 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,11 +3,11 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
-import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
+import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
-import type { AssetRef, PropValue } from '@diffusionstudio/jsx';
+import type { AssetRef } from '@diffusionstudio/jsx';
import { trait, type Entity, type World } from 'koota';
import type { ProjectDocument } from './host';
@@ -52,14 +52,13 @@ const TextParts = trait(() => new Set<TextNode>());
/**
* What the project wrote on an entity's element, in the vocabulary of the JSX
- * rather than of the traits it became: the tag, and every prop worth a value a
- * source file could spell (see `PropValue`), as last set. Kept so an editor
- * can spell an entity back out as an element — which is how a loop's
- * iterations get written down one by one — without reading the traits
+ * rather than of the traits it became: the tag, and every prop as last set.
+ * Kept so an editor can spell an entity back out as an element — which is how
+ * a loop's iterations get written down one by one — without reading the traits
* backwards. `children` is not a prop here: an element's text is `Chars`, and
* its element children are the entity's.
*/
-const Authored = trait(() => ({ tag: '', props: {} as Record<string, PropValue> }));
+const Authored = trait(() => ({ tag: '', props: {} as Record<string, unknown> }));
/** Props that address or wire an element rather than describe it. */
const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);
@@ -67,7 +66,7 @@ const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, '
export interface AuthoredElement {
/** The camelCase tag the project used. */
tag: string;
- props: Record<string, PropValue>;
+ props: Record<string, unknown>;
/** The literal text of a `<text>`, when it holds any. */
text?: string;
}
@@ -649,12 +648,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
if (!isSceneNode(node)) return;
const { entity } = node;
- // Noted before it is interpreted: what the project wrote is what an
- // editor spelling the element back out needs, however the traits pack it.
const authored = entity.get(Authored);
if (authored && !UNAUTHORED_PROPS.has(name)) {
- if (isPropValue(value)) authored.props[name] = value;
- else delete authored.props[name];
+ if (value === undefined) delete authored.props[name];
+ else authored.props[name] = value;
}
switch (name) {
@@ -985,7 +982,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
case 'src': {
- entity.remove(GenerationRequest, LoadRequest, PendingSource);
+ // Generating goes with the wait it belongs to: the resolution
+ // running for the old src will not clear it, having been
+ // superseded by whatever this one starts.
+ entity.remove(GenerationRequest, LoadRequest, PendingSource, Generating);
if (value === undefined || value === null || value === '') {
entity.remove(AssetId);
@@ -1159,13 +1159,13 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
case 'seed': {
if (!entity.has(Caption)) return;
// An authored src mounts a transcript directly; there is no
- // transcription for the seed to key. (An AssetRef src is not a
- // PropValue, so its request stands in for the authored record.)
+ // transcription for the seed to key.
if (entity.get(Authored)?.props.src !== undefined) return;
if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;
- // A resolution running for another seed must not bind late.
- entity.remove(PendingSource);
+ // A resolution running for another seed must not bind late,
+ // nor leave the wait it was marked with behind.
+ entity.remove(PendingSource, Generating);
entity.add(TranscriptionRequest);
entity.set(TranscriptionRequest, { seed: toNumber(value) ?? 0 });
return;
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index 8080d0cc..a975699c 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';
+import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';
import { bindAsset } from '../actions/assets';
import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';
@@ -25,7 +25,7 @@ export function assetSystem(world: World): void {
const ref = entity.get(GenerationRequest)!.ref;
entity.remove(GenerationRequest);
if (ref === null) continue;
- resolve(world, entity, ref, ai.resolve(ref));
+ resolve(world, entity, ref, ai.resolve(ref), true);
}
for (const entity of world.query(TranscriptionRequest)) {
@@ -34,7 +34,7 @@ export function assetSystem(world: World): void {
const seed = entity.get(TranscriptionRequest)!.seed;
entity.remove(TranscriptionRequest);
- resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed));
+ resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed), true);
}
}
}
@@ -59,19 +59,20 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean
* (PendingSource), so of overlapping resolutions only the latest binds, and
* one that outlives its element (or its src) is dropped.
*/
-function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>): void {
+function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>, generating = false): void {
entity.add(PendingSource);
entity.set(PendingSource, { value });
+ if (generating) entity.add(Generating);
const done = promise.then(
(asset) => {
if (!current(entity, value)) return;
- entity.remove(PendingSource);
+ entity.remove(PendingSource, Generating);
bindAsset(entity, asset);
},
(error: unknown) => {
if (!current(entity, value)) return;
- entity.remove(PendingSource);
+ entity.remove(PendingSource, Generating);
console.error('[runtime] could not resolve src:', error);
},
);