The VS Code of video editing, built for agents.
TypeScript
Refactor and enhance JSX editing capabilities
konstantin-paulus71a7f30a6881
diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.tsindex 72c68437..bf0adb81 100644--- a/apps/desktop/src/edit.ts+++ b/apps/desktop/src/edit.ts@@ -9,9 +9,9 @@ import { join } from "node:path";import { IndentationText, Project, SyntaxKind } from "ts-morph";-import { ID_ATTR, formatSource, isCompositionTag, isLoopTag, parseSource } from "@diffusionstudio/jsx";+import { ID_ATTR, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, parseSource } from "@diffusionstudio/jsx";-import type { PropValue } from "@diffusionstudio/jsx";+import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";import type {ArrowFunction,FunctionExpression,@@ -24,7 +24,14 @@ import type {SourceFile,} from "ts-morph";-export type { PropValue };+export type { PropValue, SerializedAssetRef };++/**+ * A value an edit can carry: what a source spells as a literal, or a+ * `generate.*` declaration in its wire form, spelled as the call that+ * reproduces it (see `setProp`).+ */+export type EditValue = PropValue | SerializedAssetRef;export interface SourceContext {/** Absolute path of the project folder. */@@ -42,7 +49,7 @@ export interface SourceContext {export interface SourceSet {kind: "set";source: string;- props: Record<string, PropValue>;+ props: Record<string, EditValue>;text?: string;}@@ -59,7 +66,7 @@ export interface SourceInsert {source: string;parent: string;tag: string;- props: Record<string, PropValue>;+ props: Record<string, EditValue>;before?: string;text?: string;}@@ -99,7 +106,7 @@ export interface SourceRemove {* one in `ids`. The first iteration keeps the body's own names, so it carries* none.*/-export type SourceIteration = Record<string, { props: Record<string, PropValue>; text?: string; pending?: string }>;+export type SourceIteration = Record<string, { props: Record<string, EditValue>; text?: string; pending?: string }>;/*** Replaces the `<For>`/`<Index>` around the element named by `source` with@@ -245,14 +252,29 @@ const initializerText = (value: PropValue): string =>? `"${value}"`: `{${literalText(value)}}`;+/** A declaration's wire form as the `generate.*` call that reproduces it. */+function generateCallText(ref: SerializedAssetRef): string {+ const { type, ...options } = ref.$generate;+ return `generate.${type}(${literalText(options as Record<string, PropValue>)})`;+}+/*** Writes a prop onto a tag as one would write it: `muted`, not `muted={true}`,* and no attribute at all rather than `muted={false}`, since absence is what a- * boolean prop's false reads as.+ * boolean prop's false reads as. A declaration is spelled as its `generate.*`+ * call — the caller makes sure `generate` is imported (see+ * `ensureGenerateImport`).*/-function setProp(tag: JsxTag, name: string, value: PropValue): void {+function setProp(tag: JsxTag, name: string, value: EditValue): void {const attribute = attributeOf(tag, name);+ if (isSerializedAssetRef(value)) {+ const initializer = `{${generateCallText(value)}}`;+ if (attribute) attribute.setInitializer(initializer);+ else tag.addAttribute({ name, initializer });+ return;+ }+if (value === true) {if (attribute) attribute.removeInitializer();else tag.addAttribute({ name });@@ -268,6 +290,40 @@ function setProp(tag: JsxTag, name: string, value: PropValue): void {else tag.addAttribute({ name, initializer: initializerText(value) });}+/** Where `generate` comes from — the module every project authors against. */+const GENERATE_MODULE = "@diffusionstudio/jsx";++/**+ * Makes sure `generate` is in scope once a declaration has been spelled into+ * the file: added to the module's existing import, or as an import of its own+ * after the last one — or above the first statement, past the file's header+ * comments, when there are no imports at all. Idempotent.+ */+function ensureGenerateImport(sourceFile: SourceFile): void {+ const declarations = sourceFile.getImportDeclarations();+ const importable = declarations.find(+ (declaration) =>+ declaration.getModuleSpecifierValue() === GENERATE_MODULE &&+ !declaration.isTypeOnly() &&+ !declaration.getNamespaceImport(),+ );++ if (importable) {+ const named = importable.getNamedImports();+ if (named.some((specifier) => specifier.getName() === "generate" && !specifier.getAliasNode())) return;+ importable.addNamedImport("generate");+ return;+ }++ const statement = `import { generate } from "${GENERATE_MODULE}";`;+ const last = declarations.at(-1);+ if (last) sourceFile.insertText(last.getEnd(), `\n${statement}`);+ else {+ const first = sourceFile.getStatements()[0];+ sourceFile.insertText(first ? first.getStart() : 0, `${statement}\n\n`);+ }+}+/*** Text content as JSX spells it: bare when JSX would read it back verbatim,* otherwise as a string expression (JSX collapses whitespace around line@@ -506,9 +562,25 @@ function isLiteral(node: Node): boolean {}/**- * A prop is only written back when the source holds a plain literal. Anything- * else — a signal, a prop of the surrounding component, an expression — is- * someone's reactivity, and a drag has no business overwriting it.+ * A `generate.*` call over a literal spec: what the writer spells a+ * declaration as, and so an initializer it may also overwrite. One whose+ * argument computes anything is authored reactivity, like any expression.+ */+function isGenerateCall(node: Node): boolean {+ const call = node.asKind(SyntaxKind.CallExpression);+ const callee = call?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);+ if (!call || !callee || !callee.getExpression().isKind(SyntaxKind.Identifier)) return false;+ if (callee.getExpression().getText() !== "generate") return false;++ const args = call.getArguments();+ return args.length === 1 && isLiteral(args[0]!);+}++/**+ * A prop is only written back when the source holds a plain literal (or a+ * declaration the writer itself spells). Anything else — a signal, a prop of+ * the surrounding component, an expression — is someone's reactivity, and a+ * drag has no business overwriting it.*/function isWritable(attribute: JsxAttribute): boolean {const initializer = attribute.getInitializer();@@ -517,7 +589,7 @@ function isWritable(attribute: JsxAttribute): boolean {if (!initializer.isKind(SyntaxKind.JsxExpression)) return false;const expression = initializer.getExpression();- return expression !== undefined && isLiteral(expression);+ return expression !== undefined && (isLiteral(expression) || isGenerateCall(expression));}// ---------------------------------------------------------------------------@@ -817,6 +889,7 @@ class SourceWriter {}setProp(tag, name, value);+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);wrote = true;}@@ -888,6 +961,7 @@ class SourceWriter {insertChild(sourceFile, parent, child, before);for (const [name, value] of Object.entries(edit.props)) {setProp(findTag(sourceFile, id)!, name, value);+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);}ids[edit.source] = formatSource(file, id);return true;@@ -1127,6 +1201,7 @@ class SourceWriter {}for (const [name, value] of Object.entries(record.props)) {setProp(found(), name, value);+ if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);}const element = elementOf(found());diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsxindex 8ee688d3..862d41c8 100644--- a/apps/web/src/components/genai/action-bar.tsx+++ b/apps/web/src/components/genai/action-bar.tsx@@ -13,15 +13,8 @@ import {DropdownMenuPortal,DropdownMenuTrigger,} from "@/components/ui/dropdown-menu";-import {- Dialog,- DialogContent,- DialogHeader,- DialogPortal,- DialogTitle,-} from "@/components/ui/dialog";import { createMemo, Show } from "solid-js";-import { PaintType, isScene as isSceneEntity, isAudio } from "@/components/engine";+import { PaintType, isAudio } from "@/components/engine";import { useEngine } from "@/context/engine";import { useRemoveBackground } from "./use-remove-background";import { useUpscaleImage } from "./use-upscale-image";@@ -73,18 +66,9 @@ export function ActionBar(props: ActionBarProps) {const { generate: generateVideo } = useGenerateVideo();const { generate: generateVoice } = useGenerateVoice();const { generate: generateAudio } = useGenerateAudio();- const { generate: autoCaptions, status: autoCaptionsStatus } = useAutoCaptions();+ const { generate: autoCaptions, hasScene } = useAutoCaptions();const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();- const autoCaptionsStatusText = createMemo(() => {- switch (autoCaptionsStatus()) {- case "encoding": return "Encoding audio…";- case "uploading": return "Uploading audio…";- case "transcribing": return "Transcribing audio…";- default: return "";- }- });-const isImage = createMemo(() => {return Array.from(selectedNodes()).some(eid => {const mediaType = getNodeMediaType(world, eid);@@ -99,12 +83,8 @@ export function ActionBar(props: ActionBarProps) {});});- const isScene = createMemo(() => {- return Array.from(selectedNodes()).some(eid => isSceneEntity(world, eid));- });-const visible = createMemo(() => {- return isImage() || isVideo() || isScene();+ return isImage() || isVideo() || hasScene();})const handleRerun = () => {@@ -151,33 +131,12 @@ export function ActionBar(props: ActionBarProps) {props.openPromptInput?.(config);};- const handleAutoCaptions = async () => {- for (const eid of selectedNodes()) {- if (isSceneEntity(world, eid)) {- await autoCaptions(eid);- }- }- };-return (<>- <Dialog open={autoCaptionsStatus() !== null}>- <DialogPortal>- <DialogContent class="sm:max-w-sm">- <DialogHeader>- <DialogTitle>Auto-Captions</DialogTitle>- </DialogHeader>- <div class="flex items-center gap-2 text-sm text-muted-foreground">- <Icon name="spinner-loader" class="size-6 animate-spin" />- {autoCaptionsStatusText()}- </div>- </DialogContent>- </DialogPortal>- </Dialog><Show when={visible()}><div class="absolute bottom-16 left-1/2 -translate-x-1/2 z-10 rounded-xl p-1 bg-background border border-border flex gap-1 items-center">- <Show when={isScene()}>- <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={handleAutoCaptions}>+ <Show when={hasScene()}>+ <Button variant="ghost" class="gap-0 pl-0.5 text-muted-foreground" onClick={autoCaptions}><Icon name="captions" />Auto-Captions</Button>diff --git a/apps/web/src/components/genai/prompt-input.tsx b/apps/web/src/components/genai/prompt-input.tsxindex 568e1671..058da27a 100644--- a/apps/web/src/components/genai/prompt-input.tsx+++ b/apps/web/src/components/genai/prompt-input.tsx@@ -51,7 +51,6 @@ import { PromptInputActions } from "./prompt-input-actions";import { useEngine } from "@/context/engine";import { showFileDialog } from "@/utils";import type {- Asset,AspectRatio,GenerationConfig,ImageGenerationConfig,@@ -64,6 +63,15 @@ import { ChildOf, PaintType, removeComponent } from "@/components/engine";import { useAssets, loadAsset } from "@/components/engine";import { useECS } from "@/context/ecs";import { Not, query } from 'bitecs';+import { AssetId } from "@diffusionstudio/runtime";+import { useLibrary } from "@/engine/library";+import { useEditor, useSelection } from "@/engine/hooks";+import { droppedFiles, importFiles, pickFiles } from "@/engine/asset-actions";+import { ASSET_DRAG_TYPE } from "@/components/sidebar-left/folder-item";++import type { Entity } from "koota";+import type { Asset as LibraryAsset, AssetCache } from "@diffusionstudio/assets";+import type { ThumbnailAsset } from "@/components/ui/asset-thumbnail";export type PromptInputMode = "IMAGE" | "VIDEO" | "VOICE" | "AUDIO";@@ -121,6 +129,11 @@ export function PromptInput(props: PromptInputProps) {const assets = useAssets(world);const { selectedNodes } = useECS();const c = world.components;+ // The runtime the IMAGE mode works against: references come from the+ // project library and the canvas selection, and land as declarations.+ const library = useLibrary();+ const editor = useEditor();+ const { nodes } = useSelection();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();const { generate: generateVoice } = useGenerateVoice();@@ -191,11 +204,30 @@ export function PromptInput(props: PromptInputProps) {selectedImageEntries().map((e) => e.assetId),);+ // The image assets of the current canvas selection, from the runtime's+ // world: entities bound to a library image count as references.+ const selectedLibraryImages = createMemo(() => {+ const lib = library();+ if (!lib) return [];++ const entries: { asset: LibraryAsset; entity: Entity }[] = [];+ for (const entity of nodes()) {+ const id = entity.get(AssetId)?.value;+ const asset = id ? lib.get(id) : undefined;+ if (asset?.type === "IMAGE") entries.push({ asset, entity });+ }+ return entries;+ });+const effectiveImageRefIds = createMemo(() => {+ const lib = library();+ if (!lib) return [];const local = imageConfig()?.imageRefIds ?? [];- const selected = selectedImageIds().filter((id) => !local.includes(id));+ const selected = selectedLibraryImages()+ .map((entry) => entry.asset.id)+ .filter((id) => !local.includes(id));return [...local, ...selected]- .filter((id) => assets.get(id))+ .filter((id) => lib.get(id)).slice(0, MAX_IMAGE_REFERENCES);});@@ -241,29 +273,27 @@ export function PromptInput(props: PromptInputProps) {patch({ imageRefIds: refs.filter((id) => id !== assetId) });return;}- const entry = selectedImageEntries().find((e) => e.assetId === assetId);- if (entry) {- removeComponent(world, entry.eid, c.Selected, false);- }+ // The reference is shown from a selected canvas entity, so removing it+ // means deselecting that entity.+ const entry = selectedLibraryImages().find((e) => e.asset.id === assetId);+ if (entry) editor.deselect(entry.entity);};const openImageReferencesPicker = async () => {if (effectiveImageRefIds().length >= MAX_IMAGE_REFERENCES) return;- const handles = await showFileDialog("image/*");- if (handles.length === 0) return;+ const lib = library();+ if (!lib) return;- try {- const loadedAssets = await Promise.all(handles.map((handle) => loadAsset(world, handle)));- const imageAssets = loadedAssets.filter((a) => a.type === "IMAGE");- const current = imageConfig()?.imageRefIds ?? [];- patch({- imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),- });- } catch (e) {- toast("Failed to import assets", {- description: (e as Error).message,- });- }+ const files = await pickFiles({ accept: "image/*" });+ if (files.length === 0) return;++ // `importFiles` reports its own failures.+ const imported = await importFiles(lib, files, "");+ const imageAssets = imported.filter((a) => a.type === "IMAGE");+ const current = imageConfig()?.imageRefIds ?? [];+ patch({+ imageRefIds: [...current, ...imageAssets.map((a) => a.id)].slice(0, MAX_IMAGE_REFERENCES),+ });};const openVideoFramePicker = async (frame: "start" | "end") => {@@ -414,9 +444,31 @@ export function PromptInput(props: PromptInputProps) {dragCounter = 0;setIsDragging(false);- const assetIdData = event.dataTransfer?.getData("application/x-asset-id");+ const assetIdData = event.dataTransfer?.getData(ASSET_DRAG_TYPE);const existingAssetIds = assetIdData?.split(",").filter(Boolean) ?? [];+ // References for an image generation live in the project library:+ // dragged library assets are taken as they are, external files imported.+ if (mode() === "IMAGE") {+ const lib = library();+ if (!lib) return;++ const files = droppedFiles(event).filter((file) => file.type.startsWith("image/"));+ if (existingAssetIds.length === 0 && files.length === 0) return;++ const newAssetIds =+ existingAssetIds.length > 0+ ? existingAssetIds+ : (await importFiles(lib, files, "")).map((a) => a.id);++ const imageIds = newAssetIds.filter((id) => lib.get(id)?.type === "IMAGE");+ const current = imageConfig()?.imageRefIds ?? [];+ patch({+ imageRefIds: [...current, ...imageIds].slice(0, MAX_IMAGE_REFERENCES),+ });+ return;+ }+const fileItems = Array.from(event.dataTransfer?.items ?? []).filter((item) => item.kind === "file" && item.type.startsWith("image/"),);@@ -439,11 +491,6 @@ export function PromptInput(props: PromptInputProps) {if (!c?.endFrameImageId && newAssetIds[1]) {patch({ endFrameImageId: newAssetIds[1] });}- } else {- const current = imageConfig()?.imageRefIds ?? [];- patch({- imageRefIds: [...current, ...newAssetIds].slice(0, MAX_IMAGE_REFERENCES),- });}} catch (e) {toast("Failed to import assets", {@@ -495,7 +542,8 @@ export function PromptInput(props: PromptInputProps) {<For each={effectiveImageRefIds()}>{(assetId) => (<PromptInputReferenceImageButton- asset={assets.get(assetId)!}+ asset={library()!.get(assetId)!}+ cache={library()!.cache}class="size-16"size={{ width: 64, height: 64 }}onRemove={() => removeImageReference(assetId)}@@ -1068,7 +1116,8 @@ function VoiceMenu(props: VoiceMenuProps) {}type PromptInputReferenceImageButtonProps = {- asset: Asset;+ asset: ThumbnailAsset;+ cache?: AssetCache;class?: string;size?: { width: number; height: number };onRemove(): void;@@ -1078,7 +1127,7 @@ function PromptInputReferenceImageButton(props: PromptInputReferenceImageButtonPreturn (<div class={cx("group relative shrink-0", props.class)}><div class="size-full overflow-hidden rounded-md bg-input outline-none">- <AssetThumbnail asset={props.asset} class="size-full" size={props.size} />+ <AssetThumbnail asset={props.asset} cache={props.cache} class="size-full" size={props.size} /></div><Show when={props.onRemove}><buttondiff --git a/apps/web/src/components/genai/use-auto-captions.ts b/apps/web/src/components/genai/use-auto-captions.tsdeleted file mode 100644index 990f3078..00000000--- a/apps/web/src/components/genai/use-auto-captions.ts+++ /dev/null@@ -1,42 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { GeometryType, createEntity, appendChild, setComponent, transcribeScene } from "@/components/engine";-import { useEngine } from "@/context/engine";-import { createSignal } from "solid-js";-import { toast } from "somoto";--import type { TranscribeStatus } from "@/components/engine";--export type AutoCaptionsStatus = TranscribeStatus | null;--export function useAutoCaptions() {- const engine = useEngine();- const world = engine.world;- const c = world.components;-- const [status, setStatus] = createSignal<AutoCaptionsStatus>(null);-- const generate = async (sceneEid: number) => {- try {- const { asset, trim } = await transcribeScene(engine, sceneEid, setStatus);-- const eid = createEntity(world);- setComponent(world, eid, c.Geometry, GeometryType.TEXT);- setComponent(world, eid, c.Caption, {});- setComponent(world, eid, c.AssetId, asset.id);- setComponent(world, eid, c.Trim, trim);- appendChild(world, eid, sceneEid);- } catch (error) {- console.error("Auto-captions failed:", error);- toast.error("Failed to generate captions", {- description: error instanceof Error ? error.message : String(error),- });- } finally {- setStatus(null);- }- };-- return { generate, status };-}diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsxnew file mode 100644index 00000000..66c0256e--- /dev/null+++ b/apps/web/src/components/genai/use-auto-captions.tsx@@ -0,0 +1,63 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++import { createMemo } from "solid-js";+import { Captions, authoredElement } from "@diffusionstudio/reconciler";+import { Caption, getEntityTree, isScene } from "@diffusionstudio/runtime";+import { useWorld } from "@diffusionstudio/koota-solid";+import { useEditor, useSelection } from "@/engine/hooks";+import { toast } from "somoto";++import type { Entity } from "koota";++export function useAutoCaptions() {+ const world = useWorld();+ const editor = useEditor();+ const { nodes } = useSelection();++ const scenes = createMemo(() => nodes().filter(isScene));+ const hasScene = () => scenes().length > 0;++ const insertFresh = (scene: Entity) => {+ const [entity] = editor.insertElement(scene, () => <Captions />);+ if (!entity) {+ toast.error("Could not add captions", {+ description: "The scene has no source to write the element into.",+ });+ return;+ }+ editor.select(entity);+ };++ const generate = () => {+ for (const scene of scenes()) {+ const existing = getEntityTree(world, scene).find(+ (entity) => entity !== scene && entity.has(Caption),+ );++ if (!existing) {+ insertFresh(scene);+ return;+ }++ const authored = authoredElement(existing);++ // Captions bound to a subtitle file say what they show; auto-captions+ // means transcribing, so the element is replaced by one that does.+ if (authored?.props.src !== undefined) {+ editor.remove(existing);+ insertFresh(scene);+ return;+ }++ // A new take: bumping the seed re-transcribes the scene.+ const seed = authored?.props.seed;+ const next = (typeof seed === "number" && Number.isFinite(seed) ? Math.trunc(seed) : 0) + 1;+ editor.editProperty(existing, "seed", next);+ editor.select(existing);+ }+ };++ return { hasScene, generate };+}diff --git a/apps/web/src/components/genai/use-generate-image.ts b/apps/web/src/components/genai/use-generate-image.tsdeleted file mode 100644index 8b743c1b..00000000--- a/apps/web/src/components/genai/use-generate-image.ts+++ /dev/null@@ -1,123 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { useEngine } from "@/context/engine";-import { trpc } from "@/lib/trpc";-import { track } from "@/lib/analytics";-import { ASPECT_RATIO_DIMENSIONS, findEmptyPlacement, recordGeneration, ensureGenerationFolder } from "@/utils/genai";-import { GeometryType, PaintType, createEntity, deleteEntity, addComponent, appendChild, removeComponent, setComponent, resizeEntity } from "@/components/engine";-import { loadAsset, uploadAssets } from "@/components/engine";--import type { ImageGenerationConfig } from "@/components/engine/db";--const IMAGE_GAP = 40;--export function useGenerateImage() {- const { world } = useEngine();- const c = world.components;-- const generate = async (config: ImageGenerationConfig) => {- let placeholders: number[] = [];- const startedAt = performance.now();- try {- track('generation_started', {- mode: 'image',- model: config.model,- aspect_ratio: config.aspectRatio,- count: config.count,- prompt_length: config.prompt.length,- reference_count: config.imageRefIds?.length ?? 0,- });-- // 1. Find a slot for the whole batch, then place placeholder rectangles with the- // Generating tag. In-flight placeholders are transient UI — build them untracked so- // they never reach the undo stack. The whole batch is recorded as one step below.- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };- const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;- const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);-- world.history.untrack(() => {- placeholders = Array.from({ length: config.count }).map(() => createEntity(world));-- for (const [index, eid] of placeholders.entries()) {- // When generating multiple variants, offset each horizontally so they- // don't stack on top of each other.- const offsetX = index * (dims.width + IMAGE_GAP) - totalWidth / 2 + dims.width / 2;-- setComponent(world, eid, c.Geometry, GeometryType.RECT);- setComponent(world, eid, c.Position, {- x: Math.round(center.x + offsetX - dims.width / 2),- y: Math.round(center.y - dims.height / 2),- });- resizeEntity(world, eid, {- width: dims.width,- height: dims.height,- });- setComponent(world, eid, c.KeepAspectRatio, {- width: dims.width,- height: dims.height,- });- addComponent(world, eid, c.Generating);- }- });-- const { name, results, generationId } = await trpc.generateImage.mutate({- model: config.model,- prompt: config.prompt,- aspectRatio: config.aspectRatio,- count: config.count,- images: await uploadAssets(world, config.imageRefIds),- });-- // 2. Apply image fills and remove Generating tag- const folderId = await ensureGenerationFolder(world, name, results.length);- await Promise.all(- results.map(async (result, i) => {- const eid = placeholders[i];- const imgName = results.length > 1 ? `${name} (${i + 1})` : name;- const asset = await loadAsset(world, result.url, { name: imgName, generationId, folderId });-- world.history.untrack(() => {- setComponent(world, eid, c.Name, imgName);-- const fid = createEntity(world);- setComponent(world, fid, c.Paint, PaintType.IMAGE);- setComponent(world, fid, c.AssetId, asset.id);- appendChild(world, fid, eid);-- removeComponent(world, eid, c.Generating);- });- }),- );-- recordGeneration(world, 'Generate image', placeholders);-- track('generation_completed', {- mode: 'image',- model: config.model,- aspect_ratio: config.aspectRatio,- count: results.length,- duration_ms: Math.round(performance.now() - startedAt),- });- } catch (err) {- // On failure, delete the placeholders so the shimmer stops. Untracked —- // a failed generation leaves no trace on the undo stack.- world.history.untrack(() => {- for (const eid of placeholders) {- deleteEntity(world, eid);- }- });- console.error(err);- track('generation_failed', {- mode: 'image',- model: config.model,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : 'unknown',- });- throw err;- }- };-- return { generate } as const;-}diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsxnew file mode 100644index 00000000..92606c80--- /dev/null+++ b/apps/web/src/components/genai/use-generate-image.tsx@@ -0,0 +1,81 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Image generation as the runtime means it: the prompt box declares a+// `generate.image` source per variant and inserts an <Image> consuming it,+// the way any drawn element is inserted. The asset system resolves the+// declaration through the world's Ai (see `EditorGenAi`), and the element+// lands in the project file with the `generate.image({...})` call that+// reproduces it — same spec, same asset, this session and the next.+//+// Generations go on the canvas rather than into a scene: they are material to+// work with, not yet part of a composition, and a scene owns a timeline they+// have no place on until someone puts them there.++import { generate } from "@diffusionstudio/jsx";+import { Image } from "@diffusionstudio/reconciler";+import { Library, Root } from "@diffusionstudio/runtime";+import { useWorld } from "@diffusionstudio/koota-solid";+import { useEditor } from "@/engine/hooks";+import { findEmptyPlacement } from "@/engine/placement";+import { ASPECT_RATIO_DIMENSIONS } from "@/utils/genai";+import { toast } from "somoto";++import type { ImageGenerationConfig } from "@/components/engine/db";+import type { Entity } from "koota";++const IMAGE_GAP = 40;++/** A declaration is keyed by its spec, so a fresh seed makes a new take. */+const randomSeed = () => Math.floor(Math.random() * 1_000_000);++export function useGenerateImage() {+ const world = useWorld();+ const editor = useEditor();++ const run = async (config: ImageGenerationConfig) => {+ const library = world.get(Library);+ const root = world.get(Root);+ if (!root) return;++ // References travel by library path: durable in the file, and readable+ // there. Ids that no longer name a library asset are dropped.+ const refs = (config.imageRefIds ?? [])+ .map((id) => library?.get(id)?.path)+ .filter((path): path is string => path !== undefined);++ const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };+ const totalWidth = config.count * dims.width + (config.count - 1) * IMAGE_GAP;+ const center = findEmptyPlacement(world, totalWidth, dims.height, IMAGE_GAP);+ const left = center.x - totalWidth / 2;+ const top = Math.round(center.y - dims.height / 2);++ const inserted: Entity[] = [];+ for (let index = 0; index < config.count; index++) {+ const src = generate.image({+ prompt: config.prompt,+ model: config.model,+ aspectRatio: config.aspectRatio,+ ...(refs.length ? { refs } : {}),+ seed: randomSeed(),+ });++ const x = Math.round(left + index * (dims.width + IMAGE_GAP));+ const [entity] = editor.insertElement(root, () => (+ <Image src={src} x={x} y={top} width={dims.width} height={dims.height} />+ ));+ if (entity) inserted.push(entity);+ }++ if (!inserted.length) {+ toast("Could not add the generation", {+ description: "Open a project to generate into.",+ });+ return;+ }+ editor.select(inserted);+ };++ return { generate: run } as const;+}diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.tsindex 84dbcbe7..f80568ec 100644--- a/apps/web/src/engine/editor.ts+++ b/apps/web/src/engine/editor.ts@@ -11,15 +11,22 @@*/import { Active, Chars, getActiveEntity, getEntityChildren, getEntityTree, getParentEntity, isText, Loop, Selected, Sequential, setActive, Source, Stage } from '@diffusionstudio/runtime';-import { isPropValue, SOURCE_ATTR } from '@diffusionstudio/jsx';+import { isAssetRef, isPropValue, serializeAssetRef, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createRoot } from 'solid-js';import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode, renderAuthored, withDocument } from '@diffusionstudio/reconciler';-import type { PropValue } from '@diffusionstudio/jsx';+import type { PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { AuthoredTree, HostNode, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';+/**+ * A value an edit can carry to the file: what a source spells as a literal,+ * or a `generate.*` declaration in its wire form (see `SerializedAssetRef`),+ * which the writer spells as the call that reproduces it.+ */+export type EditValue = PropValue | SerializedAssetRef;+/*** A property the editor changed, in the vocabulary of the JSX rather than of* the traits it was written to: `source` is the element it belongs to (its@@ -31,7 +38,7 @@ export interface PropEdit {kind: 'prop';source: string;name: string;- value: PropValue;+ value: EditValue;}/**@@ -59,7 +66,7 @@ export interface InsertEdit {source: string;parent: string;tag: string;- props: Record<string, PropValue>;+ props: Record<string, EditValue>;before?: string;text?: string;}@@ -95,7 +102,7 @@ export interface RemoveEdit {* own names — the pending source its entity has been re-stamped with, so the* write can answer with the real one (see `isPendingSource`).*/-export type LoopIteration = Record<string, { props: Record<string, PropValue>; text?: string; pending?: string }>;+export type LoopIteration = Record<string, { props: Record<string, EditValue>; text?: string; pending?: string }>;/*** A `<For>`/`<Index>` the editor needs written out as its iterations, so that@@ -132,7 +139,30 @@ export const isLooped = (entity: Entity): boolean => entity.isAlive() && entity./** What `insertElement` learns about each element while it renders. */interface Recorded {tag: string;- props: Record<string, PropValue>;+ props: Record<string, EditValue>;+}++/**+ * A prop as an edit can carry it (see `EditValue`): a value a file spells as+ * it is, a declaration in its wire form, or undefined for anything else — an+ * element is written with whatever of it the file can say, and the rest is+ * the project's to keep, not the writer's to guess at. The one place that+ * decides this, so a new kind of value is taught here once.+ */+function wireValue(value: unknown): EditValue | undefined {+ if (isPropValue(value)) return value;+ if (isAssetRef(value)) return serializeAssetRef(value);+ return undefined;+}++/** `wireValue` over a whole authored element, dropping what will not travel. */+function wireProps(props: Record<string, unknown>): Record<string, EditValue> {+ const wired: Record<string, EditValue> = {};+ for (const [name, value] of Object.entries(props)) {+ const wire = wireValue(value);+ if (wire !== undefined) wired[name] = wire;+ }+ return wired;}/**@@ -329,7 +359,7 @@ export class DocumentEditor {// know, or one the body renders twice: not spellable.if (!source || !stamped(member) || source in iteration) return;const pending = index > 0 ? nextPendingSource() : undefined;- iteration[source] = { props: authored.props, ...(authored.text === undefined ? {} : { text: authored.text }), ...(pending ? { pending } : {}) };+ iteration[source] = { props: wireProps(authored.props), ...(authored.text === undefined ? {} : { text: authored.text }), ...(pending ? { pending } : {}) };if (pending) restamp.push([member, pending]);}}@@ -770,8 +800,9 @@ export class DocumentEditor {},setProperty: (node, name, value) => {const recorded = isSceneNode(node) ? created.get(node.entity) : undefined;- if (recorded && name !== SOURCE_ATTR && name !== 'children' && name !== 'ref' && isPropValue(value)) {- recorded.props[name] = value;+ if (recorded && name !== SOURCE_ATTR && name !== 'children' && name !== 'ref') {+ const wire = wireValue(value);+ if (wire !== undefined) recorded.props[name] = wire;}document.setProperty(node, name, value);},diff --git a/apps/web/src/engine/placement.ts b/apps/web/src/engine/placement.tsnew file mode 100644index 00000000..3ab7d8fd--- /dev/null+++ b/apps/web/src/engine/placement.ts@@ -0,0 +1,102 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Where something new goes on the infinite canvas: beside the work rather+// than over it. Carried over from the engine the editor is migrating off,+// where this is what generations have always been placed by.++import {+ aabbsIntersect, getEntityBounds, getEntityChildren, getViewport, Hidden, Root, screenToWorld,+} from '@diffusionstudio/runtime';++import type { AABB, Point } from '@diffusionstudio/runtime';+import type { World } from 'koota';++/** The visible part of the canvas in document space, or null before a surface is mounted. */+function viewportBounds(world: World): AABB | null {+ const viewport = getViewport(world);+ if (!viewport) return null;++ // All four corners, so the box still holds if the camera is ever rotated.+ const corners = [+ screenToWorld(world, 0, 0),+ screenToWorld(world, viewport.width, 0),+ screenToWorld(world, viewport.width, viewport.height),+ screenToWorld(world, 0, viewport.height),+ ];++ return {+ minX: Math.min(...corners.map((corner) => corner.x)),+ minY: Math.min(...corners.map((corner) => corner.y)),+ maxX: Math.max(...corners.map((corner) => corner.x)),+ maxY: Math.max(...corners.map((corner) => corner.y)),+ };+}++/**+ * What stands on the canvas: the bounds of every visible root-level node.+ * A node the transform system has not measured yet reports nothing and is+ * left out — it has no place on the canvas to be avoided at.+ */+function occupied(world: World): AABB[] {+ const boxes: AABB[] = [];++ for (const entity of getEntityChildren(world, world.get(Root)!)) {+ if (entity.has(Hidden)) continue;+ const rect = getEntityBounds(world, [entity]);+ if (!rect) continue;+ boxes.push({ minX: rect.x, minY: rect.y, maxX: rect.x + rect.width, maxY: rect.y + rect.height });+ }++ return boxes;+}++const centerOf = (box: AABB): Point => ({ x: (box.minX + box.maxX) / 2, y: (box.minY + box.maxY) / 2 });++const distance = (a: Point, b: Point): number => Math.hypot(a.x - b.x, a.y - b.y);++/**+ * Where the middle of a `width`×`height` block goes on the canvas: beside+ * something already standing there, level with it — to its right, or to its+ * left when the right is taken — leaving `gap` between the two. Neighbours are+ * tried nearest the middle of the view first, and those in view before those+ * out of it, so a block lands next to what the user is looking at.+ *+ * The middle of the view when the canvas is empty, or when nothing has a free+ * side left; the origin when no surface is mounted yet.+ */+export function findEmptyPlacement(world: World, width: number, height: number, gap: number): Point {+ const viewport = viewportBounds(world);+ const center = viewport ? centerOf(viewport) : { x: 0, y: 0 };++ const boxes = occupied(world);+ if (boxes.length === 0) return center;++ const inView = viewport ? boxes.filter((box) => aabbsIntersect(box, viewport)) : [];+ const ordered = (inView.length > 0 ? inView : boxes)+ .slice()+ .sort((a, b) => distance(centerOf(a), center) - distance(centerOf(b), center));++ const free = (rect: AABB): boolean => !boxes.some((box) => aabbsIntersect(rect, box));++ for (const box of ordered) {+ // Level with the neighbour: the block keeps its middle, whatever the+ // two are sized like.+ const y = (box.minY + box.maxY) / 2;+ const minY = y - height / 2;+ const maxY = y + height / 2;++ const right = box.maxX + gap;+ if (free({ minX: right, minY, maxX: right + width, maxY })) {+ return { x: right + width / 2, y };+ }++ const left = box.minX - gap;+ if (free({ minX: left - width, minY, maxX: left, maxY })) {+ return { x: left - width / 2, y };+ }+ }++ return center;+}diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 880e95d6..cac4b800 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -25,6 +25,7 @@ import {} from "@/components/genai/config";import { assert } from "@/utils";import { uploadBlob } from "@/components/engine";+import { track } from "@/lib/analytics";import { trpc } from "@/lib/trpc";import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";@@ -124,9 +125,11 @@ export class EditorGenAi extends GenAi {const uploadId = crypto.randomUUID();const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });+ console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);const fileRef = await uploadBlob(audioFile, uploadId);assert(fileRef, "Failed to upload the scene audio for transcription");+ console.log(`[gen-ai] transcribing scene audio for ${key}`);const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });assert(transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),@@ -226,17 +229,43 @@ export class EditorGenAi extends GenAi {/** Runs a generation and stores its first result under `generated/`. */private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {- const { name, results, generationId } = await this.requestGeneration(spec);- assert(results.length > 0, "No results returned from the model");-- const url = results[0].url;- const response = await fetch(url);- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);- const blob = await response.blob();- const extension = url.split(/[?#]/)[0].split(".").pop();- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;+ const startedAt = performance.now();+ track("generation_started", {+ mode: spec.type,+ model: spec.model,+ prompt_length: spec.prompt.length,+ ...("aspectRatio" in spec ? { aspect_ratio: spec.aspectRatio } : {}),+ ...(spec.type === "image" ? { reference_count: spec.refIds.length } : {}),+ });- return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });+ try {+ console.log(`[gen-ai] generating ${spec.type} with ${spec.model}:`, spec);+ const { name, results, generationId } = await this.requestGeneration(spec);+ assert(results.length > 0, "No results returned from the model");++ const url = results[0].url;+ const response = await fetch(url);+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);+ const blob = await response.blob();+ const extension = url.split(/[?#]/)[0].split(".").pop();+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;++ const asset = await this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });+ track("generation_completed", {+ mode: spec.type,+ model: spec.model,+ duration_ms: Math.round(performance.now() - startedAt),+ });+ return asset;+ } catch (err) {+ track("generation_failed", {+ mode: spec.type,+ model: spec.model,+ duration_ms: Math.round(performance.now() - startedAt),+ error: err instanceof Error ? err.message.slice(0, 200) : "unknown",+ });+ throw err;+ }}private requestGeneration(spec: ResolvedSpec) {diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.tsindex 6b295012..34b6184f 100644--- a/packages/jsx/src/generate.ts+++ b/packages/jsx/src/generate.ts@@ -76,6 +76,51 @@ export function isAssetRef(value: unknown): value is AssetRef {return value instanceof AssetRef;}+/** An `AssetSpecInput` whose inputs are all strings — no nested declarations. */+export type FlatAssetSpec = AssetSpecInput & {+ refs?: string[];+ startFrame?: string;+ endFrame?: string;+};++/**+ * A declaration as data, for the edit protocol. An `AssetRef` itself cannot+ * travel as a prop value (see `isPropValue`: an object literal would not read+ * back as one), so an edit carries the spec in this tagged shape instead, and+ * the source writer spells it as the `generate.*` call that reproduces it.+ */+export type SerializedAssetRef = { $generate: FlatAssetSpec };++/**+ * The wire form of a declaration, or undefined when the spec references+ * other declarations: nested refs only exist in authored code, where the+ * call spelling them already is — the editor declares over library assets.+ */+export function serializeAssetRef(ref: AssetRef): SerializedAssetRef | undefined {+ const spec = ref.spec;+ const inputs = [+ ...(spec.type === "image" ? (spec.refs ?? []) : []),+ ...(spec.type === "video" ? [spec.startFrame, spec.endFrame] : []),+ ];+ if (inputs.some((input) => isAssetRef(input))) return undefined;++ // Without the undefined entries: they are not part of the declaration, and+ // the writer spells every entry it is handed.+ const entries = Object.entries(spec).filter(([, value]) => value !== undefined);+ return { $generate: Object.fromEntries(entries) as FlatAssetSpec };+}++const SPEC_TYPES = ["image", "video", "voice", "audio"];++export function isSerializedAssetRef(value: unknown): value is SerializedAssetRef {+ if (typeof value !== "object" || value === null || !("$generate" in value)) return false;+ const spec = (value as SerializedAssetRef).$generate;+ return (+ typeof spec === "object" && spec !== null &&+ SPEC_TYPES.includes(spec.type) && typeof spec.prompt === "string"+ );+}+/** Host-side accessor for a declaration's spec. */export function getAssetSpec(ref: AssetRef): AssetSpecInput {return ref.spec;diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 05776aa8..f1f68f20 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -10,11 +10,20 @@* @diffusionstudio/reconciler), so nothing in this package touches a host.*/-export { generate, AssetRef, isAssetRef, getAssetSpec } from "./generate";+export {+ generate,+ AssetRef,+ isAssetRef,+ getAssetSpec,+ serializeAssetRef,+ isSerializedAssetRef,+} from "./generate";export type {AspectRatio,AssetInput,AssetSpecInput,+ FlatAssetSpec,+ SerializedAssetRef,GenerateAudioOptions,GenerateImageOptions,GenerateVideoOptions,diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 623d5deb..649e88a3 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,11 +3,11 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';-import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';-import type { AssetRef, PropValue } from '@diffusionstudio/jsx';+import type { AssetRef } from '@diffusionstudio/jsx';import { trait, type Entity, type World } from 'koota';import type { ProjectDocument } from './host';@@ -52,14 +52,13 @@ const TextParts = trait(() => new Set<TextNode>());/*** What the project wrote on an entity's element, in the vocabulary of the JSX- * rather than of the traits it became: the tag, and every prop worth a value a- * source file could spell (see `PropValue`), as last set. Kept so an editor- * can spell an entity back out as an element — which is how a loop's- * iterations get written down one by one — without reading the traits+ * rather than of the traits it became: the tag, and every prop as last set.+ * Kept so an editor can spell an entity back out as an element — which is how+ * a loop's iterations get written down one by one — without reading the traits* backwards. `children` is not a prop here: an element's text is `Chars`, and* its element children are the entity's.*/-const Authored = trait(() => ({ tag: '', props: {} as Record<string, PropValue> }));+const Authored = trait(() => ({ tag: '', props: {} as Record<string, unknown> }));/** Props that address or wire an element rather than describe it. */const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);@@ -67,7 +66,7 @@ const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'export interface AuthoredElement {/** The camelCase tag the project used. */tag: string;- props: Record<string, PropValue>;+ props: Record<string, unknown>;/** The literal text of a `<text>`, when it holds any. */text?: string;}@@ -649,12 +648,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {if (!isSceneNode(node)) return;const { entity } = node;- // Noted before it is interpreted: what the project wrote is what an- // editor spelling the element back out needs, however the traits pack it.const authored = entity.get(Authored);if (authored && !UNAUTHORED_PROPS.has(name)) {- if (isPropValue(value)) authored.props[name] = value;- else delete authored.props[name];+ if (value === undefined) delete authored.props[name];+ else authored.props[name] = value;}switch (name) {@@ -985,7 +982,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return;}case 'src': {- entity.remove(GenerationRequest, LoadRequest, PendingSource);+ // Generating goes with the wait it belongs to: the resolution+ // running for the old src will not clear it, having been+ // superseded by whatever this one starts.+ entity.remove(GenerationRequest, LoadRequest, PendingSource, Generating);if (value === undefined || value === null || value === '') {entity.remove(AssetId);@@ -1159,13 +1159,13 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {case 'seed': {if (!entity.has(Caption)) return;// An authored src mounts a transcript directly; there is no- // transcription for the seed to key. (An AssetRef src is not a- // PropValue, so its request stands in for the authored record.)+ // transcription for the seed to key.if (entity.get(Authored)?.props.src !== undefined) return;if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;- // A resolution running for another seed must not bind late.- entity.remove(PendingSource);+ // A resolution running for another seed must not bind late,+ // nor leave the wait it was marked with behind.+ entity.remove(PendingSource, Generating);entity.add(TranscriptionRequest);entity.set(TranscriptionRequest, { seed: toNumber(value) ?? 0 });return;diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex 8080d0cc..a975699c 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';+import { Ai, FramePromises, Generating, GenerationRequest, Library, LoadRequest, PendingSource, TranscriptionRequest } from '../traits';import { bindAsset } from '../actions/assets';import { getEntityTree, getSceneAncestor } from '../queries/hierarchy';@@ -25,7 +25,7 @@ export function assetSystem(world: World): void {const ref = entity.get(GenerationRequest)!.ref;entity.remove(GenerationRequest);if (ref === null) continue;- resolve(world, entity, ref, ai.resolve(ref));+ resolve(world, entity, ref, ai.resolve(ref), true);}for (const entity of world.query(TranscriptionRequest)) {@@ -34,7 +34,7 @@ export function assetSystem(world: World): void {const seed = entity.get(TranscriptionRequest)!.seed;entity.remove(TranscriptionRequest);- resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed));+ resolve(world, entity, `transcript:${scene.id()}:${seed}`, ai.transcribe(world, scene, seed), true);}}}@@ -59,19 +59,20 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean* (PendingSource), so of overlapping resolutions only the latest binds, and* one that outlives its element (or its src) is dropped.*/-function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>): void {+function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>, generating = false): void {entity.add(PendingSource);entity.set(PendingSource, { value });+ if (generating) entity.add(Generating);const done = promise.then((asset) => {if (!current(entity, value)) return;- entity.remove(PendingSource);+ entity.remove(PendingSource, Generating);bindAsset(entity, asset);},(error: unknown) => {if (!current(entity, value)) return;- entity.remove(PendingSource);+ entity.remove(PendingSource, Generating);console.error('[runtime] could not resolve src:', error);},);