Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance asset management and introduce partial asset handling

konstantin-paulus28ad0372db10
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.ts
index ba96c2ff..58ecaa6a 100644
--- a/apps/desktop/src/projects.ts
+++ b/apps/desktop/src/projects.ts
@@ -598,7 +598,7 @@ a new session.
| \`index.tsx\` | The entry. Its default export renders the composition. |
| \`package.json\` | The project record: \`projectId\` (its identity, kept across renames), \`displayName\` (the name shown in the app), \`main\` (the entry), \`diffusion\` (how each scene is exported), and the dapi commands as scripts. |
| \`tsconfig.json\` | Types for the composition tags, through \`jsxImportSource\`. |
-| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be. Written by the app; hand edits are read on the next load. |
+| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be; for a generation without bytes, where it stands. Written by the app; hand edits are read on the next load. |
| \`assets/\` | The library's files: put one here and it is taken in while the app watches, and the app writes its own here too — generations under \`assets/generated/\`. Media imported through the app is linked where it lies instead, never copied. |
| \`cache/\` | Derived data (thumbnails, waveforms). Disposable, and not checked in. |
diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.ts
index 17fc24d8..829b6cc1 100644
--- a/apps/web/src/components/genai/use-modifiers.ts
+++ b/apps/web/src/components/genai/use-modifiers.ts
@@ -58,12 +58,6 @@ export function useModifiers() {
const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;
for (const entity of targets(name)) {
- // A failure is the answer to what was asked for (see `sourceErrorSystem`),
- // and asking for a different set of modifiers is a different question.
- // Taking the prop off is what asks again, and this is the user doing it.
- if (authoredElement(entity)?.props.error !== undefined) {
- editor.editProperty(entity, "error", false);
- }
editor.editProperty(entity, name, value);
}
};
diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsx
index 5210c789..825e9741 100644
--- a/apps/web/src/components/sidebar-left/assets.tsx
+++ b/apps/web/src/components/sidebar-left/assets.tsx
@@ -27,6 +27,7 @@ import {
BreadcrumbsSeparator,
} from "../ui/breadcrumbs";
import { LazyAssetItem } from "./asset-item";
+import { PartialAssetItem } from "./partial-asset-item";
import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";
import { useLibrary } from "@/engine/library";
import { useAssetSelection } from "@/engine/hooks";
@@ -56,6 +57,7 @@ export function Assets() {
const [currentFolder, setCurrentFolder] = createSignal("");
const allAssets = createMemo(() => library()?.list().filter((asset) => asset.type !== "SCRIPT") ?? []);
+ const allPartials = createMemo(() => library()?.partials() ?? []);
const allFolders = createMemo(() => [...(library()?.folders() ?? [])].sort());
const activeFilterLabel = () => {
@@ -64,17 +66,17 @@ export function Assets() {
return ASSET_FILTER_OPTIONS.find((option) => option.value === value)?.label ?? null;
};
- const filteredAssets = createMemo(() => {
+ /** Whether an entry is in view: of the filtered type, matching the search, or in the open folder. */
+ const isShown = (entry: { type: string; path: string }): boolean => {
const q = query().trim().toLowerCase();
const selectedFilter = assetFilter();
- const folder = currentFolder();
+ if (selectedFilter !== "ALL" && entry.type !== selectedFilter) return false;
+ if (q) return basename(entry.path).toLowerCase().includes(q);
+ return dirname(entry.path) === currentFolder();
+ };
- return allAssets().filter((asset) => {
- if (selectedFilter !== "ALL" && asset.type !== selectedFilter) return false;
- if (q) return basename(asset.path).toLowerCase().includes(q);
- return dirname(asset.path) === folder;
- });
- });
+ const filteredAssets = createMemo(() => allAssets().filter(isShown));
+ const filteredPartials = createMemo(() => allPartials().filter(isShown));
const visibleFolders = createMemo(() => {
const q = query().trim().toLowerCase();
@@ -85,7 +87,7 @@ export function Assets() {
});
const panelTitle = createMemo(() => (currentFolder() ? basename(currentFolder()) : "Assets"));
- const itemCount = createMemo(() => visibleFolders().length + filteredAssets().length);
+ const itemCount = createMemo(() => visibleFolders().length + filteredPartials().length + filteredAssets().length);
// Deep paths collapse like the breadcrumbs docs example:
// All assets / … / parent / current, with the hidden folders in a dropdown.
@@ -222,11 +224,11 @@ export function Assets() {
if (event.key === "Backspace" || event.key === "Delete") {
const selected = selectedAssetId();
- const asset = selected ? library()?.get(selected) : undefined;
- if (asset) {
+ const entry = selected ? (library()?.get(selected) ?? library()?.getPartial(selected)) : undefined;
+ if (entry) {
event.preventDefault();
event.stopPropagation();
- void library()?.remove([asset]);
+ void library()?.remove([entry]);
return;
}
@@ -259,9 +261,9 @@ export function Assets() {
};
const hasAssets = () => allAssets().length > 0;
- const hasContent = () => hasAssets() || allFolders().length > 0;
+ const hasContent = () => hasAssets() || allPartials().length > 0 || allFolders().length > 0;
const isFiltering = () => query().trim().length > 0 || assetFilter() !== "ALL";
- const isEmptyView = () => visibleFolders().length === 0 && filteredAssets().length === 0;
+ const isEmptyView = () => visibleFolders().length === 0 && filteredPartials().length === 0 && filteredAssets().length === 0;
const handleCreateFolder = () => withLibrary((lib) => {
const parent = currentFolder();
@@ -522,7 +524,7 @@ export function Assets() {
No matching assets
</div>
</Show>
- <Show when={visibleFolders().length > 0 || filteredAssets().length > 0}>
+ <Show when={!isEmptyView()}>
<div class="grid grid-cols-2 gap-x-2 gap-y-4">
<For each={visibleFolders()}>
{(folder) => (
@@ -535,6 +537,15 @@ export function Assets() {
/>
)}
</For>
+ <For each={filteredPartials()}>
+ {(partial) => (
+ <PartialAssetItem
+ partial={partial}
+ selected={selectedAssetId() === partial.id}
+ onSelect={() => handleSelectAsset(partial.id)}
+ />
+ )}
+ </For>
<For each={filteredAssets()}>
{(asset) => (
<LazyAssetItem
diff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsx
index 3b5efad4..a12f412b 100644
--- a/apps/web/src/components/sidebar-left/folder-item.tsx
+++ b/apps/web/src/components/sidebar-left/folder-item.tsx
@@ -67,7 +67,7 @@ export function FolderItem(props: FolderItemProps) {
const name = () => basename(props.path);
const childCount = createMemo(() => {
const children = library()?.childrenOf(props.path);
- return children ? children.folders.length + children.assets.length : 0;
+ return children ? children.folders.length + children.assets.length + children.partials.length : 0;
});
const handleDragStart = (event: DragEvent) => {
diff --git a/apps/web/src/components/sidebar-left/partial-asset-item.tsx b/apps/web/src/components/sidebar-left/partial-asset-item.tsx
new file mode 100644
index 00000000..309a56a0
--- /dev/null
+++ b/apps/web/src/components/sidebar-left/partial-asset-item.tsx
@@ -0,0 +1,109 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// A partial document in the asset panel: a generation with no bytes yet, or
+// none ever. Pending, it pulses where its asset will be; failed, it carries
+// the reason, and asking again is done from here — retrying forgets the
+// failure and re-asks for every element that got it, deleting only forgets.
+
+import { Show } from "solid-js";
+import { toast } from "somoto";
+import { useWorld } from "@diffusionstudio/koota-solid";
+import { assetName } from "@diffusionstudio/assets";
+import {
+ ContextMenu,
+ ContextMenuContent,
+ ContextMenuItem,
+ ContextMenuPortal,
+ ContextMenuSeparator,
+ ContextMenuShortcut,
+ ContextMenuTrigger,
+} from "../ui/context-menu";
+import { Icon } from "../ui/icon";
+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
+import { useLibrary } from "@/engine/library";
+import { retryGeneration } from "@/engine/generations";
+
+import type { PartialAsset } from "@diffusionstudio/assets";
+
+export type PartialAssetItemProps = {
+ partial: PartialAsset;
+ selected: boolean;
+ onSelect(): void;
+};
+
+export function PartialAssetItem(props: PartialAssetItemProps) {
+ const world = useWorld();
+ const library = useLibrary();
+
+ const failed = () => props.partial.state === "error";
+
+ const handleRetry = async () => {
+ const lib = library();
+ if (!lib) return;
+ try {
+ await retryGeneration(world, lib, props.partial);
+ } catch (e) {
+ toast.error("Failed to retry", { description: (e as Error).message });
+ }
+ };
+
+ const handleDelete = async () => {
+ try {
+ await library()?.remove([props.partial]);
+ } catch (e) {
+ toast.error("Failed to delete", { description: (e as Error).message });
+ }
+ };
+
+ return (
+ <ContextMenu>
+ <ContextMenuTrigger
+ as="div"
+ class="flex flex-col gap-1 text-left"
+ data-asset-id={props.partial.id}
+ onClick={props.onSelect}
+ onContextMenu={props.onSelect}>
+ <Tooltip placement="bottom" openDelay={400}>
+ <TooltipTrigger
+ as="div"
+ data-selected={props.selected}
+ class="relative aspect-video w-full overflow-clip rounded bg-muted after:pointer-events-none after:absolute after:inset-0 after:rounded after:opacity-0 after:ring-2 after:ring-inset after:ring-ring after:z-10 data-[selected=true]:after:opacity-100"
+ classList={{ "animate-pulse": !failed() }}
+ >
+ <div class="absolute inset-0 flex items-center justify-center text-muted-foreground">
+ <Show when={failed()} fallback={<Icon name="spinner-loader" class="size-5 animate-spin" />}>
+ <Icon name="alert-warning" class="size-5 text-destructive" />
+ </Show>
+ </div>
+ </TooltipTrigger>
+ <TooltipContent class="max-w-64">
+ {failed() ? props.partial.error || "Generation failed" : "Generating…"}
+ </TooltipContent>
+ </Tooltip>
+ <div
+ class="text-xs truncate select-none"
+ classList={{ "text-destructive": failed(), "text-muted-foreground": !failed() }}
+ title={props.partial.error}
+ >
+ {assetName(props.partial)}
+ </div>
+ </ContextMenuTrigger>
+ <ContextMenuPortal>
+ <ContextMenuContent class="w-[180px]">
+ <Show when={failed()}>
+ <ContextMenuItem onSelect={handleRetry}>
+ Retry
+ </ContextMenuItem>
+ <ContextMenuSeparator />
+ </Show>
+ <ContextMenuItem onSelect={handleDelete}>
+ Delete
+ <ContextMenuShortcut>⌫</ContextMenuShortcut>
+ </ContextMenuItem>
+ </ContextMenuContent>
+ </ContextMenuPortal>
+ </ContextMenu>
+ );
+}
diff --git a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx b/apps/web/src/components/sidebar-right/inspector/asset-info.tsx
index 21f177c6..0da21d18 100644
--- a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/asset-info.tsx
@@ -20,11 +20,86 @@ import { assetName } from "@diffusionstudio/assets";
import { useLibrary } from "@/engine/library";
import { useAssetSelection } from "@/engine/hooks";
import { insertAssetAtPlayhead, replaceAssetSource } from "@/engine/asset-actions";
+import { retryGeneration } from "@/engine/generations";
-import type { Asset } from "@diffusionstudio/assets";
+import type { Asset, PartialAsset } from "@diffusionstudio/assets";
-/** Information about the library asset picked in the assets panel. */
+/** Information about the library entry picked in the assets panel. */
export function AssetInfoPanel() {
+ const selection = useAssetSelection();
+
+ return (
+ <Show when={selection.partial()} fallback={<AssetDetails />}>
+ {(partial) => <PartialDetails partial={partial()} />}
+ </Show>
+ );
+}
+
+/**
+ * A generation without bytes: where it stands, and — once it has failed —
+ * the reason, and the way to ask again.
+ */
+function PartialDetails(props: { partial: PartialAsset }) {
+ const world = useWorld();
+ const library = useLibrary();
+
+ const failed = () => props.partial.state === "error";
+
+ const handleRetry = async () => {
+ const lib = library();
+ if (lib) await retryGeneration(world, lib, props.partial);
+ };
+
+ const handleDelete = async () => {
+ await library()?.remove([props.partial]);
+ };
+
+ return (
+ <div class="flex flex-col w-full px-4 border-t border-border">
+ <div class="h-12 flex items-center justify-between">
+ <span class="text-base font-strong">Information</span>
+ </div>
+
+ <div class="py-3 border-t border-b border-border text-xs break-all">
+ {assetName(props.partial)}
+ </div>
+
+ <div class="flex flex-col gap-1 my-2 text-xs text-muted-foreground">
+ <div class="flex h-7 items-center gap-2">
+ <span class="w-20 shrink-0">Status</span>
+ <span class="min-w-0 flex-1 text-right" classList={{ "text-destructive": failed() }}>
+ {failed() ? "Failed" : "Generating…"}
+ </span>
+ </div>
+ <Show when={props.partial.error}>
+ {(error) => (
+ <div class="flex flex-col gap-2 py-1">
+ <span class="w-20 shrink-0">Error</span>
+ <span class="min-w-0 text-left wrap-break-words text-destructive">{error()}</span>
+ </div>
+ )}
+ </Show>
+ <div class="flex h-7 items-center gap-2">
+ <span class="w-20 shrink-0">Requested</span>
+ <span class="min-w-0 flex-1 text-right truncate">{formatAssetDate(props.partial.createdAt)}</span>
+ </div>
+ </div>
+
+ <div class="flex flex-col gap-2 my-2">
+ <Show when={failed()}>
+ <Button variant="default" class="w-full" onClick={handleRetry}>
+ Retry generation
+ </Button>
+ </Show>
+ <Button variant="secondary" class="w-full" onClick={handleDelete}>
+ Delete
+ </Button>
+ </div>
+ </div>
+ );
+}
+
+function AssetDetails() {
const world = useWorld();
const library = useLibrary();
const selection = useAssetSelection();
diff --git a/apps/web/src/components/sidebar-right/inspector/inspector.tsx b/apps/web/src/components/sidebar-right/inspector/inspector.tsx
index 7a0cd250..eb05b23a 100644
--- a/apps/web/src/components/sidebar-right/inspector/inspector.tsx
+++ b/apps/web/src/components/sidebar-right/inspector/inspector.tsx
@@ -75,7 +75,7 @@ function classifyNode(entity: Entity): SelectionTarget {
export function Inspector() {
const tool = useTool();
const { nodes, keyframes, first } = useSelection();
- const { asset } = useAssetSelection();
+ const { asset, partial } = useAssetSelection();
// For ExportPanel (root scenes only) and TransitionSettings (sequence items).
const parent = createMemo(() => getParentNode(first()));
@@ -88,7 +88,7 @@ export function Inspector() {
const selectionTarget = createMemo<SelectionTarget>(() => {
if (tool() === ToolType.SCENE) return "scene-tool";
if (keyframes().length > 0) return "keyframe";
- if (asset()) return "asset";
+ if (asset() || partial()) return "asset";
const entity = first();
if (nodes().length === 1 && entity) return classifyNode(entity);
return "stage";
@@ -96,7 +96,7 @@ export function Inspector() {
// Remounts the panels when the selection changes
const selectionHash = createMemo(() => {
- return [...nodes(), ...keyframes(), asset()?.id ?? ""].join(",") + selectionTarget();
+ return [...nodes(), ...keyframes(), asset()?.id ?? partial()?.id ?? ""].join(",") + selectionTarget();
});
const includesTarget = (...targets: SelectionTarget[]) => {
diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.ts
index d431ea90..a5f23bca 100644
--- a/apps/web/src/engine/create-engine.ts
+++ b/apps/web/src/engine/create-engine.ts
@@ -9,7 +9,6 @@ import { AssetSelection, Hud, Keys, MODIFIER_KEYS, Pointer, PointerEvents, Proje
import { inputSystem } from './input/input-system';
import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';
import { shortcutSystem } from './input/shortcuts';
-import { sourceErrorSystem } from './source-errors';
import type { RuntimeWorld } from '@diffusionstudio/runtime';
import type { CanvasPointerEvent, PointerEventType } from '@diffusionstudio/runtime';
@@ -213,7 +212,6 @@ class Engine {
inputSystem(this.world);
shortcutSystem(this.world);
assetSystem(this.world);
- sourceErrorSystem(this.world);
playbackSystem(this.world);
motionSystem(this.world);
transformSystem(this.world);
diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.ts
index 84fca003..85d396ad 100644
--- a/apps/web/src/engine/editor.ts
+++ b/apps/web/src/engine/editor.ts
@@ -266,11 +266,11 @@ function nextSourceAfter(world: World, entity: Entity): string | undefined {
* The props of a `<video>`/`<image>` that only mean something while the node
* plays media, dropped when the intrinsic paint is removed (see
* `removeIntrinsicPaint`): the source, what qualifies it (its fit, its
- * window, a frames directory's rate, the modifiers put on it), its error,
- * and the audio mix a rect has no track to apply to. `start`/`end` stay:
- * they place the clip, media or not.
+ * window, a frames directory's rate, the modifiers put on it), and the audio
+ * mix a rect has no track to apply to. `start`/`end` stay: they place the
+ * clip, media or not.
*/
-const MEDIA_PROPS = ['src', 'error', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;
+const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;
/**
* What `copy` took: the subtrees as authored, and the source of the parent
diff --git a/apps/web/src/engine/generations.ts b/apps/web/src/engine/generations.ts
new file mode 100644
index 00000000..95dd97f4
--- /dev/null
+++ b/apps/web/src/engine/generations.ts
@@ -0,0 +1,48 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Asking again for a generation that failed. A failed generation stands in
+ * the library as a partial document in error, and a declaration whose key
+ * stands there resolves to the failure rather than to another run — in
+ * this session and the next. Removing the record is what asks again; this
+ * is that removal, plus making the elements still waiting on the answer ask
+ * right away rather than on the next mount.
+ */
+
+import { isAssetRef } from '@diffusionstudio/jsx';
+import { authoredElement } from '@diffusionstudio/reconciler';
+import {
+ Caption, Generating, GenerationRequest, LoadRequest, PendingSource, SourceError, TranscriptionRequest,
+} from '@diffusionstudio/runtime';
+
+import type { AssetLibrary, PartialAsset } from '@diffusionstudio/assets';
+import type { World } from 'koota';
+
+/** Forgets the failure `partial` records and re-asks for every failed source. */
+export async function retryGeneration(world: World, library: AssetLibrary, partial: PartialAsset): Promise<void> {
+ await library.remove([partial]);
+
+ for (const entity of world.query(SourceError)) {
+ if (!entity.get(SourceError)!.generated) continue;
+ const authored = authoredElement(entity);
+
+ if (!authored) continue;
+
+ const src = authored.props.src;
+ entity.remove(SourceError, PendingSource, Generating);
+
+ if (isAssetRef(src)) {
+ entity.add(GenerationRequest);
+ entity.set(GenerationRequest, { ref: src });
+ } else if (typeof src === 'string' && src !== '') {
+ // A source put through modifiers: the load is cheap, the derivation is what failed.
+ entity.add(LoadRequest);
+ entity.set(LoadRequest, { value: src });
+ } else if (entity.has(Caption)) {
+ entity.add(TranscriptionRequest);
+ entity.set(TranscriptionRequest, { seed: Number(authored.props.seed) || 0 });
+ }
+ }
+}
diff --git a/apps/web/src/engine/history.ts b/apps/web/src/engine/history.ts
index 72790c50..7b61c2ac 100644
--- a/apps/web/src/engine/history.ts
+++ b/apps/web/src/engine/history.ts
@@ -36,10 +36,9 @@ import type { CapturedNode, DocumentEditor, EntityEdit } from './editor';
/**
* Props that are pointing and viewing rather than composition: undoing them
* is not what anyone means by undo, so they pass through to the file without
- * entering the history. `error` is the runtime speaking (see source-errors),
- * not the user.
+ * entering the history.
*/
-const EXCLUDED_PROPS: ReadonlySet<string> = new Set(['selected', 'active', 'camera', 'expanded', 'clipHeight', 'timeline', 'error']);
+const EXCLUDED_PROPS: ReadonlySet<string> = new Set(['selected', 'active', 'camera', 'expanded', 'clipHeight', 'timeline']);
/**
* How long after one step a same-shaped step still merges into it. Edits
diff --git a/apps/web/src/engine/hooks/use-asset-selection.ts b/apps/web/src/engine/hooks/use-asset-selection.ts
index 6f1bee81..bc31a2a0 100644
--- a/apps/web/src/engine/hooks/use-asset-selection.ts
+++ b/apps/web/src/engine/hooks/use-asset-selection.ts
@@ -8,12 +8,13 @@ import { createMemo } from 'solid-js';
import { AssetSelection } from '../traits';
import { useLibrary } from '../library';
-import type { Asset } from '@diffusionstudio/assets';
+import type { Asset, PartialAsset } from '@diffusionstudio/assets';
/**
- * Reactive read and write of the AssetSelection trait: the one library asset
- * the assets panel has picked and the inspector describes. Shared between
- * the two sidebars, so it lives on the world rather than in either panel.
+ * Reactive read and write of the AssetSelection trait: the one library entry
+ * the assets panel has picked and the inspector describes — an asset, or the
+ * partial document of a generation without bytes. Shared between the two
+ * sidebars, so it lives on the world rather than in either panel.
*/
export function useAssetSelection() {
const world = useWorld();
@@ -31,9 +32,17 @@ export function useAssetSelection() {
return lib.list().find((entry) => entry.id === current);
});
- const select = (next: Asset | string | null) => {
+ /** The picked partial document, when the pick is one; drops out as its bytes land. */
+ const partial = createMemo(() => {
+ const lib = library();
+ const current = id();
+ if (!lib || !current) return undefined;
+ return lib.partials().find((entry) => entry.id === current);
+ });
+
+ const select = (next: Asset | PartialAsset | string | null) => {
world.set(AssetSelection, { id: typeof next === 'string' ? next : (next?.id ?? null) });
};
- return { id, asset, select };
+ return { id, asset, partial, select };
}
diff --git a/apps/web/src/engine/source-errors.ts b/apps/web/src/engine/source-errors.ts
deleted file mode 100644
index 75b13a08..00000000
--- a/apps/web/src/engine/source-errors.ts
+++ /dev/null
@@ -1,47 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-/**
- * A failed generation, written back to the JSX that asked for it. The runtime
- * leaves the reason on the entity (`SourceError`); this is what turns it into
- * the `error` prop of the element, so the failure lives where the rest of the
- * document lives — in the file — rather than only in the session that saw it.
- *
- * That is what keeps a generation from being run again: the project remounts
- * on every edit and is compiled afresh on every reload, and each time the
- * element carries the answer its declaration already got (see the asset
- * system, which leaves such an element alone). Taking the prop off the
- * element — in the file, like any other authored change — is what asks for
- * the run again; nothing else in the editor does it silently.
- *
- * Only generations are written: a load that failed is cheap to try again, and
- * an asset that has since been put back should simply load.
- */
-
-import { SourceError } from '@diffusionstudio/runtime';
-import { authoredElement } from '@diffusionstudio/reconciler';
-
-import { getDocumentEditor, isLooped } from './editor';
-
-import type { World } from 'koota';
-
-export function sourceErrorSystem(world: World): void {
- for (const entity of world.query(SourceError)) {
- const failure = entity.get(SourceError)!;
- if (!failure.generated) continue;
-
- const authored = authoredElement(entity);
- // Nothing to write it to (an entity no element produced), or nothing
- // new to say.
- if (!authored || authored.props.error === failure.value) continue;
-
- // One element renders every iteration of a loop, so writing a prop to
- // one of them means unrolling the loop into the file. That is a change
- // to the project worth making when the user asks for one, and not
- // worth making behind their back over a generation that failed.
- if (isLooped(entity)) continue;
-
- getDocumentEditor(world).editProperty(entity, 'error', failure.value);
- }
-}
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 2b8df4f1..71e72bf6 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -10,7 +10,7 @@ import {
import type { SourceModifierValues } from "@diffusionstudio/runtime";
import { createEncoder } from "@diffusionstudio/encoder";
import { createCapture } from "@/engine/capture";
-import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";
+import { assetName, GENERATED_DIR, isPartialAsset } from "@diffusionstudio/assets";
import {
PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
@@ -25,7 +25,7 @@ import { trpc } from "@/lib/trpc";
import { toast } from "somoto";
import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
-import type { Asset, AssetLibrary, AssetType } from "@diffusionstudio/assets";
+import type { Asset, AssetLibrary, AssetType, PartialAsset, ReserveOptions } from "@diffusionstudio/assets";
import type { FileRef } from "@diffusionstudio/api-contract";
import type { ExportResult } from "@diffusionstudio/encoder";
import type { Entity, World } from "koota";
@@ -39,6 +39,14 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
transcript: "Caption generation failed",
};
+/** What each declaration is to become, for the partial that stands for it. */
+const ASSET_TYPES: Record<AssetSpecInput["type"], AssetType> = {
+ image: "IMAGE",
+ video: "VIDEO",
+ voice: "AUDIO",
+ audio: "AUDIO",
+};
+
/**
* One step of a source modifier (see the runtime's `SourceModifiers`): a
* model call that makes a new asset out of one the library already holds.
@@ -74,6 +82,17 @@ type ResolvedSpec =
| { type: "voice"; model: string; prompt: string; voice: string; seed?: number }
| { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };
+/** What the partial standing for a run is called and is to become, and what its failure is called. */
+type RunDescription = Omit<ReserveOptions, "key" | "folder"> & { title: string };
+
+/**
+ * A failure the user has already been told of — as a toast when it happened,
+ * and in the library's record from then on. It goes up the dependency graph
+ * as it is: a declaration whose input failed is not a second thing gone
+ * wrong.
+ */
+class ReportedError extends Error {}
+
/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */
export function attachAi(world: World, library: AssetLibrary, dir?: string): EditorGenAi {
const ai = new EditorGenAi(library, world.get(Project)?.id ?? "project", dir);
@@ -90,12 +109,12 @@ export class EditorGenAi extends GenAi {
/**
* Declarations already resolved, keyed by ref identity — a ref consumed by
- * several elements resolves (and validates) once. A failed one is forgotten:
- * from there on the element that asked is what carries the failure, in its
- * `error` prop, and that is what keeps the generation from running again.
+ * several elements resolves (and validates) once. A failed one is
+ * forgotten, so that asking again — once its record is gone from the
+ * library — asks the library again rather than this map.
*/
private readonly memo = new Map<AssetRef, Promise<Asset>>();
- /** In-flight generations and transcriptions keyed by `generationKey`. */
+ /** Runs in flight, keyed by generation key. */
private readonly inflight = new Map<string, Promise<Asset>>();
public constructor(library: AssetLibrary, projectId: string, dir?: string) {
@@ -120,120 +139,114 @@ export class EditorGenAi extends GenAi {
/**
* Transcribes the scene's audible mix for a `<captions>` element (see the
- * runtime's asset system). Cached by scene id + seed: the same pair is the
- * same transcript asset across sessions (the transcript lands in the
- * library under `generated/` with that key in the manifest), and a new
- * seed transcribes the scene again.
+ * runtime's asset system). Keyed by scene id + seed: the same pair is the
+ * same transcript asset across sessions, and a new seed transcribes the
+ * scene again.
*/
- public async transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {
+ public transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {
const key = transcriptKey(scene, seed);
+ return this.generated(
+ key,
+ () => ({ type: "TRANSCRIPT", name: `${this.nextCaptionsName()}.json`, title: FAILURE_TITLES.transcript }),
+ (partial) => this.runTranscription(world, scene, key, partial),
+ );
+ }
+
+ /**
+ * `asset` through every modifier the element asked for, one step at a time
+ * and in a fixed order. Each step is cached in its own right, so turning
+ * one on leaves what the others already made alone — adding `upscale` to a
+ * cut-out picture pays for the enlarging, not for the matte again.
+ */
+ public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {
+ let derived = asset;
+ for (const kind of steps(modifiers)) {
+ derived = await this.transform(kind, derived);
+ }
+ return derived;
+ }
- const cached = this.library.list().find((asset) => asset.generation?.key === key);
- if (cached) return cached;
+ /**
+ * Runs one step over `asset` and returns what it produced, stored under
+ * `generated/` like any other model output. Keyed by step and input, so the
+ * same call on the same asset is the same result in this session and the
+ * next: upscaling a picture twice costs what upscaling it once did.
+ */
+ public transform(kind: TransformKind, asset: Asset): Promise<Asset> {
+ // No upscale factor in the key: the endpoint takes none, so every factor
+ // is the same call and would otherwise be billed once per number asked
+ // for. It belongs here the moment the API can be told one.
+ const key = `transform:v1:${kind}:${asset.id}`;
+ const base = assetName(asset).replace(/\.[^.]+$/, "");
+ return this.generated(
+ key,
+ () => ({ type: asset.type, name: `${base} (${TRANSFORMS[kind].suffix})`, title: TRANSFORMS[kind].title }),
+ () => this.runTransform(kind, asset, key),
+ );
+ }
+
+ /**
+ * The library's answer for `key`, or the run that produces one. An asset
+ * the key landed as is returned; a key standing in error rejects with the
+ * recorded reason — answered, not run or paid for again, and not toasted
+ * again either; a run in flight is joined. Otherwise the run starts, with
+ * a partial document reserved for it first (a pending one the last session
+ * never finished is taken over), and ends either as the asset that
+ * replaces the partial or as the partial's recorded failure.
+ */
+ private async generated(
+ key: string,
+ describe: () => RunDescription,
+ run: (partial: PartialAsset) => Promise<Asset>,
+ ): Promise<Asset> {
+ const known = this.library.generated(key);
+ if (known && !isPartialAsset(known)) return known;
+ if (known?.state === "error") throw new ReportedError(known.error || "Generation failed");
const running = this.inflight.get(key);
if (running) return await running;
- const promise = this.runTranscription(world, scene, key);
+ const { title, ...partial } = describe();
+ const promise = this.library.reserve({ key, folder: GENERATED_DIR, ...partial }).then(async (reserved) => {
+ try {
+ return await run(reserved);
+ } catch (error) {
+ const failure = reportFailure(error, title);
+ this.library.fail(reserved, failure.message);
+ throw failure;
+ }
+ });
this.inflight.set(key, promise);
try {
return await promise;
- } catch (error) {
- throw reportFailure(error, FAILURE_TITLES.transcript);
} finally {
this.inflight.delete(key);
}
}
- /** Encodes the scene's audio, transcribes it, and stores the transcript. */
- private async runTranscription(world: World, scene: Entity, key: string): Promise<Asset> {
- assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");
+ private async generateFromRef(ref: AssetRef): Promise<Asset> {
+ const spec = getAssetSpec(ref);
- // The scene's own capture world: the project rendered again, reduced to
- // this scene, with nothing drawn — see `createCapture`.
- const capture = await createCapture(world, scene, { mode: "offline-audio", dir: this.dir });
- let result: ExportResult;
+ // Inputs first: their ids are part of the key. An input that failed has
+ // said so already; anything else wrong with them is said here.
+ let resolved: ResolvedSpec;
try {
- const encoder = await createEncoder(capture.world, {
- format: "ogg",
- video: { enabled: false },
- audio: { enabled: true, codec: "opus", sampleRate: 24000 },
- });
- result = await encoder.render();
- } finally {
- capture.dispose();
+ resolved = await this.resolveSpec(spec);
+ } catch (error) {
+ throw error instanceof ReportedError ? error : reportFailure(error, FAILURE_TITLES[spec.type]);
}
- assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");
- const uploadId = crypto.randomUUID();
- const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });
- console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);
- const fileRef = await uploadBlob(audioFile, uploadId);
- assert(fileRef, "Failed to upload the scene audio for transcription");
-
- console.log(`[gen-ai] transcribing scene audio for ${key}`);
- const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });
- assert(
- transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),
- "No speech detected. The audio does not appear to contain recognizable speech.",
+ return this.generated(
+ JSON.stringify(resolved),
+ () => ({ type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt), title: FAILURE_TITLES[spec.type] }),
+ (partial) => this.runGeneration(resolved, partial),
);
-
- const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });
- const asset = await this.library.store(blob, {
- name: `${this.nextCaptionsName()}.json`,
- folder: GENERATED_DIR,
- generation: { key },
- });
-
- // A re-take of unchanged speech comes back byte-identical, and the library
- // dedups by content: `store` then hands back the earlier take still keyed
- // by its old seed. Re-key it, or the authored seed misses the cache and
- // transcribes again on every load.
- if (asset.generation?.key !== key) {
- this.library.update(asset, { generation: { key } });
- }
-
- return asset;
- }
-
- private nextCaptionsName(): string {
- let max = 0;
- for (const asset of this.library.list()) {
- const match = assetName(asset).match(/^Captions (\d+)\.json$/);
- if (match) max = Math.max(max, Number(match[1]));
- }
- return `Captions ${max + 1}`;
}
private resolveInput(input: AssetInput): Promise<Asset> {
return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);
}
- private async generateFromRef(ref: AssetRef): Promise<Asset> {
- const spec = getAssetSpec(ref);
-
- try {
- const resolved = await this.resolveSpec(spec);
- const generationKey = JSON.stringify(resolved);
-
- const cached = this.library.list().find((asset) => asset.generation?.key === generationKey);
- if (cached) return cached;
-
- const running = this.inflight.get(generationKey);
- if (running) return await running;
-
- const promise = this.runGeneration(resolved, generationKey);
- this.inflight.set(generationKey, promise);
- try {
- return await promise;
- } finally {
- this.inflight.delete(generationKey);
- }
- } catch (error) {
- throw reportFailure(error, FAILURE_TITLES[spec.type]);
- }
- }
-
private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {
switch (spec.type) {
case "image": {
@@ -252,7 +265,7 @@ export class EditorGenAi extends GenAi {
spec.startFrame !== undefined ? this.resolveInput(spec.startFrame) : undefined,
spec.endFrame !== undefined ? this.resolveInput(spec.endFrame) : undefined,
]);
- const resolved = {
+ return {
type: "video",
model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,
prompt: spec.prompt,
@@ -262,9 +275,7 @@ export class EditorGenAi extends GenAi {
seed: spec.seed,
startFrameId: startFrame?.id,
endFrameId: endFrame?.id,
- } satisfies ResolvedSpec;
- checkVideoConstraints(resolved);
- return resolved;
+ };
}
case "voice": {
return {
@@ -287,8 +298,14 @@ export class EditorGenAi extends GenAi {
}
}
- /** Runs a generation and stores its first result under `generated/`. */
- private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {
+ /**
+ * Runs a generation and stores its first result in place of `partial`. A
+ * spec a model cannot take fails here rather than before the partial is
+ * reserved, so the refusal is recorded like any other and not asked again.
+ */
+ private async runGeneration(spec: ResolvedSpec, partial: PartialAsset): Promise<Asset> {
+ if (spec.type === "video") checkVideoConstraints(spec);
+
const startedAt = performance.now();
track("generation_started", {
mode: spec.type,
@@ -303,7 +320,7 @@ export class EditorGenAi extends GenAi {
const { name, results, generationId } = await this.requestGeneration(spec);
assert(results.length > 0, "No results returned from the model");
- const asset = await this.store(results[0].url, name, { key: generationKey, id: generationId });
+ const asset = await this.store(results[0].url, name, { key: partial.generation.key, id: generationId });
track("generation_completed", {
mode: spec.type,
model: spec.model,
@@ -321,47 +338,55 @@ export class EditorGenAi extends GenAi {
}
}
- /**
- * `asset` through every modifier the element asked for, one step at a time
- * and in a fixed order. Each step is cached in its own right, so turning
- * one on leaves what the others already made alone — adding `upscale` to a
- * cut-out picture pays for the enlarging, not for the matte again.
- */
- public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {
- let derived = asset;
- for (const kind of steps(modifiers)) {
- derived = await this.transform(kind, derived);
+ /** Encodes the scene's audio, transcribes it, and stores the transcript in place of `partial`. */
+ private async runTranscription(world: World, scene: Entity, key: string, partial: PartialAsset): Promise<Asset> {
+ assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");
+
+ // The scene's own capture world: the project rendered again, reduced to
+ // this scene, with nothing drawn — see `createCapture`.
+ const capture = await createCapture(world, scene, { mode: "offline-audio", dir: this.dir });
+ let result: ExportResult;
+ try {
+ const encoder = await createEncoder(capture.world, {
+ format: "ogg",
+ video: { enabled: false },
+ audio: { enabled: true, codec: "opus", sampleRate: 24000 },
+ });
+ result = await encoder.render();
+ } finally {
+ capture.dispose();
}
- return derived;
- }
+ assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");
- /**
- * Runs one step over `asset` and returns what it produced, stored under
- * `generated/` like any other model output. Keyed by step and input, so the
- * same call on the same asset is the same result in this session and the
- * next: upscaling a picture twice costs what upscaling it once did.
- */
- public async transform(kind: TransformKind, asset: Asset): Promise<Asset> {
- // No upscale factor in the key: the endpoint takes none, so every factor
- // is the same call and would otherwise be billed once per number asked
- // for. It belongs here the moment the API can be told one.
- const key = `transform:v1:${kind}:${asset.id}`;
+ const uploadId = crypto.randomUUID();
+ const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });
+ console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);
+ const fileRef = await uploadBlob(audioFile, uploadId);
+ assert(fileRef, "Failed to upload the scene audio for transcription");
- const cached = this.library.list().find((entry) => entry.generation?.key === key);
- if (cached) return cached;
+ console.log(`[gen-ai] transcribing scene audio for ${key}`);
+ const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });
+ assert(
+ transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),
+ "No speech detected. The audio does not appear to contain recognizable speech.",
+ );
- const running = this.inflight.get(key);
- if (running) return await running;
+ const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });
+ return this.library.store(blob, {
+ name: assetName(partial),
+ folder: GENERATED_DIR,
+ generation: { key },
+ });
+ }
- const promise = this.runTransform(kind, asset, key);
- this.inflight.set(key, promise);
- try {
- return await promise;
- } catch (error) {
- throw reportFailure(error, TRANSFORMS[kind].title);
- } finally {
- this.inflight.delete(key);
+ /** The next free `Captions N`, counting the takes still in flight. */
+ private nextCaptionsName(): string {
+ let max = 0;
+ for (const entry of [...this.library.list(), ...this.library.partials()]) {
+ const match = assetName(entry).match(/^Captions (\d+)\.json$/);
+ if (match) max = Math.max(max, Number(match[1]));
}
+ return `Captions ${max + 1}`;
}
/** Uploads the input, runs the call, and stores the result beside the generations. */
@@ -493,6 +518,21 @@ export class EditorGenAi extends GenAi {
/** Whether an asset type is footage, for the calls that only take footage. */
const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";
+/** How much of a prompt names the partial standing for its generation. */
+const PROVISIONAL_NAME_LENGTH = 48;
+
+/**
+ * What a generation is called before its result names it: the head of its
+ * prompt, cut at a word. The model's own name takes over when the bytes
+ * land (see `store`).
+ */
+function provisionalName(prompt: string): string {
+ const text = prompt.trim().replace(/\s+/g, " ").replace(/\//g, "-");
+ if (text.length <= PROVISIONAL_NAME_LENGTH) return text || "Generation";
+ const cut = text.lastIndexOf(" ", PROVISIONAL_NAME_LENGTH);
+ return `${text.slice(0, cut > 0 ? cut : PROVISIONAL_NAME_LENGTH)}…`;
+}
+
/**
* What to call a generated result: the extension its type implies, falling
* back to the one its URL carries. Neither is guaranteed — some models hand
@@ -512,23 +552,24 @@ function resultExtension(url: string, blob: Blob): string {
}
/**
- * Says what a generation failed with, and hands the error on. The message is
- * what the element ends up carrying too (the runtime's `SourceError`, which
- * the editor writes into its `error` prop), so the sentence in the file, the
- * one on the canvas and the one in the toast are the same. The toast is keyed
- * by it: variants that failed the same way are one thing gone wrong rather
- * than four, and a render that ran into the same wall does not stack up.
+ * Says what a generation failed with, and hands the failure on as one
+ * already reported. The message is what the library records on the partial
+ * and what the element carries (the runtime's `SourceError`), so the
+ * sentence in the asset panel, the one on the canvas and the one in the
+ * toast are the same. The toast is keyed by it: variants that failed the
+ * same way are one thing gone wrong rather than four, and a render that ran
+ * into the same wall does not stack up.
*/
-function reportFailure(error: unknown, title: string): Error {
- const failure = error instanceof Error ? error : new Error(String(error));
+function reportFailure(error: unknown, title: string): ReportedError {
+ const message = error instanceof Error ? error.message : String(error);
- console.error(`[gen-ai] ${title}:`, failure);
+ console.error(`[gen-ai] ${title}:`, error);
toast.error(title, {
- id: `gen-ai:${title}:${failure.message}`,
- description: failure.message,
+ id: `gen-ai:${title}:${message}`,
+ description: message,
});
- return failure;
+ return new ReportedError(message);
}
/**
diff --git a/docs/examples/02-genai.tsx b/docs/examples/02-genai.tsx
index 68012163..b121e16a 100644
--- a/docs/examples/02-genai.tsx
+++ b/docs/examples/02-genai.tsx
@@ -9,7 +9,8 @@
* motion clips) that generates in topological order, blocking the command
* until every asset lands. <captions> reads the finished scene audio after
* generation, so caption timing matches the voiceover. Consumes credits;
- * results are cached per session, so re-mounting unchanged specs is free.
+ * results are cached in the project's library, so re-mounting unchanged
+ * specs is free.
* Discover models and voices with `dapi models <type>` and `dapi voices`;
* omitted here, so each stage uses the default model.
*/
diff --git a/docs/reference/jsx/captions.md b/docs/reference/jsx/captions.md
index 08a1585f..9e700294 100644
--- a/docs/reference/jsx/captions.md
+++ b/docs/reference/jsx/captions.md
@@ -9,7 +9,7 @@
</scene>
```
-Transcription is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once ready. Because it reads the scene's audible mix, it waits until **every other source in the scene has landed** — a generated `voice` or `audio` track is transcribed at its final placement, not at the placeholder. The scene must contain an unmuted, unhidden audio or video source; without one the node carries an [`error`](./errors.md#failed-sources) saying so.
+Transcription is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once ready. Because it reads the scene's audible mix, it waits until **every other source in the scene has landed** — a generated `voice` or `audio` track is transcribed at its final placement, not at the placeholder. The scene must contain an unmuted, unhidden audio or video source; without one the node carries an error saying so, recorded in the library like any failed generation (see [errors.md](./errors.md#failed-sources)).
## Bringing your own transcript
@@ -42,6 +42,8 @@ A transcript is cached under **the scene's id and the `seed`**, and the cached a
The audio itself is not part of the key. **Recutting a scene does not re-transcribe it**: to pick up changed audio, bump `seed`. A value used before replays that take from cache; a value that has not been used for this scene transcribes it again, which costs credits.
+A transcription that failed is recorded under the same key, as a library entry in error, and is not run again until that entry is removed — see [errors.md](./errors.md#failed-sources).
+
## Trimming
Captions carry the same [timing](./timing.md) props as media nodes, and the transcript is source content that must stay aligned to the audio — so advance `start` and `sourceIn` together (and `end`/`sourceOut`), never `start` alone. To show only from 15 s onward, set both `sourceIn={15}` and `start={15}`. One node can't skip a gap, so to blank captions out for a middle stretch — e.g. under an overlay — use two nodes that meet at the gap:
diff --git a/docs/reference/jsx/errors.md b/docs/reference/jsx/errors.md
index 5b2148f6..44afc16c 100644
--- a/docs/reference/jsx/errors.md
+++ b/docs/reference/jsx/errors.md
@@ -12,17 +12,23 @@ Runtime errors are reported against the compiled module. Since types are strippe
## Failed sources
-A failed **generation** is written back into your file as the element's `error` prop:
-
-```tsx
-<image src={generate.image({ prompt: "a red fox" })} error="Model refused the prompt" />
+A failed **generation** is recorded in the library, never in your file. The library holds a partial document for every generation the project has asked for: `pending` while the model runs, replaced by the asset when it lands, and kept as `error` — with the reason — when it does not. In `assets.yml`:
+
+```yaml
+- id: 3f9c1a7e2b8d4c05
+ path: generated/a red fox
+ type: IMAGE
+ generation:
+ key: '{"type":"image","model":"…","prompt":"a red fox",…}'
+ state: error
+ error: Model refused the prompt
```
-That is deliberate, and it is what keeps a refused or impossible generation from being run — or paid for — again by every reopen of the project. An element holding an `error` is not resolved a second time, and nothing in the editor takes the prop off silently.
+That record is what keeps a refused or impossible generation from being run — or paid for — again by every reopen of the project: a declaration whose key stands in error resolves to that error, and its element carries the message without anything being requested. The source that declared it is not written to.
-**Removing the `error` attribute is what asks for the run again**, and it is the only thing that does. Not another take, not another prompt.
+**Removing the record is what asks for the run again**, and it is the only thing that does — not another take, not another prompt. Delete the entry in the asset panel (or its record in `assets.yml`), or use *Retry* on it, which deletes the record and re-asks at once for every element still waiting on the answer. Changing the declaration is a new key, and so a new generation; the old record stays until removed.
-Only generations are written down this way: a load that failed is cheap to try again, and an asset that has since been put back should simply load. An element rendered inside a `<For>` is left alone too — writing a prop to one iteration would mean unrolling the loop into the file, which is not a change to make behind your back over a generation that failed.
+Only generations are recorded this way: a load that failed is cheap to try again, and an asset that has since been put back should simply load.
## Blank or partial `<html>` content in captures
diff --git a/docs/reference/jsx/generate.md b/docs/reference/jsx/generate.md
index f2b7286a..fe0326da 100644
--- a/docs/reference/jsx/generate.md
+++ b/docs/reference/jsx/generate.md
@@ -34,7 +34,7 @@ export default function Project() {
Declarations are **pure**: calling `generate.*` validates its options and returns a ref; nothing is requested until an element carrying it mounts. A ref that is never used by a mounted element (directly or as an input to another asset) is never generated. Declarations may live at module scope or inside components.
-Generation is **asynchronous and non-blocking**: the element is on the canvas immediately, showing a generating state, and its paint attaches when the asset lands. [`context`](../tools/context.md) reports where each one stands — generating, failed with the reason, or done with the library path it landed as — and a declaration that fails leaves its element carrying an [`error`](./errors.md#failed-sources).
+Generation is **asynchronous and non-blocking**: the element is on the canvas immediately, showing a generating state, and its paint attaches when the asset lands. [`context`](../tools/context.md) reports where each one stands — generating, failed with the reason, or done with the library path it landed as — and a declaration that fails leaves its element carrying the reason, recorded in the library (see [errors.md](./errors.md#failed-sources)).
## Declaration options
@@ -85,3 +85,5 @@ generate.audio(opts: {
Generation is long-running and consumes credits, so results are **cached by content**: a key derived from the fully-resolved spec — `type`, `model`, `prompt`, the resolved ids of any references, `seed`, and the rest of the options, with the defaults filled in. Re-mounting an unchanged project reuses cached assets instead of regenerating, two declarations with identical specs collapse to a single asset, and identical concurrent declarations share one request; changing any option produces a new asset. Set `seed` to make a spec reproducible.
Finished generations are stored in the project's library under `generated/` with their spec key, so the cache survives app restarts — and a save that only touched an unrelated part of the file regenerates nothing. Deleting the asset from the library regenerates it on the next mount.
+
+Where a generation stands is recorded there too: a run in flight is a `pending` entry of the library, and one that failed stays as an `error` entry carrying its reason, which answers the declaration — instead of another run — until the entry is removed (see [errors.md](./errors.md#failed-sources)).
diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.md
index 3d68c203..641ac896 100644
--- a/docs/reference/jsx/media.md
+++ b/docs/reference/jsx/media.md
@@ -8,7 +8,7 @@
- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.
- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is mounted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.
-Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts, showing a generating state until its source lands. A source that never lands leaves the element carrying an [`error`](./errors.md#failed-sources).
+Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts, showing a generating state until its source lands. A source that never lands leaves the element carrying the reason (see [errors.md](./errors.md#failed-sources)).
An `<img>` inside [`<html>`](./html.md) additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.
@@ -46,6 +46,8 @@ Modifiers compose with declarations: `<image src={generate.image({ prompt: "a re
A project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/`, whether the app wrote the bytes there or a symlink points at them), and what it was found to be. Folders are listed too, so an empty one survives a reload. Renaming or moving an asset in the panel rewrites the `src` props that named it.
+Generations are listed from the moment they are asked for: a partial record (`state: pending`) with no `source` while the model runs, the asset once it lands, and a record in `state: error` carrying the reason when it fails, which stands until removed (see [errors.md](./errors.md#failed-sources)).
+
## Adding an asset
Symlink the file into the project's `assets/` folder and it is in the library. The app is watching, so it lands while the project is open; a project that is closed takes it in on the next open. Its library path is where the link sits under `assets/` — folder and name, so `assets/b-roll/drone.mp4` is `"b-roll/drone.mp4"` and `assets/logo.png` is `"logo.png"`.
diff --git a/docs/reference/tools/context.md b/docs/reference/tools/context.md
index 311e2a8b..e73e5c42 100644
--- a/docs/reference/tools/context.md
+++ b/docs/reference/tools/context.md
@@ -46,4 +46,4 @@ With no project open (the app sits at the dashboard) the report is just `{ rootD
`fontFamilies` is what text can be drawn with right now — loaded into the world, not merely named in the source — and always includes the editor default. For every family installed on the machine, see [`fonts`](./fonts.md).
-`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the same message the element carries as its `error` prop, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).
+`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the message the library recorded for the generation, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).
diff --git a/packages/assets/src/hash.ts b/packages/assets/src/hash.ts
index 13b05267..7abe7ea3 100644
--- a/packages/assets/src/hash.ts
+++ b/packages/assets/src/hash.ts
@@ -42,6 +42,16 @@ export async function hashSequence(frames: { name: string; size: number }[]): Pr
return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);
}
+/**
+ * The id of a partial document: the hash of its generation key, since there
+ * are no bytes yet. Stable for the key, so the same generation reserved twice
+ * is one record, and unlike any content hash the bytes could come to.
+ */
+export async function hashKey(key: string): Promise<string> {
+ const bytes = new TextEncoder().encode(`key\n${key}`);
+ return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);
+}
+
/** Whether a string could be an asset id (and not, say, a library path). */
export const looksLikeId = (value: string): boolean =>
value.length === ID_LENGTH && /^[0-9a-f]+$/.test(value);
diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.ts
index 73af34c4..4a03cdc0 100644
--- a/packages/assets/src/library.ts
+++ b/packages/assets/src/library.ts
@@ -16,22 +16,27 @@
// path, rebind whatever is bound to a relinked asset's old id) it asks of
// `LibraryOptions`. Generation is not its business either — a generated
// file is stored like any other bytes the app produced, with its
-// `generation` record along. Its state is solid signals (`assets()`,
+// `generation` record along — but where a generation stands is: a run that
+// has started is a partial document here (`reserve`), one that failed stays
+// as one with its reason (`fail`), and one that landed replaces its partial
+// (`store`). Its state is solid signals (`assets()`, `partials()`,
// `folders()`), so readers inside a tracking scope follow it.
import { createSignal, type Accessor } from 'solid-js';
import { AssetCache } from './cache';
-import { hashBlob, hashSequence } from './hash';
+import { hashBlob, hashKey, hashSequence } from './hash';
import {
- ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,
+ ASSETS_DIR, isAbsoluteSource, isPartialRecord, isProjectSource, isUrlSource, normalizeManifest, toRecord,
} from './manifest';
import { detectMimeType, DEFAULT_SEQUENCE_FPS, isSequenceListing, probeMedia, sortFrames } from './probe';
-import { assetFolder, assetName, basename, dirname, joinPath, normalizePath } from './types';
+import { assetFolder, assetName, basename, dirname, isPartialAsset, joinPath, normalizePath } from './types';
import type { FsEntry, ProjectFS } from './fs';
import type { AssetRecord, Manifest } from './manifest';
-import type { Asset, AssetDirectoryHandle, AssetFileHandle, AssetGeneration, SequenceAsset } from './types';
+import type {
+ Asset, AssetDirectoryHandle, AssetEntry, AssetFileHandle, AssetGeneration, AssetType, PartialAsset, SequenceAsset,
+} from './types';
/** How long changes pile up before the manifest is written. */
const SAVE_DEBOUNCE = 200;
@@ -64,21 +69,38 @@ export interface ImportOptions {
generation?: AssetGeneration;
}
+export interface ReserveOptions {
+ /** The generation key the partial stands for (see `AssetGeneration`). */
+ key: string;
+ /** What the generation is to become. */
+ type: AssetType;
+ /** A provisional library name; the bytes, once they land, are named by whoever stores them. */
+ name: string;
+ /** Library folder to place it in; root by default. */
+ folder?: string;
+}
+
export class AssetLibrary {
public readonly fs: ProjectFS;
/** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */
public readonly cache: AssetCache;
/** Library assets, newest first; transient ones excluded. Reactive. */
public readonly assets: Accessor<Asset[]>;
+ /** Partial documents — generations pending or failed — newest first. Reactive. */
+ public readonly partials: Accessor<PartialAsset[]>;
/** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */
public readonly folders: Accessor<ReadonlySet<string>>;
private readonly setAssets: (assets: Asset[]) => void;
+ private readonly setPartials: (partials: PartialAsset[]) => void;
private readonly setFolders: (folders: ReadonlySet<string>) => void;
/** Folders declared in the manifest (the empty ones need declaring). */
private declared = new Set<string>();
- /** Every asset by id: the library's, and transient ones resolved from outside it. */
- private readonly map = new Map<string, Asset>();
+ /**
+ * Every entry by id: the library's assets, transient ones resolved from
+ * outside it, and the partial documents of generations without bytes.
+ */
+ private readonly map = new Map<string, AssetEntry>();
private readonly onRename: LibraryOptions['onRename'];
private readonly onRelink: LibraryOptions['onRelink'];
private inflight = new Map<string, Promise<Asset>>();
@@ -93,6 +115,7 @@ export class AssetLibrary {
this.onRename = options.onRename;
this.onRelink = options.onRelink;
[this.assets, this.setAssets] = createSignal<Asset[]>([]);
+ [this.partials, this.setPartials] = createSignal<PartialAsset[]>([]);
[this.folders, this.setFolders] = createSignal<ReadonlySet<string>>(new Set());
}
@@ -106,30 +129,63 @@ export class AssetLibrary {
/** The library's assets as of now, straight from the map. */
private listNow(): Asset[] {
- return Array.from(this.map.values()).filter((asset) => !asset.transient);
+ const assets: Asset[] = [];
+ for (const entry of this.map.values()) {
+ if (!isPartialAsset(entry) && !entry.transient) assets.push(entry);
+ }
+ return assets;
+ }
+
+ /** The partial documents as of now, straight from the map. */
+ private partialsNow(): PartialAsset[] {
+ return Array.from(this.map.values()).filter(isPartialAsset);
+ }
+
+ /** Everything at a library path as of now: assets and partials, transient ones excluded. */
+ private entriesNow(): AssetEntry[] {
+ return Array.from(this.map.values()).filter((entry) => isPartialAsset(entry) || !entry.transient);
}
/** The asset with `id`, or at library path `path`; undefined otherwise. */
public get(pathOrId: string): Asset | undefined {
const byId = this.map.get(pathOrId);
- if (byId) return byId;
+ if (byId && !isPartialAsset(byId)) return byId;
const path = normalizePath(pathOrId);
- for (const asset of this.map.values()) {
- if (asset.path === path && !asset.transient) return asset;
+ for (const asset of this.listNow()) {
+ if (asset.path === path) return asset;
}
return undefined;
}
+ /** The partial document with `id`; undefined when there is none, or the id is an asset's. */
+ public getPartial(id: string): PartialAsset | undefined {
+ const entry = this.map.get(id);
+ return entry && isPartialAsset(entry) ? entry : undefined;
+ }
+
/** The asset whose bytes are `source`, if the library links to it. */
public bySource(source: string): Asset | undefined {
- for (const asset of this.map.values()) {
- if (asset.source === source && !asset.transient) return asset;
+ for (const asset of this.listNow()) {
+ if (asset.source === source) return asset;
+ }
+ return undefined;
+ }
+
+ /**
+ * What the library holds for a generation key: the asset the generation
+ * landed as, or the partial document standing for it while it runs or
+ * after it failed. Undefined when the generation was never asked for
+ * here — or its record was removed, which is how one is asked for again.
+ */
+ public generated(key: string): AssetEntry | undefined {
+ for (const entry of this.entriesNow()) {
+ if (entry.generation?.key === key) return entry;
}
return undefined;
}
- /** Direct children of a folder ('' for the root): its subfolders and assets. Reactive. */
- public childrenOf(folder: string): { folders: string[]; assets: Asset[] } {
+ /** Direct children of a folder ('' for the root): its subfolders, assets and partials. Reactive. */
+ public childrenOf(folder: string): { folders: string[]; assets: Asset[]; partials: PartialAsset[] } {
const prefix = folder ? `${folder}/` : '';
const folders = new Set<string>();
for (const path of this.folders()) {
@@ -138,23 +194,25 @@ export class AssetLibrary {
}
}
const assets = this.assets().filter((asset) => assetFolder(asset) === folder);
- return { folders: [...folders].sort(), assets };
+ const partials = this.partials().filter((partial) => assetFolder(partial) === folder);
+ return { folders: [...folders].sort(), assets, partials };
}
- /** Every folder as of now: declared, implied by asset paths, and all their ancestors. */
+ /** Every folder as of now: declared, implied by entry paths, and all their ancestors. */
private foldersNow(): Set<string> {
const folders = new Set<string>();
const declare = (path: string): void => {
for (let folder = path; folder && !folders.has(folder); folder = dirname(folder)) folders.add(folder);
};
for (const folder of this.declared) declare(folder);
- for (const asset of this.listNow()) declare(assetFolder(asset));
+ for (const entry of this.entriesNow()) declare(assetFolder(entry));
return folders;
}
/** Publishes the map's state to the signals. */
private publish(): void {
this.setAssets(this.listNow());
+ this.setPartials(this.partialsNow());
this.setFolders(this.foldersNow());
}
@@ -171,8 +229,13 @@ export class AssetLibrary {
const manifest = normalizeManifest(await this.fs.readManifest());
this.declared = new Set(manifest.folders);
- const next = new Map<string, Asset>();
+ const next = new Map<string, AssetEntry>();
await Promise.all(manifest.assets.map(async (record) => {
+ // A partial has no bytes to attach or re-examine; it is its record.
+ if (isPartialRecord(record)) {
+ next.set(record.id, { ...record });
+ return;
+ }
const asset = await this.revive(record).catch((error: unknown) => {
console.warn(`[assets] could not load ${record.path}:`, error);
return this.attach(record);
@@ -181,8 +244,8 @@ export class AssetLibrary {
}));
// Keep transient assets and the same-id instances entities already hold.
- for (const [id, asset] of this.map) {
- if (asset.transient && !next.has(id)) next.set(id, asset);
+ for (const [id, entry] of this.map) {
+ if (!isPartialAsset(entry) && entry.transient && !next.has(id)) next.set(id, entry);
}
this.map.clear();
for (const [id, asset] of next) {
@@ -294,7 +357,7 @@ export class AssetLibrary {
/** Resolves a source outside the library without adding it to the manifest. */
private transient(source: string): Promise<Asset> {
- const existing = Array.from(this.map.values()).find((asset) => asset.transient && asset.source === source);
+ const existing = this.transientBySource(source);
if (existing) return Promise.resolve(existing);
return this.once(`transient:${source}`, async () => {
@@ -304,12 +367,19 @@ export class AssetLibrary {
asset.transient = true;
// A transient asset that turns out to be one the library has is that one.
const owned = this.map.get(asset.id);
- if (owned && !owned.transient) return owned;
+ if (owned && !isPartialAsset(owned) && !owned.transient) return owned;
this.map.set(asset.id, asset);
return asset;
});
}
+ private transientBySource(source: string): Asset | undefined {
+ for (const entry of this.map.values()) {
+ if (!isPartialAsset(entry) && entry.transient && entry.source === source) return entry;
+ }
+ return undefined;
+ }
+
/** The File backing an asset. */
public file(asset: Asset): Promise<File> {
return asset.handle.getFile();
@@ -373,23 +443,88 @@ export class AssetLibrary {
/**
* Writes bytes the app produced into `assets/<folder>/<name>` and takes
- * the file into the library at the same library path.
+ * the file into the library at the same library path. Stored under a
+ * generation key, the asset takes the place of the partial document
+ * reserved for that key, and answers for the key from here on — even
+ * when the bytes turn out to be ones the library already had under
+ * another key, as a re-take of unchanged speech does.
*/
public async store(blob: Blob, options: ImportOptions & { name: string }): Promise<Asset> {
const path = this.uniquePath(joinPath(options.folder ?? '', options.name));
const source = joinPath(ASSETS_DIR, path);
await this.fs.write(source, blob);
const asset = await this.describeFile(source, { path, generation: options.generation });
- return this.add(asset);
+ const key = options.generation?.key;
+ if (key !== undefined) this.dropPartial(key);
+ const stored = this.add(asset);
+ if (options.generation && stored.generation?.key !== key) {
+ this.update(stored, { generation: options.generation });
+ }
+ return stored;
+ }
+
+ /**
+ * Puts a partial document in the library for a generation about to run:
+ * `pending`, at a path of its own under `folder`. A partial already
+ * standing for the key — a run the last session never finished, or one
+ * that failed and is being asked for again — is set pending again rather
+ * than doubled. An asset the key already landed as is left alone: this
+ * is not the place to ask whether to run (see `generated`).
+ */
+ public async reserve(options: ReserveOptions): Promise<PartialAsset> {
+ const id = await hashKey(options.key);
+ const existing = this.getPartial(id);
+ if (existing) {
+ existing.state = 'pending';
+ delete existing.error;
+ this.changed();
+ return existing;
+ }
+
+ const partial: PartialAsset = {
+ id,
+ path: this.uniquePath(joinPath(options.folder ?? '', options.name)),
+ type: options.type,
+ createdAt: new Date().toISOString(),
+ generation: { key: options.key },
+ state: 'pending',
+ };
+ this.reorder(partial);
+ this.changed();
+ return partial;
+ }
+
+ /**
+ * Records that the generation `partial` stands for failed, and why. The
+ * record stays: it is what answers for the key from here on, until it
+ * is removed. A partial no longer in the library (removed while the run
+ * went on) is not brought back — that removal was the answer.
+ */
+ public fail(partial: PartialAsset, error: string): void {
+ const current = this.getPartial(partial.id);
+ if (!current) return;
+ current.state = 'error';
+ current.error = error;
+ this.changed();
+ }
+
+ /** Takes the partial standing for `key` out, its bytes having landed. */
+ private dropPartial(key: string): void {
+ for (const partial of this.partialsNow()) {
+ if (partial.generation.key === key) this.map.delete(partial.id);
+ }
}
/** Puts an asset into the library, deduplicating by content. */
private add(asset: Asset): Asset {
const existing = this.map.get(asset.id);
- if (existing && !existing.transient) return existing;
- // A transient asset the library now takes in becomes a real one; the
- // instance entities hold stays valid, so mutate rather than replace.
- if (existing) {
+ if (existing && isPartialAsset(existing)) {
+ // A content hash colliding with a key hash: the bytes win the id.
+ this.map.delete(existing.id);
+ } else if (existing) {
+ if (!existing.transient) return existing;
+ // A transient asset the library now takes in becomes a real one; the
+ // instance entities hold stays valid, so mutate rather than replace.
Object.assign(existing, asset);
delete existing.transient;
this.reorder(existing);
@@ -401,11 +536,11 @@ export class AssetLibrary {
return asset;
}
- /** Puts `asset` at the front of the map (newest first). */
- private reorder(asset: Asset): void {
- const rest = Array.from(this.map.values()).filter((other) => other.id !== asset.id);
+ /** Puts `entry` at the front of the map (newest first). */
+ private reorder(entry: AssetEntry): void {
+ const rest = Array.from(this.map.values()).filter((other) => other.id !== entry.id);
this.map.clear();
- this.map.set(asset.id, asset);
+ this.map.set(entry.id, entry);
for (const other of rest) this.map.set(other.id, other);
}
@@ -444,39 +579,45 @@ export class AssetLibrary {
return next;
}
- /** Renames an asset within its folder; the source file is untouched. */
- public rename(asset: Asset, name: string): void {
+ /** Renames an entry within its folder; an asset's source file is untouched. */
+ public rename(entry: AssetEntry, name: string): void {
const clean = normalizePath(name).replace(/\//g, '-');
- if (!clean || clean === assetName(asset)) return;
- this.setPath(asset, this.uniquePath(joinPath(assetFolder(asset), clean), asset));
+ if (!clean || clean === assetName(entry)) return;
+ this.setPath(entry, this.uniquePath(joinPath(assetFolder(entry), clean), entry));
}
- /** Moves assets into a folder ('' for the root). */
- public move(assets: Asset[], folder: string): void {
+ /** Moves entries into a folder ('' for the root). */
+ public move(entries: AssetEntry[], folder: string): void {
const target = normalizePath(folder);
if (target) this.declared.add(target);
- for (const asset of assets) {
- if (assetFolder(asset) === target) continue;
- this.setPath(asset, this.uniquePath(joinPath(target, assetName(asset)), asset));
+ for (const entry of entries) {
+ if (assetFolder(entry) === target) continue;
+ this.setPath(entry, this.uniquePath(joinPath(target, assetName(entry)), entry));
}
}
- private setPath(asset: Asset, path: string): void {
- const from = asset.path;
+ private setPath(entry: AssetEntry, path: string): void {
+ const from = entry.path;
if (from === path) return;
- asset.path = path;
+ entry.path = path;
this.changed();
- this.onRename?.(asset, from);
+ // Nothing names a partial by path: it is not a source yet.
+ if (!isPartialAsset(entry)) this.onRename?.(entry, from);
}
- /** Removes assets from the library. Bytes the app wrote into `assets/` go too. */
- public async remove(assets: Asset[]): Promise<void> {
- for (const asset of assets) {
- this.map.delete(asset.id);
- if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {
- await this.fs.remove(asset.source).catch(() => { });
+ /**
+ * Removes entries from the library. Bytes the app wrote into `assets/`
+ * go too. Removing a partial forgets where its generation stood — for a
+ * failed one, that is what asks for the run again.
+ */
+ public async remove(entries: AssetEntry[]): Promise<void> {
+ for (const entry of entries) {
+ this.map.delete(entry.id);
+ if (isPartialAsset(entry)) continue;
+ if (isProjectSource(entry.source) && entry.source.startsWith(`${ASSETS_DIR}/`)) {
+ await this.fs.remove(entry.source).catch(() => { });
}
- this.cache.remove(asset);
+ this.cache.remove(entry);
}
this.changed();
}
@@ -527,20 +668,20 @@ export class AssetLibrary {
}
}
this.declared.add(to);
- for (const asset of this.listNow()) {
- if (asset.path.startsWith(prefix)) this.setPath(asset, to + asset.path.slice(from.length));
+ for (const entry of this.entriesNow()) {
+ if (entry.path.startsWith(prefix)) this.setPath(entry, to + entry.path.slice(from.length));
}
this.changed();
return to;
}
- /** Deletes a folder and everything in it. Returns the removed assets. */
- public async deleteFolder(path: string): Promise<Asset[]> {
+ /** Deletes a folder and everything in it. Returns the removed entries. */
+ public async deleteFolder(path: string): Promise<AssetEntry[]> {
const prefix = `${path}/`;
for (const folder of [...this.declared]) {
if (folder === path || folder.startsWith(prefix)) this.declared.delete(folder);
}
- const doomed = this.listNow().filter((asset) => asset.path.startsWith(prefix));
+ const doomed = this.entriesNow().filter((entry) => entry.path.startsWith(prefix));
await this.remove(doomed);
return doomed;
}
@@ -561,7 +702,7 @@ export class AssetLibrary {
return {
version: 1,
folders: [...this.declared].sort(),
- assets: this.listNow().map(toRecord),
+ assets: this.entriesNow().map(toRecord),
};
}
@@ -586,9 +727,9 @@ export class AssetLibrary {
// -----------------------------------------------------------------------
// Describing
- /** A library path not taken by any other asset: `name`, `name 2`, … */
- private uniquePath(path: string, except?: Asset): string {
- const taken = new Set(this.listNow().filter((asset) => asset !== except).map((asset) => asset.path));
+ /** A library path not taken by any other entry: `name`, `name 2`, … */
+ private uniquePath(path: string, except?: AssetEntry): string {
+ const taken = new Set(this.entriesNow().filter((entry) => entry !== except).map((entry) => entry.path));
if (!taken.has(path)) return path;
const folder = dirname(path);
const name = basename(path);
diff --git a/packages/assets/src/manifest.ts b/packages/assets/src/manifest.ts
index 5f77d1b9..f6ac2422 100644
--- a/packages/assets/src/manifest.ts
+++ b/packages/assets/src/manifest.ts
@@ -2,17 +2,10 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// The asset manifest: the project's record of its library. On desktop it is
-// `assets.yml` at the project root (the main process converts to and from
-// YAML); elsewhere the same shape as JSON. It holds what cannot be derived
-// from the files themselves — which files belong to the project, where the
-// project keeps them (`path`), where their bytes are (`source`), and what
-// they were found to be — so opening a project never re-probes footage that
-// has not changed.
import { normalizePath } from './types';
-import type { Asset } from './types';
+import type { Asset, PartialAsset } from './types';
export const MANIFEST_VERSION = 1;
@@ -30,6 +23,11 @@ type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K>
/** An asset as written to the manifest: the model minus its runtime handles. */
export type AssetRecord = DistributiveOmit<Asset, 'handle' | 'directoryHandle' | 'transient'>;
+/** A partial document as written to the manifest: it has no handles to lose. */
+export type PartialRecord = PartialAsset;
+
+export type ManifestRecord = AssetRecord | PartialRecord;
+
export interface Manifest {
version: number;
/**
@@ -37,19 +35,20 @@ export interface Manifest {
* in it is also implied by their paths). `/`-separated, no leading slash.
*/
folders: string[];
- assets: AssetRecord[];
+ /** Assets and the partial records of generations without bytes, newest first. */
+ assets: ManifestRecord[];
}
const RECORD_KEYS = new Set(['handle', 'directoryHandle', 'transient']);
-/** The manifest record of an asset: its data, none of its handles. */
-export function toRecord(asset: Asset): AssetRecord {
+/** The manifest record of a library entry: its data, none of its handles. */
+export function toRecord(entry: Asset | PartialAsset): ManifestRecord {
const record: Record<string, unknown> = {};
- for (const [key, value] of Object.entries(asset)) {
+ for (const [key, value] of Object.entries(entry)) {
if (RECORD_KEYS.has(key) || value === undefined) continue;
record[key] = value;
}
- return record as AssetRecord;
+ return record as ManifestRecord;
}
/** Whether `record` has what every asset needs; unrecognized fields are kept. */
@@ -65,6 +64,21 @@ export function isAssetRecord(record: unknown): record is AssetRecord {
);
}
+/** Whether `record` is a partial document: a generation's place and state, no bytes. */
+export function isPartialRecord(record: unknown): record is PartialRecord {
+ if (!record || typeof record !== 'object') return false;
+ const r = record as Record<string, unknown>;
+ const generation = r.generation as Record<string, unknown> | undefined;
+ return (
+ typeof r.id === 'string' && r.id.length > 0 &&
+ typeof r.path === 'string' && r.path.length > 0 &&
+ typeof r.type === 'string' &&
+ (r.state === 'pending' || r.state === 'error') &&
+ !!generation && typeof generation === 'object' && typeof generation.key === 'string' &&
+ (r.error === undefined || typeof r.error === 'string')
+ );
+}
+
/**
* Reads whatever the host handed back as a manifest into a well-formed one.
* Tolerant: a missing or malformed file is an empty library, and a broken
@@ -86,7 +100,7 @@ export function normalizeManifest(input: unknown): Manifest {
if (Array.isArray(raw.assets)) {
const seen = new Set<string>();
for (const record of raw.assets) {
- if (!isAssetRecord(record)) continue;
+ if (!isAssetRecord(record) && !isPartialRecord(record)) continue;
const path = normalizePath(record.path);
if (!path || seen.has(record.id)) continue;
seen.add(record.id);
diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.ts
index f302acd8..6e7d6f6d 100644
--- a/packages/assets/src/types.ts
+++ b/packages/assets/src/types.ts
@@ -2,13 +2,6 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// The asset model. An asset is a manifest record (see ./manifest.ts) plus a
-// runtime handle to its bytes, attached by the AssetLibrary. Everything an
-// asset *is* lives in the record: what the file was found to be (probe
-// results), where its bytes are (`source`), and where the project keeps it in
-// the library (`path`). Its `id` is a content hash, so the same bytes imported
-// twice are one asset, and a relinked file keeps its metadata.
-
/** Whatever hands out the asset's bytes: a real File on desktop, an OPFS
* file on the web, a fetched blob for a URL. */
export interface AssetFileHandle {
@@ -123,11 +116,45 @@ export type Asset =
export type AssetType = Asset['type'];
+/** Where a generation stands while the library has no bytes for it. */
+export type PartialAssetState = 'pending' | 'error';
+
+/**
+ * A partial document: the library's record of a generation before — or
+ * instead of — its bytes. Written `pending` when a run starts, so the
+ * generation is in the library from the moment it is asked for; replaced by
+ * the asset when the run lands; kept as `error`, with what went wrong, when
+ * it does not. The error record is what keeps a refused or impossible
+ * generation from being run — or paid for — again on every reopen: a
+ * declaration whose key stands in error resolves to that error, and removing
+ * the record is what asks for the run again. Never bound to an entity;
+ * elements see the state through their resolution.
+ */
+export interface PartialAsset {
+ /** Hash of the generation key: there are no bytes to hash. */
+ id: string;
+ /** Library path, like any asset's; provisional until the bytes name themselves. */
+ path: string;
+ /** What the generation is to become. */
+ type: AssetType;
+ createdAt: string;
+ generation: AssetGeneration;
+ state: PartialAssetState;
+ /** What the run failed with, on an `error` record. */
+ error?: string;
+}
+
+/** Anything the library holds at a path: an asset, or a partial standing for one. */
+export type AssetEntry = Asset | PartialAsset;
+
+/** Whether a library entry is a partial document rather than an asset with bytes. */
+export const isPartialAsset = (entry: AssetEntry): entry is PartialAsset => 'state' in entry;
+
/** The file name of an asset: the last segment of its library path. */
-export const assetName = (asset: Pick<Asset, 'path'>): string => basename(asset.path);
+export const assetName = (asset: Pick<AssetEntry, 'path'>): string => basename(asset.path);
/** The folder of an asset: its library path without the name, '' at root. */
-export const assetFolder = (asset: Pick<Asset, 'path'>): string => dirname(asset.path);
+export const assetFolder = (asset: Pick<AssetEntry, 'path'>): string => dirname(asset.path);
export function basename(path: string): string {
const at = path.lastIndexOf('/');
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index a62187e2..9acc1ec7 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -294,16 +294,6 @@ type MediaProps = {
* scene; `generate.*` is not accepted there.
*/
src: string | AssetRef;
- /**
- * Why this element's source never became an asset. Editor state carried by
- * the source like `selected`, and the answer to the `src` it was written
- * for: an element holding one is not resolved again, so a generation the
- * model refused is not run — or paid for — a second time by every reopen of
- * the project. The editor writes it when a generation or a transcription
- * fails; taking it off the element is what asks for the run again, and
- * nothing else does — not another take, not another prompt.
- */
- error?: string;
};
/**
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 38ab1c2f..6375ae42 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import { createSignal } from 'solid-js';
import { SVGElements } from 'solid-js/web';
@@ -1156,18 +1156,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
entity.set(LoadRequest, { value });
return;
}
- case 'error': {
- const message = typeof value === 'string' && value !== '' ? value : undefined;
-
- if (message === undefined) {
- entity.remove(SourceError);
- return;
- }
-
- entity.add(SourceError);
- entity.set(SourceError, { value: message, generated: true });
- return;
- }
case 'objectFit': {
const mode = typeof value === 'string' ? SCALE_MODES[value] : undefined;
if (mode === undefined) {
diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.ts
index 25e6516c..f0a2ae2c 100644
--- a/packages/runtime/src/ai.ts
+++ b/packages/runtime/src/ai.ts
@@ -15,12 +15,22 @@ import type {
GenerateVoiceOptions,
} from '@diffusionstudio/jsx';
+/**
+ * The generation service a host attaches to its world. Where a generation
+ * stands is the library's to hold, as partial documents (see
+ * `PartialAsset` in @diffusionstudio/assets): a run that starts is
+ * `pending` there, one that lands is the asset, one that fails stays as
+ * `error` with its reason — and a key standing in error resolves to that
+ * error rather than to another run, in this session and the next. Nothing
+ * here writes to the source that declared the generation.
+ */
export abstract class GenAi {
/**
* The declaration, made real: an asset whose bytes the model produced.
* Content-addressed — the fully-resolved spec's hash is the asset's
* `generation.key`, so the same spec is the same asset in this session
* and the next, and identical concurrent declarations share one run.
+ * Rejects with the recorded reason when the key stands in error.
*/
public abstract resolve(ref: AssetRef): Promise<Asset>;
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index 05f0bf26..cc81aa7f 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -39,7 +39,7 @@ export function assetSystem(world: World): void {
for (const entity of world.query(GenerationRequest)) {
const ref = entity.get(GenerationRequest)!.ref;
entity.remove(GenerationRequest);
- if (ref === null || entity.has(SourceError)) continue;
+ if (ref === null) continue;
if (isDomImage(entity)) pointDomImageAt(entity, null);
start(world, entity, ai.resolve(ref), true);
}
@@ -50,7 +50,6 @@ export function assetSystem(world: World): void {
const seed = entity.get(TranscriptionRequest)!.seed;
entity.remove(TranscriptionRequest);
- if (entity.has(SourceError)) continue;
resolve(world, entity, ai.transcribe(world, scene, seed), true);
}
}
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index a7960d92..7c90436d 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -121,16 +121,15 @@ export const PendingSync = trait(() => ({ value: undefined as unknown }));
export const PendingSource = trait(() => ({ value: undefined as unknown }));
// Why the entity's src never became an asset: the message of the rejection
-// the asset system saw. It stands until the element is authored without it
-// (the `error` prop) or a resolution for it starts — which, for a generation,
-// only happens once that prop is gone.
+// the asset system saw. Session state, like the requests above: it stands
+// until a resolution for the src starts again, and is never written back to
+// the source. What outlives the session is the library's: a generation that
+// failed stands there as a partial document carrying this same message, and
+// answers the next request for it with the failure rather than another run.
//
-// `generated` tells a failed generation from a failed load, which are worth
-// different things: a load is cheap and idempotent, so it is simply tried
-// again next render, while a generation is neither. An element carrying one
-// of those is not resolved again (see the asset system), and the host is
-// expected to write it back to the source the element came from — which is
-// what makes the failure outlive the session that saw it.
+// `generated` tells a failed generation from a failed load, for whoever
+// reports them: the one is a model refusing or a spec a model cannot take,
+// the other a path that is not there.
export const SourceError = trait({ value: '', generated: false });
// Sibling order under a ChildOf parent.