The VS Code of video editing, built for agents.
TypeScript
Enhance asset management and introduce partial asset handling
konstantin-paulus28ad0372db10
diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex ba96c2ff..58ecaa6a 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -598,7 +598,7 @@ a new session.| \`index.tsx\` | The entry. Its default export renders the composition. || \`package.json\` | The project record: \`projectId\` (its identity, kept across renames), \`displayName\` (the name shown in the app), \`main\` (the entry), \`diffusion\` (how each scene is exported), and the dapi commands as scripts. || \`tsconfig.json\` | Types for the composition tags, through \`jsxImportSource\`. |-| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be. Written by the app; hand edits are read on the next load. |+| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be; for a generation without bytes, where it stands. Written by the app; hand edits are read on the next load. || \`assets/\` | The library's files: put one here and it is taken in while the app watches, and the app writes its own here too — generations under \`assets/generated/\`. Media imported through the app is linked where it lies instead, never copied. || \`cache/\` | Derived data (thumbnails, waveforms). Disposable, and not checked in. |diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.tsindex 17fc24d8..829b6cc1 100644--- a/apps/web/src/components/genai/use-modifiers.ts+++ b/apps/web/src/components/genai/use-modifiers.ts@@ -58,12 +58,6 @@ export function useModifiers() {const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;for (const entity of targets(name)) {- // A failure is the answer to what was asked for (see `sourceErrorSystem`),- // and asking for a different set of modifiers is a different question.- // Taking the prop off is what asks again, and this is the user doing it.- if (authoredElement(entity)?.props.error !== undefined) {- editor.editProperty(entity, "error", false);- }editor.editProperty(entity, name, value);}};diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsxindex 5210c789..825e9741 100644--- a/apps/web/src/components/sidebar-left/assets.tsx+++ b/apps/web/src/components/sidebar-left/assets.tsx@@ -27,6 +27,7 @@ import {BreadcrumbsSeparator,} from "../ui/breadcrumbs";import { LazyAssetItem } from "./asset-item";+import { PartialAssetItem } from "./partial-asset-item";import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";@@ -56,6 +57,7 @@ export function Assets() {const [currentFolder, setCurrentFolder] = createSignal("");const allAssets = createMemo(() => library()?.list().filter((asset) => asset.type !== "SCRIPT") ?? []);+ const allPartials = createMemo(() => library()?.partials() ?? []);const allFolders = createMemo(() => [...(library()?.folders() ?? [])].sort());const activeFilterLabel = () => {@@ -64,17 +66,17 @@ export function Assets() {return ASSET_FILTER_OPTIONS.find((option) => option.value === value)?.label ?? null;};- const filteredAssets = createMemo(() => {+ /** Whether an entry is in view: of the filtered type, matching the search, or in the open folder. */+ const isShown = (entry: { type: string; path: string }): boolean => {const q = query().trim().toLowerCase();const selectedFilter = assetFilter();- const folder = currentFolder();+ if (selectedFilter !== "ALL" && entry.type !== selectedFilter) return false;+ if (q) return basename(entry.path).toLowerCase().includes(q);+ return dirname(entry.path) === currentFolder();+ };- return allAssets().filter((asset) => {- if (selectedFilter !== "ALL" && asset.type !== selectedFilter) return false;- if (q) return basename(asset.path).toLowerCase().includes(q);- return dirname(asset.path) === folder;- });- });+ const filteredAssets = createMemo(() => allAssets().filter(isShown));+ const filteredPartials = createMemo(() => allPartials().filter(isShown));const visibleFolders = createMemo(() => {const q = query().trim().toLowerCase();@@ -85,7 +87,7 @@ export function Assets() {});const panelTitle = createMemo(() => (currentFolder() ? basename(currentFolder()) : "Assets"));- const itemCount = createMemo(() => visibleFolders().length + filteredAssets().length);+ const itemCount = createMemo(() => visibleFolders().length + filteredPartials().length + filteredAssets().length);// Deep paths collapse like the breadcrumbs docs example:// All assets / … / parent / current, with the hidden folders in a dropdown.@@ -222,11 +224,11 @@ export function Assets() {if (event.key === "Backspace" || event.key === "Delete") {const selected = selectedAssetId();- const asset = selected ? library()?.get(selected) : undefined;- if (asset) {+ const entry = selected ? (library()?.get(selected) ?? library()?.getPartial(selected)) : undefined;+ if (entry) {event.preventDefault();event.stopPropagation();- void library()?.remove([asset]);+ void library()?.remove([entry]);return;}@@ -259,9 +261,9 @@ export function Assets() {};const hasAssets = () => allAssets().length > 0;- const hasContent = () => hasAssets() || allFolders().length > 0;+ const hasContent = () => hasAssets() || allPartials().length > 0 || allFolders().length > 0;const isFiltering = () => query().trim().length > 0 || assetFilter() !== "ALL";- const isEmptyView = () => visibleFolders().length === 0 && filteredAssets().length === 0;+ const isEmptyView = () => visibleFolders().length === 0 && filteredPartials().length === 0 && filteredAssets().length === 0;const handleCreateFolder = () => withLibrary((lib) => {const parent = currentFolder();@@ -522,7 +524,7 @@ export function Assets() {No matching assets</div></Show>- <Show when={visibleFolders().length > 0 || filteredAssets().length > 0}>+ <Show when={!isEmptyView()}><div class="grid grid-cols-2 gap-x-2 gap-y-4"><For each={visibleFolders()}>{(folder) => (@@ -535,6 +537,15 @@ export function Assets() {/>)}</For>+ <For each={filteredPartials()}>+ {(partial) => (+ <PartialAssetItem+ partial={partial}+ selected={selectedAssetId() === partial.id}+ onSelect={() => handleSelectAsset(partial.id)}+ />+ )}+ </For><For each={filteredAssets()}>{(asset) => (<LazyAssetItemdiff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsxindex 3b5efad4..a12f412b 100644--- a/apps/web/src/components/sidebar-left/folder-item.tsx+++ b/apps/web/src/components/sidebar-left/folder-item.tsx@@ -67,7 +67,7 @@ export function FolderItem(props: FolderItemProps) {const name = () => basename(props.path);const childCount = createMemo(() => {const children = library()?.childrenOf(props.path);- return children ? children.folders.length + children.assets.length : 0;+ return children ? children.folders.length + children.assets.length + children.partials.length : 0;});const handleDragStart = (event: DragEvent) => {diff --git a/apps/web/src/components/sidebar-left/partial-asset-item.tsx b/apps/web/src/components/sidebar-left/partial-asset-item.tsxnew file mode 100644index 00000000..309a56a0--- /dev/null+++ b/apps/web/src/components/sidebar-left/partial-asset-item.tsx@@ -0,0 +1,109 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// A partial document in the asset panel: a generation with no bytes yet, or+// none ever. Pending, it pulses where its asset will be; failed, it carries+// the reason, and asking again is done from here — retrying forgets the+// failure and re-asks for every element that got it, deleting only forgets.++import { Show } from "solid-js";+import { toast } from "somoto";+import { useWorld } from "@diffusionstudio/koota-solid";+import { assetName } from "@diffusionstudio/assets";+import {+ ContextMenu,+ ContextMenuContent,+ ContextMenuItem,+ ContextMenuPortal,+ ContextMenuSeparator,+ ContextMenuShortcut,+ ContextMenuTrigger,+} from "../ui/context-menu";+import { Icon } from "../ui/icon";+import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";+import { useLibrary } from "@/engine/library";+import { retryGeneration } from "@/engine/generations";++import type { PartialAsset } from "@diffusionstudio/assets";++export type PartialAssetItemProps = {+ partial: PartialAsset;+ selected: boolean;+ onSelect(): void;+};++export function PartialAssetItem(props: PartialAssetItemProps) {+ const world = useWorld();+ const library = useLibrary();++ const failed = () => props.partial.state === "error";++ const handleRetry = async () => {+ const lib = library();+ if (!lib) return;+ try {+ await retryGeneration(world, lib, props.partial);+ } catch (e) {+ toast.error("Failed to retry", { description: (e as Error).message });+ }+ };++ const handleDelete = async () => {+ try {+ await library()?.remove([props.partial]);+ } catch (e) {+ toast.error("Failed to delete", { description: (e as Error).message });+ }+ };++ return (+ <ContextMenu>+ <ContextMenuTrigger+ as="div"+ class="flex flex-col gap-1 text-left"+ data-asset-id={props.partial.id}+ onClick={props.onSelect}+ onContextMenu={props.onSelect}>+ <Tooltip placement="bottom" openDelay={400}>+ <TooltipTrigger+ as="div"+ data-selected={props.selected}+ class="relative aspect-video w-full overflow-clip rounded bg-muted after:pointer-events-none after:absolute after:inset-0 after:rounded after:opacity-0 after:ring-2 after:ring-inset after:ring-ring after:z-10 data-[selected=true]:after:opacity-100"+ classList={{ "animate-pulse": !failed() }}+ >+ <div class="absolute inset-0 flex items-center justify-center text-muted-foreground">+ <Show when={failed()} fallback={<Icon name="spinner-loader" class="size-5 animate-spin" />}>+ <Icon name="alert-warning" class="size-5 text-destructive" />+ </Show>+ </div>+ </TooltipTrigger>+ <TooltipContent class="max-w-64">+ {failed() ? props.partial.error || "Generation failed" : "Generating…"}+ </TooltipContent>+ </Tooltip>+ <div+ class="text-xs truncate select-none"+ classList={{ "text-destructive": failed(), "text-muted-foreground": !failed() }}+ title={props.partial.error}+ >+ {assetName(props.partial)}+ </div>+ </ContextMenuTrigger>+ <ContextMenuPortal>+ <ContextMenuContent class="w-[180px]">+ <Show when={failed()}>+ <ContextMenuItem onSelect={handleRetry}>+ Retry+ </ContextMenuItem>+ <ContextMenuSeparator />+ </Show>+ <ContextMenuItem onSelect={handleDelete}>+ Delete+ <ContextMenuShortcut>⌫</ContextMenuShortcut>+ </ContextMenuItem>+ </ContextMenuContent>+ </ContextMenuPortal>+ </ContextMenu>+ );+}diff --git a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx b/apps/web/src/components/sidebar-right/inspector/asset-info.tsxindex 21f177c6..0da21d18 100644--- a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx+++ b/apps/web/src/components/sidebar-right/inspector/asset-info.tsx@@ -20,11 +20,86 @@ import { assetName } from "@diffusionstudio/assets";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";import { insertAssetAtPlayhead, replaceAssetSource } from "@/engine/asset-actions";+import { retryGeneration } from "@/engine/generations";-import type { Asset } from "@diffusionstudio/assets";+import type { Asset, PartialAsset } from "@diffusionstudio/assets";-/** Information about the library asset picked in the assets panel. */+/** Information about the library entry picked in the assets panel. */export function AssetInfoPanel() {+ const selection = useAssetSelection();++ return (+ <Show when={selection.partial()} fallback={<AssetDetails />}>+ {(partial) => <PartialDetails partial={partial()} />}+ </Show>+ );+}++/**+ * A generation without bytes: where it stands, and — once it has failed —+ * the reason, and the way to ask again.+ */+function PartialDetails(props: { partial: PartialAsset }) {+ const world = useWorld();+ const library = useLibrary();++ const failed = () => props.partial.state === "error";++ const handleRetry = async () => {+ const lib = library();+ if (lib) await retryGeneration(world, lib, props.partial);+ };++ const handleDelete = async () => {+ await library()?.remove([props.partial]);+ };++ return (+ <div class="flex flex-col w-full px-4 border-t border-border">+ <div class="h-12 flex items-center justify-between">+ <span class="text-base font-strong">Information</span>+ </div>++ <div class="py-3 border-t border-b border-border text-xs break-all">+ {assetName(props.partial)}+ </div>++ <div class="flex flex-col gap-1 my-2 text-xs text-muted-foreground">+ <div class="flex h-7 items-center gap-2">+ <span class="w-20 shrink-0">Status</span>+ <span class="min-w-0 flex-1 text-right" classList={{ "text-destructive": failed() }}>+ {failed() ? "Failed" : "Generating…"}+ </span>+ </div>+ <Show when={props.partial.error}>+ {(error) => (+ <div class="flex flex-col gap-2 py-1">+ <span class="w-20 shrink-0">Error</span>+ <span class="min-w-0 text-left wrap-break-words text-destructive">{error()}</span>+ </div>+ )}+ </Show>+ <div class="flex h-7 items-center gap-2">+ <span class="w-20 shrink-0">Requested</span>+ <span class="min-w-0 flex-1 text-right truncate">{formatAssetDate(props.partial.createdAt)}</span>+ </div>+ </div>++ <div class="flex flex-col gap-2 my-2">+ <Show when={failed()}>+ <Button variant="default" class="w-full" onClick={handleRetry}>+ Retry generation+ </Button>+ </Show>+ <Button variant="secondary" class="w-full" onClick={handleDelete}>+ Delete+ </Button>+ </div>+ </div>+ );+}++function AssetDetails() {const world = useWorld();const library = useLibrary();const selection = useAssetSelection();diff --git a/apps/web/src/components/sidebar-right/inspector/inspector.tsx b/apps/web/src/components/sidebar-right/inspector/inspector.tsxindex 7a0cd250..eb05b23a 100644--- a/apps/web/src/components/sidebar-right/inspector/inspector.tsx+++ b/apps/web/src/components/sidebar-right/inspector/inspector.tsx@@ -75,7 +75,7 @@ function classifyNode(entity: Entity): SelectionTarget {export function Inspector() {const tool = useTool();const { nodes, keyframes, first } = useSelection();- const { asset } = useAssetSelection();+ const { asset, partial } = useAssetSelection();// For ExportPanel (root scenes only) and TransitionSettings (sequence items).const parent = createMemo(() => getParentNode(first()));@@ -88,7 +88,7 @@ export function Inspector() {const selectionTarget = createMemo<SelectionTarget>(() => {if (tool() === ToolType.SCENE) return "scene-tool";if (keyframes().length > 0) return "keyframe";- if (asset()) return "asset";+ if (asset() || partial()) return "asset";const entity = first();if (nodes().length === 1 && entity) return classifyNode(entity);return "stage";@@ -96,7 +96,7 @@ export function Inspector() {// Remounts the panels when the selection changesconst selectionHash = createMemo(() => {- return [...nodes(), ...keyframes(), asset()?.id ?? ""].join(",") + selectionTarget();+ return [...nodes(), ...keyframes(), asset()?.id ?? partial()?.id ?? ""].join(",") + selectionTarget();});const includesTarget = (...targets: SelectionTarget[]) => {diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.tsindex d431ea90..a5f23bca 100644--- a/apps/web/src/engine/create-engine.ts+++ b/apps/web/src/engine/create-engine.ts@@ -9,7 +9,6 @@ import { AssetSelection, Hud, Keys, MODIFIER_KEYS, Pointer, PointerEvents, Projeimport { inputSystem } from './input/input-system';import { clearClipFrames, clearClipPeaks, clearMedia, clearPeaks, timelineSystem, TimelineSurface } from './timeline';import { shortcutSystem } from './input/shortcuts';-import { sourceErrorSystem } from './source-errors';import type { RuntimeWorld } from '@diffusionstudio/runtime';import type { CanvasPointerEvent, PointerEventType } from '@diffusionstudio/runtime';@@ -213,7 +212,6 @@ class Engine {inputSystem(this.world);shortcutSystem(this.world);assetSystem(this.world);- sourceErrorSystem(this.world);playbackSystem(this.world);motionSystem(this.world);transformSystem(this.world);diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.tsindex 84fca003..85d396ad 100644--- a/apps/web/src/engine/editor.ts+++ b/apps/web/src/engine/editor.ts@@ -266,11 +266,11 @@ function nextSourceAfter(world: World, entity: Entity): string | undefined {* The props of a `<video>`/`<image>` that only mean something while the node* plays media, dropped when the intrinsic paint is removed (see* `removeIntrinsicPaint`): the source, what qualifies it (its fit, its- * window, a frames directory's rate, the modifiers put on it), its error,- * and the audio mix a rect has no track to apply to. `start`/`end` stay:- * they place the clip, media or not.+ * window, a frames directory's rate, the modifiers put on it), and the audio+ * mix a rect has no track to apply to. `start`/`end` stay: they place the+ * clip, media or not.*/-const MEDIA_PROPS = ['src', 'error', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;+const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;/*** What `copy` took: the subtrees as authored, and the source of the parentdiff --git a/apps/web/src/engine/generations.ts b/apps/web/src/engine/generations.tsnew file mode 100644index 00000000..95dd97f4--- /dev/null+++ b/apps/web/src/engine/generations.ts@@ -0,0 +1,48 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Asking again for a generation that failed. A failed generation stands in+ * the library as a partial document in error, and a declaration whose key+ * stands there resolves to the failure rather than to another run — in+ * this session and the next. Removing the record is what asks again; this+ * is that removal, plus making the elements still waiting on the answer ask+ * right away rather than on the next mount.+ */++import { isAssetRef } from '@diffusionstudio/jsx';+import { authoredElement } from '@diffusionstudio/reconciler';+import {+ Caption, Generating, GenerationRequest, LoadRequest, PendingSource, SourceError, TranscriptionRequest,+} from '@diffusionstudio/runtime';++import type { AssetLibrary, PartialAsset } from '@diffusionstudio/assets';+import type { World } from 'koota';++/** Forgets the failure `partial` records and re-asks for every failed source. */+export async function retryGeneration(world: World, library: AssetLibrary, partial: PartialAsset): Promise<void> {+ await library.remove([partial]);++ for (const entity of world.query(SourceError)) {+ if (!entity.get(SourceError)!.generated) continue;+ const authored = authoredElement(entity);++ if (!authored) continue;++ const src = authored.props.src;+ entity.remove(SourceError, PendingSource, Generating);++ if (isAssetRef(src)) {+ entity.add(GenerationRequest);+ entity.set(GenerationRequest, { ref: src });+ } else if (typeof src === 'string' && src !== '') {+ // A source put through modifiers: the load is cheap, the derivation is what failed.+ entity.add(LoadRequest);+ entity.set(LoadRequest, { value: src });+ } else if (entity.has(Caption)) {+ entity.add(TranscriptionRequest);+ entity.set(TranscriptionRequest, { seed: Number(authored.props.seed) || 0 });+ }+ }+}diff --git a/apps/web/src/engine/history.ts b/apps/web/src/engine/history.tsindex 72790c50..7b61c2ac 100644--- a/apps/web/src/engine/history.ts+++ b/apps/web/src/engine/history.ts@@ -36,10 +36,9 @@ import type { CapturedNode, DocumentEditor, EntityEdit } from './editor';/*** Props that are pointing and viewing rather than composition: undoing them* is not what anyone means by undo, so they pass through to the file without- * entering the history. `error` is the runtime speaking (see source-errors),- * not the user.+ * entering the history.*/-const EXCLUDED_PROPS: ReadonlySet<string> = new Set(['selected', 'active', 'camera', 'expanded', 'clipHeight', 'timeline', 'error']);+const EXCLUDED_PROPS: ReadonlySet<string> = new Set(['selected', 'active', 'camera', 'expanded', 'clipHeight', 'timeline']);/*** How long after one step a same-shaped step still merges into it. Editsdiff --git a/apps/web/src/engine/hooks/use-asset-selection.ts b/apps/web/src/engine/hooks/use-asset-selection.tsindex 6f1bee81..bc31a2a0 100644--- a/apps/web/src/engine/hooks/use-asset-selection.ts+++ b/apps/web/src/engine/hooks/use-asset-selection.ts@@ -8,12 +8,13 @@ import { createMemo } from 'solid-js';import { AssetSelection } from '../traits';import { useLibrary } from '../library';-import type { Asset } from '@diffusionstudio/assets';+import type { Asset, PartialAsset } from '@diffusionstudio/assets';/**- * Reactive read and write of the AssetSelection trait: the one library asset- * the assets panel has picked and the inspector describes. Shared between- * the two sidebars, so it lives on the world rather than in either panel.+ * Reactive read and write of the AssetSelection trait: the one library entry+ * the assets panel has picked and the inspector describes — an asset, or the+ * partial document of a generation without bytes. Shared between the two+ * sidebars, so it lives on the world rather than in either panel.*/export function useAssetSelection() {const world = useWorld();@@ -31,9 +32,17 @@ export function useAssetSelection() {return lib.list().find((entry) => entry.id === current);});- const select = (next: Asset | string | null) => {+ /** The picked partial document, when the pick is one; drops out as its bytes land. */+ const partial = createMemo(() => {+ const lib = library();+ const current = id();+ if (!lib || !current) return undefined;+ return lib.partials().find((entry) => entry.id === current);+ });++ const select = (next: Asset | PartialAsset | string | null) => {world.set(AssetSelection, { id: typeof next === 'string' ? next : (next?.id ?? null) });};- return { id, asset, select };+ return { id, asset, partial, select };}diff --git a/apps/web/src/engine/source-errors.ts b/apps/web/src/engine/source-errors.tsdeleted file mode 100644index 75b13a08..00000000--- a/apps/web/src/engine/source-errors.ts+++ /dev/null@@ -1,47 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--/**- * A failed generation, written back to the JSX that asked for it. The runtime- * leaves the reason on the entity (`SourceError`); this is what turns it into- * the `error` prop of the element, so the failure lives where the rest of the- * document lives — in the file — rather than only in the session that saw it.- *- * That is what keeps a generation from being run again: the project remounts- * on every edit and is compiled afresh on every reload, and each time the- * element carries the answer its declaration already got (see the asset- * system, which leaves such an element alone). Taking the prop off the- * element — in the file, like any other authored change — is what asks for- * the run again; nothing else in the editor does it silently.- *- * Only generations are written: a load that failed is cheap to try again, and- * an asset that has since been put back should simply load.- */--import { SourceError } from '@diffusionstudio/runtime';-import { authoredElement } from '@diffusionstudio/reconciler';--import { getDocumentEditor, isLooped } from './editor';--import type { World } from 'koota';--export function sourceErrorSystem(world: World): void {- for (const entity of world.query(SourceError)) {- const failure = entity.get(SourceError)!;- if (!failure.generated) continue;-- const authored = authoredElement(entity);- // Nothing to write it to (an entity no element produced), or nothing- // new to say.- if (!authored || authored.props.error === failure.value) continue;-- // One element renders every iteration of a loop, so writing a prop to- // one of them means unrolling the loop into the file. That is a change- // to the project worth making when the user asks for one, and not- // worth making behind their back over a generation that failed.- if (isLooped(entity)) continue;-- getDocumentEditor(world).editProperty(entity, 'error', failure.value);- }-}diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 2b8df4f1..71e72bf6 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -10,7 +10,7 @@ import {import type { SourceModifierValues } from "@diffusionstudio/runtime";import { createEncoder } from "@diffusionstudio/encoder";import { createCapture } from "@/engine/capture";-import { assetName, GENERATED_DIR } from "@diffusionstudio/assets";+import { assetName, GENERATED_DIR, isPartialAsset } from "@diffusionstudio/assets";import {PROMPT_INPUT_AUDIO_MODEL_OPTIONS,PROMPT_INPUT_IMAGE_MODEL_OPTIONS,@@ -25,7 +25,7 @@ import { trpc } from "@/lib/trpc";import { toast } from "somoto";import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";-import type { Asset, AssetLibrary, AssetType } from "@diffusionstudio/assets";+import type { Asset, AssetLibrary, AssetType, PartialAsset, ReserveOptions } from "@diffusionstudio/assets";import type { FileRef } from "@diffusionstudio/api-contract";import type { ExportResult } from "@diffusionstudio/encoder";import type { Entity, World } from "koota";@@ -39,6 +39,14 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {transcript: "Caption generation failed",};+/** What each declaration is to become, for the partial that stands for it. */+const ASSET_TYPES: Record<AssetSpecInput["type"], AssetType> = {+ image: "IMAGE",+ video: "VIDEO",+ voice: "AUDIO",+ audio: "AUDIO",+};+/*** One step of a source modifier (see the runtime's `SourceModifiers`): a* model call that makes a new asset out of one the library already holds.@@ -74,6 +82,17 @@ type ResolvedSpec =| { type: "voice"; model: string; prompt: string; voice: string; seed?: number }| { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };+/** What the partial standing for a run is called and is to become, and what its failure is called. */+type RunDescription = Omit<ReserveOptions, "key" | "folder"> & { title: string };++/**+ * A failure the user has already been told of — as a toast when it happened,+ * and in the library's record from then on. It goes up the dependency graph+ * as it is: a declaration whose input failed is not a second thing gone+ * wrong.+ */+class ReportedError extends Error {}+/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */export function attachAi(world: World, library: AssetLibrary, dir?: string): EditorGenAi {const ai = new EditorGenAi(library, world.get(Project)?.id ?? "project", dir);@@ -90,12 +109,12 @@ export class EditorGenAi extends GenAi {/*** Declarations already resolved, keyed by ref identity — a ref consumed by- * several elements resolves (and validates) once. A failed one is forgotten:- * from there on the element that asked is what carries the failure, in its- * `error` prop, and that is what keeps the generation from running again.+ * several elements resolves (and validates) once. A failed one is+ * forgotten, so that asking again — once its record is gone from the+ * library — asks the library again rather than this map.*/private readonly memo = new Map<AssetRef, Promise<Asset>>();- /** In-flight generations and transcriptions keyed by `generationKey`. */+ /** Runs in flight, keyed by generation key. */private readonly inflight = new Map<string, Promise<Asset>>();public constructor(library: AssetLibrary, projectId: string, dir?: string) {@@ -120,120 +139,114 @@ export class EditorGenAi extends GenAi {/*** Transcribes the scene's audible mix for a `<captions>` element (see the- * runtime's asset system). Cached by scene id + seed: the same pair is the- * same transcript asset across sessions (the transcript lands in the- * library under `generated/` with that key in the manifest), and a new- * seed transcribes the scene again.+ * runtime's asset system). Keyed by scene id + seed: the same pair is the+ * same transcript asset across sessions, and a new seed transcribes the+ * scene again.*/- public async transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {+ public transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {const key = transcriptKey(scene, seed);+ return this.generated(+ key,+ () => ({ type: "TRANSCRIPT", name: `${this.nextCaptionsName()}.json`, title: FAILURE_TITLES.transcript }),+ (partial) => this.runTranscription(world, scene, key, partial),+ );+ }++ /**+ * `asset` through every modifier the element asked for, one step at a time+ * and in a fixed order. Each step is cached in its own right, so turning+ * one on leaves what the others already made alone — adding `upscale` to a+ * cut-out picture pays for the enlarging, not for the matte again.+ */+ public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {+ let derived = asset;+ for (const kind of steps(modifiers)) {+ derived = await this.transform(kind, derived);+ }+ return derived;+ }- const cached = this.library.list().find((asset) => asset.generation?.key === key);- if (cached) return cached;+ /**+ * Runs one step over `asset` and returns what it produced, stored under+ * `generated/` like any other model output. Keyed by step and input, so the+ * same call on the same asset is the same result in this session and the+ * next: upscaling a picture twice costs what upscaling it once did.+ */+ public transform(kind: TransformKind, asset: Asset): Promise<Asset> {+ // No upscale factor in the key: the endpoint takes none, so every factor+ // is the same call and would otherwise be billed once per number asked+ // for. It belongs here the moment the API can be told one.+ const key = `transform:v1:${kind}:${asset.id}`;+ const base = assetName(asset).replace(/\.[^.]+$/, "");+ return this.generated(+ key,+ () => ({ type: asset.type, name: `${base} (${TRANSFORMS[kind].suffix})`, title: TRANSFORMS[kind].title }),+ () => this.runTransform(kind, asset, key),+ );+ }++ /**+ * The library's answer for `key`, or the run that produces one. An asset+ * the key landed as is returned; a key standing in error rejects with the+ * recorded reason — answered, not run or paid for again, and not toasted+ * again either; a run in flight is joined. Otherwise the run starts, with+ * a partial document reserved for it first (a pending one the last session+ * never finished is taken over), and ends either as the asset that+ * replaces the partial or as the partial's recorded failure.+ */+ private async generated(+ key: string,+ describe: () => RunDescription,+ run: (partial: PartialAsset) => Promise<Asset>,+ ): Promise<Asset> {+ const known = this.library.generated(key);+ if (known && !isPartialAsset(known)) return known;+ if (known?.state === "error") throw new ReportedError(known.error || "Generation failed");const running = this.inflight.get(key);if (running) return await running;- const promise = this.runTranscription(world, scene, key);+ const { title, ...partial } = describe();+ const promise = this.library.reserve({ key, folder: GENERATED_DIR, ...partial }).then(async (reserved) => {+ try {+ return await run(reserved);+ } catch (error) {+ const failure = reportFailure(error, title);+ this.library.fail(reserved, failure.message);+ throw failure;+ }+ });this.inflight.set(key, promise);try {return await promise;- } catch (error) {- throw reportFailure(error, FAILURE_TITLES.transcript);} finally {this.inflight.delete(key);}}- /** Encodes the scene's audio, transcribes it, and stores the transcript. */- private async runTranscription(world: World, scene: Entity, key: string): Promise<Asset> {- assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");+ private async generateFromRef(ref: AssetRef): Promise<Asset> {+ const spec = getAssetSpec(ref);- // The scene's own capture world: the project rendered again, reduced to- // this scene, with nothing drawn — see `createCapture`.- const capture = await createCapture(world, scene, { mode: "offline-audio", dir: this.dir });- let result: ExportResult;+ // Inputs first: their ids are part of the key. An input that failed has+ // said so already; anything else wrong with them is said here.+ let resolved: ResolvedSpec;try {- const encoder = await createEncoder(capture.world, {- format: "ogg",- video: { enabled: false },- audio: { enabled: true, codec: "opus", sampleRate: 24000 },- });- result = await encoder.render();- } finally {- capture.dispose();+ resolved = await this.resolveSpec(spec);+ } catch (error) {+ throw error instanceof ReportedError ? error : reportFailure(error, FAILURE_TITLES[spec.type]);}- assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");- const uploadId = crypto.randomUUID();- const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });- console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);- const fileRef = await uploadBlob(audioFile, uploadId);- assert(fileRef, "Failed to upload the scene audio for transcription");-- console.log(`[gen-ai] transcribing scene audio for ${key}`);- const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });- assert(- transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),- "No speech detected. The audio does not appear to contain recognizable speech.",+ return this.generated(+ JSON.stringify(resolved),+ () => ({ type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt), title: FAILURE_TITLES[spec.type] }),+ (partial) => this.runGeneration(resolved, partial),);-- const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });- const asset = await this.library.store(blob, {- name: `${this.nextCaptionsName()}.json`,- folder: GENERATED_DIR,- generation: { key },- });-- // A re-take of unchanged speech comes back byte-identical, and the library- // dedups by content: `store` then hands back the earlier take still keyed- // by its old seed. Re-key it, or the authored seed misses the cache and- // transcribes again on every load.- if (asset.generation?.key !== key) {- this.library.update(asset, { generation: { key } });- }-- return asset;- }-- private nextCaptionsName(): string {- let max = 0;- for (const asset of this.library.list()) {- const match = assetName(asset).match(/^Captions (\d+)\.json$/);- if (match) max = Math.max(max, Number(match[1]));- }- return `Captions ${max + 1}`;}private resolveInput(input: AssetInput): Promise<Asset> {return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);}- private async generateFromRef(ref: AssetRef): Promise<Asset> {- const spec = getAssetSpec(ref);-- try {- const resolved = await this.resolveSpec(spec);- const generationKey = JSON.stringify(resolved);-- const cached = this.library.list().find((asset) => asset.generation?.key === generationKey);- if (cached) return cached;-- const running = this.inflight.get(generationKey);- if (running) return await running;-- const promise = this.runGeneration(resolved, generationKey);- this.inflight.set(generationKey, promise);- try {- return await promise;- } finally {- this.inflight.delete(generationKey);- }- } catch (error) {- throw reportFailure(error, FAILURE_TITLES[spec.type]);- }- }-private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {switch (spec.type) {case "image": {@@ -252,7 +265,7 @@ export class EditorGenAi extends GenAi {spec.startFrame !== undefined ? this.resolveInput(spec.startFrame) : undefined,spec.endFrame !== undefined ? this.resolveInput(spec.endFrame) : undefined,]);- const resolved = {+ return {type: "video",model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,prompt: spec.prompt,@@ -262,9 +275,7 @@ export class EditorGenAi extends GenAi {seed: spec.seed,startFrameId: startFrame?.id,endFrameId: endFrame?.id,- } satisfies ResolvedSpec;- checkVideoConstraints(resolved);- return resolved;+ };}case "voice": {return {@@ -287,8 +298,14 @@ export class EditorGenAi extends GenAi {}}- /** Runs a generation and stores its first result under `generated/`. */- private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {+ /**+ * Runs a generation and stores its first result in place of `partial`. A+ * spec a model cannot take fails here rather than before the partial is+ * reserved, so the refusal is recorded like any other and not asked again.+ */+ private async runGeneration(spec: ResolvedSpec, partial: PartialAsset): Promise<Asset> {+ if (spec.type === "video") checkVideoConstraints(spec);+const startedAt = performance.now();track("generation_started", {mode: spec.type,@@ -303,7 +320,7 @@ export class EditorGenAi extends GenAi {const { name, results, generationId } = await this.requestGeneration(spec);assert(results.length > 0, "No results returned from the model");- const asset = await this.store(results[0].url, name, { key: generationKey, id: generationId });+ const asset = await this.store(results[0].url, name, { key: partial.generation.key, id: generationId });track("generation_completed", {mode: spec.type,model: spec.model,@@ -321,47 +338,55 @@ export class EditorGenAi extends GenAi {}}- /**- * `asset` through every modifier the element asked for, one step at a time- * and in a fixed order. Each step is cached in its own right, so turning- * one on leaves what the others already made alone — adding `upscale` to a- * cut-out picture pays for the enlarging, not for the matte again.- */- public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {- let derived = asset;- for (const kind of steps(modifiers)) {- derived = await this.transform(kind, derived);+ /** Encodes the scene's audio, transcribes it, and stores the transcript in place of `partial`. */+ private async runTranscription(world: World, scene: Entity, key: string, partial: PartialAsset): Promise<Asset> {+ assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");++ // The scene's own capture world: the project rendered again, reduced to+ // this scene, with nothing drawn — see `createCapture`.+ const capture = await createCapture(world, scene, { mode: "offline-audio", dir: this.dir });+ let result: ExportResult;+ try {+ const encoder = await createEncoder(capture.world, {+ format: "ogg",+ video: { enabled: false },+ audio: { enabled: true, codec: "opus", sampleRate: 24000 },+ });+ result = await encoder.render();+ } finally {+ capture.dispose();}- return derived;- }+ assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");- /**- * Runs one step over `asset` and returns what it produced, stored under- * `generated/` like any other model output. Keyed by step and input, so the- * same call on the same asset is the same result in this session and the- * next: upscaling a picture twice costs what upscaling it once did.- */- public async transform(kind: TransformKind, asset: Asset): Promise<Asset> {- // No upscale factor in the key: the endpoint takes none, so every factor- // is the same call and would otherwise be billed once per number asked- // for. It belongs here the moment the API can be told one.- const key = `transform:v1:${kind}:${asset.id}`;+ const uploadId = crypto.randomUUID();+ const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });+ console.log(`[gen-ai] uploading scene audio for ${key} (${audioFile.size} bytes)`);+ const fileRef = await uploadBlob(audioFile, uploadId);+ assert(fileRef, "Failed to upload the scene audio for transcription");- const cached = this.library.list().find((entry) => entry.generation?.key === key);- if (cached) return cached;+ console.log(`[gen-ai] transcribing scene audio for ${key}`);+ const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });+ assert(+ transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),+ "No speech detected. The audio does not appear to contain recognizable speech.",+ );- const running = this.inflight.get(key);- if (running) return await running;+ const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });+ return this.library.store(blob, {+ name: assetName(partial),+ folder: GENERATED_DIR,+ generation: { key },+ });+ }- const promise = this.runTransform(kind, asset, key);- this.inflight.set(key, promise);- try {- return await promise;- } catch (error) {- throw reportFailure(error, TRANSFORMS[kind].title);- } finally {- this.inflight.delete(key);+ /** The next free `Captions N`, counting the takes still in flight. */+ private nextCaptionsName(): string {+ let max = 0;+ for (const entry of [...this.library.list(), ...this.library.partials()]) {+ const match = assetName(entry).match(/^Captions (\d+)\.json$/);+ if (match) max = Math.max(max, Number(match[1]));}+ return `Captions ${max + 1}`;}/** Uploads the input, runs the call, and stores the result beside the generations. */@@ -493,6 +518,21 @@ export class EditorGenAi extends GenAi {/** Whether an asset type is footage, for the calls that only take footage. */const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";+/** How much of a prompt names the partial standing for its generation. */+const PROVISIONAL_NAME_LENGTH = 48;++/**+ * What a generation is called before its result names it: the head of its+ * prompt, cut at a word. The model's own name takes over when the bytes+ * land (see `store`).+ */+function provisionalName(prompt: string): string {+ const text = prompt.trim().replace(/\s+/g, " ").replace(/\//g, "-");+ if (text.length <= PROVISIONAL_NAME_LENGTH) return text || "Generation";+ const cut = text.lastIndexOf(" ", PROVISIONAL_NAME_LENGTH);+ return `${text.slice(0, cut > 0 ? cut : PROVISIONAL_NAME_LENGTH)}…`;+}+/*** What to call a generated result: the extension its type implies, falling* back to the one its URL carries. Neither is guaranteed — some models hand@@ -512,23 +552,24 @@ function resultExtension(url: string, blob: Blob): string {}/**- * Says what a generation failed with, and hands the error on. The message is- * what the element ends up carrying too (the runtime's `SourceError`, which- * the editor writes into its `error` prop), so the sentence in the file, the- * one on the canvas and the one in the toast are the same. The toast is keyed- * by it: variants that failed the same way are one thing gone wrong rather- * than four, and a render that ran into the same wall does not stack up.+ * Says what a generation failed with, and hands the failure on as one+ * already reported. The message is what the library records on the partial+ * and what the element carries (the runtime's `SourceError`), so the+ * sentence in the asset panel, the one on the canvas and the one in the+ * toast are the same. The toast is keyed by it: variants that failed the+ * same way are one thing gone wrong rather than four, and a render that ran+ * into the same wall does not stack up.*/-function reportFailure(error: unknown, title: string): Error {- const failure = error instanceof Error ? error : new Error(String(error));+function reportFailure(error: unknown, title: string): ReportedError {+ const message = error instanceof Error ? error.message : String(error);- console.error(`[gen-ai] ${title}:`, failure);+ console.error(`[gen-ai] ${title}:`, error);toast.error(title, {- id: `gen-ai:${title}:${failure.message}`,- description: failure.message,+ id: `gen-ai:${title}:${message}`,+ description: message,});- return failure;+ return new ReportedError(message);}/**diff --git a/docs/examples/02-genai.tsx b/docs/examples/02-genai.tsxindex 68012163..b121e16a 100644--- a/docs/examples/02-genai.tsx+++ b/docs/examples/02-genai.tsx@@ -9,7 +9,8 @@* motion clips) that generates in topological order, blocking the command* until every asset lands. <captions> reads the finished scene audio after* generation, so caption timing matches the voiceover. Consumes credits;- * results are cached per session, so re-mounting unchanged specs is free.+ * results are cached in the project's library, so re-mounting unchanged+ * specs is free.* Discover models and voices with `dapi models <type>` and `dapi voices`;* omitted here, so each stage uses the default model.*/diff --git a/docs/reference/jsx/captions.md b/docs/reference/jsx/captions.mdindex 08a1585f..9e700294 100644--- a/docs/reference/jsx/captions.md+++ b/docs/reference/jsx/captions.md@@ -9,7 +9,7 @@</scene>```-Transcription is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once ready. Because it reads the scene's audible mix, it waits until **every other source in the scene has landed** — a generated `voice` or `audio` track is transcribed at its final placement, not at the placeholder. The scene must contain an unmuted, unhidden audio or video source; without one the node carries an [`error`](./errors.md#failed-sources) saying so.+Transcription is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once ready. Because it reads the scene's audible mix, it waits until **every other source in the scene has landed** — a generated `voice` or `audio` track is transcribed at its final placement, not at the placeholder. The scene must contain an unmuted, unhidden audio or video source; without one the node carries an error saying so, recorded in the library like any failed generation (see [errors.md](./errors.md#failed-sources)).## Bringing your own transcript@@ -42,6 +42,8 @@ A transcript is cached under **the scene's id and the `seed`**, and the cached aThe audio itself is not part of the key. **Recutting a scene does not re-transcribe it**: to pick up changed audio, bump `seed`. A value used before replays that take from cache; a value that has not been used for this scene transcribes it again, which costs credits.+A transcription that failed is recorded under the same key, as a library entry in error, and is not run again until that entry is removed — see [errors.md](./errors.md#failed-sources).+## TrimmingCaptions carry the same [timing](./timing.md) props as media nodes, and the transcript is source content that must stay aligned to the audio — so advance `start` and `sourceIn` together (and `end`/`sourceOut`), never `start` alone. To show only from 15 s onward, set both `sourceIn={15}` and `start={15}`. One node can't skip a gap, so to blank captions out for a middle stretch — e.g. under an overlay — use two nodes that meet at the gap:diff --git a/docs/reference/jsx/errors.md b/docs/reference/jsx/errors.mdindex 5b2148f6..44afc16c 100644--- a/docs/reference/jsx/errors.md+++ b/docs/reference/jsx/errors.md@@ -12,17 +12,23 @@ Runtime errors are reported against the compiled module. Since types are strippe## Failed sources-A failed **generation** is written back into your file as the element's `error` prop:--```tsx-<image src={generate.image({ prompt: "a red fox" })} error="Model refused the prompt" />+A failed **generation** is recorded in the library, never in your file. The library holds a partial document for every generation the project has asked for: `pending` while the model runs, replaced by the asset when it lands, and kept as `error` — with the reason — when it does not. In `assets.yml`:++```yaml+- id: 3f9c1a7e2b8d4c05+ path: generated/a red fox+ type: IMAGE+ generation:+ key: '{"type":"image","model":"…","prompt":"a red fox",…}'+ state: error+ error: Model refused the prompt```-That is deliberate, and it is what keeps a refused or impossible generation from being run — or paid for — again by every reopen of the project. An element holding an `error` is not resolved a second time, and nothing in the editor takes the prop off silently.+That record is what keeps a refused or impossible generation from being run — or paid for — again by every reopen of the project: a declaration whose key stands in error resolves to that error, and its element carries the message without anything being requested. The source that declared it is not written to.-**Removing the `error` attribute is what asks for the run again**, and it is the only thing that does. Not another take, not another prompt.+**Removing the record is what asks for the run again**, and it is the only thing that does — not another take, not another prompt. Delete the entry in the asset panel (or its record in `assets.yml`), or use *Retry* on it, which deletes the record and re-asks at once for every element still waiting on the answer. Changing the declaration is a new key, and so a new generation; the old record stays until removed.-Only generations are written down this way: a load that failed is cheap to try again, and an asset that has since been put back should simply load. An element rendered inside a `<For>` is left alone too — writing a prop to one iteration would mean unrolling the loop into the file, which is not a change to make behind your back over a generation that failed.+Only generations are recorded this way: a load that failed is cheap to try again, and an asset that has since been put back should simply load.## Blank or partial `<html>` content in capturesdiff --git a/docs/reference/jsx/generate.md b/docs/reference/jsx/generate.mdindex f2b7286a..fe0326da 100644--- a/docs/reference/jsx/generate.md+++ b/docs/reference/jsx/generate.md@@ -34,7 +34,7 @@ export default function Project() {Declarations are **pure**: calling `generate.*` validates its options and returns a ref; nothing is requested until an element carrying it mounts. A ref that is never used by a mounted element (directly or as an input to another asset) is never generated. Declarations may live at module scope or inside components.-Generation is **asynchronous and non-blocking**: the element is on the canvas immediately, showing a generating state, and its paint attaches when the asset lands. [`context`](../tools/context.md) reports where each one stands — generating, failed with the reason, or done with the library path it landed as — and a declaration that fails leaves its element carrying an [`error`](./errors.md#failed-sources).+Generation is **asynchronous and non-blocking**: the element is on the canvas immediately, showing a generating state, and its paint attaches when the asset lands. [`context`](../tools/context.md) reports where each one stands — generating, failed with the reason, or done with the library path it landed as — and a declaration that fails leaves its element carrying the reason, recorded in the library (see [errors.md](./errors.md#failed-sources)).## Declaration options@@ -85,3 +85,5 @@ generate.audio(opts: {Generation is long-running and consumes credits, so results are **cached by content**: a key derived from the fully-resolved spec — `type`, `model`, `prompt`, the resolved ids of any references, `seed`, and the rest of the options, with the defaults filled in. Re-mounting an unchanged project reuses cached assets instead of regenerating, two declarations with identical specs collapse to a single asset, and identical concurrent declarations share one request; changing any option produces a new asset. Set `seed` to make a spec reproducible.Finished generations are stored in the project's library under `generated/` with their spec key, so the cache survives app restarts — and a save that only touched an unrelated part of the file regenerates nothing. Deleting the asset from the library regenerates it on the next mount.++Where a generation stands is recorded there too: a run in flight is a `pending` entry of the library, and one that failed stays as an `error` entry carrying its reason, which answers the declaration — instead of another run — until the entry is removed (see [errors.md](./errors.md#failed-sources)).diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.mdindex 3d68c203..641ac896 100644--- a/docs/reference/jsx/media.md+++ b/docs/reference/jsx/media.md@@ -8,7 +8,7 @@- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is mounted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.-Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts, showing a generating state until its source lands. A source that never lands leaves the element carrying an [`error`](./errors.md#failed-sources).+Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts, showing a generating state until its source lands. A source that never lands leaves the element carrying the reason (see [errors.md](./errors.md#failed-sources)).An `<img>` inside [`<html>`](./html.md) additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.@@ -46,6 +46,8 @@ Modifiers compose with declarations: `<image src={generate.image({ prompt: "a reA project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/`, whether the app wrote the bytes there or a symlink points at them), and what it was found to be. Folders are listed too, so an empty one survives a reload. Renaming or moving an asset in the panel rewrites the `src` props that named it.+Generations are listed from the moment they are asked for: a partial record (`state: pending`) with no `source` while the model runs, the asset once it lands, and a record in `state: error` carrying the reason when it fails, which stands until removed (see [errors.md](./errors.md#failed-sources)).+## Adding an assetSymlink the file into the project's `assets/` folder and it is in the library. The app is watching, so it lands while the project is open; a project that is closed takes it in on the next open. Its library path is where the link sits under `assets/` — folder and name, so `assets/b-roll/drone.mp4` is `"b-roll/drone.mp4"` and `assets/logo.png` is `"logo.png"`.diff --git a/docs/reference/tools/context.md b/docs/reference/tools/context.mdindex 311e2a8b..e73e5c42 100644--- a/docs/reference/tools/context.md+++ b/docs/reference/tools/context.md@@ -46,4 +46,4 @@ With no project open (the app sits at the dashboard) the report is just `{ rootD`fontFamilies` is what text can be drawn with right now — loaded into the world, not merely named in the source — and always includes the editor default. For every family installed on the machine, see [`fonts`](./fonts.md).-`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the same message the element carries as its `error` prop, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).+`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the message the library recorded for the generation, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).diff --git a/packages/assets/src/hash.ts b/packages/assets/src/hash.tsindex 13b05267..7abe7ea3 100644--- a/packages/assets/src/hash.ts+++ b/packages/assets/src/hash.ts@@ -42,6 +42,16 @@ export async function hashSequence(frames: { name: string; size: number }[]): Prreturn hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);}+/**+ * The id of a partial document: the hash of its generation key, since there+ * are no bytes yet. Stable for the key, so the same generation reserved twice+ * is one record, and unlike any content hash the bytes could come to.+ */+export async function hashKey(key: string): Promise<string> {+ const bytes = new TextEncoder().encode(`key\n${key}`);+ return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);+}+/** Whether a string could be an asset id (and not, say, a library path). */export const looksLikeId = (value: string): boolean =>value.length === ID_LENGTH && /^[0-9a-f]+$/.test(value);diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.tsindex 73af34c4..4a03cdc0 100644--- a/packages/assets/src/library.ts+++ b/packages/assets/src/library.ts@@ -16,22 +16,27 @@// path, rebind whatever is bound to a relinked asset's old id) it asks of// `LibraryOptions`. Generation is not its business either — a generated// file is stored like any other bytes the app produced, with its-// `generation` record along. Its state is solid signals (`assets()`,+// `generation` record along — but where a generation stands is: a run that+// has started is a partial document here (`reserve`), one that failed stays+// as one with its reason (`fail`), and one that landed replaces its partial+// (`store`). Its state is solid signals (`assets()`, `partials()`,// `folders()`), so readers inside a tracking scope follow it.import { createSignal, type Accessor } from 'solid-js';import { AssetCache } from './cache';-import { hashBlob, hashSequence } from './hash';+import { hashBlob, hashKey, hashSequence } from './hash';import {- ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,+ ASSETS_DIR, isAbsoluteSource, isPartialRecord, isProjectSource, isUrlSource, normalizeManifest, toRecord,} from './manifest';import { detectMimeType, DEFAULT_SEQUENCE_FPS, isSequenceListing, probeMedia, sortFrames } from './probe';-import { assetFolder, assetName, basename, dirname, joinPath, normalizePath } from './types';+import { assetFolder, assetName, basename, dirname, isPartialAsset, joinPath, normalizePath } from './types';import type { FsEntry, ProjectFS } from './fs';import type { AssetRecord, Manifest } from './manifest';-import type { Asset, AssetDirectoryHandle, AssetFileHandle, AssetGeneration, SequenceAsset } from './types';+import type {+ Asset, AssetDirectoryHandle, AssetEntry, AssetFileHandle, AssetGeneration, AssetType, PartialAsset, SequenceAsset,+} from './types';/** How long changes pile up before the manifest is written. */const SAVE_DEBOUNCE = 200;@@ -64,21 +69,38 @@ export interface ImportOptions {generation?: AssetGeneration;}+export interface ReserveOptions {+ /** The generation key the partial stands for (see `AssetGeneration`). */+ key: string;+ /** What the generation is to become. */+ type: AssetType;+ /** A provisional library name; the bytes, once they land, are named by whoever stores them. */+ name: string;+ /** Library folder to place it in; root by default. */+ folder?: string;+}+export class AssetLibrary {public readonly fs: ProjectFS;/** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */public readonly cache: AssetCache;/** Library assets, newest first; transient ones excluded. Reactive. */public readonly assets: Accessor<Asset[]>;+ /** Partial documents — generations pending or failed — newest first. Reactive. */+ public readonly partials: Accessor<PartialAsset[]>;/** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */public readonly folders: Accessor<ReadonlySet<string>>;private readonly setAssets: (assets: Asset[]) => void;+ private readonly setPartials: (partials: PartialAsset[]) => void;private readonly setFolders: (folders: ReadonlySet<string>) => void;/** Folders declared in the manifest (the empty ones need declaring). */private declared = new Set<string>();- /** Every asset by id: the library's, and transient ones resolved from outside it. */- private readonly map = new Map<string, Asset>();+ /**+ * Every entry by id: the library's assets, transient ones resolved from+ * outside it, and the partial documents of generations without bytes.+ */+ private readonly map = new Map<string, AssetEntry>();private readonly onRename: LibraryOptions['onRename'];private readonly onRelink: LibraryOptions['onRelink'];private inflight = new Map<string, Promise<Asset>>();@@ -93,6 +115,7 @@ export class AssetLibrary {this.onRename = options.onRename;this.onRelink = options.onRelink;[this.assets, this.setAssets] = createSignal<Asset[]>([]);+ [this.partials, this.setPartials] = createSignal<PartialAsset[]>([]);[this.folders, this.setFolders] = createSignal<ReadonlySet<string>>(new Set());}@@ -106,30 +129,63 @@ export class AssetLibrary {/** The library's assets as of now, straight from the map. */private listNow(): Asset[] {- return Array.from(this.map.values()).filter((asset) => !asset.transient);+ const assets: Asset[] = [];+ for (const entry of this.map.values()) {+ if (!isPartialAsset(entry) && !entry.transient) assets.push(entry);+ }+ return assets;+ }++ /** The partial documents as of now, straight from the map. */+ private partialsNow(): PartialAsset[] {+ return Array.from(this.map.values()).filter(isPartialAsset);+ }++ /** Everything at a library path as of now: assets and partials, transient ones excluded. */+ private entriesNow(): AssetEntry[] {+ return Array.from(this.map.values()).filter((entry) => isPartialAsset(entry) || !entry.transient);}/** The asset with `id`, or at library path `path`; undefined otherwise. */public get(pathOrId: string): Asset | undefined {const byId = this.map.get(pathOrId);- if (byId) return byId;+ if (byId && !isPartialAsset(byId)) return byId;const path = normalizePath(pathOrId);- for (const asset of this.map.values()) {- if (asset.path === path && !asset.transient) return asset;+ for (const asset of this.listNow()) {+ if (asset.path === path) return asset;}return undefined;}+ /** The partial document with `id`; undefined when there is none, or the id is an asset's. */+ public getPartial(id: string): PartialAsset | undefined {+ const entry = this.map.get(id);+ return entry && isPartialAsset(entry) ? entry : undefined;+ }+/** The asset whose bytes are `source`, if the library links to it. */public bySource(source: string): Asset | undefined {- for (const asset of this.map.values()) {- if (asset.source === source && !asset.transient) return asset;+ for (const asset of this.listNow()) {+ if (asset.source === source) return asset;+ }+ return undefined;+ }++ /**+ * What the library holds for a generation key: the asset the generation+ * landed as, or the partial document standing for it while it runs or+ * after it failed. Undefined when the generation was never asked for+ * here — or its record was removed, which is how one is asked for again.+ */+ public generated(key: string): AssetEntry | undefined {+ for (const entry of this.entriesNow()) {+ if (entry.generation?.key === key) return entry;}return undefined;}- /** Direct children of a folder ('' for the root): its subfolders and assets. Reactive. */- public childrenOf(folder: string): { folders: string[]; assets: Asset[] } {+ /** Direct children of a folder ('' for the root): its subfolders, assets and partials. Reactive. */+ public childrenOf(folder: string): { folders: string[]; assets: Asset[]; partials: PartialAsset[] } {const prefix = folder ? `${folder}/` : '';const folders = new Set<string>();for (const path of this.folders()) {@@ -138,23 +194,25 @@ export class AssetLibrary {}}const assets = this.assets().filter((asset) => assetFolder(asset) === folder);- return { folders: [...folders].sort(), assets };+ const partials = this.partials().filter((partial) => assetFolder(partial) === folder);+ return { folders: [...folders].sort(), assets, partials };}- /** Every folder as of now: declared, implied by asset paths, and all their ancestors. */+ /** Every folder as of now: declared, implied by entry paths, and all their ancestors. */private foldersNow(): Set<string> {const folders = new Set<string>();const declare = (path: string): void => {for (let folder = path; folder && !folders.has(folder); folder = dirname(folder)) folders.add(folder);};for (const folder of this.declared) declare(folder);- for (const asset of this.listNow()) declare(assetFolder(asset));+ for (const entry of this.entriesNow()) declare(assetFolder(entry));return folders;}/** Publishes the map's state to the signals. */private publish(): void {this.setAssets(this.listNow());+ this.setPartials(this.partialsNow());this.setFolders(this.foldersNow());}@@ -171,8 +229,13 @@ export class AssetLibrary {const manifest = normalizeManifest(await this.fs.readManifest());this.declared = new Set(manifest.folders);- const next = new Map<string, Asset>();+ const next = new Map<string, AssetEntry>();await Promise.all(manifest.assets.map(async (record) => {+ // A partial has no bytes to attach or re-examine; it is its record.+ if (isPartialRecord(record)) {+ next.set(record.id, { ...record });+ return;+ }const asset = await this.revive(record).catch((error: unknown) => {console.warn(`[assets] could not load ${record.path}:`, error);return this.attach(record);@@ -181,8 +244,8 @@ export class AssetLibrary {}));// Keep transient assets and the same-id instances entities already hold.- for (const [id, asset] of this.map) {- if (asset.transient && !next.has(id)) next.set(id, asset);+ for (const [id, entry] of this.map) {+ if (!isPartialAsset(entry) && entry.transient && !next.has(id)) next.set(id, entry);}this.map.clear();for (const [id, asset] of next) {@@ -294,7 +357,7 @@ export class AssetLibrary {/** Resolves a source outside the library without adding it to the manifest. */private transient(source: string): Promise<Asset> {- const existing = Array.from(this.map.values()).find((asset) => asset.transient && asset.source === source);+ const existing = this.transientBySource(source);if (existing) return Promise.resolve(existing);return this.once(`transient:${source}`, async () => {@@ -304,12 +367,19 @@ export class AssetLibrary {asset.transient = true;// A transient asset that turns out to be one the library has is that one.const owned = this.map.get(asset.id);- if (owned && !owned.transient) return owned;+ if (owned && !isPartialAsset(owned) && !owned.transient) return owned;this.map.set(asset.id, asset);return asset;});}+ private transientBySource(source: string): Asset | undefined {+ for (const entry of this.map.values()) {+ if (!isPartialAsset(entry) && entry.transient && entry.source === source) return entry;+ }+ return undefined;+ }+/** The File backing an asset. */public file(asset: Asset): Promise<File> {return asset.handle.getFile();@@ -373,23 +443,88 @@ export class AssetLibrary {/*** Writes bytes the app produced into `assets/<folder>/<name>` and takes- * the file into the library at the same library path.+ * the file into the library at the same library path. Stored under a+ * generation key, the asset takes the place of the partial document+ * reserved for that key, and answers for the key from here on — even+ * when the bytes turn out to be ones the library already had under+ * another key, as a re-take of unchanged speech does.*/public async store(blob: Blob, options: ImportOptions & { name: string }): Promise<Asset> {const path = this.uniquePath(joinPath(options.folder ?? '', options.name));const source = joinPath(ASSETS_DIR, path);await this.fs.write(source, blob);const asset = await this.describeFile(source, { path, generation: options.generation });- return this.add(asset);+ const key = options.generation?.key;+ if (key !== undefined) this.dropPartial(key);+ const stored = this.add(asset);+ if (options.generation && stored.generation?.key !== key) {+ this.update(stored, { generation: options.generation });+ }+ return stored;+ }++ /**+ * Puts a partial document in the library for a generation about to run:+ * `pending`, at a path of its own under `folder`. A partial already+ * standing for the key — a run the last session never finished, or one+ * that failed and is being asked for again — is set pending again rather+ * than doubled. An asset the key already landed as is left alone: this+ * is not the place to ask whether to run (see `generated`).+ */+ public async reserve(options: ReserveOptions): Promise<PartialAsset> {+ const id = await hashKey(options.key);+ const existing = this.getPartial(id);+ if (existing) {+ existing.state = 'pending';+ delete existing.error;+ this.changed();+ return existing;+ }++ const partial: PartialAsset = {+ id,+ path: this.uniquePath(joinPath(options.folder ?? '', options.name)),+ type: options.type,+ createdAt: new Date().toISOString(),+ generation: { key: options.key },+ state: 'pending',+ };+ this.reorder(partial);+ this.changed();+ return partial;+ }++ /**+ * Records that the generation `partial` stands for failed, and why. The+ * record stays: it is what answers for the key from here on, until it+ * is removed. A partial no longer in the library (removed while the run+ * went on) is not brought back — that removal was the answer.+ */+ public fail(partial: PartialAsset, error: string): void {+ const current = this.getPartial(partial.id);+ if (!current) return;+ current.state = 'error';+ current.error = error;+ this.changed();+ }++ /** Takes the partial standing for `key` out, its bytes having landed. */+ private dropPartial(key: string): void {+ for (const partial of this.partialsNow()) {+ if (partial.generation.key === key) this.map.delete(partial.id);+ }}/** Puts an asset into the library, deduplicating by content. */private add(asset: Asset): Asset {const existing = this.map.get(asset.id);- if (existing && !existing.transient) return existing;- // A transient asset the library now takes in becomes a real one; the- // instance entities hold stays valid, so mutate rather than replace.- if (existing) {+ if (existing && isPartialAsset(existing)) {+ // A content hash colliding with a key hash: the bytes win the id.+ this.map.delete(existing.id);+ } else if (existing) {+ if (!existing.transient) return existing;+ // A transient asset the library now takes in becomes a real one; the+ // instance entities hold stays valid, so mutate rather than replace.Object.assign(existing, asset);delete existing.transient;this.reorder(existing);@@ -401,11 +536,11 @@ export class AssetLibrary {return asset;}- /** Puts `asset` at the front of the map (newest first). */- private reorder(asset: Asset): void {- const rest = Array.from(this.map.values()).filter((other) => other.id !== asset.id);+ /** Puts `entry` at the front of the map (newest first). */+ private reorder(entry: AssetEntry): void {+ const rest = Array.from(this.map.values()).filter((other) => other.id !== entry.id);this.map.clear();- this.map.set(asset.id, asset);+ this.map.set(entry.id, entry);for (const other of rest) this.map.set(other.id, other);}@@ -444,39 +579,45 @@ export class AssetLibrary {return next;}- /** Renames an asset within its folder; the source file is untouched. */- public rename(asset: Asset, name: string): void {+ /** Renames an entry within its folder; an asset's source file is untouched. */+ public rename(entry: AssetEntry, name: string): void {const clean = normalizePath(name).replace(/\//g, '-');- if (!clean || clean === assetName(asset)) return;- this.setPath(asset, this.uniquePath(joinPath(assetFolder(asset), clean), asset));+ if (!clean || clean === assetName(entry)) return;+ this.setPath(entry, this.uniquePath(joinPath(assetFolder(entry), clean), entry));}- /** Moves assets into a folder ('' for the root). */- public move(assets: Asset[], folder: string): void {+ /** Moves entries into a folder ('' for the root). */+ public move(entries: AssetEntry[], folder: string): void {const target = normalizePath(folder);if (target) this.declared.add(target);- for (const asset of assets) {- if (assetFolder(asset) === target) continue;- this.setPath(asset, this.uniquePath(joinPath(target, assetName(asset)), asset));+ for (const entry of entries) {+ if (assetFolder(entry) === target) continue;+ this.setPath(entry, this.uniquePath(joinPath(target, assetName(entry)), entry));}}- private setPath(asset: Asset, path: string): void {- const from = asset.path;+ private setPath(entry: AssetEntry, path: string): void {+ const from = entry.path;if (from === path) return;- asset.path = path;+ entry.path = path;this.changed();- this.onRename?.(asset, from);+ // Nothing names a partial by path: it is not a source yet.+ if (!isPartialAsset(entry)) this.onRename?.(entry, from);}- /** Removes assets from the library. Bytes the app wrote into `assets/` go too. */- public async remove(assets: Asset[]): Promise<void> {- for (const asset of assets) {- this.map.delete(asset.id);- if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {- await this.fs.remove(asset.source).catch(() => { });+ /**+ * Removes entries from the library. Bytes the app wrote into `assets/`+ * go too. Removing a partial forgets where its generation stood — for a+ * failed one, that is what asks for the run again.+ */+ public async remove(entries: AssetEntry[]): Promise<void> {+ for (const entry of entries) {+ this.map.delete(entry.id);+ if (isPartialAsset(entry)) continue;+ if (isProjectSource(entry.source) && entry.source.startsWith(`${ASSETS_DIR}/`)) {+ await this.fs.remove(entry.source).catch(() => { });}- this.cache.remove(asset);+ this.cache.remove(entry);}this.changed();}@@ -527,20 +668,20 @@ export class AssetLibrary {}}this.declared.add(to);- for (const asset of this.listNow()) {- if (asset.path.startsWith(prefix)) this.setPath(asset, to + asset.path.slice(from.length));+ for (const entry of this.entriesNow()) {+ if (entry.path.startsWith(prefix)) this.setPath(entry, to + entry.path.slice(from.length));}this.changed();return to;}- /** Deletes a folder and everything in it. Returns the removed assets. */- public async deleteFolder(path: string): Promise<Asset[]> {+ /** Deletes a folder and everything in it. Returns the removed entries. */+ public async deleteFolder(path: string): Promise<AssetEntry[]> {const prefix = `${path}/`;for (const folder of [...this.declared]) {if (folder === path || folder.startsWith(prefix)) this.declared.delete(folder);}- const doomed = this.listNow().filter((asset) => asset.path.startsWith(prefix));+ const doomed = this.entriesNow().filter((entry) => entry.path.startsWith(prefix));await this.remove(doomed);return doomed;}@@ -561,7 +702,7 @@ export class AssetLibrary {return {version: 1,folders: [...this.declared].sort(),- assets: this.listNow().map(toRecord),+ assets: this.entriesNow().map(toRecord),};}@@ -586,9 +727,9 @@ export class AssetLibrary {// -----------------------------------------------------------------------// Describing- /** A library path not taken by any other asset: `name`, `name 2`, … */- private uniquePath(path: string, except?: Asset): string {- const taken = new Set(this.listNow().filter((asset) => asset !== except).map((asset) => asset.path));+ /** A library path not taken by any other entry: `name`, `name 2`, … */+ private uniquePath(path: string, except?: AssetEntry): string {+ const taken = new Set(this.entriesNow().filter((entry) => entry !== except).map((entry) => entry.path));if (!taken.has(path)) return path;const folder = dirname(path);const name = basename(path);diff --git a/packages/assets/src/manifest.ts b/packages/assets/src/manifest.tsindex 5f77d1b9..f6ac2422 100644--- a/packages/assets/src/manifest.ts+++ b/packages/assets/src/manifest.ts@@ -2,17 +2,10 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// The asset manifest: the project's record of its library. On desktop it is-// `assets.yml` at the project root (the main process converts to and from-// YAML); elsewhere the same shape as JSON. It holds what cannot be derived-// from the files themselves — which files belong to the project, where the-// project keeps them (`path`), where their bytes are (`source`), and what-// they were found to be — so opening a project never re-probes footage that-// has not changed.import { normalizePath } from './types';-import type { Asset } from './types';+import type { Asset, PartialAsset } from './types';export const MANIFEST_VERSION = 1;@@ -30,6 +23,11 @@ type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K>/** An asset as written to the manifest: the model minus its runtime handles. */export type AssetRecord = DistributiveOmit<Asset, 'handle' | 'directoryHandle' | 'transient'>;+/** A partial document as written to the manifest: it has no handles to lose. */+export type PartialRecord = PartialAsset;++export type ManifestRecord = AssetRecord | PartialRecord;+export interface Manifest {version: number;/**@@ -37,19 +35,20 @@ export interface Manifest {* in it is also implied by their paths). `/`-separated, no leading slash.*/folders: string[];- assets: AssetRecord[];+ /** Assets and the partial records of generations without bytes, newest first. */+ assets: ManifestRecord[];}const RECORD_KEYS = new Set(['handle', 'directoryHandle', 'transient']);-/** The manifest record of an asset: its data, none of its handles. */-export function toRecord(asset: Asset): AssetRecord {+/** The manifest record of a library entry: its data, none of its handles. */+export function toRecord(entry: Asset | PartialAsset): ManifestRecord {const record: Record<string, unknown> = {};- for (const [key, value] of Object.entries(asset)) {+ for (const [key, value] of Object.entries(entry)) {if (RECORD_KEYS.has(key) || value === undefined) continue;record[key] = value;}- return record as AssetRecord;+ return record as ManifestRecord;}/** Whether `record` has what every asset needs; unrecognized fields are kept. */@@ -65,6 +64,21 @@ export function isAssetRecord(record: unknown): record is AssetRecord {);}+/** Whether `record` is a partial document: a generation's place and state, no bytes. */+export function isPartialRecord(record: unknown): record is PartialRecord {+ if (!record || typeof record !== 'object') return false;+ const r = record as Record<string, unknown>;+ const generation = r.generation as Record<string, unknown> | undefined;+ return (+ typeof r.id === 'string' && r.id.length > 0 &&+ typeof r.path === 'string' && r.path.length > 0 &&+ typeof r.type === 'string' &&+ (r.state === 'pending' || r.state === 'error') &&+ !!generation && typeof generation === 'object' && typeof generation.key === 'string' &&+ (r.error === undefined || typeof r.error === 'string')+ );+}+/*** Reads whatever the host handed back as a manifest into a well-formed one.* Tolerant: a missing or malformed file is an empty library, and a broken@@ -86,7 +100,7 @@ export function normalizeManifest(input: unknown): Manifest {if (Array.isArray(raw.assets)) {const seen = new Set<string>();for (const record of raw.assets) {- if (!isAssetRecord(record)) continue;+ if (!isAssetRecord(record) && !isPartialRecord(record)) continue;const path = normalizePath(record.path);if (!path || seen.has(record.id)) continue;seen.add(record.id);diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.tsindex f302acd8..6e7d6f6d 100644--- a/packages/assets/src/types.ts+++ b/packages/assets/src/types.ts@@ -2,13 +2,6 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// The asset model. An asset is a manifest record (see ./manifest.ts) plus a-// runtime handle to its bytes, attached by the AssetLibrary. Everything an-// asset *is* lives in the record: what the file was found to be (probe-// results), where its bytes are (`source`), and where the project keeps it in-// the library (`path`). Its `id` is a content hash, so the same bytes imported-// twice are one asset, and a relinked file keeps its metadata.-/** Whatever hands out the asset's bytes: a real File on desktop, an OPFS* file on the web, a fetched blob for a URL. */export interface AssetFileHandle {@@ -123,11 +116,45 @@ export type Asset =export type AssetType = Asset['type'];+/** Where a generation stands while the library has no bytes for it. */+export type PartialAssetState = 'pending' | 'error';++/**+ * A partial document: the library's record of a generation before — or+ * instead of — its bytes. Written `pending` when a run starts, so the+ * generation is in the library from the moment it is asked for; replaced by+ * the asset when the run lands; kept as `error`, with what went wrong, when+ * it does not. The error record is what keeps a refused or impossible+ * generation from being run — or paid for — again on every reopen: a+ * declaration whose key stands in error resolves to that error, and removing+ * the record is what asks for the run again. Never bound to an entity;+ * elements see the state through their resolution.+ */+export interface PartialAsset {+ /** Hash of the generation key: there are no bytes to hash. */+ id: string;+ /** Library path, like any asset's; provisional until the bytes name themselves. */+ path: string;+ /** What the generation is to become. */+ type: AssetType;+ createdAt: string;+ generation: AssetGeneration;+ state: PartialAssetState;+ /** What the run failed with, on an `error` record. */+ error?: string;+}++/** Anything the library holds at a path: an asset, or a partial standing for one. */+export type AssetEntry = Asset | PartialAsset;++/** Whether a library entry is a partial document rather than an asset with bytes. */+export const isPartialAsset = (entry: AssetEntry): entry is PartialAsset => 'state' in entry;+/** The file name of an asset: the last segment of its library path. */-export const assetName = (asset: Pick<Asset, 'path'>): string => basename(asset.path);+export const assetName = (asset: Pick<AssetEntry, 'path'>): string => basename(asset.path);/** The folder of an asset: its library path without the name, '' at root. */-export const assetFolder = (asset: Pick<Asset, 'path'>): string => dirname(asset.path);+export const assetFolder = (asset: Pick<AssetEntry, 'path'>): string => dirname(asset.path);export function basename(path: string): string {const at = path.lastIndexOf('/');diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex a62187e2..9acc1ec7 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -294,16 +294,6 @@ type MediaProps = {* scene; `generate.*` is not accepted there.*/src: string | AssetRef;- /**- * Why this element's source never became an asset. Editor state carried by- * the source like `selected`, and the answer to the `src` it was written- * for: an element holding one is not resolved again, so a generation the- * model refused is not run — or paid for — a second time by every reopen of- * the project. The editor writes it when a generation or a transcription- * fails; taking it off the element is what asks for the run again, and- * nothing else does — not another take, not another prompt.- */- error?: string;};/**diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 38ab1c2f..6375ae42 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';import { SVGElements } from 'solid-js/web';@@ -1156,18 +1156,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {entity.set(LoadRequest, { value });return;}- case 'error': {- const message = typeof value === 'string' && value !== '' ? value : undefined;-- if (message === undefined) {- entity.remove(SourceError);- return;- }-- entity.add(SourceError);- entity.set(SourceError, { value: message, generated: true });- return;- }case 'objectFit': {const mode = typeof value === 'string' ? SCALE_MODES[value] : undefined;if (mode === undefined) {diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsindex 25e6516c..f0a2ae2c 100644--- a/packages/runtime/src/ai.ts+++ b/packages/runtime/src/ai.ts@@ -15,12 +15,22 @@ import type {GenerateVoiceOptions,} from '@diffusionstudio/jsx';+/**+ * The generation service a host attaches to its world. Where a generation+ * stands is the library's to hold, as partial documents (see+ * `PartialAsset` in @diffusionstudio/assets): a run that starts is+ * `pending` there, one that lands is the asset, one that fails stays as+ * `error` with its reason — and a key standing in error resolves to that+ * error rather than to another run, in this session and the next. Nothing+ * here writes to the source that declared the generation.+ */export abstract class GenAi {/*** The declaration, made real: an asset whose bytes the model produced.* Content-addressed — the fully-resolved spec's hash is the asset's* `generation.key`, so the same spec is the same asset in this session* and the next, and identical concurrent declarations share one run.+ * Rejects with the recorded reason when the key stands in error.*/public abstract resolve(ref: AssetRef): Promise<Asset>;diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex 05f0bf26..cc81aa7f 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -39,7 +39,7 @@ export function assetSystem(world: World): void {for (const entity of world.query(GenerationRequest)) {const ref = entity.get(GenerationRequest)!.ref;entity.remove(GenerationRequest);- if (ref === null || entity.has(SourceError)) continue;+ if (ref === null) continue;if (isDomImage(entity)) pointDomImageAt(entity, null);start(world, entity, ai.resolve(ref), true);}@@ -50,7 +50,6 @@ export function assetSystem(world: World): void {const seed = entity.get(TranscriptionRequest)!.seed;entity.remove(TranscriptionRequest);- if (entity.has(SourceError)) continue;resolve(world, entity, ai.transcribe(world, scene, seed), true);}}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex a7960d92..7c90436d 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -121,16 +121,15 @@ export const PendingSync = trait(() => ({ value: undefined as unknown }));export const PendingSource = trait(() => ({ value: undefined as unknown }));// Why the entity's src never became an asset: the message of the rejection-// the asset system saw. It stands until the element is authored without it-// (the `error` prop) or a resolution for it starts — which, for a generation,-// only happens once that prop is gone.+// the asset system saw. Session state, like the requests above: it stands+// until a resolution for the src starts again, and is never written back to+// the source. What outlives the session is the library's: a generation that+// failed stands there as a partial document carrying this same message, and+// answers the next request for it with the failure rather than another run.//-// `generated` tells a failed generation from a failed load, which are worth-// different things: a load is cheap and idempotent, so it is simply tried-// again next render, while a generation is neither. An element carrying one-// of those is not resolved again (see the asset system), and the host is-// expected to write it back to the source the element came from — which is-// what makes the failure outlive the session that saw it.+// `generated` tells a failed generation from a failed load, for whoever+// reports them: the one is a model refusing or a spec a model cannot take,+// the other a path that is not there.export const SourceError = trait({ value: '', generated: false });// Sibling order under a ChildOf parent.