The VS Code of video editing, built for agents.
TypeScript
Enhance asset management and integration with AI generation
konstantin-paulus2fafd8668c0e
diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.tsindex 3c21dde6..29a2b2fd 100644--- a/apps/web/src/engine/create-engine.ts+++ b/apps/web/src/engine/create-engine.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';+import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';import { hudSystem } from './hud';import { createSignal, type Accessor, type Setter } from 'solid-js';import { AssetSelection, Hud, Keys, Pointer, PointerEvents, ProjectConfig, SnapLines } from './traits';@@ -198,6 +198,7 @@ class Engine {private runSystems(): void {inputSystem(this.world);shortcutSystem(this.world);+ assetSystem(this.world);playbackSystem(this.world);motionSystem(this.world);transformSystem(this.world);diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.tsindex f28e7265..e6ab122c 100644--- a/apps/web/src/engine/library.ts+++ b/apps/web/src/engine/library.ts@@ -6,14 +6,13 @@// disk and kept in step with the JSX (a rename in the library is a `src`// edit in the file).-import { AssetId, Library, Project } from '@diffusionstudio/runtime';+import { AssetId, Library } from '@diffusionstudio/runtime';import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';import { useTrait, useWorld } from '@diffusionstudio/koota-solid';import { authoredElement } from '@diffusionstudio/reconciler';import type { Accessor } from 'solid-js';import { createProjectFS } from '@/projects/fs';-import { generateAsset } from '@/utils/jsx-generation';import { getDocumentEditor } from './editor';import type { Asset } from '@diffusionstudio/assets';@@ -26,10 +25,7 @@ import type { World } from 'koota';* disposer that flushes the manifest and detaches it.*/export function attachLibrary(world: World, dir: string) {- const projectId = world.get(Project)?.id ?? 'project';-const library = new AssetLibrary(createProjectFS(dir), {- generate: (ref, library) => generateAsset(ref, library, projectId),onRename: (asset, from) => followRename(world, asset, from),onRelink: (asset, from) => followRelink(world, asset, from),});diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsxindex d834cf84..4a667db2 100644--- a/apps/web/src/pages/editor.tsx+++ b/apps/web/src/pages/editor.tsx@@ -16,6 +16,7 @@ import { useWorld } from '@diffusionstudio/koota-solid';import { mount } from '@diffusionstudio/reconciler';import { getDocumentEditor } from '@/engine/editor';import { attachLibrary, isLibraryFile } from '@/engine/library';+import { attachAi } from '@/utils/gen-ai';import { attachProjectConfig, isProjectConfigFile } from '@/engine/project-config';import { isCacheFile } from '@diffusionstudio/assets';import { createEditWriter } from '@/projects/edits';@@ -52,6 +53,8 @@ export function EditorPage() {// The library first: a mounted project's `src` values name its assets.const library = attachLibrary(world, dir);+ // The generation service over it: what `generate.*` sources resolve through.+ attachAi(world, library);// The project's own settings (package.json `diffusion`), next to the scene.const config = attachProjectConfig(world, dir);diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsnew file mode 100644index 00000000..6196f7bb--- /dev/null+++ b/apps/web/src/utils/gen-ai.ts@@ -0,0 +1,247 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The app's GenAi: `generate.*` declarations made real against this+// project's library and the dapi backend. Attached to the world as the `Ai`+// trait, which is what the runtime's asset system (and `ai.generate.image`+// callers) resolve through. A declaration resolves to an asset by content —+// the same spec is the same asset, in this session and the next, since the+// finished file lands in the library under `generated/` with the spec's key+// in the manifest.++import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";+import { Ai, GenAi, getAssetFile, Project } from "@diffusionstudio/runtime";+import { GENERATED_DIR } from "@diffusionstudio/assets";+import {+ PROMPT_INPUT_AUDIO_MODEL_OPTIONS,+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS,+ PROMPT_INPUT_VIDEO_MODEL_OPTIONS,+ PROMPT_INPUT_VOICE_OPTIONS,+} from "@/components/genai/config";+import { assert } from "@/utils";+import { uploadBlob } from "@/components/engine";+import { trpc } from "@/lib/trpc";++import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";+import type { World } from "koota";++// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).+const VOICE_MODEL = "elevenlabs-v3";++/**+ * A spec with defaults applied and every `AssetInput` reduced to an asset id.+ * Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.+ */+type ResolvedSpec =+ | { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }+ | { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }+ | { type: "voice"; model: string; prompt: string; voice: string }+ | { type: "audio"; model: string; prompt: string; duration?: number };++/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */+export function attachAi(world: World, library: AssetLibrary): EditorGenAi {+ const ai = new EditorGenAi(library, world.get(Project)?.id ?? "project");+ world.set(Ai, ai);+ return ai;+}++export class EditorGenAi extends GenAi {+ private readonly library: AssetLibrary;+ /** Prefixes upload keys, so referenced assets land project-unique in the bucket. */+ private readonly projectId: string;++ /**+ * Declarations already resolved, keyed by ref identity — a ref consumed by+ * several elements resolves (and validates) once. A failed one is+ * forgotten, so a remount tries again.+ */+ private readonly memo = new Map<AssetRef, Promise<Asset>>();+ /** In-flight generations keyed by `generationKey`. */+ private readonly inflight = new Map<string, Promise<Asset>>();++ public constructor(library: AssetLibrary, projectId: string) {+ super();+ this.library = library;+ this.projectId = projectId;+ }++ /** Identical concurrent declarations collapse to one request. */+ public resolve(ref: AssetRef): Promise<Asset> {+ assert(isAssetRef(ref), "Not a generate.* declaration");++ let promise = this.memo.get(ref);+ if (!promise) {+ promise = this.generateFromRef(ref);+ promise.catch(() => this.memo.delete(ref));+ this.memo.set(ref, promise);+ }+ return promise;+ }++ private resolveInput(input: AssetInput): Promise<Asset> {+ return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);+ }++ private async generateFromRef(ref: AssetRef): Promise<Asset> {+ const resolved = await this.resolveSpec(getAssetSpec(ref));+ const generationKey = JSON.stringify(resolved);++ const cached = this.library.list().find((asset) => asset.generation?.key === generationKey);+ if (cached) return cached;++ const running = this.inflight.get(generationKey);+ if (running) return await running;++ const promise = this.runGeneration(resolved, generationKey);+ this.inflight.set(generationKey, promise);+ try {+ return await promise;+ } finally {+ this.inflight.delete(generationKey);+ }+ }++ private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {+ switch (spec.type) {+ case "image": {+ const refs = await Promise.all((spec.refs ?? []).map((ref) => this.resolveInput(ref)));+ return {+ type: "image",+ model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio ?? "16:9",+ seed: spec.seed,+ refIds: refs.map((asset) => asset.id),+ };+ }+ case "video": {+ const [startFrame, endFrame] = await Promise.all([+ spec.startFrame !== undefined ? this.resolveInput(spec.startFrame) : undefined,+ spec.endFrame !== undefined ? this.resolveInput(spec.endFrame) : undefined,+ ]);+ const resolved = {+ type: "video",+ model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio ?? "16:9",+ duration: spec.duration ?? 5,+ audio: spec.audio ?? false,+ seed: spec.seed,+ startFrameId: startFrame?.id,+ endFrameId: endFrame?.id,+ } satisfies ResolvedSpec;+ checkVideoConstraints(resolved);+ return resolved;+ }+ case "voice": {+ return {+ type: "voice",+ model: VOICE_MODEL,+ prompt: spec.prompt,+ voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,+ };+ }+ case "audio": {+ return {+ type: "audio",+ model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,+ prompt: spec.prompt,+ duration: spec.duration,+ };+ }+ }+ }++ /** Runs a generation and stores its first result under `generated/`. */+ private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {+ const { name, results, generationId } = await this.requestGeneration(spec);+ assert(results.length > 0, "No results returned from the model");++ const url = results[0].url;+ const response = await fetch(url);+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);+ const blob = await response.blob();+ const extension = url.split(/[?#]/)[0].split(".").pop();+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;++ return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });+ }++ private requestGeneration(spec: ResolvedSpec) {+ switch (spec.type) {+ case "image": {+ return (async () => {+ const images = spec.refIds.length > 0+ ? await Promise.all(spec.refIds.map((id) => this.uploadInput(id)))+ : undefined;++ return await trpc.generateImage.mutate({+ model: spec.model,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio,+ count: 1,+ seed: spec.seed,+ images,+ });+ })();+ }+ case "video": {+ return (async () => {+ const [startFrame, endFrame] = await Promise.all([+ spec.startFrameId ? this.uploadInput(spec.startFrameId) : undefined,+ spec.endFrameId ? this.uploadInput(spec.endFrameId) : undefined,+ ]);++ return await trpc.generateVideo.mutate({+ model: spec.model,+ prompt: spec.prompt,+ aspectRatio: spec.aspectRatio,+ duration: spec.duration,+ generateAudio: spec.audio,+ seed: spec.seed,+ startFrame,+ endFrame,+ });+ })();+ }+ case "voice": {+ return trpc.textToSpeech.mutate({+ model: spec.model,+ prompt: spec.prompt,+ voice: spec.voice,+ });+ }+ case "audio": {+ return trpc.generateSound.mutate({+ model: spec.model,+ prompt: spec.prompt,+ duration: spec.duration,+ });+ }+ }+ }++ /** Uploads a referenced asset for the model to read; the bucket key is project-unique. */+ private async uploadInput(assetId: string) {+ const asset = this.library.get(assetId);+ assert(asset, `Referenced asset ${assetId} not found`);+ const uploaded = await uploadBlob(await getAssetFile(asset), `${this.projectId}-${assetId}`);+ assert(uploaded, `Failed to upload referenced asset ${assetId}`);+ return uploaded;+ }+}++/**+ * Per-model constraints (`dapi models video`); unknown models are left to the server.+ */+function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {+ const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);+ if (!model) return;++ assert(model.aspectRatios.includes(spec.aspectRatio), `${spec.model} does not support aspect ratio ${spec.aspectRatio}`);+ assert(model.durations.includes(`${spec.duration}s`), `${spec.model} does not support a duration of ${spec.duration}s`);+ assert(!spec.audio || model.features.includes("audio"), `${spec.model} does not support audio generation`);+ assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);+}diff --git a/apps/web/src/utils/jsx-generation.ts b/apps/web/src/utils/jsx-generation.tsdeleted file mode 100644index 1bc56f99..00000000--- a/apps/web/src/utils/jsx-generation.ts+++ /dev/null@@ -1,239 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// `generate.*` declarations in a project's JSX, made real: the asset-// library's generator (see `attachLibrary`). A declaration resolves to an-// asset by content — the same spec is the same asset, in this session and-// the next, since the finished file lands in the library under-// `generated/` with the spec's key in the manifest.--import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";-import { getAssetFile } from "@diffusionstudio/runtime";-import { GENERATED_DIR } from "@diffusionstudio/assets";-import {- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,- PROMPT_INPUT_VOICE_OPTIONS,-} from "@/components/genai/config";-import { assert } from "@/utils";-import { uploadBlob } from "@/components/engine";-import { trpc } from "@/lib/trpc";--import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";-import type { Asset, AssetLibrary } from "@diffusionstudio/assets";--const memo: GenerationMemo = new Map();-const inflight: Inflight = new Map();--/**- * A library's `generate` option for the project `projectId` (which prefixes- * upload keys). Identical concurrent declarations collapse to one request.- */-export function generateAsset(ref: object, library: AssetLibrary, projectId: string): Promise<Asset> {- assert(isAssetRef(ref), "Not a generate.* declaration");-- return resolveGeneratedAsset({ library, projectId, memo, inflight }, ref);-}--/** What one generation run has to hand: the library it fills and its caches. */-type Generation = { library: AssetLibrary; projectId: string; memo: GenerationMemo; inflight: Inflight };--// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).-const VOICE_MODEL = "elevenlabs-v3";--/**- * A spec with defaults applied and every `AssetInput` reduced to an asset id.- * Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.- */-type ResolvedSpec =- | { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }- | { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }- | { type: "voice"; model: string; prompt: string; voice: string }- | { type: "audio"; model: string; prompt: string; duration?: number };--/**- * Declarations already resolved, keyed by ref identity — a ref consumed by- * several elements resolves (and validates) once. A failed one is forgotten,- * so a remount tries again.- */-type GenerationMemo = Map<AssetRef, Promise<Asset>>;--/** In-flight generations keyed by `generationKey`. */-type Inflight = Map<string, Promise<Asset>>;--function resolveGeneratedAsset(gen: Generation, ref: AssetRef): Promise<Asset> {- let promise = gen.memo.get(ref);- if (!promise) {- promise = generateFromRef(gen, ref);- promise.catch(() => gen.memo.delete(ref));- gen.memo.set(ref, promise);- }- return promise;-}--function resolveInput(gen: Generation, input: AssetInput): Promise<Asset> {- return isAssetRef(input) ? resolveGeneratedAsset(gen, input) : gen.library.resolve(input);-}--async function generateFromRef(gen: Generation, ref: AssetRef): Promise<Asset> {- const resolved = await resolveSpec(gen, getAssetSpec(ref));- const generationKey = JSON.stringify(resolved);-- const cached = gen.library.list().find((asset) => asset.generation?.key === generationKey);- if (cached) return cached;-- const running = gen.inflight.get(generationKey);- if (running) return await running;-- const promise = runGeneration(gen, resolved, generationKey);- gen.inflight.set(generationKey, promise);- try {- return await promise;- } finally {- gen.inflight.delete(generationKey);- }-}--async function resolveSpec(gen: Generation, spec: AssetSpecInput): Promise<ResolvedSpec> {- switch (spec.type) {- case "image": {- const refs = await Promise.all((spec.refs ?? []).map((ref) => resolveInput(gen, ref)));- return {- type: "image",- model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- seed: spec.seed,- refIds: refs.map((asset) => asset.id),- };- }- case "video": {- const [startFrame, endFrame] = await Promise.all([- spec.startFrame !== undefined ? resolveInput(gen, spec.startFrame) : undefined,- spec.endFrame !== undefined ? resolveInput(gen, spec.endFrame) : undefined,- ]);- const resolved = {- type: "video",- model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- duration: spec.duration ?? 5,- audio: spec.audio ?? false,- seed: spec.seed,- startFrameId: startFrame?.id,- endFrameId: endFrame?.id,- } satisfies ResolvedSpec;- checkVideoConstraints(resolved);- return resolved;- }- case "voice": {- return {- type: "voice",- model: VOICE_MODEL,- prompt: spec.prompt,- voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,- };- }- case "audio": {- return {- type: "audio",- model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- duration: spec.duration,- };- }- }-}--/**- * Per-model constraints (`dapi models video`); unknown models are left to the server.- */-function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {- const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);- if (!model) return;-- assert(model.aspectRatios.includes(spec.aspectRatio), `${spec.model} does not support aspect ratio ${spec.aspectRatio}`);- assert(model.durations.includes(`${spec.duration}s`), `${spec.model} does not support a duration of ${spec.duration}s`);- assert(!spec.audio || model.features.includes("audio"), `${spec.model} does not support audio generation`);- assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);-}--/** Runs a generation and stores its first result under `generated/`. */-async function runGeneration(gen: Generation, spec: ResolvedSpec, generationKey: string): Promise<Asset> {- const { name, results, generationId } = await requestGeneration(gen, spec);- assert(results.length > 0, "No results returned from the model");-- const url = results[0].url;- const response = await fetch(url);- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);- const blob = await response.blob();- const extension = url.split(/[?#]/)[0].split(".").pop();- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;-- return gen.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });-}--function requestGeneration(gen: Generation, spec: ResolvedSpec) {- switch (spec.type) {- case "image": {- return (async () => {- const images = spec.refIds.length > 0- ? await Promise.all(spec.refIds.map((id) => uploadInput(gen, id)))- : undefined;-- return await trpc.generateImage.mutate({- model: spec.model,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio,- count: 1,- seed: spec.seed,- images,- });- })();- }- case "video": {- return (async () => {- const [startFrame, endFrame] = await Promise.all([- spec.startFrameId ? uploadInput(gen, spec.startFrameId) : undefined,- spec.endFrameId ? uploadInput(gen, spec.endFrameId) : undefined,- ]);-- return await trpc.generateVideo.mutate({- model: spec.model,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio,- duration: spec.duration,- generateAudio: spec.audio,- seed: spec.seed,- startFrame,- endFrame,- });- })();- }- case "voice": {- return trpc.textToSpeech.mutate({- model: spec.model,- prompt: spec.prompt,- voice: spec.voice,- });- }- case "audio": {- return trpc.generateSound.mutate({- model: spec.model,- prompt: spec.prompt,- duration: spec.duration,- });- }- }-}--/** Uploads a referenced asset for the model to read; the bucket key is project-unique. */-async function uploadInput(gen: Generation, assetId: string) {- const asset = gen.library.get(assetId);- assert(asset, `Referenced asset ${assetId} not found`);- const uploaded = await uploadBlob(await getAssetFile(asset), `${gen.projectId}-${assetId}`);- assert(uploaded, `Failed to upload referenced asset ${assetId}`);- return uploaded;-}diff --git a/examples/06-three.tsx b/examples/06-three.tsxindex 164275c2..4d1b54a5 100644--- a/examples/06-three.tsx+++ b/examples/06-three.tsx@@ -11,9 +11,9 @@* createEffect, so scrubbing and exports stay frame-accurate, not wall-clock.** The "Battle Damaged Sci-Fi Helmet" is the Khronos DamagedHelmet sample, a- * single-file .glb GLTFLoader fetches directly over the network (useFile is the- * host-resolved route for project-local assets). RoomEnvironment bakes an- * in-memory PBR reflection map, so the metal reads without shipping an HDR.+ * single-file .glb GLTFLoader fetches directly over the network.+ * RoomEnvironment bakes an in-memory PBR reflection map, so the metal reads+ * without shipping an HDR.*/import { createEffect, createSignal, onCleanup } from "solid-js";diff --git a/package-lock.json b/package-lock.jsonindex c679d21f..42ff1dbe 100644--- a/package-lock.json+++ b/package-lock.json@@ -12477,6 +12477,7 @@"license": "MPL-2.0","dependencies": {"@diffusionstudio/assets": "*",+ "@diffusionstudio/jsx": "*","animejs": "^4.4.1","colord": "^2.9.3","koota": "^0.6.6",diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.tsindex 180408e1..5097d77a 100644--- a/packages/assets/src/library.ts+++ b/packages/assets/src/library.ts@@ -12,9 +12,11 @@// keeping (thumbnails, waveforms) is the `cache`'s, under `cache/`.//// Knows nothing of the host: files come and go through a `ProjectFS`, and-// what it cannot do itself (generate an asset, follow a rename into whatever-// names assets by path, rebind whatever is bound to a relinked asset's old-// id) it asks of `LibraryOptions`. Its state is solid signals (`assets()`,+// what it cannot do itself (follow a rename into whatever names assets by+// path, rebind whatever is bound to a relinked asset's old id) it asks of+// `LibraryOptions`. Generation is not its business either — a generated+// file is stored like any other bytes the app produced, with its+// `generation` record along. Its state is solid signals (`assets()`,// `folders()`), so readers inside a tracking scope follow it.import { createSignal, type Accessor } from 'solid-js';@@ -35,12 +37,6 @@ import type { Asset, AssetDirectoryHandle, AssetFileHandle, AssetGeneration, Seqconst SAVE_DEBOUNCE = 200;export interface LibraryOptions {- /**- * Turns what the runtime cannot — a `generate.*` declaration — into an- * asset of this library; `resolve` hands it any non-string source. Without- * one, such sources fail to resolve.- */- generate?: AssetGenerator;/*** Called when an asset's library path changes (renamed or moved), so the* host can follow in whatever names assets by path — the JSX `src` props.@@ -68,9 +64,6 @@ export interface ImportOptions {generation?: AssetGeneration;}-/** See `LibraryOptions.generate`. */-export type AssetGenerator = (ref: object, library: AssetLibrary) => Promise<Asset>;-export class AssetLibrary {public readonly fs: ProjectFS;/** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */@@ -86,7 +79,6 @@ export class AssetLibrary {private declared = new Set<string>();/** Every asset by id: the library's, and transient ones resolved from outside it. */private readonly map = new Map<string, Asset>();- private readonly generator: LibraryOptions['generate'];private readonly onRename: LibraryOptions['onRename'];private readonly onRelink: LibraryOptions['onRelink'];private inflight = new Map<string, Promise<Asset>>();@@ -98,7 +90,6 @@ export class AssetLibrary {public constructor(fs: ProjectFS, options: LibraryOptions = {}) {this.fs = fs;this.cache = new AssetCache(fs);- this.generator = options.generate;this.onRename = options.onRename;this.onRelink = options.onRelink;[this.assets, this.setAssets] = createSignal<Asset[]>([]);@@ -252,16 +243,12 @@ export class AssetLibrary {// Resolving/**- * The asset a `src` names: a library path or id, an absolute path or URL- * (resolved on the fly, kept in memory only), or a `generate.*` ref (left- * to the host's generator).+ * The asset a source string names: a library path or id, or an absolute+ * path or URL (resolved on the fly, kept in memory only). Anything the+ * library cannot name — a `generate.*` declaration — is not a source; it+ * is the host's Ai's to resolve.*/- public async resolve(input: unknown): Promise<Asset> {- if (typeof input !== 'string') {- if (!this.generator) throw new Error('This host cannot generate assets');- return this.generator(input as object, this);- }-+ public async resolve(input: string): Promise<Asset> {const value = input.trim();const known = this.get(value);if (known) return known;diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.tsindex 97a5835c..ef18a665 100644--- a/packages/encoder/src/encoder.ts+++ b/packages/encoder/src/encoder.ts@@ -13,7 +13,7 @@ import {createRuntimeWorld, setActive, serializeEntity, cloneFromRecords,getEntityTree, propagateTimeRangeDown, framesToSeconds,assert, store, disposeDecoders,- playbackSystem, motionSystem, transformSystem, renderSystem,+ assetSystem, playbackSystem, motionSystem, transformSystem, renderSystem,AudioBus, AudioBusHandle,ChildOf, Geometry, Group, Paint, Workarea, Playback,AudioPlayback, Computed, Start, End, SourceIn, SourceOut,@@ -21,7 +21,7 @@ import {Position, Offset, Rotation, Scale, Skew,Project, Mode, Time, FrameRate, RenderSurface, AudioEngine, Root,resetCamera,- Assets, Fonts, FramePromises,+ Library, Ai, Fonts, FramePromises,} from '@diffusionstudio/runtime';import { TargetBuffer } from './buffer';@@ -95,7 +95,8 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {// loaded so the new world doesn't try to re-add them.const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');world.set(Mode, { value: videoEnabled ? 'offline-video' : 'offline-audio' });- world.set(Assets, sourceWorld.get(Assets)!);+ world.set(Library, sourceWorld.get(Library) ?? null);+ world.set(Ai, sourceWorld.get(Ai) ?? null);world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: scale });world.set(AudioEngine, { context: offlineAudioCtx });@@ -273,6 +274,7 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {world.set(Time, { delta: frameDuration * 1000, now: time.now + frameDuration * 1000 });{+ assetSystem(world);playbackSystem(world);await resolverSystem(world);motionSystem(world);diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.tsindex ad3533fe..e42ce535 100644--- a/packages/encoder/src/image-encoder.ts+++ b/packages/encoder/src/image-encoder.ts@@ -7,10 +7,10 @@ import {createRuntimeWorld, createEntity, appendChild, serializeEntity,cloneFromRecords, getEntityTree, framesToSeconds,formatTimecode, assert, store, disposeDecoders,- playbackSystem, motionSystem, transformSystem, renderSystem,+ assetSystem, playbackSystem, motionSystem, transformSystem, renderSystem,ChildOf, Geometry, Group, Hidden, IsMask, Muted, Culled,ClipsContent, Playback, Computed, WorldBounds,- Project, Mode, RenderSurface, AudioEngine, Assets, Fonts, Root,+ Project, Mode, RenderSurface, AudioEngine, Library, Ai, Fonts, Root,resetCamera, setCamera,FramePromises, Time, FrameRate,} from '@diffusionstudio/runtime';@@ -65,7 +65,8 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncodeconst sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');world.set(Mode, { value: 'offline-video' });- world.set(Assets, sourceWorld.get(Assets)!);+ world.set(Library, sourceWorld.get(Library) ?? null);+ world.set(Ai, sourceWorld.get(Ai) ?? null);world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: 1 });world.set(AudioEngine, { context: offlineAudioCtx });@@ -197,6 +198,7 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode}seek(frame);+ assetSystem(world);await resolverSystem(world);motionSystem(world);transformSystem(world);diff --git a/packages/jsx/src/hooks.ts b/packages/jsx/src/hooks.tsindex 3788c987..cf3f6323 100644--- a/packages/jsx/src/hooks.ts+++ b/packages/jsx/src/hooks.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. *//**- * The authoring hooks, as signatures. Both read the host a project is mounted+ * The authoring hooks, as signatures. They read the host a project is mounted* into, which only exists inside the editor: when a project is mounted, the* renderer substitutes its own "@diffusionstudio/jsx" module (see* @diffusionstudio/reconciler) and these implementations are replaced by ones@@ -12,8 +12,7 @@* throws rather than silently returning a dead value.*/-import type { Accessor, ResourceReturn } from "solid-js";-import type { AssetInput } from "./generate";+import type { Accessor } from "solid-js";export type Ticker = {time: Accessor<number>;@@ -39,28 +38,3 @@ function hostOnly(name: string): never {export function useTicker(): Ticker {return hostOnly("useTicker");}--/**- * Resolves a source (path, asset id, URL, or a `generate.*` ref) to its `File`,- * for reading raw bytes inside effects. Returns Solid's `createResource` tuple- * unchanged: the resource accessor reads `undefined` until it resolves, then- * the `File` (with `.loading` / `.error`), plus `mutate` / `refetch`.- * Resolution is async (fetch a URL, read a path or library asset, await a- * `generate.*` ref) and, like `src`, needs the host — a sandboxed module can't- * fetch a path or asset id itself.- *- * ```tsx- * const [canvas, setCanvas] = createSignal<HTMLCanvasElement>();- * const [file] = useFile("/assets/logo.png");- * createEffect(async () => {- * const el = canvas();- * const f = file();- * if (!el || !f) return;- * el.getContext("2d")!.drawImage(await createImageBitmap(f), 0, 0);- * });- * // <surface ref={setCanvas} width={640} height={360} />- * ```- */-export function useFile(_src: AssetInput): ResourceReturn<File> {- return hostOnly("useFile");-}diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 01ab1b51..05776aa8 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -34,7 +34,7 @@ export {parseSource,} from "./source";export type { CompositionTag, PropValue } from "./source";-export { useTicker, useFile } from "./hooks";+export { useTicker } from "./hooks";export type { Ticker } from "./hooks";export type {AdjustmentLayerProps,diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 3a35b458..a82011d9 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,12 +3,11 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Loop, Geometry, GeometryType, Library, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getAssetFile, getEntityChildren, getLibrary, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';-import type { Asset } from '@diffusionstudio/assets';-import type { AssetInput, PropValue } from '@diffusionstudio/jsx';+import type { AssetRef, PropValue } from '@diffusionstudio/jsx';import { trait, type Entity, type World } from 'koota';import type { ProjectDocument } from './host';@@ -62,13 +61,6 @@ const TextParts = trait(() => new Set<TextNode>());*/const Authored = trait(() => ({ tag: '', props: {} as Record<string, PropValue> }));-/**- * The `src` an element is waiting on: set while the library resolves a- * source the world does not know yet, so a resolution that arrives after the- * element was given another source (or none) is dropped.- */-const SourceRequest = trait(() => ({ value: undefined as unknown }));-/** Props that address or wire an element rather than describe it. */const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);@@ -992,7 +984,29 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return;}case 'src': {- this.setSource(entity, value);+ entity.remove(GenerationRequest, LoadRequest, PendingSource);++ if (value === undefined || value === null || value === '') {+ entity.remove(AssetId);+ return;+ }++ if (typeof value !== 'string') {+ entity.remove(AssetId);+ entity.add(GenerationRequest);+ entity.set(GenerationRequest, { ref: value as AssetRef });+ return;+ }++ const known = getAsset(this.world, value);+ if (known) {+ bindAsset(entity, known);+ return;+ }++ entity.remove(AssetId);+ entity.add(LoadRequest);+ entity.set(LoadRequest, { value });return;}case 'objectFit': {@@ -1314,70 +1328,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return next === undefined ? undefined : { entity: next };}- /**- * Binds an element to the asset its `src` names. A library path or id the- * world already knows binds at once; anything else — a path or URL outside- * the library, a `generate.*` ref — is resolved through the library, and- * binds when it arrives, unless the element has moved on to another source- * or gone away in the meantime.- */- private setSource(entity: Entity, value: unknown): void {- const library = this.world.get(Library);- const known = typeof value === 'string' ? getAsset(this.world, value) : undefined;-- if (known) {- entity.remove(SourceRequest);- this.bindAsset(entity, known);- return;- }-- if (!library || value === undefined || value === null || value === '') {- entity.remove(SourceRequest);- entity.remove(AssetId);- return;- }-- // Whatever it showed before is wrong until the new source arrives.- entity.remove(AssetId);- entity.add(SourceRequest);- entity.set(SourceRequest, { value });-- void library.resolve(value).then(- (asset) => {- if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;- entity.remove(SourceRequest);- this.bindAsset(entity, asset);- },- (error: unknown) => {- if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;- entity.remove(SourceRequest);- console.error(`[reconciler] could not resolve src:`, error);- },- );- }-- /**- * Binds an element to an asset. A frames directory on a `<video>` or- * `<image>` plays as a sequence: the element's paint follows the asset.- */- private bindAsset(entity: Entity, asset: Asset): void {- entity.add(AssetId);- entity.set(AssetId, { value: asset.id });-- const paint = entity.get(Paint)?.value;- if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {- entity.set(Paint, { value: PaintType.SEQUENCE });- } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {- entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });- }- }-- /** The `useFile` of a project: what its `src` would bind to, as a File. */- public async loadFile(input: AssetInput): Promise<File> {- const known = typeof input === 'string' ? getAsset(this.world, input) : undefined;- const asset = known ?? await getLibrary(this.world).resolve(input);- return getAssetFile(asset);- }public dispose(): void {this.removeNode(this.stage, this.stage);diff --git a/packages/reconciler/src/host.ts b/packages/reconciler/src/host.tsindex 1d225b9e..f0244a81 100644--- a/packages/reconciler/src/host.ts+++ b/packages/reconciler/src/host.ts@@ -10,8 +10,6 @@* opaque to the renderer.*/-import type { AssetInput } from "@diffusionstudio/jsx";-/*** One reading of the host's timeline clock, local to the mounted project:* the playhead of the scene the mount's root lives in (or is). Values only@@ -43,5 +41,4 @@ export interface ProjectDocument<N = unknown> {getNextSibling(node: N): N | undefined;tick?(): ProjectTick;applyRef?(node: N, ref: (target: unknown) => void): void;- loadFile?(input: AssetInput): Promise<File>;}diff --git a/packages/reconciler/src/renderer.ts b/packages/reconciler/src/renderer.tsindex 02320774..ea7f291c 100644--- a/packages/reconciler/src/renderer.ts+++ b/packages/reconciler/src/renderer.ts@@ -2,11 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { createContext, createMemo, createResource, useContext } from "solid-js";+import { createContext, createMemo, useContext } from "solid-js";import { createRenderer } from "solid-js/universal";-import type { JSX, ResourceReturn } from "solid-js";-import type { AssetInput, Ticker } from "@diffusionstudio/jsx";+import type { JSX } from "solid-js";+import type { Ticker } from "@diffusionstudio/jsx";import type { ProjectDocument } from "./host";// Compiled project modules call the static runtime exports below@@ -107,22 +107,6 @@ export function useTicker(): Ticker {};}-/**- * The live `useFile` — substituted for the throwing declaration in- * @diffusionstudio/jsx (see "./runtime"). Resolution is async (fetch a URL,- * read a path or library asset, await a `generate.*` ref) and needs the host,- * so it is a `createResource` over the document's `loadFile`.- */-export function useFile(src: AssetInput): ResourceReturn<File> {- const document = doc();- if (!document.loadFile) {- throw new Error("useFile: this host does not provide file resolution");- }-- const load = document.loadFile.bind(document);- return createResource(() => src, (input) => load(input));-}-/*** Runs `fn` with `document` as the one the static runtime exports write into.* Anything creating elements outside a project render — a host insertingdiff --git a/packages/reconciler/src/runtime.ts b/packages/reconciler/src/runtime.tsindex 2a8d8e6e..ba28ddd0 100644--- a/packages/reconciler/src/runtime.ts+++ b/packages/reconciler/src/runtime.ts@@ -12,7 +12,7 @@* `evaluate` binds it to this object instead. So the module a project imports* is the authoring surface with the host-bound implementations layered over* it: the renderer's element operations, the compile-target components, and- * live `useTicker` / `useFile` in place of the throwing declarations.+ * live `useTicker` in place of the throwing declaration.** Order matters — the renderer bindings must come last so they win.*/@@ -33,7 +33,6 @@ import {setProp,spread,use,- useFile,useTicker,} from './renderer';@@ -53,5 +52,4 @@ export const JSX_RUNTIME = {mergeProps,use,useTicker,- useFile,};diff --git a/packages/runtime/package.json b/packages/runtime/package.jsonindex 8d616b23..df2ae955 100644--- a/packages/runtime/package.json+++ b/packages/runtime/package.json@@ -22,6 +22,7 @@],"dependencies": {"@diffusionstudio/assets": "*",+ "@diffusionstudio/jsx": "*","animejs": "^4.4.1","colord": "^2.9.3","koota": "^0.6.6",diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex d270b2de..58f16e7b 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -6,10 +6,12 @@// goes through the AssetLibrary (@diffusionstudio/assets); these are the// lookups the decoders and hosts share.-import { Library } from '../traits';+import { Ai, AssetId, Library, Paint } from '../traits';+import { PaintType } from '../constants';-import type { World } from 'koota';+import type { Entity, World } from 'koota';import type { Asset, AssetLibrary } from '@diffusionstudio/assets';+import type { GenAi } from '../ai';/** The world's asset library; throws when the host attached none. */export function getLibrary(world: World): AssetLibrary {@@ -18,6 +20,30 @@ export function getLibrary(world: World): AssetLibrary {return library;}+/** The world's generation service; throws when the host attached none. */+export function getAi(world: World): GenAi {+ const ai = world.get(Ai);+ if (!ai) throw new Error('This world cannot generate assets (no Ai attached)');+ return ai;+}++/**+ * Binds an entity to an asset: stamps its AssetId, and follows the asset+ * with the paint — a frames directory on a `<video>` or `<image>` plays as+ * a sequence, and a sequence paint goes back when the asset is not one.+ */+export function bindAsset(entity: Entity, asset: Asset): void {+ entity.add(AssetId);+ entity.set(AssetId, { value: asset.id });++ const paint = entity.get(Paint)?.value;+ if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {+ entity.set(Paint, { value: PaintType.SEQUENCE });+ } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {+ entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });+ }+}+/** The asset with `id` (or at that library path), or undefined without a library. */export function getAsset(world: World, idOrPath: string): Asset | undefined {return world.get(Library)?.get(idOrPath);diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsnew file mode 100644index 00000000..6dc653db--- /dev/null+++ b/packages/runtime/src/ai.ts@@ -0,0 +1,42 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// The generation service a host may attach to the world (the `Ai` trait):+// what turns a `generate.*` declaration into an asset of the project's+// library. The runtime itself cannot generate — it only knows the contract —+// so a world without an Ai never resolves such sources (see the asset+// system). The asset library knows nothing of generation either; an+// implementation *uses* a library to look up and store what it makes.++import { generate } from '@diffusionstudio/jsx';++import type { Asset } from '@diffusionstudio/assets';+import type {+ AssetRef,+ GenerateAudioOptions,+ GenerateImageOptions,+ GenerateVideoOptions,+ GenerateVoiceOptions,+} from '@diffusionstudio/jsx';++export abstract class GenAi {+ /**+ * The declaration, made real: an asset whose bytes the model produced.+ * Content-addressed — the fully-resolved spec's hash is the asset's+ * `generation.key`, so the same spec is the same asset in this session+ * and the next, and identical concurrent declarations share one run.+ */+ public abstract resolve(ref: AssetRef): Promise<Asset>;++ /**+ * The imperative surface: `ai.generate.image({...})` declares and resolves+ * in one call, returning the finished asset.+ */+ public readonly generate = {+ image: (options: GenerateImageOptions): Promise<Asset> => this.resolve(generate.image(options)),+ video: (options: GenerateVideoOptions): Promise<Asset> => this.resolve(generate.video(options)),+ voice: (options: GenerateVoiceOptions): Promise<Asset> => this.resolve(generate.voice(options)),+ audio: (options: GenerateAudioOptions): Promise<Asset> => this.resolve(generate.audio(options)),+ };+}diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.tsindex 2a3827d2..f777bd2a 100644--- a/packages/runtime/src/index.ts+++ b/packages/runtime/src/index.ts@@ -1,4 +1,5 @@export * from './constants';+export * from './ai';export * from './traits';export * from './world/create-world';export * from './world/observers';diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsnew file mode 100644index 00000000..f6a5a84a--- /dev/null+++ b/packages/runtime/src/systems/assets.ts@@ -0,0 +1,71 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Binds entities to the assets their `src` names. The document is+// synchronous and only leaves requests behind (LoadRequest for a source+// outside the library, GenerationRequest for a `generate.*` declaration);+// this system consumes them each frame, starts the async work, and stamps+// AssetId when the asset lands. Loads go through the world's library;+// generations through the world's Ai — deduplication (a spec whose hash+// already produced a library asset binds to it, identical inflight specs+// share one run) is the Ai implementation's contract (see GenAi). A request+// the world cannot serve yet — no library, no Ai — is left in place for a+// frame that can. In offline mode the started work joins FramePromises, so+// the encoder's barrier waits for the binding before it samples a frame.++import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource } from '../traits';+import { bindAsset } from '../actions/assets';++import type { Entity, World } from 'koota';+import type { Asset } from '@diffusionstudio/assets';++export function assetSystem(world: World): void {+ const library = world.get(Library);+ if (library) {+ for (const entity of world.query(LoadRequest)) {+ const source = entity.get(LoadRequest)!.value;+ entity.remove(LoadRequest);+ resolve(world, entity, source, library.resolve(source));+ }+ }++ const ai = world.get(Ai);+ if (ai) {+ for (const entity of world.query(GenerationRequest)) {+ const ref = entity.get(GenerationRequest)!.ref;+ entity.remove(GenerationRequest);+ if (ref === null) continue;+ resolve(world, entity, ref, ai.resolve(ref));+ }+ }+}++/**+ * Tracks one started resolution: the entity remembers what it is waiting on+ * (PendingSource), so of overlapping resolutions only the latest binds, and+ * one that outlives its element (or its src) is dropped.+ */+function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>): void {+ entity.add(PendingSource);+ entity.set(PendingSource, { value });++ const done = promise.then(+ (asset) => {+ if (!current(entity, value)) return;+ entity.remove(PendingSource);+ bindAsset(entity, asset);+ },+ (error: unknown) => {+ if (!current(entity, value)) return;+ entity.remove(PendingSource);+ console.error('[runtime] could not resolve src:', error);+ },+ );++ world.get(FramePromises)?.list?.push(done);+}++function current(entity: Entity, value: unknown): boolean {+ return entity.isAlive() && entity.get(PendingSource)?.value === value;+}diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.tsindex 626c0744..6911d34a 100644--- a/packages/runtime/src/systems/index.ts+++ b/packages/runtime/src/systems/index.ts@@ -1,4 +1,5 @@// Deterministic systems. Browser-only systems (input, keyboard, camera, hud, dom) stay in apps/web.+export * from './assets';export * from './transform';export * from './motion';export * from './gradients';diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex f0101095..e7dee2f7 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -6,6 +6,8 @@ import { trait } from 'koota';import { GeometryType, PaintType, CaptionType, CaptionAlign } from '../constants';+import type { AssetRef } from '@diffusionstudio/jsx';+// Geometric primitive: RECT or TEXT (see GeometryType). Other node-like roles// (group, audio, scene, caption) are layered on top via tag traits.export const Geometry = trait({ value: GeometryType.RECT as GeometryType });@@ -71,6 +73,23 @@ export const Loop = trait({ value: '' });export const AssetId = trait({ value: '' });+// A `src` the document could not bind synchronously. The document only+// leaves one of these requests behind; the asset system does the async work+// and stamps AssetId when the asset lands. Never serialized: a request is+// this world's business, re-derived from the src on any re-render.++// The `generate.*` declaration a src names, to run through the world's Ai.+export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }));++// A source outside the library — a path or URL — to load into memory+// through the library.+export const LoadRequest = trait({ value: '' });++// The src whose resolution is inflight, kept while the asset system works,+// so a resolution that arrives after the element was given another source+// (or none) is dropped. The document clears it whenever a new src arrives.+export const PendingSource = trait(() => ({ value: undefined as unknown }));+// Sibling order under a ChildOf parent.export const ItemIndex = trait({ value: 0 });diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.tsindex d99290c2..edf7fac2 100644--- a/packages/runtime/src/traits/world.ts+++ b/packages/runtime/src/traits/world.ts@@ -8,6 +8,7 @@ import { createLiveMounts } from '../utils/live-mounts';import type { FontSource } from '../fonts/types';import type { AssetLibrary } from '@diffusionstudio/assets';+import type { GenAi } from '../ai';import type { Quad } from '../math/aabb';// Singleton state attached to the world itself (world.get/world.set).@@ -69,6 +70,12 @@ export const Fonts = trait({ list: () => [] as FontSource[] });// and transient alike. Null in a world without a project.export const Library = trait(() => null as AssetLibrary | null);+// The generation service, once the host has attached one (it needs a+// backend): what the asset system runs `generate.*` sources through, and+// what `getAi(world).generate.image(...)` calls. Optional — in a world+// without one, such sources stay pending and never resolve.+export const Ai = trait(() => null as GenAi | null);+// Live JSX mount registry; running graphs advance once per frame.export const Mounts = trait(() => createLiveMounts());diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.tsindex 1834b7d0..98771a21 100644--- a/packages/runtime/src/world/create-world.ts+++ b/packages/runtime/src/world/create-world.ts@@ -18,6 +18,7 @@ import {AudioEngine,Fonts,Library,+ Ai,Mounts,FramePromises,HitRegions,@@ -40,6 +41,7 @@ export function createRuntimeWorld(projectId: string) {AudioEngine,Fonts,Library,+ Ai,Mounts,FramePromises,HitRegions,