Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance asset management and integration with AI generation

konstantin-paulus2fafd8668c0e
diff --git a/apps/web/src/engine/create-engine.ts b/apps/web/src/engine/create-engine.ts
index 3c21dde6..29a2b2fd 100644
--- a/apps/web/src/engine/create-engine.ts
+++ b/apps/web/src/engine/create-engine.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';
+import { assetSystem, renderSystem, transformSystem, playbackSystem, motionSystem, AudioEngine, createRuntimeWorld, Mode, RenderSurface, Time, ChildOf, syncInteractiveState } from '@diffusionstudio/runtime';
import { hudSystem } from './hud';
import { createSignal, type Accessor, type Setter } from 'solid-js';
import { AssetSelection, Hud, Keys, Pointer, PointerEvents, ProjectConfig, SnapLines } from './traits';
@@ -198,6 +198,7 @@ class Engine {
private runSystems(): void {
inputSystem(this.world);
shortcutSystem(this.world);
+ assetSystem(this.world);
playbackSystem(this.world);
motionSystem(this.world);
transformSystem(this.world);
diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.ts
index f28e7265..e6ab122c 100644
--- a/apps/web/src/engine/library.ts
+++ b/apps/web/src/engine/library.ts
@@ -6,14 +6,13 @@
// disk and kept in step with the JSX (a rename in the library is a `src`
// edit in the file).
-import { AssetId, Library, Project } from '@diffusionstudio/runtime';
+import { AssetId, Library } from '@diffusionstudio/runtime';
import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';
import { useTrait, useWorld } from '@diffusionstudio/koota-solid';
import { authoredElement } from '@diffusionstudio/reconciler';
import type { Accessor } from 'solid-js';
import { createProjectFS } from '@/projects/fs';
-import { generateAsset } from '@/utils/jsx-generation';
import { getDocumentEditor } from './editor';
import type { Asset } from '@diffusionstudio/assets';
@@ -26,10 +25,7 @@ import type { World } from 'koota';
* disposer that flushes the manifest and detaches it.
*/
export function attachLibrary(world: World, dir: string) {
- const projectId = world.get(Project)?.id ?? 'project';
-
const library = new AssetLibrary(createProjectFS(dir), {
- generate: (ref, library) => generateAsset(ref, library, projectId),
onRename: (asset, from) => followRename(world, asset, from),
onRelink: (asset, from) => followRelink(world, asset, from),
});
diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsx
index d834cf84..4a667db2 100644
--- a/apps/web/src/pages/editor.tsx
+++ b/apps/web/src/pages/editor.tsx
@@ -16,6 +16,7 @@ import { useWorld } from '@diffusionstudio/koota-solid';
import { mount } from '@diffusionstudio/reconciler';
import { getDocumentEditor } from '@/engine/editor';
import { attachLibrary, isLibraryFile } from '@/engine/library';
+import { attachAi } from '@/utils/gen-ai';
import { attachProjectConfig, isProjectConfigFile } from '@/engine/project-config';
import { isCacheFile } from '@diffusionstudio/assets';
import { createEditWriter } from '@/projects/edits';
@@ -52,6 +53,8 @@ export function EditorPage() {
// The library first: a mounted project's `src` values name its assets.
const library = attachLibrary(world, dir);
+ // The generation service over it: what `generate.*` sources resolve through.
+ attachAi(world, library);
// The project's own settings (package.json `diffusion`), next to the scene.
const config = attachProjectConfig(world, dir);
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
new file mode 100644
index 00000000..6196f7bb
--- /dev/null
+++ b/apps/web/src/utils/gen-ai.ts
@@ -0,0 +1,247 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The app's GenAi: `generate.*` declarations made real against this
+// project's library and the dapi backend. Attached to the world as the `Ai`
+// trait, which is what the runtime's asset system (and `ai.generate.image`
+// callers) resolve through. A declaration resolves to an asset by content —
+// the same spec is the same asset, in this session and the next, since the
+// finished file lands in the library under `generated/` with the spec's key
+// in the manifest.
+
+import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
+import { Ai, GenAi, getAssetFile, Project } from "@diffusionstudio/runtime";
+import { GENERATED_DIR } from "@diffusionstudio/assets";
+import {
+ PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
+ PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
+ PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
+ PROMPT_INPUT_VOICE_OPTIONS,
+} from "@/components/genai/config";
+import { assert } from "@/utils";
+import { uploadBlob } from "@/components/engine";
+import { trpc } from "@/lib/trpc";
+
+import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
+import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
+import type { World } from "koota";
+
+// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).
+const VOICE_MODEL = "elevenlabs-v3";
+
+/**
+ * A spec with defaults applied and every `AssetInput` reduced to an asset id.
+ * Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.
+ */
+type ResolvedSpec =
+ | { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }
+ | { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }
+ | { type: "voice"; model: string; prompt: string; voice: string }
+ | { type: "audio"; model: string; prompt: string; duration?: number };
+
+/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */
+export function attachAi(world: World, library: AssetLibrary): EditorGenAi {
+ const ai = new EditorGenAi(library, world.get(Project)?.id ?? "project");
+ world.set(Ai, ai);
+ return ai;
+}
+
+export class EditorGenAi extends GenAi {
+ private readonly library: AssetLibrary;
+ /** Prefixes upload keys, so referenced assets land project-unique in the bucket. */
+ private readonly projectId: string;
+
+ /**
+ * Declarations already resolved, keyed by ref identity — a ref consumed by
+ * several elements resolves (and validates) once. A failed one is
+ * forgotten, so a remount tries again.
+ */
+ private readonly memo = new Map<AssetRef, Promise<Asset>>();
+ /** In-flight generations keyed by `generationKey`. */
+ private readonly inflight = new Map<string, Promise<Asset>>();
+
+ public constructor(library: AssetLibrary, projectId: string) {
+ super();
+ this.library = library;
+ this.projectId = projectId;
+ }
+
+ /** Identical concurrent declarations collapse to one request. */
+ public resolve(ref: AssetRef): Promise<Asset> {
+ assert(isAssetRef(ref), "Not a generate.* declaration");
+
+ let promise = this.memo.get(ref);
+ if (!promise) {
+ promise = this.generateFromRef(ref);
+ promise.catch(() => this.memo.delete(ref));
+ this.memo.set(ref, promise);
+ }
+ return promise;
+ }
+
+ private resolveInput(input: AssetInput): Promise<Asset> {
+ return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);
+ }
+
+ private async generateFromRef(ref: AssetRef): Promise<Asset> {
+ const resolved = await this.resolveSpec(getAssetSpec(ref));
+ const generationKey = JSON.stringify(resolved);
+
+ const cached = this.library.list().find((asset) => asset.generation?.key === generationKey);
+ if (cached) return cached;
+
+ const running = this.inflight.get(generationKey);
+ if (running) return await running;
+
+ const promise = this.runGeneration(resolved, generationKey);
+ this.inflight.set(generationKey, promise);
+ try {
+ return await promise;
+ } finally {
+ this.inflight.delete(generationKey);
+ }
+ }
+
+ private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {
+ switch (spec.type) {
+ case "image": {
+ const refs = await Promise.all((spec.refs ?? []).map((ref) => this.resolveInput(ref)));
+ return {
+ type: "image",
+ model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio ?? "16:9",
+ seed: spec.seed,
+ refIds: refs.map((asset) => asset.id),
+ };
+ }
+ case "video": {
+ const [startFrame, endFrame] = await Promise.all([
+ spec.startFrame !== undefined ? this.resolveInput(spec.startFrame) : undefined,
+ spec.endFrame !== undefined ? this.resolveInput(spec.endFrame) : undefined,
+ ]);
+ const resolved = {
+ type: "video",
+ model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio ?? "16:9",
+ duration: spec.duration ?? 5,
+ audio: spec.audio ?? false,
+ seed: spec.seed,
+ startFrameId: startFrame?.id,
+ endFrameId: endFrame?.id,
+ } satisfies ResolvedSpec;
+ checkVideoConstraints(resolved);
+ return resolved;
+ }
+ case "voice": {
+ return {
+ type: "voice",
+ model: VOICE_MODEL,
+ prompt: spec.prompt,
+ voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,
+ };
+ }
+ case "audio": {
+ return {
+ type: "audio",
+ model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,
+ prompt: spec.prompt,
+ duration: spec.duration,
+ };
+ }
+ }
+ }
+
+ /** Runs a generation and stores its first result under `generated/`. */
+ private async runGeneration(spec: ResolvedSpec, generationKey: string): Promise<Asset> {
+ const { name, results, generationId } = await this.requestGeneration(spec);
+ assert(results.length > 0, "No results returned from the model");
+
+ const url = results[0].url;
+ const response = await fetch(url);
+ assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
+ const blob = await response.blob();
+ const extension = url.split(/[?#]/)[0].split(".").pop();
+ const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
+
+ return this.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
+ }
+
+ private requestGeneration(spec: ResolvedSpec) {
+ switch (spec.type) {
+ case "image": {
+ return (async () => {
+ const images = spec.refIds.length > 0
+ ? await Promise.all(spec.refIds.map((id) => this.uploadInput(id)))
+ : undefined;
+
+ return await trpc.generateImage.mutate({
+ model: spec.model,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio,
+ count: 1,
+ seed: spec.seed,
+ images,
+ });
+ })();
+ }
+ case "video": {
+ return (async () => {
+ const [startFrame, endFrame] = await Promise.all([
+ spec.startFrameId ? this.uploadInput(spec.startFrameId) : undefined,
+ spec.endFrameId ? this.uploadInput(spec.endFrameId) : undefined,
+ ]);
+
+ return await trpc.generateVideo.mutate({
+ model: spec.model,
+ prompt: spec.prompt,
+ aspectRatio: spec.aspectRatio,
+ duration: spec.duration,
+ generateAudio: spec.audio,
+ seed: spec.seed,
+ startFrame,
+ endFrame,
+ });
+ })();
+ }
+ case "voice": {
+ return trpc.textToSpeech.mutate({
+ model: spec.model,
+ prompt: spec.prompt,
+ voice: spec.voice,
+ });
+ }
+ case "audio": {
+ return trpc.generateSound.mutate({
+ model: spec.model,
+ prompt: spec.prompt,
+ duration: spec.duration,
+ });
+ }
+ }
+ }
+
+ /** Uploads a referenced asset for the model to read; the bucket key is project-unique. */
+ private async uploadInput(assetId: string) {
+ const asset = this.library.get(assetId);
+ assert(asset, `Referenced asset ${assetId} not found`);
+ const uploaded = await uploadBlob(await getAssetFile(asset), `${this.projectId}-${assetId}`);
+ assert(uploaded, `Failed to upload referenced asset ${assetId}`);
+ return uploaded;
+ }
+}
+
+/**
+ * Per-model constraints (`dapi models video`); unknown models are left to the server.
+ */
+function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {
+ const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);
+ if (!model) return;
+
+ assert(model.aspectRatios.includes(spec.aspectRatio), `${spec.model} does not support aspect ratio ${spec.aspectRatio}`);
+ assert(model.durations.includes(`${spec.duration}s`), `${spec.model} does not support a duration of ${spec.duration}s`);
+ assert(!spec.audio || model.features.includes("audio"), `${spec.model} does not support audio generation`);
+ assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);
+}
diff --git a/apps/web/src/utils/jsx-generation.ts b/apps/web/src/utils/jsx-generation.ts
deleted file mode 100644
index 1bc56f99..00000000
--- a/apps/web/src/utils/jsx-generation.ts
+++ /dev/null
@@ -1,239 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// `generate.*` declarations in a project's JSX, made real: the asset
-// library's generator (see `attachLibrary`). A declaration resolves to an
-// asset by content — the same spec is the same asset, in this session and
-// the next, since the finished file lands in the library under
-// `generated/` with the spec's key in the manifest.
-
-import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
-import { getAssetFile } from "@diffusionstudio/runtime";
-import { GENERATED_DIR } from "@diffusionstudio/assets";
-import {
- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,
- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,
- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,
- PROMPT_INPUT_VOICE_OPTIONS,
-} from "@/components/genai/config";
-import { assert } from "@/utils";
-import { uploadBlob } from "@/components/engine";
-import { trpc } from "@/lib/trpc";
-
-import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
-import type { Asset, AssetLibrary } from "@diffusionstudio/assets";
-
-const memo: GenerationMemo = new Map();
-const inflight: Inflight = new Map();
-
-/**
- * A library's `generate` option for the project `projectId` (which prefixes
- * upload keys). Identical concurrent declarations collapse to one request.
- */
-export function generateAsset(ref: object, library: AssetLibrary, projectId: string): Promise<Asset> {
- assert(isAssetRef(ref), "Not a generate.* declaration");
-
- return resolveGeneratedAsset({ library, projectId, memo, inflight }, ref);
-}
-
-/** What one generation run has to hand: the library it fills and its caches. */
-type Generation = { library: AssetLibrary; projectId: string; memo: GenerationMemo; inflight: Inflight };
-
-// The UI offers a fixed set of voices on a fixed model (see prompt-input.tsx).
-const VOICE_MODEL = "elevenlabs-v3";
-
-/**
- * A spec with defaults applied and every `AssetInput` reduced to an asset id.
- * Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.
- */
-type ResolvedSpec =
- | { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }
- | { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }
- | { type: "voice"; model: string; prompt: string; voice: string }
- | { type: "audio"; model: string; prompt: string; duration?: number };
-
-/**
- * Declarations already resolved, keyed by ref identity — a ref consumed by
- * several elements resolves (and validates) once. A failed one is forgotten,
- * so a remount tries again.
- */
-type GenerationMemo = Map<AssetRef, Promise<Asset>>;
-
-/** In-flight generations keyed by `generationKey`. */
-type Inflight = Map<string, Promise<Asset>>;
-
-function resolveGeneratedAsset(gen: Generation, ref: AssetRef): Promise<Asset> {
- let promise = gen.memo.get(ref);
- if (!promise) {
- promise = generateFromRef(gen, ref);
- promise.catch(() => gen.memo.delete(ref));
- gen.memo.set(ref, promise);
- }
- return promise;
-}
-
-function resolveInput(gen: Generation, input: AssetInput): Promise<Asset> {
- return isAssetRef(input) ? resolveGeneratedAsset(gen, input) : gen.library.resolve(input);
-}
-
-async function generateFromRef(gen: Generation, ref: AssetRef): Promise<Asset> {
- const resolved = await resolveSpec(gen, getAssetSpec(ref));
- const generationKey = JSON.stringify(resolved);
-
- const cached = gen.library.list().find((asset) => asset.generation?.key === generationKey);
- if (cached) return cached;
-
- const running = gen.inflight.get(generationKey);
- if (running) return await running;
-
- const promise = runGeneration(gen, resolved, generationKey);
- gen.inflight.set(generationKey, promise);
- try {
- return await promise;
- } finally {
- gen.inflight.delete(generationKey);
- }
-}
-
-async function resolveSpec(gen: Generation, spec: AssetSpecInput): Promise<ResolvedSpec> {
- switch (spec.type) {
- case "image": {
- const refs = await Promise.all((spec.refs ?? []).map((ref) => resolveInput(gen, ref)));
- return {
- type: "image",
- model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,
- prompt: spec.prompt,
- aspectRatio: spec.aspectRatio ?? "16:9",
- seed: spec.seed,
- refIds: refs.map((asset) => asset.id),
- };
- }
- case "video": {
- const [startFrame, endFrame] = await Promise.all([
- spec.startFrame !== undefined ? resolveInput(gen, spec.startFrame) : undefined,
- spec.endFrame !== undefined ? resolveInput(gen, spec.endFrame) : undefined,
- ]);
- const resolved = {
- type: "video",
- model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,
- prompt: spec.prompt,
- aspectRatio: spec.aspectRatio ?? "16:9",
- duration: spec.duration ?? 5,
- audio: spec.audio ?? false,
- seed: spec.seed,
- startFrameId: startFrame?.id,
- endFrameId: endFrame?.id,
- } satisfies ResolvedSpec;
- checkVideoConstraints(resolved);
- return resolved;
- }
- case "voice": {
- return {
- type: "voice",
- model: VOICE_MODEL,
- prompt: spec.prompt,
- voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,
- };
- }
- case "audio": {
- return {
- type: "audio",
- model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,
- prompt: spec.prompt,
- duration: spec.duration,
- };
- }
- }
-}
-
-/**
- * Per-model constraints (`dapi models video`); unknown models are left to the server.
- */
-function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {
- const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);
- if (!model) return;
-
- assert(model.aspectRatios.includes(spec.aspectRatio), `${spec.model} does not support aspect ratio ${spec.aspectRatio}`);
- assert(model.durations.includes(`${spec.duration}s`), `${spec.model} does not support a duration of ${spec.duration}s`);
- assert(!spec.audio || model.features.includes("audio"), `${spec.model} does not support audio generation`);
- assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);
-}
-
-/** Runs a generation and stores its first result under `generated/`. */
-async function runGeneration(gen: Generation, spec: ResolvedSpec, generationKey: string): Promise<Asset> {
- const { name, results, generationId } = await requestGeneration(gen, spec);
- assert(results.length > 0, "No results returned from the model");
-
- const url = results[0].url;
- const response = await fetch(url);
- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);
- const blob = await response.blob();
- const extension = url.split(/[?#]/)[0].split(".").pop();
- const fileName = extension && extension.length <= 5 ? `${name}.${extension}` : name;
-
- return gen.library.store(blob, { name: fileName, folder: GENERATED_DIR, generation: { key: generationKey, id: generationId } });
-}
-
-function requestGeneration(gen: Generation, spec: ResolvedSpec) {
- switch (spec.type) {
- case "image": {
- return (async () => {
- const images = spec.refIds.length > 0
- ? await Promise.all(spec.refIds.map((id) => uploadInput(gen, id)))
- : undefined;
-
- return await trpc.generateImage.mutate({
- model: spec.model,
- prompt: spec.prompt,
- aspectRatio: spec.aspectRatio,
- count: 1,
- seed: spec.seed,
- images,
- });
- })();
- }
- case "video": {
- return (async () => {
- const [startFrame, endFrame] = await Promise.all([
- spec.startFrameId ? uploadInput(gen, spec.startFrameId) : undefined,
- spec.endFrameId ? uploadInput(gen, spec.endFrameId) : undefined,
- ]);
-
- return await trpc.generateVideo.mutate({
- model: spec.model,
- prompt: spec.prompt,
- aspectRatio: spec.aspectRatio,
- duration: spec.duration,
- generateAudio: spec.audio,
- seed: spec.seed,
- startFrame,
- endFrame,
- });
- })();
- }
- case "voice": {
- return trpc.textToSpeech.mutate({
- model: spec.model,
- prompt: spec.prompt,
- voice: spec.voice,
- });
- }
- case "audio": {
- return trpc.generateSound.mutate({
- model: spec.model,
- prompt: spec.prompt,
- duration: spec.duration,
- });
- }
- }
-}
-
-/** Uploads a referenced asset for the model to read; the bucket key is project-unique. */
-async function uploadInput(gen: Generation, assetId: string) {
- const asset = gen.library.get(assetId);
- assert(asset, `Referenced asset ${assetId} not found`);
- const uploaded = await uploadBlob(await getAssetFile(asset), `${gen.projectId}-${assetId}`);
- assert(uploaded, `Failed to upload referenced asset ${assetId}`);
- return uploaded;
-}
diff --git a/examples/06-three.tsx b/examples/06-three.tsx
index 164275c2..4d1b54a5 100644
--- a/examples/06-three.tsx
+++ b/examples/06-three.tsx
@@ -11,9 +11,9 @@
* createEffect, so scrubbing and exports stay frame-accurate, not wall-clock.
*
* The "Battle Damaged Sci-Fi Helmet" is the Khronos DamagedHelmet sample, a
- * single-file .glb GLTFLoader fetches directly over the network (useFile is the
- * host-resolved route for project-local assets). RoomEnvironment bakes an
- * in-memory PBR reflection map, so the metal reads without shipping an HDR.
+ * single-file .glb GLTFLoader fetches directly over the network.
+ * RoomEnvironment bakes an in-memory PBR reflection map, so the metal reads
+ * without shipping an HDR.
*/
import { createEffect, createSignal, onCleanup } from "solid-js";
diff --git a/package-lock.json b/package-lock.json
index c679d21f..42ff1dbe 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -12477,6 +12477,7 @@
"license": "MPL-2.0",
"dependencies": {
"@diffusionstudio/assets": "*",
+ "@diffusionstudio/jsx": "*",
"animejs": "^4.4.1",
"colord": "^2.9.3",
"koota": "^0.6.6",
diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.ts
index 180408e1..5097d77a 100644
--- a/packages/assets/src/library.ts
+++ b/packages/assets/src/library.ts
@@ -12,9 +12,11 @@
// keeping (thumbnails, waveforms) is the `cache`'s, under `cache/`.
//
// Knows nothing of the host: files come and go through a `ProjectFS`, and
-// what it cannot do itself (generate an asset, follow a rename into whatever
-// names assets by path, rebind whatever is bound to a relinked asset's old
-// id) it asks of `LibraryOptions`. Its state is solid signals (`assets()`,
+// what it cannot do itself (follow a rename into whatever names assets by
+// path, rebind whatever is bound to a relinked asset's old id) it asks of
+// `LibraryOptions`. Generation is not its business either — a generated
+// file is stored like any other bytes the app produced, with its
+// `generation` record along. Its state is solid signals (`assets()`,
// `folders()`), so readers inside a tracking scope follow it.
import { createSignal, type Accessor } from 'solid-js';
@@ -35,12 +37,6 @@ import type { Asset, AssetDirectoryHandle, AssetFileHandle, AssetGeneration, Seq
const SAVE_DEBOUNCE = 200;
export interface LibraryOptions {
- /**
- * Turns what the runtime cannot — a `generate.*` declaration — into an
- * asset of this library; `resolve` hands it any non-string source. Without
- * one, such sources fail to resolve.
- */
- generate?: AssetGenerator;
/**
* Called when an asset's library path changes (renamed or moved), so the
* host can follow in whatever names assets by path — the JSX `src` props.
@@ -68,9 +64,6 @@ export interface ImportOptions {
generation?: AssetGeneration;
}
-/** See `LibraryOptions.generate`. */
-export type AssetGenerator = (ref: object, library: AssetLibrary) => Promise<Asset>;
-
export class AssetLibrary {
public readonly fs: ProjectFS;
/** Thumbnails, rough waveforms: what is derived from assets and kept in `cache/`. */
@@ -86,7 +79,6 @@ export class AssetLibrary {
private declared = new Set<string>();
/** Every asset by id: the library's, and transient ones resolved from outside it. */
private readonly map = new Map<string, Asset>();
- private readonly generator: LibraryOptions['generate'];
private readonly onRename: LibraryOptions['onRename'];
private readonly onRelink: LibraryOptions['onRelink'];
private inflight = new Map<string, Promise<Asset>>();
@@ -98,7 +90,6 @@ export class AssetLibrary {
public constructor(fs: ProjectFS, options: LibraryOptions = {}) {
this.fs = fs;
this.cache = new AssetCache(fs);
- this.generator = options.generate;
this.onRename = options.onRename;
this.onRelink = options.onRelink;
[this.assets, this.setAssets] = createSignal<Asset[]>([]);
@@ -252,16 +243,12 @@ export class AssetLibrary {
// Resolving
/**
- * The asset a `src` names: a library path or id, an absolute path or URL
- * (resolved on the fly, kept in memory only), or a `generate.*` ref (left
- * to the host's generator).
+ * The asset a source string names: a library path or id, or an absolute
+ * path or URL (resolved on the fly, kept in memory only). Anything the
+ * library cannot name — a `generate.*` declaration — is not a source; it
+ * is the host's Ai's to resolve.
*/
- public async resolve(input: unknown): Promise<Asset> {
- if (typeof input !== 'string') {
- if (!this.generator) throw new Error('This host cannot generate assets');
- return this.generator(input as object, this);
- }
-
+ public async resolve(input: string): Promise<Asset> {
const value = input.trim();
const known = this.get(value);
if (known) return known;
diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.ts
index 97a5835c..ef18a665 100644
--- a/packages/encoder/src/encoder.ts
+++ b/packages/encoder/src/encoder.ts
@@ -13,7 +13,7 @@ import {
createRuntimeWorld, setActive, serializeEntity, cloneFromRecords,
getEntityTree, propagateTimeRangeDown, framesToSeconds,
assert, store, disposeDecoders,
- playbackSystem, motionSystem, transformSystem, renderSystem,
+ assetSystem, playbackSystem, motionSystem, transformSystem, renderSystem,
AudioBus, AudioBusHandle,
ChildOf, Geometry, Group, Paint, Workarea, Playback,
AudioPlayback, Computed, Start, End, SourceIn, SourceOut,
@@ -21,7 +21,7 @@ import {
Position, Offset, Rotation, Scale, Skew,
Project, Mode, Time, FrameRate, RenderSurface, AudioEngine, Root,
resetCamera,
- Assets, Fonts, FramePromises,
+ Library, Ai, Fonts, FramePromises,
} from '@diffusionstudio/runtime';
import { TargetBuffer } from './buffer';
@@ -95,7 +95,8 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
// loaded so the new world doesn't try to re-add them.
const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');
world.set(Mode, { value: videoEnabled ? 'offline-video' : 'offline-audio' });
- world.set(Assets, sourceWorld.get(Assets)!);
+ world.set(Library, sourceWorld.get(Library) ?? null);
+ world.set(Ai, sourceWorld.get(Ai) ?? null);
world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });
world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: scale });
world.set(AudioEngine, { context: offlineAudioCtx });
@@ -273,6 +274,7 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
world.set(Time, { delta: frameDuration * 1000, now: time.now + frameDuration * 1000 });
{
+ assetSystem(world);
playbackSystem(world);
await resolverSystem(world);
motionSystem(world);
diff --git a/packages/encoder/src/image-encoder.ts b/packages/encoder/src/image-encoder.ts
index ad3533fe..e42ce535 100644
--- a/packages/encoder/src/image-encoder.ts
+++ b/packages/encoder/src/image-encoder.ts
@@ -7,10 +7,10 @@ import {
createRuntimeWorld, createEntity, appendChild, serializeEntity,
cloneFromRecords, getEntityTree, framesToSeconds,
formatTimecode, assert, store, disposeDecoders,
- playbackSystem, motionSystem, transformSystem, renderSystem,
+ assetSystem, playbackSystem, motionSystem, transformSystem, renderSystem,
ChildOf, Geometry, Group, Hidden, IsMask, Muted, Culled,
ClipsContent, Playback, Computed, WorldBounds,
- Project, Mode, RenderSurface, AudioEngine, Assets, Fonts, Root,
+ Project, Mode, RenderSurface, AudioEngine, Library, Ai, Fonts, Root,
resetCamera, setCamera,
FramePromises, Time, FrameRate,
} from '@diffusionstudio/runtime';
@@ -65,7 +65,8 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode
const sourceFrameRate = sourceWorld.get(FrameRate)?.value ?? 30;
const world = createRuntimeWorld(sourceWorld.get(Project)?.id ?? '');
world.set(Mode, { value: 'offline-video' });
- world.set(Assets, sourceWorld.get(Assets)!);
+ world.set(Library, sourceWorld.get(Library) ?? null);
+ world.set(Ai, sourceWorld.get(Ai) ?? null);
world.set(Fonts, { list: [...(sourceWorld.get(Fonts)?.list ?? [])] });
world.set(RenderSurface, { canvas: offscreenCanvas, ctx: offscreenCtx, resolution: 1 });
world.set(AudioEngine, { context: offlineAudioCtx });
@@ -197,6 +198,7 @@ export async function createImageEncoder(sourceWorld: World, config: ImageEncode
}
seek(frame);
+ assetSystem(world);
await resolverSystem(world);
motionSystem(world);
transformSystem(world);
diff --git a/packages/jsx/src/hooks.ts b/packages/jsx/src/hooks.ts
index 3788c987..cf3f6323 100644
--- a/packages/jsx/src/hooks.ts
+++ b/packages/jsx/src/hooks.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
- * The authoring hooks, as signatures. Both read the host a project is mounted
+ * The authoring hooks, as signatures. They read the host a project is mounted
* into, which only exists inside the editor: when a project is mounted, the
* renderer substitutes its own "@diffusionstudio/jsx" module (see
* @diffusionstudio/reconciler) and these implementations are replaced by ones
@@ -12,8 +12,7 @@
* throws rather than silently returning a dead value.
*/
-import type { Accessor, ResourceReturn } from "solid-js";
-import type { AssetInput } from "./generate";
+import type { Accessor } from "solid-js";
export type Ticker = {
time: Accessor<number>;
@@ -39,28 +38,3 @@ function hostOnly(name: string): never {
export function useTicker(): Ticker {
return hostOnly("useTicker");
}
-
-/**
- * Resolves a source (path, asset id, URL, or a `generate.*` ref) to its `File`,
- * for reading raw bytes inside effects. Returns Solid's `createResource` tuple
- * unchanged: the resource accessor reads `undefined` until it resolves, then
- * the `File` (with `.loading` / `.error`), plus `mutate` / `refetch`.
- * Resolution is async (fetch a URL, read a path or library asset, await a
- * `generate.*` ref) and, like `src`, needs the host — a sandboxed module can't
- * fetch a path or asset id itself.
- *
- * ```tsx
- * const [canvas, setCanvas] = createSignal<HTMLCanvasElement>();
- * const [file] = useFile("/assets/logo.png");
- * createEffect(async () => {
- * const el = canvas();
- * const f = file();
- * if (!el || !f) return;
- * el.getContext("2d")!.drawImage(await createImageBitmap(f), 0, 0);
- * });
- * // <surface ref={setCanvas} width={640} height={360} />
- * ```
- */
-export function useFile(_src: AssetInput): ResourceReturn<File> {
- return hostOnly("useFile");
-}
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.ts
index 01ab1b51..05776aa8 100644
--- a/packages/jsx/src/index.ts
+++ b/packages/jsx/src/index.ts
@@ -34,7 +34,7 @@ export {
parseSource,
} from "./source";
export type { CompositionTag, PropValue } from "./source";
-export { useTicker, useFile } from "./hooks";
+export { useTicker } from "./hooks";
export type { Ticker } from "./hooks";
export type {
AdjustmentLayerProps,
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 3a35b458..a82011d9 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,12 +3,11 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Loop, Geometry, GeometryType, Library, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getAssetFile, getEntityChildren, getLibrary, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceIn, SourceOut, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';
import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
-import type { Asset } from '@diffusionstudio/assets';
-import type { AssetInput, PropValue } from '@diffusionstudio/jsx';
+import type { AssetRef, PropValue } from '@diffusionstudio/jsx';
import { trait, type Entity, type World } from 'koota';
import type { ProjectDocument } from './host';
@@ -62,13 +61,6 @@ const TextParts = trait(() => new Set<TextNode>());
*/
const Authored = trait(() => ({ tag: '', props: {} as Record<string, PropValue> }));
-/**
- * The `src` an element is waiting on: set while the library resolves a
- * source the world does not know yet, so a resolution that arrives after the
- * element was given another source (or none) is dropped.
- */
-const SourceRequest = trait(() => ({ value: undefined as unknown }));
-
/** Props that address or wire an element rather than describe it. */
const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, 'children', 'ref']);
@@ -992,7 +984,29 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
case 'src': {
- this.setSource(entity, value);
+ entity.remove(GenerationRequest, LoadRequest, PendingSource);
+
+ if (value === undefined || value === null || value === '') {
+ entity.remove(AssetId);
+ return;
+ }
+
+ if (typeof value !== 'string') {
+ entity.remove(AssetId);
+ entity.add(GenerationRequest);
+ entity.set(GenerationRequest, { ref: value as AssetRef });
+ return;
+ }
+
+ const known = getAsset(this.world, value);
+ if (known) {
+ bindAsset(entity, known);
+ return;
+ }
+
+ entity.remove(AssetId);
+ entity.add(LoadRequest);
+ entity.set(LoadRequest, { value });
return;
}
case 'objectFit': {
@@ -1314,70 +1328,6 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return next === undefined ? undefined : { entity: next };
}
- /**
- * Binds an element to the asset its `src` names. A library path or id the
- * world already knows binds at once; anything else — a path or URL outside
- * the library, a `generate.*` ref — is resolved through the library, and
- * binds when it arrives, unless the element has moved on to another source
- * or gone away in the meantime.
- */
- private setSource(entity: Entity, value: unknown): void {
- const library = this.world.get(Library);
- const known = typeof value === 'string' ? getAsset(this.world, value) : undefined;
-
- if (known) {
- entity.remove(SourceRequest);
- this.bindAsset(entity, known);
- return;
- }
-
- if (!library || value === undefined || value === null || value === '') {
- entity.remove(SourceRequest);
- entity.remove(AssetId);
- return;
- }
-
- // Whatever it showed before is wrong until the new source arrives.
- entity.remove(AssetId);
- entity.add(SourceRequest);
- entity.set(SourceRequest, { value });
-
- void library.resolve(value).then(
- (asset) => {
- if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;
- entity.remove(SourceRequest);
- this.bindAsset(entity, asset);
- },
- (error: unknown) => {
- if (!entity.isAlive() || entity.get(SourceRequest)?.value !== value) return;
- entity.remove(SourceRequest);
- console.error(`[reconciler] could not resolve src:`, error);
- },
- );
- }
-
- /**
- * Binds an element to an asset. A frames directory on a `<video>` or
- * `<image>` plays as a sequence: the element's paint follows the asset.
- */
- private bindAsset(entity: Entity, asset: Asset): void {
- entity.add(AssetId);
- entity.set(AssetId, { value: asset.id });
-
- const paint = entity.get(Paint)?.value;
- if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {
- entity.set(Paint, { value: PaintType.SEQUENCE });
- } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {
- entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });
- }
- }
-
- /** The `useFile` of a project: what its `src` would bind to, as a File. */
- public async loadFile(input: AssetInput): Promise<File> {
- const known = typeof input === 'string' ? getAsset(this.world, input) : undefined;
- const asset = known ?? await getLibrary(this.world).resolve(input);
- return getAssetFile(asset);
- }
public dispose(): void {
this.removeNode(this.stage, this.stage);
diff --git a/packages/reconciler/src/host.ts b/packages/reconciler/src/host.ts
index 1d225b9e..f0244a81 100644
--- a/packages/reconciler/src/host.ts
+++ b/packages/reconciler/src/host.ts
@@ -10,8 +10,6 @@
* opaque to the renderer.
*/
-import type { AssetInput } from "@diffusionstudio/jsx";
-
/**
* One reading of the host's timeline clock, local to the mounted project:
* the playhead of the scene the mount's root lives in (or is). Values only
@@ -43,5 +41,4 @@ export interface ProjectDocument<N = unknown> {
getNextSibling(node: N): N | undefined;
tick?(): ProjectTick;
applyRef?(node: N, ref: (target: unknown) => void): void;
- loadFile?(input: AssetInput): Promise<File>;
}
diff --git a/packages/reconciler/src/renderer.ts b/packages/reconciler/src/renderer.ts
index 02320774..ea7f291c 100644
--- a/packages/reconciler/src/renderer.ts
+++ b/packages/reconciler/src/renderer.ts
@@ -2,11 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { createContext, createMemo, createResource, useContext } from "solid-js";
+import { createContext, createMemo, useContext } from "solid-js";
import { createRenderer } from "solid-js/universal";
-import type { JSX, ResourceReturn } from "solid-js";
-import type { AssetInput, Ticker } from "@diffusionstudio/jsx";
+import type { JSX } from "solid-js";
+import type { Ticker } from "@diffusionstudio/jsx";
import type { ProjectDocument } from "./host";
// Compiled project modules call the static runtime exports below
@@ -107,22 +107,6 @@ export function useTicker(): Ticker {
};
}
-/**
- * The live `useFile` — substituted for the throwing declaration in
- * @diffusionstudio/jsx (see "./runtime"). Resolution is async (fetch a URL,
- * read a path or library asset, await a `generate.*` ref) and needs the host,
- * so it is a `createResource` over the document's `loadFile`.
- */
-export function useFile(src: AssetInput): ResourceReturn<File> {
- const document = doc();
- if (!document.loadFile) {
- throw new Error("useFile: this host does not provide file resolution");
- }
-
- const load = document.loadFile.bind(document);
- return createResource(() => src, (input) => load(input));
-}
-
/**
* Runs `fn` with `document` as the one the static runtime exports write into.
* Anything creating elements outside a project render — a host inserting
diff --git a/packages/reconciler/src/runtime.ts b/packages/reconciler/src/runtime.ts
index 2a8d8e6e..ba28ddd0 100644
--- a/packages/reconciler/src/runtime.ts
+++ b/packages/reconciler/src/runtime.ts
@@ -12,7 +12,7 @@
* `evaluate` binds it to this object instead. So the module a project imports
* is the authoring surface with the host-bound implementations layered over
* it: the renderer's element operations, the compile-target components, and
- * live `useTicker` / `useFile` in place of the throwing declarations.
+ * live `useTicker` in place of the throwing declaration.
*
* Order matters — the renderer bindings must come last so they win.
*/
@@ -33,7 +33,6 @@ import {
setProp,
spread,
use,
- useFile,
useTicker,
} from './renderer';
@@ -53,5 +52,4 @@ export const JSX_RUNTIME = {
mergeProps,
use,
useTicker,
- useFile,
};
diff --git a/packages/runtime/package.json b/packages/runtime/package.json
index 8d616b23..df2ae955 100644
--- a/packages/runtime/package.json
+++ b/packages/runtime/package.json
@@ -22,6 +22,7 @@
],
"dependencies": {
"@diffusionstudio/assets": "*",
+ "@diffusionstudio/jsx": "*",
"animejs": "^4.4.1",
"colord": "^2.9.3",
"koota": "^0.6.6",
diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.ts
index d270b2de..58f16e7b 100644
--- a/packages/runtime/src/actions/assets.ts
+++ b/packages/runtime/src/actions/assets.ts
@@ -6,10 +6,12 @@
// goes through the AssetLibrary (@diffusionstudio/assets); these are the
// lookups the decoders and hosts share.
-import { Library } from '../traits';
+import { Ai, AssetId, Library, Paint } from '../traits';
+import { PaintType } from '../constants';
-import type { World } from 'koota';
+import type { Entity, World } from 'koota';
import type { Asset, AssetLibrary } from '@diffusionstudio/assets';
+import type { GenAi } from '../ai';
/** The world's asset library; throws when the host attached none. */
export function getLibrary(world: World): AssetLibrary {
@@ -18,6 +20,30 @@ export function getLibrary(world: World): AssetLibrary {
return library;
}
+/** The world's generation service; throws when the host attached none. */
+export function getAi(world: World): GenAi {
+ const ai = world.get(Ai);
+ if (!ai) throw new Error('This world cannot generate assets (no Ai attached)');
+ return ai;
+}
+
+/**
+ * Binds an entity to an asset: stamps its AssetId, and follows the asset
+ * with the paint — a frames directory on a `<video>` or `<image>` plays as
+ * a sequence, and a sequence paint goes back when the asset is not one.
+ */
+export function bindAsset(entity: Entity, asset: Asset): void {
+ entity.add(AssetId);
+ entity.set(AssetId, { value: asset.id });
+
+ const paint = entity.get(Paint)?.value;
+ if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {
+ entity.set(Paint, { value: PaintType.SEQUENCE });
+ } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {
+ entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });
+ }
+}
+
/** The asset with `id` (or at that library path), or undefined without a library. */
export function getAsset(world: World, idOrPath: string): Asset | undefined {
return world.get(Library)?.get(idOrPath);
diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.ts
new file mode 100644
index 00000000..6dc653db
--- /dev/null
+++ b/packages/runtime/src/ai.ts
@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// The generation service a host may attach to the world (the `Ai` trait):
+// what turns a `generate.*` declaration into an asset of the project's
+// library. The runtime itself cannot generate — it only knows the contract —
+// so a world without an Ai never resolves such sources (see the asset
+// system). The asset library knows nothing of generation either; an
+// implementation *uses* a library to look up and store what it makes.
+
+import { generate } from '@diffusionstudio/jsx';
+
+import type { Asset } from '@diffusionstudio/assets';
+import type {
+ AssetRef,
+ GenerateAudioOptions,
+ GenerateImageOptions,
+ GenerateVideoOptions,
+ GenerateVoiceOptions,
+} from '@diffusionstudio/jsx';
+
+export abstract class GenAi {
+ /**
+ * The declaration, made real: an asset whose bytes the model produced.
+ * Content-addressed — the fully-resolved spec's hash is the asset's
+ * `generation.key`, so the same spec is the same asset in this session
+ * and the next, and identical concurrent declarations share one run.
+ */
+ public abstract resolve(ref: AssetRef): Promise<Asset>;
+
+ /**
+ * The imperative surface: `ai.generate.image({...})` declares and resolves
+ * in one call, returning the finished asset.
+ */
+ public readonly generate = {
+ image: (options: GenerateImageOptions): Promise<Asset> => this.resolve(generate.image(options)),
+ video: (options: GenerateVideoOptions): Promise<Asset> => this.resolve(generate.video(options)),
+ voice: (options: GenerateVoiceOptions): Promise<Asset> => this.resolve(generate.voice(options)),
+ audio: (options: GenerateAudioOptions): Promise<Asset> => this.resolve(generate.audio(options)),
+ };
+}
diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts
index 2a3827d2..f777bd2a 100644
--- a/packages/runtime/src/index.ts
+++ b/packages/runtime/src/index.ts
@@ -1,4 +1,5 @@
export * from './constants';
+export * from './ai';
export * from './traits';
export * from './world/create-world';
export * from './world/observers';
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
new file mode 100644
index 00000000..f6a5a84a
--- /dev/null
+++ b/packages/runtime/src/systems/assets.ts
@@ -0,0 +1,71 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Binds entities to the assets their `src` names. The document is
+// synchronous and only leaves requests behind (LoadRequest for a source
+// outside the library, GenerationRequest for a `generate.*` declaration);
+// this system consumes them each frame, starts the async work, and stamps
+// AssetId when the asset lands. Loads go through the world's library;
+// generations through the world's Ai — deduplication (a spec whose hash
+// already produced a library asset binds to it, identical inflight specs
+// share one run) is the Ai implementation's contract (see GenAi). A request
+// the world cannot serve yet — no library, no Ai — is left in place for a
+// frame that can. In offline mode the started work joins FramePromises, so
+// the encoder's barrier waits for the binding before it samples a frame.
+
+import { Ai, FramePromises, GenerationRequest, Library, LoadRequest, PendingSource } from '../traits';
+import { bindAsset } from '../actions/assets';
+
+import type { Entity, World } from 'koota';
+import type { Asset } from '@diffusionstudio/assets';
+
+export function assetSystem(world: World): void {
+ const library = world.get(Library);
+ if (library) {
+ for (const entity of world.query(LoadRequest)) {
+ const source = entity.get(LoadRequest)!.value;
+ entity.remove(LoadRequest);
+ resolve(world, entity, source, library.resolve(source));
+ }
+ }
+
+ const ai = world.get(Ai);
+ if (ai) {
+ for (const entity of world.query(GenerationRequest)) {
+ const ref = entity.get(GenerationRequest)!.ref;
+ entity.remove(GenerationRequest);
+ if (ref === null) continue;
+ resolve(world, entity, ref, ai.resolve(ref));
+ }
+ }
+}
+
+/**
+ * Tracks one started resolution: the entity remembers what it is waiting on
+ * (PendingSource), so of overlapping resolutions only the latest binds, and
+ * one that outlives its element (or its src) is dropped.
+ */
+function resolve(world: World, entity: Entity, value: unknown, promise: Promise<Asset>): void {
+ entity.add(PendingSource);
+ entity.set(PendingSource, { value });
+
+ const done = promise.then(
+ (asset) => {
+ if (!current(entity, value)) return;
+ entity.remove(PendingSource);
+ bindAsset(entity, asset);
+ },
+ (error: unknown) => {
+ if (!current(entity, value)) return;
+ entity.remove(PendingSource);
+ console.error('[runtime] could not resolve src:', error);
+ },
+ );
+
+ world.get(FramePromises)?.list?.push(done);
+}
+
+function current(entity: Entity, value: unknown): boolean {
+ return entity.isAlive() && entity.get(PendingSource)?.value === value;
+}
diff --git a/packages/runtime/src/systems/index.ts b/packages/runtime/src/systems/index.ts
index 626c0744..6911d34a 100644
--- a/packages/runtime/src/systems/index.ts
+++ b/packages/runtime/src/systems/index.ts
@@ -1,4 +1,5 @@
// Deterministic systems. Browser-only systems (input, keyboard, camera, hud, dom) stay in apps/web.
+export * from './assets';
export * from './transform';
export * from './motion';
export * from './gradients';
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index f0101095..e7dee2f7 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -6,6 +6,8 @@ import { trait } from 'koota';
import { GeometryType, PaintType, CaptionType, CaptionAlign } from '../constants';
+import type { AssetRef } from '@diffusionstudio/jsx';
+
// Geometric primitive: RECT or TEXT (see GeometryType). Other node-like roles
// (group, audio, scene, caption) are layered on top via tag traits.
export const Geometry = trait({ value: GeometryType.RECT as GeometryType });
@@ -71,6 +73,23 @@ export const Loop = trait({ value: '' });
export const AssetId = trait({ value: '' });
+// A `src` the document could not bind synchronously. The document only
+// leaves one of these requests behind; the asset system does the async work
+// and stamps AssetId when the asset lands. Never serialized: a request is
+// this world's business, re-derived from the src on any re-render.
+
+// The `generate.*` declaration a src names, to run through the world's Ai.
+export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }));
+
+// A source outside the library — a path or URL — to load into memory
+// through the library.
+export const LoadRequest = trait({ value: '' });
+
+// The src whose resolution is inflight, kept while the asset system works,
+// so a resolution that arrives after the element was given another source
+// (or none) is dropped. The document clears it whenever a new src arrives.
+export const PendingSource = trait(() => ({ value: undefined as unknown }));
+
// Sibling order under a ChildOf parent.
export const ItemIndex = trait({ value: 0 });
diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.ts
index d99290c2..edf7fac2 100644
--- a/packages/runtime/src/traits/world.ts
+++ b/packages/runtime/src/traits/world.ts
@@ -8,6 +8,7 @@ import { createLiveMounts } from '../utils/live-mounts';
import type { FontSource } from '../fonts/types';
import type { AssetLibrary } from '@diffusionstudio/assets';
+import type { GenAi } from '../ai';
import type { Quad } from '../math/aabb';
// Singleton state attached to the world itself (world.get/world.set).
@@ -69,6 +70,12 @@ export const Fonts = trait({ list: () => [] as FontSource[] });
// and transient alike. Null in a world without a project.
export const Library = trait(() => null as AssetLibrary | null);
+// The generation service, once the host has attached one (it needs a
+// backend): what the asset system runs `generate.*` sources through, and
+// what `getAi(world).generate.image(...)` calls. Optional — in a world
+// without one, such sources stay pending and never resolve.
+export const Ai = trait(() => null as GenAi | null);
+
// Live JSX mount registry; running graphs advance once per frame.
export const Mounts = trait(() => createLiveMounts());
diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.ts
index 1834b7d0..98771a21 100644
--- a/packages/runtime/src/world/create-world.ts
+++ b/packages/runtime/src/world/create-world.ts
@@ -18,6 +18,7 @@ import {
AudioEngine,
Fonts,
Library,
+ Ai,
Mounts,
FramePromises,
HitRegions,
@@ -40,6 +41,7 @@ export function createRuntimeWorld(projectId: string) {
AudioEngine,
Fonts,
Library,
+ Ai,
Mounts,
FramePromises,
HitRegions,