The VS Code of video editing, built for agents.
TypeScript
Refactor generative AI components and update dependencies
konstantin-paulus1a8968a396b2
diff --git a/README.md b/README.mdindex 562ef34c..50e2f6be 100644--- a/README.md+++ b/README.md@@ -49,7 +49,6 @@ The agent handles everything from there:Beyond finishing a cut, it covers:- **Motion graphics**: explainers, promos, and title sequences-- **Generative assets**: images, video, and voiceover, declared in code and composited into the timeline- **Clipping**: highlights from a long video, reformatted for social- **Video understanding**: summaries, scene search, quotes with timestamps@@ -141,10 +140,6 @@ The root is a `<stage>` holding one `<scene>` per frame you cut in:```tsximport { For } from "solid-js";-import { generate } from "@diffusionstudio/jsx";--const hero = generate.image({ prompt: "A neon city at night, cinematic", aspectRatio: "16:9" });-const motion = generate.video({ prompt: "slow camera push-in", startFrame: hero, duration: 5 });const TITLES = [{ text: "The Grid", start: 0, end: 2.5 },@@ -155,7 +150,7 @@ export default function Project() {return (<stage camera={[0.3, 0, 0, 0.3, 85, 150]}><scene name="Intro" width={1920} height={1080} fill="black" active>- <video src={motion} start={0} end={5} width={1920} height={1080} />+ <video src="footage/city.mp4" start={0} end={5} width={1920} height={1080} /><For each={TITLES}>{(t) => (<text@@ -203,7 +198,7 @@ Each command is the MCP tool of the same name: `diffusion media grab` is `media_| `diffusion capture` | Render frames of a scene, as an export would, to a labelled contact sheet or one PNG per position || `diffusion check` | Check a node's subtree for structural mistakes (black-frame gaps, never-visible nodes, failed sources) and report subtree stats || `diffusion media …` | Inspect a file by id or path: `probe`, `grab`, `filmstrip`, `waveform`, `transcribe`, `listen` |-| `diffusion models` / `diffusion voices` / `diffusion fonts` | Discover generation models, speech voices, local fonts |+| `diffusion fonts` | Discover Google Fonts and local font families || `diffusion screenshot` / `diffusion logs` | The app itself: capture the window, read recent console output || `diffusion report` | Report a bug in the tools or the app: diagnostics bundled, filed as a GitHub issue via `gh` |@@ -212,8 +207,8 @@ Conventions throughout: every result is one JSON object, the same structured con## Documentation- [Tool reference](docs/reference/tools/README.md): every tool and CLI command, its options, and its output-- [JSX reference](docs/reference/jsx/README.md): the composition markup with elements, timing, paints, generative assets, and captions-- [Examples](docs/examples/README.md): runnable compositions, from basic scenes and generative assets to three.js and raw WebGPU+- [JSX reference](docs/reference/jsx/README.md): the composition markup with elements, timing, paints, and captions+- [Examples](docs/examples/README.md): runnable compositions, from basic scenes to three.js and raw WebGPU## Repository layout@@ -224,7 +219,7 @@ Conventions throughout: every result is one JSON object, the same structured con| `apps/cli` | `@diffusionstudio/cli` | The `diffusion` CLI (`dapi` alias): a client of the app's MCP server || `packages/runtime` | `@diffusionstudio/runtime` | Headless editor runtime: the koota world, traits, actions, systems, media decoding, capture. No DOM, no Solid || `packages/reconciler` | `@diffusionstudio/reconciler` | Evaluates a compiled project bundle and reconciles its element tree onto runtime entities, via Solid's universal renderer |-| `packages/jsx` | `@diffusionstudio/jsx` | The authoring API: element vocabulary, types, and generated assets (`generate.*`) |+| `packages/jsx` | `@diffusionstudio/jsx` | The authoring API: element vocabulary and types || `packages/assets` | `@diffusionstudio/assets` | A project's asset library: the `assets.yml` manifest, content hashing, probing, resolution || `packages/encoder` | `@diffusionstudio/encoder` | Offline video/audio/image encoding over runtime worlds (mediabunny) || `packages/koota-solid` | `@diffusionstudio/koota-solid` | Solid bindings for koota, ported from `@koota/react` |diff --git a/apps/cli/src/cli-client.ts b/apps/cli/src/cli-client.tsindex 9aab75fb..f5ef8ef7 100644--- a/apps/cli/src/cli-client.ts+++ b/apps/cli/src/cli-client.ts@@ -12,7 +12,7 @@ import type { ToolInput, ToolName, ToolOutput } from "@diffusionstudio/dapi";export const APP_NAME = "Diffusion Studio";-// Renders and AI generation outlive the 60s default.+// Renders and media analysis outlive the 60s default.const TIMEOUTS: Record<string, number> = {export: 3_600_000,capture: 600_000,diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.tsindex 2ad078e9..3db03d84 100644--- a/apps/cli/src/index.ts+++ b/apps/cli/src/index.ts@@ -92,9 +92,9 @@ const program = new Command();program.name("diffusion").description(- `The Diffusion Studio CLI: understand, generate, and edit footage.-Analyze video/audio/images, generate them with AI, and compose assets.-Use for any media analysis, media generation, or video editing task. No ffmpeg needed.`)+ `The Diffusion Studio CLI: understand and edit footage.+Analyze video/audio/images and compose assets.+Use for any media analysis or video editing task. No ffmpeg needed.`).version(version);program@@ -252,17 +252,6 @@ mediareturn run("media_segment", { path: assetPath(ref), ...rest, points, output: opts.output && maskOutput(opts.output) });});-program- .command("models")- .description(describe("models"))- .argument("[type]", field("models", "type"))- .action((type: ToolInput<"models">["type"]) => run("models", { type }));--program- .command("voices")- .description(describe("voices"))- .action(() => run("voices", {}));-program.command("logs").description(describe("logs"))diff --git a/apps/desktop/src/dapi/present.test.ts b/apps/desktop/src/dapi/present.test.tsindex 52bb7d1b..840777fe 100644--- a/apps/desktop/src/dapi/present.test.ts+++ b/apps/desktop/src/dapi/present.test.ts@@ -103,7 +103,7 @@ describe("present", () => {id: "track-1", src: "masks/a.mask", video: "a.mp4", state: "done" as const, progress: 1,model: "tiny" as const, frameRate: 30, start: 0, end: 2, frames: 60, bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [],};- const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [], generations: [] };+ const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [] };const first = await present("context", {}, { ...base, masks: [{ ...row, png: png(8) }] });const image = (first.output as { masks: { image: string }[] }).masks[0]!.image;expect(image).toMatch(/dapi-segment-.*\.png$/);diff --git a/apps/desktop/src/dapi/tools-session.test.ts b/apps/desktop/src/dapi/tools-session.test.tsindex 2eff82f2..d52840ed 100644--- a/apps/desktop/src/dapi/tools-session.test.ts+++ b/apps/desktop/src/dapi/tools-session.test.ts@@ -9,6 +9,8 @@ import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";import { JSON_SCHEMA_DIALECT, tools } from "@diffusionstudio/dapi";import { serveCatalog } from "./tools-session";+const CONTEXT = { rootDir: null, projectDir: null, currentTime: null, fontFamilies: [], masks: [] };+let client: Client;const calls: Array<{ name: string; args: unknown }> = [];@@ -17,8 +19,8 @@ beforeAll(async () => {serveCatalog(session, async (tool, args) => {calls.push({ name: tool.name, args });switch (tool.name) {- case "voices":- return { voices: [] };+ case "context":+ return CONTEXT;case "check":throw new Error("No project open — run open first");default:@@ -54,9 +56,9 @@ describe("serveCatalog", () => {expect(JSON.stringify(bad.content)).toContain("limit");expect(calls.length).toBe(before);- const result = await client.callTool({ name: "voices", arguments: {} });- expect(result.structuredContent).toEqual({ voices: [] });- expect(calls.at(-1)).toEqual({ name: "voices", args: {} });+ const result = await client.callTool({ name: "context", arguments: {} });+ expect(result.structuredContent).toEqual(CONTEXT);+ expect(calls.at(-1)).toEqual({ name: "context", args: {} });});it("returns a handler's failure as a readable error result, not a protocol error", async () => {diff --git a/apps/desktop/src/edit-types.ts b/apps/desktop/src/edit-types.tsindex 535df7d8..ccd6a738 100644--- a/apps/desktop/src/edit-types.ts+++ b/apps/desktop/src/edit-types.ts@@ -6,16 +6,12 @@// canvas (renderer) and the edit writer (main). Types only, with no Node// imports, so the renderer's program can include it without Node's types.-import type { InspectValue, PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";+import type { InspectValue, PropValue } from "@diffusionstudio/jsx";-export type { PropValue, SerializedAssetRef };+export type { PropValue };-/**- * A value an edit can carry: what a source spells as a literal, or a- * `generate.*` declaration in its wire form, spelled as the call that- * reproduces it (see `setProp`).- */-export type EditValue = PropValue | SerializedAssetRef;+/** A value an edit can carry: what a source spells as a literal (see `setProp`). */+export type EditValue = PropValue;export interface SourceContext {/** Absolute path of the project folder. */diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.tsindex 3a9c1522..59400cfa 100644--- a/apps/desktop/src/edit.ts+++ b/apps/desktop/src/edit.ts@@ -9,11 +9,11 @@ import { join } from "node:path";import { IndentationText, Project, SyntaxKind } from "ts-morph";-import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, isTransformType, parseSource } from "@diffusionstudio/jsx";+import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, parseSource } from "@diffusionstudio/jsx";import { writeFileAtomic } from "./atomic";-import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";+import type { PropValue } from "@diffusionstudio/jsx";import type {ArrowFunction,FunctionExpression,@@ -121,8 +121,7 @@ const round = (value: number): number => Math.round(value * 100) / 100;const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;/** A prop value as the JavaScript that would have produced it. */-function literalText(value: PropValue | SerializedAssetRef): string {- if (isSerializedAssetRef(value)) return declarationText(value);+function literalText(value: PropValue): string {if (typeof value === "string") return JSON.stringify(value);if (typeof value === "number") return String(round(value));if (typeof value === "boolean" || value === null) return String(value);@@ -152,46 +151,14 @@ const initializerText = (value: PropValue): string =>? `"${value}"`: `{${literalText(value)}}`;-/**- * A declaration's wire form as the call that reproduces it: `generate.<type>`- * over its options, or `transform.<type>` over its input, with declared- * inputs spelled as the nested calls they are.- */-function declarationText(ref: SerializedAssetRef): string {- const { type, ...options } = ref.$asset;- if (isTransformType(type)) return `transform.${type}(${literalText((options as { input: EditValue }).input)})`;- return `generate.${type}(${literalText(options as Record<string, PropValue>)})`;-}--/** The namespaces a declaration is spelled with: what its file has to import. */-function declarationImports(ref: SerializedAssetRef, names = new Set<string>()): Set<string> {- const { type, ...options } = ref.$asset;- names.add(isTransformType(type) ? "transform" : "generate");- for (const value of Object.values(options)) {- for (const input of Array.isArray(value) ? value : [value]) {- if (isSerializedAssetRef(input)) declarationImports(input, names);- }- }- return names;-}-/*** Writes a prop onto a tag as one would write it: `muted`, not `muted={true}`,* and no attribute at all rather than `muted={false}`, since absence is what a- * boolean prop's false reads as. A declaration is spelled as the call that- * reproduces it — the caller makes sure its namespaces are imported (see- * `ensureDeclarationImports`).+ * boolean prop's false reads as.*/function setProp(tag: JsxTag, name: string, value: EditValue): void {const attribute = attributeOf(tag, name);- if (isSerializedAssetRef(value)) {- const initializer = `{${declarationText(value)}}`;- if (attribute) attribute.setInitializer(initializer);- else tag.addAttribute({ name, initializer });- return;- }-if (value === true) {if (attribute) attribute.removeInitializer();else tag.addAttribute({ name });@@ -207,45 +174,6 @@ function setProp(tag: JsxTag, name: string, value: EditValue): void {else tag.addAttribute({ name, initializer: initializerText(value) });}-/** Where `generate` and `transform` come from — the module every project authors against. */-const DECLARATION_MODULE = "@diffusionstudio/jsx";--/**- * Makes sure the namespaces a declaration is spelled with are in scope once- * it has been written into the file: added to the module's existing import,- * or as an import of their own after the last one — or above the first- * statement, past the file's header comments, when there are no imports at- * all. Idempotent.- */-function ensureDeclarationImports(sourceFile: SourceFile, ref: SerializedAssetRef): void {- for (const name of declarationImports(ref)) ensureNamedImport(sourceFile, name);-}--function ensureNamedImport(sourceFile: SourceFile, name: string): void {- const declarations = sourceFile.getImportDeclarations();- const importable = declarations.find(- (declaration) =>- declaration.getModuleSpecifierValue() === DECLARATION_MODULE &&- !declaration.isTypeOnly() &&- !declaration.getNamespaceImport(),- );-- if (importable) {- const named = importable.getNamedImports();- if (named.some((specifier) => specifier.getName() === name && !specifier.getAliasNode())) return;- importable.addNamedImport(name);- return;- }-- const statement = `import { ${name} } from "${DECLARATION_MODULE}";`;- const last = declarations.at(-1);- if (last) sourceFile.insertText(last.getEnd(), `\n${statement}`);- else {- const first = sourceFile.getStatements()[0];- sourceFile.insertText(first ? first.getStart() : 0, `${statement}\n\n`);- }-}-/*** Text content as JSX spells it: bare when JSX would read it back verbatim,* otherwise as a string expression (JSX collapses whitespace around line@@ -487,14 +415,8 @@ function reindent(text: string, from: string, to: string): string {.join("\n");}-/**- * Whether an expression is a value rather than a way of computing one. A- * declaration call over values is one too: it is what the writer spells a- * declaration as, so it is an initializer the writer may also overwrite.- */+/** Whether an expression is a value rather than a way of computing one. */function isLiteral(node: Node): boolean {- if (isDeclarationCall(node)) return true;-if (node.isKind(SyntaxKind.StringLiteral) ||node.isKind(SyntaxKind.NumericLiteral) ||@@ -528,24 +450,6 @@ function isLiteral(node: Node): boolean {return false;}-/** The namespaces a declaration call is made through. */-const DECLARATION_NAMESPACES = new Set(["generate", "transform"]);--/**- * A `generate.*` or `transform.*` call over literals — a nested declaration- * among them. One whose argument computes anything is authored reactivity,- * like any expression.- */-function isDeclarationCall(node: Node): boolean {- const call = node.asKind(SyntaxKind.CallExpression);- const callee = call?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);- if (!call || !callee || !callee.getExpression().isKind(SyntaxKind.Identifier)) return false;- if (!DECLARATION_NAMESPACES.has(callee.getExpression().getText())) return false;-- const args = call.getArguments();- return args.length === 1 && isLiteral(args[0]!);-}-// ---------------------------------------------------------------------------// Files@@ -858,7 +762,6 @@ class SourceWriter {// the canvas already shows the edit, and an export and the next open// render the file, so a prop left as it was would move back.setProp(tag, name, value);- if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);wrote = true;}@@ -955,7 +858,6 @@ class SourceWriter {insertChild(sourceFile, parent, child, before);for (const [name, value] of Object.entries(edit.props)) {setProp(findTag(sourceFile, id)!, name, value);- if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);}ids[edit.source] = formatSource(file, id);return true;@@ -1195,7 +1097,6 @@ class SourceWriter {}for (const [name, value] of Object.entries(record.props)) {setProp(found(), name, value);- if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);}const element = elementOf(found());diff --git a/apps/desktop/src/projects.ts b/apps/desktop/src/projects.tsindex 6f22e520..6db90bcf 100644--- a/apps/desktop/src/projects.ts+++ b/apps/desktop/src/projects.ts@@ -464,8 +464,6 @@ const SCRIPTS: Record<string, string> = {filmstrip: "diffusion media filmstrip",waveform: "diffusion media waveform",listen: "diffusion media listen",- models: "diffusion models",- voices: "diffusion voices",fonts: "diffusion fonts",logs: "diffusion logs",screenshot: "diffusion screenshot",@@ -552,8 +550,8 @@ a new session.| \`index.tsx\` | The entry. Its default export renders the composition. || \`package.json\` | The project record: \`projectId\` (its identity, kept across renames), \`displayName\` (the name shown in the app), \`main\` (the entry), \`diffusion\` (how each scene is exported), and the diffusion commands as scripts. || \`tsconfig.json\` | Types for the composition tags, through \`jsxImportSource\`. |-| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be; for a generation without bytes, where it stands. Written by the app; hand edits are read on the next load. |-| \`assets/\` | The library's files: put one here and it is taken in while the app watches, and the app writes its own here too — generations under \`assets/generated/\`. Media imported through the app is linked where it lies instead, never copied. |+| \`assets.yml\` | The asset library: for every asset its library path, where its bytes are, and what it was found to be. Written by the app; hand edits are read on the next load. |+| \`assets/\` | The library's files: put one here and it is taken in while the app watches, and the app writes its own here too. Media imported through the app is linked where it lies instead, never copied. || \`cache/\` | Derived data (thumbnails, waveforms). Disposable, and not checked in. |## Authoring@@ -599,10 +597,6 @@ export default function Project() {and the gradient paints, \`<stroke>\`, \`<shadow>\`, \`<effect>\`.- \`src\` takes a library path (\`"b-roll/drone.mp4"\` — the portable form, itsurvives the file being relinked), an asset id, a URL, or an absolute path.-- Generated assets are declared rather than fetched: \`src={generate.image({ prompt })}\`,- and \`generate.video\`, \`generate.voice\`, \`generate.audio\`. They are produced on- mount, in dependency order. \`diffusion context\` reports where each stands:- generating, failed with the reason, or done with the asset path it landed as.- Solid is fully available while mounting: \`<For>\`, \`<Show>\`, \`createMemo\`, and\`useTicker()\` for values that follow the playhead.- npm packages work as they normally do. The folder is a real npm package, so@@ -622,7 +616,7 @@ All of them talk to the running app, except \`fonts\`.| Script | Command | What it does || ------ | ------- | ------------ || \`open\` | \`diffusion open .\` | Launch the app with this project open. |-| \`context\` | \`diffusion context\` | Which project the app has open, where its playhead sits, its fonts, where its generations stand. |+| \`context\` | \`diffusion context\` | Which project the app has open, where its playhead sits, its fonts, its background mask tracks. || \`capture\` | \`diffusion capture <id>\` | Render frames of a scene, as an export would, to labelled PNG contact sheets. || \`probe\` | \`diffusion media probe <id\\|path>\` | Container and per-track metadata, without decoding. || \`transcribe\` | \`diffusion media transcribe <id\\|path>\` | Timed speech transcript, word by word. |@@ -630,8 +624,6 @@ All of them talk to the running app, except \`fonts\`.| \`filmstrip\` | \`diffusion media filmstrip <id\\|path>\` | Thumbnail grid across a window of a video. || \`waveform\` | \`diffusion media waveform <id\\|path>\` | Loudness over time, with the silences marked. || \`listen\` | \`diffusion media listen <id\\|path>\` | Ask a multimodal model what is in an audio track. |-| \`models\` | \`diffusion models [type]\` | Generation models and their per-model constraints. |-| \`voices\` | \`diffusion voices\` | Speech voices for \`generate.voice\`. || \`fonts\` | \`diffusion fonts\` | Google Fonts and local font families, valid as \`fontFamily\`. || \`logs\` | \`diffusion logs\` | Recent console output from the app. || \`screenshot\` | \`diffusion screenshot\` | The whole app window as a PNG. |@@ -639,7 +631,7 @@ All of them talk to the running app, except \`fonts\`.## Reference-- [JSX reference](https://github.com/diffusionstudio/editor/blob/main/docs/reference/jsx/README.md): elements, timing, paints, generation, captions+- [JSX reference](https://github.com/diffusionstudio/editor/blob/main/docs/reference/jsx/README.md): elements, timing, paints, captions- [Tool reference](https://github.com/diffusionstudio/editor/blob/main/docs/reference/tools/README.md): every tool and command, its options and its output- [Examples](https://github.com/diffusionstudio/editor/tree/main/docs/examples): runnable compositions to read`;diff --git a/apps/web/.env.example b/apps/web/.env.exampleindex 1c7ccd9e..9c88f685 100644--- a/apps/web/.env.example+++ b/apps/web/.env.example@@ -4,5 +4,3 @@ VITE_SUPABASE_ANON_KEY=sb_publishable_V-K9coUWXVy8dsZPGKDw-g_d1JzJf8uVITE_UMAMI_WEBSITE_ID=e898e381-6ce3-4f5f-a485-8327ec9aa88bVITE_UMAMI_SCRIPT_URL=https://cloud.umami.is/script.jsVITE_UMAMI_DOMAINS=diffusion.studio,app.diffusion.studio--VITE_API_URL=https://api.diffusion.studiodiff --git a/apps/web/package.json b/apps/web/package.jsonindex 6dfd77c2..716b930a 100644--- a/apps/web/package.json+++ b/apps/web/package.json@@ -14,7 +14,6 @@},"dependencies": {"@diffusionstudio/agent-chat": "*",- "@diffusionstudio/api-contract": "^0.1.0","@diffusionstudio/assets": "*","@diffusionstudio/dapi": "*","@diffusionstudio/encoder": "*",@@ -30,8 +29,6 @@"@solid-primitives/resize-observer": "^2.1.3","@solidjs/router": "^0.15.4","@supabase/supabase-js": "^2.99.2",- "@trpc/client": "^11.10.0",- "@trpc/server": "^11.10.0","animejs": "^4.4.1","cmdk-solid": "^1.1.2","colord": "^2.9.3",diff --git a/apps/web/src/components/dashboard/account-view.tsx b/apps/web/src/components/dashboard/account-view.tsxindex cce1a549..a389ee67 100644--- a/apps/web/src/components/dashboard/account-view.tsx+++ b/apps/web/src/components/dashboard/account-view.tsx@@ -27,7 +27,7 @@ import {import { TextField, TextFieldInput, TextFieldLabel } from "@/components/ui/text-field";import { useAuth } from "@/context/auth";import { supabase } from "@/lib/supabase";-import { trpc } from "@/lib/trpc";+import { backend } from "@/lib/backend";import {DashboardDividedStack,@@ -593,7 +593,7 @@ function DashboardAccountEmailPreferencesSection() {const handleChange = (column: EmailPreferenceColumn) => async (v: boolean) => {setOptimistic({ ...optimistic(), [column]: v });try {- await trpc.updateEmailPreference.mutate({ column, value: v });+ await backend.updateEmailPreference({ column, value: v });} catch (err) {const next = { ...optimistic() };delete next[column];diff --git a/apps/web/src/components/dashboard/ai-credits-view.tsx b/apps/web/src/components/dashboard/ai-credits-view.tsxindex e23b3e36..5f9df3f7 100644--- a/apps/web/src/components/dashboard/ai-credits-view.tsx+++ b/apps/web/src/components/dashboard/ai-credits-view.tsx@@ -22,7 +22,7 @@ import {} from "./shared";import { DashboardProPlanFeatures } from "./plans-view";-import type { TopupCredits } from "@diffusionstudio/api-contract";+import type { TopupCredits } from "@/lib/backend";function formatCredits(value: number): string {return value.toLocaleString();diff --git a/apps/web/src/components/dashboard/billing-info-view.tsx b/apps/web/src/components/dashboard/billing-info-view.tsxindex 0edf1560..704fbc10 100644--- a/apps/web/src/components/dashboard/billing-info-view.tsx+++ b/apps/web/src/components/dashboard/billing-info-view.tsx@@ -8,7 +8,7 @@ import { Button } from "@/components/ui/button";import { Icon } from "@/components/ui/icon";import { openBillingPortal } from "@/lib/checkout";import { useAuth } from "@/context/auth";-import { trpc } from "@/lib/trpc";+import { backend } from "@/lib/backend";import {DashboardDividedStack,@@ -220,7 +220,7 @@ export function DashboardBillingInfoView(props: DashboardBillingInfoViewProps) {() => auth.isPro(),async (isPro) => {if (!isPro) return undefined;- return (await trpc.getBillingInfo.query()) as BillingInfo;+ return (await backend.getBillingInfo()) ?? undefined;},);diff --git a/apps/web/src/components/dashboard/invoices-view.tsx b/apps/web/src/components/dashboard/invoices-view.tsxindex 8a240f40..d3bc61f5 100644--- a/apps/web/src/components/dashboard/invoices-view.tsx+++ b/apps/web/src/components/dashboard/invoices-view.tsx@@ -8,7 +8,7 @@ import { Badge } from "@/components/ui/badge";import { Button } from "@/components/ui/button";import { Icon } from "@/components/ui/icon";import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";-import { trpc } from "@/lib/trpc";+import { backend } from "@/lib/backend";import { DashboardDividedStack, DashboardSurfaceCard } from "./shared";@@ -145,7 +145,7 @@ function DashboardInvoicesList(props: { invoices: InvoiceItem[] }) {export function DashboardInvoicesView() {const [invoices] = createResource<InvoiceItem[]>(async () => {try {- const raw = await trpc.listInvoices.query();+ const raw = await backend.listInvoices();return raw.map(toInvoiceItem);} catch (err) {console.error("Failed to load invoices", err);diff --git a/apps/web/src/components/dashboard/plans-view.tsx b/apps/web/src/components/dashboard/plans-view.tsxindex 17d8e368..9b5c1507 100644--- a/apps/web/src/components/dashboard/plans-view.tsx+++ b/apps/web/src/components/dashboard/plans-view.tsx@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { Show, createSignal } from "solid-js";-import type { SubscriptionCredits } from "@diffusionstudio/api-contract";+import type { SubscriptionCredits } from "@/lib/backend";import { Button } from "@/components/ui/button";import {diff --git a/apps/web/src/components/dashboard/shared.tsx b/apps/web/src/components/dashboard/shared.tsxindex b355ca56..c0ee3401 100644--- a/apps/web/src/components/dashboard/shared.tsx+++ b/apps/web/src/components/dashboard/shared.tsx@@ -7,7 +7,7 @@ import { Dialog, DialogContent, DialogPortal } from "@/components/ui/dialog";import { Icon } from "@/components/ui/icon";import { useAuth } from "@/context/auth";import { cx } from "@/lib/cva";-import { trpc } from "@/lib/trpc";+import { backend } from "@/lib/backend";import {For,Show,@@ -249,7 +249,7 @@ function formatCurrency(amountCents: number, currency: string) {export function DashboardProPlanDetails() {const auth = useAuth();- const [summary] = createResource(() => trpc.getSubscriptionSummary.query());+ const [summary] = createResource(() => backend.getSubscriptionSummary());const priceLabel = () => {const s = summary();diff --git a/apps/web/src/components/genai/insert.tsx b/apps/web/src/components/genai/insert.tsxdeleted file mode 100644index 1d7aa7b4..00000000--- a/apps/web/src/components/genai/insert.tsx+++ /dev/null@@ -1,91 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { Ai, getNextName, Name, Root } from "@diffusionstudio/runtime";-import { assetName } from "@diffusionstudio/assets";-import { findEmptyPlacement } from "@/engine/placement";-import { toast } from "somoto";--import type { AssetRef } from "@diffusionstudio/jsx";-import type { DocumentEditor } from "@/engine/editor";-import type { Entity, World } from "koota";--/** Between the variants of one batch, and between the batch and its neighbour. */-const GAP = 40;--/** A declaration is keyed by its spec, so a fresh seed makes a new take. */-export const randomSeed = () => Math.floor(Math.random() * 1_000_000);--/** The box an inserted element is given, in canvas space. */-export interface GenerationBox {- x: number;- y: number;- width: number;- height: number;-}--/**- * Inserts `count` elements of `size` at the root of the canvas, side by side- * as one block, and selects them. `element` is called once per variant with- * the box it goes in; each call declares its own source, so the variants of a- * batch are separate takes rather than one asset shown four times.- *- * Returns the entities, or nothing (having said so) when there is no project- * to insert into.- *- * `sources[index]` is the declaration variant `index` consumes; each element- * is named after the asset its generation produces, once it lands.- */-export function insertGenerated(- world: World,- editor: DocumentEditor,- size: { width: number; height: number },- element: (box: GenerationBox, index: number) => unknown,- count = 1,- sources: AssetRef[] = [],-): Entity[] {- const root = world.get(Root);- const inserted: Entity[] = [];-- if (root) {- const totalWidth = count * size.width + (count - 1) * GAP;- const center = findEmptyPlacement(world, totalWidth, size.height, GAP);- const left = center.x - totalWidth / 2;- const y = Math.round(center.y - size.height / 2);-- for (let index = 0; index < count; index++) {- const x = Math.round(left + index * (size.width + GAP));- const [entity] = editor.insertElement(root, () => element({ x, y, ...size }, index));- if (entity) {- inserted.push(entity);- const ref = sources[index];- if (ref) nameAfterGeneration(world, editor, entity, ref);- }- }- }-- if (!inserted.length) {- toast("Could not add the generation", {- description: "Open a project to generate into.",- });- return inserted;- }-- editor.select(inserted);- return inserted;-}--/**- * Names the element after the asset its declaration resolves to — the same- * name a library insert would give it — unless it got one in the meantime.- * The resolution is memoized by ref, so this rides the request the element's- * own src makes rather than asking for another; a failure is the asset- * system's to surface (see SourceError), not this one's.- */-function nameAfterGeneration(world: World, editor: DocumentEditor, entity: Entity, ref: AssetRef): void {- world.get(Ai)?.resolve(ref).then((asset) => {- if (!entity.isAlive() || entity.get(Name)?.value) return;- editor.editProperty(entity, 'name', getNextName(world, assetName(asset).replace(/\.[^.]+$/, '')));- }).catch(() => undefined);-}diff --git a/apps/web/src/components/genai/placeholder.ts b/apps/web/src/components/genai/placeholder.tsnew file mode 100644index 00000000..201f6189--- /dev/null+++ b/apps/web/src/components/genai/placeholder.ts@@ -0,0 +1,16 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Placeholders for the generative AI features, which are being rebuilt. The+// prompt box and the action bar stay in place; every action that would have reached a model lands here instead.++import { toast } from "somoto";++/** Tells the user `feature` is not wired up yet. */+export function notAvailable(feature: string): void {+ toast(`${feature} is not available yet`, {+ id: `genai-placeholder:${feature}`,+ description: "Generative AI features are being rebuilt.",+ });+}diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsxindex 8c606030..b3cacf90 100644--- a/apps/web/src/components/genai/prompt-input-actions.tsx+++ b/apps/web/src/components/genai/prompt-input-actions.tsx@@ -25,7 +25,7 @@ import { useGenerateAudio } from "./use-generate-audio";import { useMediaSelection } from "./selection";import { useTransforms } from "./use-transforms";-import type { TransformType } from "@diffusionstudio/jsx";+import type { TransformType } from "./schemas";export function PromptInputActions() {const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();diff --git a/apps/web/src/components/genai/schemas.ts b/apps/web/src/components/genai/schemas.tsindex 7df5a7e9..4705baa2 100644--- a/apps/web/src/components/genai/schemas.ts+++ b/apps/web/src/components/genai/schemas.ts@@ -4,11 +4,7 @@import { z } from "zod";-/**- * What a generation request says, in the vocabulary the prompt input speaks.- * The same shape the server-side adapters take, so a stored `usage_records`- * config parses back into one (see `use-generation-records.ts`).- */+/** What a generation request says, in the vocabulary the prompt input speaks. */export const aspectRatioSchema = z.enum(["1:1", "4:3", "3:4", "16:9", "9:16"]);export type AspectRatio = z.infer<typeof aspectRatioSchema>;@@ -58,3 +54,6 @@ export type VideoGenerationConfig = z.infer<typeof videoGenerationConfigSchema>;export type VoiceGenerationConfig = z.infer<typeof voiceGenerationConfigSchema>;export type AudioGenerationConfig = z.infer<typeof audioGenerationConfigSchema>;export type GenerationConfig = z.infer<typeof generationConfigSchema>;++/** What can be done to an asset the selection shows. */+export type TransformType = "upscale" | "removeBackground" | "addAudio";diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.tsindex df6a01c9..54f37e73 100644--- a/apps/web/src/components/genai/selection.ts+++ b/apps/web/src/components/genai/selection.ts@@ -58,9 +58,6 @@ export function useMediaSelection() {const imageNodes = painted(PaintType.IMAGE);const videoNodes = painted(PaintType.VIDEO);- /** Every selected node as the element its source props live on. */- const sources = createMemo(() => selected().map((node) => node.source));-const bound = createMemo(() => {const lib = library();if (!lib) return [];@@ -76,5 +73,5 @@ export function useMediaSelection() {const images = createMemo(() => bound().filter((entry) => entry.asset.type === "IMAGE"));- return { bound, images, imageNodes, videoNodes, sources };+ return { bound, images, imageNodes, videoNodes };}diff --git a/apps/web/src/components/genai/use-auto-captions.tsx b/apps/web/src/components/genai/use-auto-captions.tsxindex 385cdfa5..f874b43a 100644--- a/apps/web/src/components/genai/use-auto-captions.tsx+++ b/apps/web/src/components/genai/use-auto-captions.tsx@@ -3,60 +3,15 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createMemo } from "solid-js";-import { Captions, authoredElement } from "@diffusionstudio/reconciler";-import { Caption, getEntityTree, isScene } from "@diffusionstudio/runtime";-import { useWorld } from "@diffusionstudio/koota-solid";-import { useEditor, useSelection } from "@/engine/hooks";-import { toast } from "somoto";--import type { Entity } from "koota";+import { isScene } from "@diffusionstudio/runtime";+import { useSelection } from "@/engine/hooks";+import { notAvailable } from "./placeholder";export function useAutoCaptions() {- const world = useWorld();- const editor = useEditor();const { nodes } = useSelection();- const scenes = createMemo(() => nodes().filter(isScene));- const hasScene = () => scenes().length > 0;-- const insertFresh = (scene: Entity) => {- const [entity] = editor.insertElement(scene, () => (- <Captions seed={Date.now()} />- ));- if (!entity) {- toast.error("Could not add captions", {- description: "The scene has no source to write the element into.",- });- return;- }- };-- const generate = () => {- for (const scene of scenes()) {- const existing = getEntityTree(world, scene).find(- (entity) => entity !== scene && entity.has(Caption),- );-- if (!existing) {- insertFresh(scene);- return;- }-- const authored = authoredElement(existing);-- // Captions bound to a subtitle file say what they show; auto-captions- // means transcribing, so the element is replaced by one that does.- if (authored?.props.src !== undefined) {- editor.remove(existing);- insertFresh(scene);- return;- }-- // A new take: a fresh seed re-transcribes the scene.- editor.editProperty(existing, "seed", Date.now());- editor.select(existing);- }- };+ const hasScene = createMemo(() => nodes().some(isScene));+ const generate = () => notAvailable("Auto captions");return { hasScene, generate };}diff --git a/apps/web/src/components/genai/use-generate-audio.tsx b/apps/web/src/components/genai/use-generate-audio.tsxindex 1656bdb5..77ac9429 100644--- a/apps/web/src/components/genai/use-generate-audio.tsx+++ b/apps/web/src/components/genai/use-generate-audio.tsx@@ -2,33 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Music and sound effects, declared: the prompt box writes a `generate.audio`-// source and inserts an <Audio> consuming it (see ./insert).--import { generate } from "@diffusionstudio/jsx";-import { Audio } from "@diffusionstudio/reconciler";-import { useWorld } from "@diffusionstudio/koota-solid";-import { useEditor } from "@/engine/hooks";-import { AUDIO_SIZE } from "@/engine/insert-asset";-import { insertGenerated, randomSeed } from "./insert";+import { notAvailable } from "./placeholder";import type { AudioGenerationConfig } from "./schemas";export function useGenerateAudio() {- const world = useWorld();- const editor = useEditor();-- const run = async (config: AudioGenerationConfig) => {- const src = generate.audio({- prompt: config.prompt,- model: config.model,- seed: randomSeed(),- });-- insertGenerated(world, editor, AUDIO_SIZE, (box) => (- <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />- ), 1, [src]);- };-+ const run = async (_config: AudioGenerationConfig) => notAvailable("Audio generation");return { generate: run } as const;}diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsxindex 7214bf4d..882cca78 100644--- a/apps/web/src/components/genai/use-generate-image.tsx+++ b/apps/web/src/components/genai/use-generate-image.tsx@@ -2,48 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { generate } from "@diffusionstudio/jsx";-import { ImagePaint, Rect } from "@diffusionstudio/reconciler";-import { Library } from "@diffusionstudio/runtime";-import { useWorld } from "@diffusionstudio/koota-solid";-import { useEditor } from "@/engine/hooks";-import { ASPECT_RATIO_DIMENSIONS } from "./config";-import { insertGenerated, randomSeed } from "./insert";+import { notAvailable } from "./placeholder";import type { ImageGenerationConfig } from "./schemas";export function useGenerateImage() {- const world = useWorld();- const editor = useEditor();-- const run = async (config: ImageGenerationConfig) => {- const library = world.get(Library);-- // References travel by library path: durable in the file, and readable- // there. Ids that no longer name a library asset are dropped.- const refs = (config.imageRefIds ?? [])- .map((id) => library?.get(id)?.path)- .filter((path): path is string => path !== undefined);-- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };-- // One declaration per variant, each with its own seed: a batch is four- // takes of the prompt rather than one asset shown four times.- const sources = Array.from({ length: config.count }, () => generate.image({- prompt: config.prompt,- model: config.model,- aspectRatio: config.aspectRatio,- ...(refs.length ? { refs } : {}),- seed: randomSeed(),- }));-- insertGenerated(world, editor, dims, (box, index) => (- <Rect keepAspectRatio x={box.x} y={box.y} width={box.width} height={box.height}>- <ImagePaint src={sources[index]} />- </Rect>- ), config.count, sources);- };-+ const run = async (_config: ImageGenerationConfig) => notAvailable("Image generation");return { generate: run } as const;}diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsxindex fb4f906d..48b5111c 100644--- a/apps/web/src/components/genai/use-generate-video.tsx+++ b/apps/web/src/components/genai/use-generate-video.tsx@@ -2,45 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { generate } from "@diffusionstudio/jsx";-import { Rect, VideoPaint } from "@diffusionstudio/reconciler";-import { Library } from "@diffusionstudio/runtime";-import { useWorld } from "@diffusionstudio/koota-solid";-import { useEditor } from "@/engine/hooks";-import { ASPECT_RATIO_DIMENSIONS } from "./config";-import { insertGenerated, randomSeed } from "./insert";+import { notAvailable } from "./placeholder";import type { VideoGenerationConfig } from "./schemas";export function useGenerateVideo() {- const world = useWorld();- const editor = useEditor();-- const run = async (config: VideoGenerationConfig) => {- const library = world.get(Library);- const frame = (id: string | undefined) => (id ? library?.get(id)?.path : undefined);-- const startFrame = frame(config.startFrameImageId);- const endFrame = frame(config.endFrameImageId);- const dims = ASPECT_RATIO_DIMENSIONS[config.aspectRatio] ?? { width: 1920, height: 1080 };-- const src = generate.video({- prompt: config.prompt,- model: config.model,- aspectRatio: config.aspectRatio,- duration: config.duration,- audio: config.generateAudio ?? false,- ...(startFrame ? { startFrame } : {}),- ...(endFrame ? { endFrame } : {}),- seed: randomSeed(),- });-- insertGenerated(world, editor, dims, (box) => (- <Rect keepAspectRatio x={box.x} y={box.y} width={box.width} height={box.height}>- <VideoPaint src={src} />- </Rect>- ), 1, [src]);- };-+ const run = async (_config: VideoGenerationConfig) => notAvailable("Video generation");return { generate: run } as const;}diff --git a/apps/web/src/components/genai/use-generate-voice.tsx b/apps/web/src/components/genai/use-generate-voice.tsxindex 931f2d1f..bb999a55 100644--- a/apps/web/src/components/genai/use-generate-voice.tsx+++ b/apps/web/src/components/genai/use-generate-voice.tsx@@ -2,34 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Spoken lines, declared: the prompt box writes a `generate.voice` source and-// inserts an <Audio> consuming it (see ./insert), so the file holds the line-// and the voice that reads it rather than a sound nobody can trace back.--import { generate } from "@diffusionstudio/jsx";-import { Audio } from "@diffusionstudio/reconciler";-import { useWorld } from "@diffusionstudio/koota-solid";-import { useEditor } from "@/engine/hooks";-import { AUDIO_SIZE } from "@/engine/insert-asset";-import { insertGenerated, randomSeed } from "./insert";+import { notAvailable } from "./placeholder";import type { VoiceGenerationConfig } from "./schemas";export function useGenerateVoice() {- const world = useWorld();- const editor = useEditor();-- const run = async (config: VoiceGenerationConfig) => {- const src = generate.voice({- prompt: config.prompt,- voice: config.voice,- seed: randomSeed(),- });-- insertGenerated(world, editor, AUDIO_SIZE, (box) => (- <Audio src={src} x={box.x} y={box.y} width={box.width} height={box.height} />- ), 1, [src]);- };-+ const run = async (_config: VoiceGenerationConfig) => notAvailable("Voice generation");return { generate: run } as const;}diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.tsindex 766e99ee..381ef80f 100644--- a/apps/web/src/components/genai/use-generation-records.ts+++ b/apps/web/src/components/genai/use-generation-records.ts@@ -2,177 +2,15 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { createMemo, createResource } from "solid-js";-import { getAssetSpec, isAssetRef, isTransformSpec } from "@diffusionstudio/jsx";-import { authoredElement } from "@diffusionstudio/reconciler";-import { useLibrary } from "@/engine/library";-import { supabase } from "@/lib/supabase";-import { useMediaSelection } from "./selection";-import {- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,- PROMPT_INPUT_VOICE_MODEL,- PROMPT_INPUT_VOICE_OPTIONS,-} from "./config";-import { generationConfigSchema, type GenerationConfig } from "./schemas";+// How the selection was generated and what it cost: a placeholder until the+// generative features are back (see ./placeholder). Nothing reads as+// generated, so "Rerun", "Reuse" and the credits line stay hidden.-import type { AssetInput, AssetRef, GenerateSpec } from "@diffusionstudio/jsx";-import type { AssetLibrary } from "@diffusionstudio/assets";--/**- * The stored `usage_records.config` IS the server-side adapter input, which- * shares its schema with the client's `GenerationConfig` — except that media- * fields carry `fileRef`s with embedded `assetId`s instead of raw asset IDs.- * This normalizes those fields so the object validates as a `GenerationConfig`.- */-export function toClientConfig(stored: unknown): GenerationConfig | undefined {- if (!stored || typeof stored !== "object") return undefined;- const c = stored as Record<string, unknown> & {- images?: Array<{ assetId?: string }>;- startFrame?: { assetId?: string };- endFrame?: { assetId?: string };- };-- const candidate: Record<string, unknown> = { ...c };-- if (c.images) {- const imageRefIds = c.images- .map((r) => r.assetId)- .filter((id): id is string => typeof id === "string");- delete candidate.images;- if (imageRefIds.length > 0) candidate.imageRefIds = imageRefIds;- }- if (c.startFrame?.assetId) {- candidate.startFrameImageId = c.startFrame.assetId;- }- delete candidate.startFrame;- if (c.endFrame?.assetId) {- candidate.endFrameImageId = c.endFrame.assetId;- }- delete candidate.endFrame;-- const parsed = generationConfigSchema.safeParse(candidate);- return parsed.success ? parsed.data : undefined;-}--/**- * A declaration read back as what the prompt box would have to be set to to- * produce it — the element's own account of how it was made, which is what- * "Rerun" and "Reuse" work from. Inputs are named by library path in the- * file and by asset id in the prompt box, so they are looked up on the way.- */-function toPromptConfig(spec: GenerateSpec, library: AssetLibrary): GenerationConfig {- const idOf = (input: AssetInput | undefined): string | undefined =>- typeof input === "string" ? library.get(input)?.id : undefined;-- switch (spec.type) {- case "image":- return {- mode: "IMAGE",- model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- count: 1,- imageRefIds: (spec.refs ?? []).map(idOf).filter((id): id is string => id !== undefined),- };- case "video":- return {- mode: "VIDEO",- model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- duration: spec.duration ?? 5,- generateAudio: spec.audio ?? false,- startFrameImageId: idOf(spec.startFrame),- endFrameImageId: idOf(spec.endFrame),- };- case "voice":- return {- mode: "VOICE",- model: PROMPT_INPUT_VOICE_MODEL,- prompt: spec.prompt,- voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,- };- case "audio":- return {- mode: "AUDIO",- model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- };- }-}--/** The generation a declaration rests on: itself, or what its transforms were put over. */-function generationUnder(ref: AssetRef): GenerateSpec | undefined {- const spec = getAssetSpec(ref);- if (!isTransformSpec(spec)) return spec;- return isAssetRef(spec.input) ? generationUnder(spec.input) : undefined;-}+import type { GenerationConfig } from "./schemas";export function useGenerationRecords() {- const library = useLibrary();- const { bound, sources } = useMediaSelection();-- // What the selected elements declare their source to be. An element made by- // the prompt box carries the whole spec, so this answers before the asset- // exists — and without asking the server what it was asked for. Transforms- // are looked through: the generation under them is what the box made.- const declarations = createMemo(() =>- sources()- .map((entity) => authoredElement(entity)?.props.src)- .filter(isAssetRef)- .map(generationUnder)- .filter((spec): spec is GenerateSpec => spec !== undefined),- );-- // Credits are the server's to know, and it knows them per generation.- const generationIds = createMemo(() => {- const ids = new Set<string>();- for (const { asset } of bound()) {- if (asset.generation?.id) ids.add(asset.generation.id);- }- return [...ids];- });-- const [records] = createResource(() => generationIds(), async (ids) => {- if (ids.length === 0 || !supabase) return [];- const { data, error } = await supabase- .from("usage_records")- .select("id,credits,config")- .in("id", ids);-- if (error) {- console.error("[credits] Failed to load usage records", error);- return [];- }-- return (data ?? []).map((row) => ({- id: row.id as string,- credits: (row.credits as number | null) ?? 0,- config: row.config as unknown,- }));- },- );-- const totalCredits = createMemo(() => {- return (records() ?? []).reduce((sum, r) => sum + r.credits, 0);- });-- const isGenerated = createMemo(() => declarations().length > 0 || generationIds().length > 0);-- const firstConfig = (): GenerationConfig | undefined => {- const lib = library();- const declared = declarations()[0];- if (declared && lib) return toPromptConfig(declared, lib);-- // Nothing declared: an asset generated before the project was written in- // JSX, whose settings only the record of the run still has.- const ids = generationIds();- if (ids.length === 0) return undefined;- const record = (records() ?? []).find((r) => r.id === ids[0]);- return toClientConfig(record?.config);- };-+ const isGenerated = (): boolean => false;+ const totalCredits = (): number => 0;+ const firstConfig = (): GenerationConfig | undefined => undefined;return { isGenerated, totalCredits, firstConfig };}diff --git a/apps/web/src/components/genai/use-transforms.ts b/apps/web/src/components/genai/use-transforms.tsindex e06b87de..8f6f991f 100644--- a/apps/web/src/components/genai/use-transforms.ts+++ b/apps/web/src/components/genai/use-transforms.ts@@ -2,85 +2,21 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Upscaling, taking a background out, scoring footage: the editor's side of-// `transform.*` (see reference/jsx/generate.md). A toggle wraps the-// element's `src` in the transform — `transform.upscale(<what it was>)` —-// or takes that layer out again, and stops there: the asset system is what-// notices the new declaration, runs it and binds the result. Turning one-// off gives the inner declaration back rather than running anything.+// Upscaling, taking a background out, scoring footage: placeholders until the+// generative features are back (see ./placeholder).-import { getAssetSpec, isAssetRef, isTransformSpec, mapAssetInputs, transform } from "@diffusionstudio/jsx";-import { authoredElement } from "@diffusionstudio/reconciler";-import { useEditor } from "@/engine/hooks";-import { useMediaSelection } from "./selection";+import { notAvailable } from "./placeholder";-import type { AssetInput, TransformType } from "@diffusionstudio/jsx";-import type { Entity } from "koota";+import type { TransformType } from "./schemas";-/** Whether `input` is, somewhere down its chain of transforms, put through `type`. */-function hasTransform(input: AssetInput, type: TransformType): boolean {- if (!isAssetRef(input)) return false;- const spec = getAssetSpec(input);- if (!isTransformSpec(spec)) return false;- return spec.type === type || hasTransform(spec.input, type);-}--/**- * `input` with every `type` layer taken out of its chain of transforms. A- * generation under the chain is left as it is: its own inputs are its- * business, not the element's.- */-function withoutTransform(input: AssetInput, type: TransformType): AssetInput {- if (!isAssetRef(input)) return input;- const spec = getAssetSpec(input);- if (!isTransformSpec(spec)) return input;- if (spec.type === type) return withoutTransform(spec.input, type);- return mapAssetInputs(input, (inner) => withoutTransform(inner, type));-}--/** The element's `src` as authored, when it is something a transform can be put over. */-function sourceOf(entity: Entity): AssetInput | undefined {- const src = authoredElement(entity)?.props.src;- return typeof src === "string" && src !== "" ? src : isAssetRef(src) ? src : undefined;-}+const LABELS: Record<TransformType, string> = {+ upscale: "Upscaling",+ removeBackground: "Background removal",+ addAudio: "Adding audio",+};export function useTransforms() {- const editor = useEditor();- const { imageNodes, videoNodes } = useMediaSelection();-- /** The selected elements a transform can be asked of. */- const targets = (type: TransformType): Entity[] => {- if (type === "removeBackground") return imageNodes();- if (type === "addAudio") return videoNodes();- return [...imageNodes(), ...videoNodes()];- };-- /** On when every element it applies to asks for it, so the toggle turns the odd one on. */- const isOn = (type: TransformType): boolean => {- const selected = targets(type);- return selected.length > 0 && selected.every((entity) => {- const src = sourceOf(entity);- return src !== undefined && hasTransform(src, type);- });- };-- /**- * Puts `type` over the selection's sources, or takes it off when they all- * have it. On goes outermost — the last thing done to the source — and off- * takes every layer of it out, wherever it sits in the chain.- */- const toggle = (type: TransformType): void => {- const next = !isOn(type);-- for (const entity of targets(type)) {- const src = sourceOf(entity);- if (src === undefined) continue;- const declared = next- ? (hasTransform(src, type) ? src : transform[type](src))- : withoutTransform(src, type);- if (declared !== src) editor.editProperty(entity, "src", declared);- }- };-+ const isOn = (_type: TransformType): boolean => false;+ const toggle = (type: TransformType): void => notAvailable(LABELS[type]);return { isOn, toggle };}diff --git a/apps/web/src/components/sidebar-left/assets.tsx b/apps/web/src/components/sidebar-left/assets.tsxindex e29a95f5..1768eb33 100644--- a/apps/web/src/components/sidebar-left/assets.tsx+++ b/apps/web/src/components/sidebar-left/assets.tsx@@ -28,7 +28,6 @@ import {BreadcrumbsSeparator,} from "../ui/breadcrumbs";import { LazyAssetItem } from "./asset-item";-import { PartialAssetItem } from "./partial-asset-item";import { FolderItem, handleFolderDrop, isAssetOrFolderDrag, ASSET_DRAG_TYPE, FOLDER_DRAG_TYPE } from "./folder-item";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";@@ -58,7 +57,6 @@ export function Assets() {const [currentFolder, setCurrentFolder] = createSignal("");const allAssets = createMemo(() => library()?.list().filter((asset) => asset.type !== "SCRIPT") ?? []);- const allPartials = createMemo(() => library()?.partials() ?? []);const allFolders = createMemo(() => [...(library()?.folders() ?? [])].sort());const activeFilterLabel = () => {@@ -77,7 +75,6 @@ export function Assets() {};const filteredAssets = createMemo(() => allAssets().filter(isShown));- const filteredPartials = createMemo(() => allPartials().filter(isShown));const visibleFolders = createMemo(() => {const q = query().trim().toLowerCase();@@ -87,7 +84,7 @@ export function Assets() {return library()?.childrenOf(currentFolder()).folders ?? [];});- const itemCount = createMemo(() => visibleFolders().length + filteredPartials().length + filteredAssets().length);+ const itemCount = createMemo(() => visibleFolders().length + filteredAssets().length);// Deep paths collapse like the breadcrumbs docs example:// All assets / … / parent / current, with the hidden folders in a dropdown.@@ -224,7 +221,7 @@ export function Assets() {if (event.key === "Backspace" || event.key === "Delete") {const selected = selectedAssetId();- const entry = selected ? (library()?.get(selected) ?? library()?.getPartial(selected)) : undefined;+ const entry = selected ? library()?.get(selected) : undefined;if (entry) {event.preventDefault();event.stopPropagation();@@ -261,9 +258,9 @@ export function Assets() {};const hasAssets = () => allAssets().length > 0;- const hasContent = () => hasAssets() || allPartials().length > 0 || allFolders().length > 0;+ const hasContent = () => hasAssets() || allFolders().length > 0;const isFiltering = () => query().trim().length > 0 || assetFilter() !== "ALL";- const isEmptyView = () => visibleFolders().length === 0 && filteredPartials().length === 0 && filteredAssets().length === 0;+ const isEmptyView = () => visibleFolders().length === 0 && filteredAssets().length === 0;const handleCreateFolder = () => withLibrary((lib) => {const parent = currentFolder();@@ -524,15 +521,6 @@ export function Assets() {/>)}</For>- <For each={filteredPartials()}>- {(partial) => (- <PartialAssetItem- partial={partial}- selected={selectedAssetId() === partial.id}- onSelect={() => handleSelectAsset(partial.id)}- />- )}- </For><For each={filteredAssets()}>{(asset) => (<LazyAssetItemdiff --git a/apps/web/src/components/sidebar-left/folder-item.tsx b/apps/web/src/components/sidebar-left/folder-item.tsxindex a12f412b..3b5efad4 100644--- a/apps/web/src/components/sidebar-left/folder-item.tsx+++ b/apps/web/src/components/sidebar-left/folder-item.tsx@@ -67,7 +67,7 @@ export function FolderItem(props: FolderItemProps) {const name = () => basename(props.path);const childCount = createMemo(() => {const children = library()?.childrenOf(props.path);- return children ? children.folders.length + children.assets.length + children.partials.length : 0;+ return children ? children.folders.length + children.assets.length : 0;});const handleDragStart = (event: DragEvent) => {diff --git a/apps/web/src/components/sidebar-left/partial-asset-item.tsx b/apps/web/src/components/sidebar-left/partial-asset-item.tsxdeleted file mode 100644index 4e990c92..00000000--- a/apps/web/src/components/sidebar-left/partial-asset-item.tsx+++ /dev/null@@ -1,109 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// A partial document in the asset panel: a generation with no bytes yet, or-// none ever. Pending, it pulses where its asset will be; failed, it carries-// the reason, and asking again is done from here — retrying forgets the-// failure and re-asks for every element that got it, deleting only forgets.--import { Show } from "solid-js";-import { toast } from "somoto";-import { useWorld } from "@diffusionstudio/koota-solid";-import { assetName } from "@diffusionstudio/assets";-import {- ContextMenu,- ContextMenuContent,- ContextMenuItem,- ContextMenuPortal,- ContextMenuSeparator,- ContextMenuShortcut,- ContextMenuTrigger,-} from "../ui/context-menu";-import { Icon } from "../ui/icon";-import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";-import { useLibrary } from "@/engine/library";-import { retryGeneration } from "@/engine/generations";--import type { PartialAsset } from "@diffusionstudio/assets";--export type PartialAssetItemProps = {- partial: PartialAsset;- selected: boolean;- onSelect(): void;-};--export function PartialAssetItem(props: PartialAssetItemProps) {- const world = useWorld();- const library = useLibrary();-- const failed = () => props.partial.state === "error";-- const handleRetry = async () => {- const lib = library();- if (!lib) return;- try {- await retryGeneration(world, lib, props.partial);- } catch (e) {- toast.error("Failed to retry", { description: (e as Error).message });- }- };-- const handleDelete = async () => {- try {- await library()?.remove([props.partial]);- } catch (e) {- toast.error("Failed to delete", { description: (e as Error).message });- }- };-- return (- <ContextMenu>- <ContextMenuTrigger- as="div"- class="flex flex-col gap-1 text-left"- data-asset-id={props.partial.id}- onClick={props.onSelect}- onContextMenu={props.onSelect}>- <Tooltip placement="bottom" openDelay={400}>- <TooltipTrigger- as="div"- data-selected={props.selected}- class="relative aspect-video w-full overflow-clip rounded bg-muted after:pointer-events-none after:absolute after:inset-0 after:rounded after:opacity-0 after:ring-2 after:ring-inset after:ring-ring after:z-10 data-[selected=true]:after:opacity-100"- classList={{ "animate-pulse": !failed() }}- >- <div class="absolute inset-0 flex items-center justify-center text-muted-foreground">- <Show when={failed()} fallback={<Icon name="spinner-loader" class="size-6 animate-spin" />}>- <Icon name="alert-warning" class="size-6 text-destructive" />- </Show>- </div>- </TooltipTrigger>- <TooltipContent class="max-w-64">- {failed() ? props.partial.error || "Generation failed" : "Generating…"}- </TooltipContent>- </Tooltip>- <div- class="text-xs truncate select-none"- classList={{ "text-destructive": failed(), "text-muted-foreground": !failed() }}- title={props.partial.error}- >- {assetName(props.partial)}- </div>- </ContextMenuTrigger>- <ContextMenuPortal>- <ContextMenuContent class="w-[180px]">- <Show when={failed()}>- <ContextMenuItem onSelect={handleRetry}>- Retry- </ContextMenuItem>- <ContextMenuSeparator />- </Show>- <ContextMenuItem onSelect={handleDelete}>- Delete- <ContextMenuShortcut>⌫</ContextMenuShortcut>- </ContextMenuItem>- </ContextMenuContent>- </ContextMenuPortal>- </ContextMenu>- );-}diff --git a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx b/apps/web/src/components/sidebar-right/inspector/asset-info.tsxindex 0da21d18..6bad4463 100644--- a/apps/web/src/components/sidebar-right/inspector/asset-info.tsx+++ b/apps/web/src/components/sidebar-right/inspector/asset-info.tsx@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { Button } from "@/components/ui/button";-import { For, Show, Match, Switch, createMemo, createResource, createSignal, type Accessor } from "solid-js";+import { For, Show, Match, Switch, createMemo, createSignal, type Accessor } from "solid-js";import {formatAspectRatio,formatBytes,@@ -13,93 +13,16 @@ import {formatAssetDate} from "@/utils/formatters";import { AssetInfoPreview } from "./asset-info-preview";-import { supabase } from "@/lib/supabase";-import { toClientConfig } from "@/components/genai/use-generation-records";import { useWorld } from "@diffusionstudio/koota-solid";import { assetName } from "@diffusionstudio/assets";import { useLibrary } from "@/engine/library";import { useAssetSelection } from "@/engine/hooks";import { insertAssetAtPlayhead, replaceAssetSource } from "@/engine/asset-actions";-import { retryGeneration } from "@/engine/generations";-import type { Asset, PartialAsset } from "@diffusionstudio/assets";+import type { Asset } from "@diffusionstudio/assets";-/** Information about the library entry picked in the assets panel. */-export function AssetInfoPanel() {- const selection = useAssetSelection();-- return (- <Show when={selection.partial()} fallback={<AssetDetails />}>- {(partial) => <PartialDetails partial={partial()} />}- </Show>- );-}--/**- * A generation without bytes: where it stands, and — once it has failed —- * the reason, and the way to ask again.- */-function PartialDetails(props: { partial: PartialAsset }) {- const world = useWorld();- const library = useLibrary();-- const failed = () => props.partial.state === "error";- const handleRetry = async () => {- const lib = library();- if (lib) await retryGeneration(world, lib, props.partial);- };-- const handleDelete = async () => {- await library()?.remove([props.partial]);- };-- return (- <div class="flex flex-col w-full px-4 border-t border-border">- <div class="h-12 flex items-center justify-between">- <span class="text-base font-strong">Information</span>- </div>-- <div class="py-3 border-t border-b border-border text-xs break-all">- {assetName(props.partial)}- </div>-- <div class="flex flex-col gap-1 my-2 text-xs text-muted-foreground">- <div class="flex h-7 items-center gap-2">- <span class="w-20 shrink-0">Status</span>- <span class="min-w-0 flex-1 text-right" classList={{ "text-destructive": failed() }}>- {failed() ? "Failed" : "Generating…"}- </span>- </div>- <Show when={props.partial.error}>- {(error) => (- <div class="flex flex-col gap-2 py-1">- <span class="w-20 shrink-0">Error</span>- <span class="min-w-0 text-left wrap-break-words text-destructive">{error()}</span>- </div>- )}- </Show>- <div class="flex h-7 items-center gap-2">- <span class="w-20 shrink-0">Requested</span>- <span class="min-w-0 flex-1 text-right truncate">{formatAssetDate(props.partial.createdAt)}</span>- </div>- </div>-- <div class="flex flex-col gap-2 my-2">- <Show when={failed()}>- <Button variant="default" class="w-full" onClick={handleRetry}>- Retry generation- </Button>- </Show>- <Button variant="secondary" class="w-full" onClick={handleDelete}>- Delete- </Button>- </div>- </div>- );-}--function AssetDetails() {+export function AssetInfoPanel() {const world = useWorld();const library = useLibrary();const selection = useAssetSelection();@@ -188,22 +111,6 @@ function AssetDetails() {}export function useAssetMetadataRows(asset: Accessor<Asset | undefined>) {- const generationId = createMemo(() => asset()?.generation?.id ?? null);-- const [config] = createResource(() => generationId(), async (id) => {- if (!id || !supabase) return undefined;- const { data, error } = await supabase- .from("usage_records")- .select("config")- .eq("id", id)- .maybeSingle();- if (error) {- console.error("[asset-info] Failed to load generation record", error);- return undefined;- }- return toClientConfig(data?.config);- });-return createMemo(() => {const a = asset();if (!a) return [];@@ -227,9 +134,9 @@ export function useAssetMetadataRows(asset: Accessor<Asset | undefined>) {: null;const imported = formatAssetDate(a.createdAt);const modified = a.stat ? formatAssetDate(a.stat.mtime) : null;- const c = config();- const prompt = c?.prompt ?? null;- const model = c?.model ?? null;+ // How the asset was generated: placeholders until generation is back.+ const prompt: string | null = null;+ const model: string | null = null;return [{ label: "Dimensions", value: dimensions },diff --git a/apps/web/src/components/sidebar-right/inspector/inspector.tsx b/apps/web/src/components/sidebar-right/inspector/inspector.tsxindex 47f7c785..d6de4759 100644--- a/apps/web/src/components/sidebar-right/inspector/inspector.tsx+++ b/apps/web/src/components/sidebar-right/inspector/inspector.tsx@@ -74,7 +74,7 @@ function classifyNode(entity: Entity): SelectionTarget {export function Inspector() {const tool = useTool();const { nodes, keyframes, first } = useSelection();- const { asset, partial } = useAssetSelection();+ const { asset } = useAssetSelection();// For ExportPanel (root scenes only) and TransitionSettings (sequence items).const parent = createMemo(() => getParentNode(first()));@@ -87,7 +87,7 @@ export function Inspector() {const selectionTarget = createMemo<SelectionTarget>(() => {if (tool() === ToolType.SCENE) return "scene-tool";if (keyframes().length > 0) return "keyframe";- if (asset() || partial()) return "asset";+ if (asset()) return "asset";const entity = first();if (nodes().length === 1 && entity) return classifyNode(entity);return "stage";@@ -95,7 +95,7 @@ export function Inspector() {// Remounts the panels when the selection changesconst selectionHash = createMemo(() => {- return [...nodes(), ...keyframes(), asset()?.id ?? partial()?.id ?? ""].join(",") + selectionTarget();+ return [...nodes(), ...keyframes(), asset()?.id ?? ""].join(",") + selectionTarget();});const includesTarget = (...targets: SelectionTarget[]) => {diff --git a/apps/web/src/components/upgrade-dialog.tsx b/apps/web/src/components/upgrade-dialog.tsxindex 8eceb85d..df329478 100644--- a/apps/web/src/components/upgrade-dialog.tsx+++ b/apps/web/src/components/upgrade-dialog.tsx@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { For, Show, createSignal, onMount } from "solid-js";-import type { SubscriptionCredits, TopupCredits } from "@diffusionstudio/api-contract";+import type { SubscriptionCredits, TopupCredits } from "@/lib/backend";import { Button } from "@/components/ui/button";import { Dialog, DialogContent, DialogPortal } from "@/components/ui/dialog";diff --git a/apps/web/src/context/auth.tsx b/apps/web/src/context/auth.tsxindex 6d2cf938..daebf0e0 100644--- a/apps/web/src/context/auth.tsx+++ b/apps/web/src/context/auth.tsx@@ -15,10 +15,9 @@ import {} from 'solid-js';import { toast } from 'somoto';import type { Session, User } from '@supabase/supabase-js';-import { FREE_CREDITS_QUOTA, type UserData } from '@diffusionstudio/api-contract';import { supabase } from '@/lib/supabase';-import { trpc } from '@/lib/trpc';+import { backend, FREE_CREDITS_QUOTA, type UserData } from '@/lib/backend';import { identify, resetIdentity, track } from '@/lib/analytics';import { mainBridge } from '@/lib/ipc';import { MAIN_CHANNELS } from '@desktop/main-channels';@@ -247,12 +246,8 @@ export function AuthProvider(props: { children: JSX.Element }) {return { error: 'Not authenticated' };}- if (!trpc) {- return { error: 'API is not configured' };- }-try {- await trpc.deleteAccount.mutate();+ await backend.deleteAccount();track('account_deleted');// Sign out locally after successful server-side deletiondiff --git a/apps/web/src/dapi/api.tsx b/apps/web/src/dapi/api.tsxindex 2d6a170d..bf2c2ef2 100644--- a/apps/web/src/dapi/api.tsx+++ b/apps/web/src/dapi/api.tsx@@ -6,9 +6,7 @@ import { createEffect, createContext, useContext, onCleanup } from "solid-js";import { useNavigate } from "@solidjs/router";import { useWorld } from "@diffusionstudio/koota-solid";import { Project } from "@diffusionstudio/runtime";-import { DapiError } from "@diffusionstudio/dapi";import { useProject } from "@/context/project";-import { useAuth } from "@/context/auth";import { useEngineContext } from "@/engine";import { openProjectFolder } from "@/projects";import { projectRoute } from "@/hooks/use-project-route";@@ -33,11 +31,10 @@ const EditorApiContext = createContext<EditorApiContextValue>();* reachable whether or not a project is open; the ones that need one read* the session slot (see ./session) per call and fail with a clear error —* or, for `context`, report that nothing is open. Renders nothing; must sit- * inside the router tree for `useNavigate` and inside the auth provider.+ * inside the router tree for `useNavigate`.*/export function EditorApi() {const navigate = useNavigate();- const auth = useAuth();const context: ContextFactory = (signal) => ({session: editorSession,@@ -49,11 +46,6 @@ export function EditorApi() {navigate(projectRoute(project.id || project.name));return { id: project.id, name: project.displayName, dir: project.dir };},- requireUser() {- const user = auth.user();- if (!user) throw new DapiError("sign-in-required", "Sign in required: AI generation needs a Diffusion Studio account.");- return user;- },},});diff --git a/apps/web/src/dapi/handler.ts b/apps/web/src/dapi/handler.tsindex b7c946dc..077e3df7 100644--- a/apps/web/src/dapi/handler.ts+++ b/apps/web/src/dapi/handler.ts@@ -3,7 +3,6 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import type { Accessor } from "solid-js";-import type { User } from "@supabase/supabase-js";import type { ToolArgs, ToolName, ToolResult } from "@diffusionstudio/dapi";import type { EditorSession } from "./session";@@ -19,11 +18,9 @@ export type ToolContext = {requireSession(): EditorSession;/** Fires when the caller cancels or goes away before the reply. */signal: AbortSignal;- /** What only the app shell can do: navigate, and know who is signed in. */+ /** What only the app shell can do: navigate. */app: {openProject(dir: string): Promise<ToolResult<"open">>;- /** The signed-in user, or a `sign-in-required` error. */- requireUser(): User;};};diff --git a/apps/web/src/dapi/handlers/check.ts b/apps/web/src/dapi/handlers/check.tsindex 5aa12f67..8e4f6488 100644--- a/apps/web/src/dapi/handlers/check.ts+++ b/apps/web/src/dapi/handlers/check.ts@@ -76,7 +76,7 @@ function visit(entity: Entity, window: Interval | null, depth: number, state: Wacode: "source-error",severity: "error",node: stamp,- message: `Source failed to ${failure.generated ? "generate" : "load"}: ${failure.value}`,+ message: `Source failed to load: ${failure.value}`,});}diff --git a/apps/web/src/dapi/handlers/context.ts b/apps/web/src/dapi/handlers/context.tsindex cef1cb06..81cbd149 100644--- a/apps/web/src/dapi/handlers/context.ts+++ b/apps/web/src/dapi/handlers/context.ts@@ -2,12 +2,10 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { AssetId, Computed, Fonts, FrameRate, Generating, getActiveEntity, Library, Name, PendingSource, Source, SourceError } from "@diffusionstudio/runtime";+import { Computed, Fonts, FrameRate, getActiveEntity } from "@diffusionstudio/runtime";import { getProjectsRoot } from "@/projects";import { maskTrackRows } from "./media-segment";-import type { Entity, World } from "koota";-import type { GenerationRow } from "@diffusionstudio/dapi";import type { ToolHandler } from "../handler";/**@@ -16,14 +14,14 @@ import type { ToolHandler } from "../handler";* the file, and a caller that wants them reads it. What is left over is which* folder new projects go in, which project folder the app has open, where its* playhead sits, which font families are actually registered in the world- * drawing it, and how generations and background mask tracks are getting on.+ * drawing it, and how background mask tracks are getting on.* With no project open only the root is left to report.*/export const context: ToolHandler<"context"> = async (_, ctx) => {const rootDir = await getProjectsRoot();const open = ctx.session();- if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], generations: [], masks: [] };+ if (!open) return { rootDir, projectDir: null, currentTime: null, fontFamilies: [], masks: [] };const { world, project } = open;const frameRate = world.get(FrameRate)?.value || 30;@@ -38,48 +36,6 @@ export const context: ToolHandler<"context"> = async (_, ctx) => {// What text can be drawn with right now: registered in the world, not// merely named in the source. The editor default is always among them.fontFamilies: [...new Set(["Inter", ...(world.get(Fonts)?.list ?? []).map((f) => f.family)])],- generations: collectGenerations(world),masks: maskTrackRows(world),};};--function collectGenerations(world: World): GenerationRow[] {- const library = world.get(Library);- const rows: GenerationRow[] = [];- const seen = new Set<string>();-- const push = (row: GenerationRow) => {- const key = JSON.stringify(row);- if (seen.has(key)) return;- seen.add(key);- rows.push(row);- };-- for (const entity of world.query(Generating)) {- push({ ...describeElement(entity), state: "generating" });- }-- for (const entity of world.query(SourceError)) {- const failure = entity.get(SourceError)!;- if (!failure.generated) continue;- push({ ...describeElement(entity), state: "failed", error: failure.value });- }-- for (const entity of world.query(AssetId)) {- // A stale binding — the element is off generating a new answer or failed- // getting one — is not a generation that is done.- if (entity.has(Generating) || entity.has(PendingSource) || entity.has(SourceError)) continue;- const asset = library?.get(entity.get(AssetId)!.value);- if (!asset?.generation) continue;- push({ ...describeElement(entity), state: "done", asset: asset.path });- }-- return rows;-}--function describeElement(entity: Entity): Pick<GenerationRow, "element" | "name"> {- return {- element: entity.get(Source)?.value || null,- name: entity.get(Name)?.value || null,- };-}diff --git a/apps/web/src/dapi/handlers/index.ts b/apps/web/src/dapi/handlers/index.tsindex ce1b8489..55437dd5 100644--- a/apps/web/src/dapi/handlers/index.ts+++ b/apps/web/src/dapi/handlers/index.ts@@ -7,8 +7,6 @@ import { context } from "./context";import { capture } from "./capture";import { check } from "./check";import { exportScene } from "./export";-import { models } from "./models";-import { voices } from "./voices";import { fonts } from "./fonts";import { screenshot } from "./screenshot";import { mediaProbe } from "./media-probe";@@ -28,8 +26,6 @@ export const handlers: Handlers = {capture,check,export: exportScene,- models,- voices,fonts,screenshot,media_probe: mediaProbe,diff --git a/apps/web/src/dapi/handlers/media-listen.ts b/apps/web/src/dapi/handlers/media-listen.tsindex eb3fc85a..733d5bf3 100644--- a/apps/web/src/dapi/handlers/media-listen.ts+++ b/apps/web/src/dapi/handlers/media-listen.ts@@ -2,39 +2,15 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Project, transcodeForAnalysis } from "@diffusionstudio/runtime";-import { trpc } from "@/lib/trpc";-import { startResumableSession, uploadResumableStream } from "@/lib/uploads";+import { DapiError } from "@diffusionstudio/dapi";import { requireAssetType, resolveAsset } from "../lib/assets";import type { ToolHandler } from "../handler";-export const mediaListen: ToolHandler<"media_listen"> = async ({ path, prompt, start, end }, ctx) => {- ctx.app.requireUser();+// Placeholder: the analysis ran on the backend, which is being rebuilt. The+// tool keeps its place in the catalog so callers get a clear answer.+export const mediaListen: ToolHandler<"media_listen"> = async ({ path }, ctx) => {const asset = await resolveAsset(ctx, path);requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");-- const hasWindow = start !== undefined || end !== undefined;- // A video is stripped to its audio track: the model listens, it does not watch.- const audioOnly = asset.type === "VIDEO";- const contentType = "audio/ogg";-- // One upload per distinct analysis input, so a repeated question about the- // same span reuses the transcode.- const window = hasWindow ? `-${start ?? 0}-${end ?? "end"}` : "";- const uploadId = `${ctx.session()?.world.get(Project)?.id ?? "project"}-${asset.id}-analyze${audioOnly ? "-audio" : ""}${window}`- .replace(/[^A-Za-z0-9._-]/g, "_");- const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({ action: "resumable", id: uploadId, contentType });-- // An upload URL means the server does not have this input yet.- if (uploadUrl) {- const transcoder = await transcodeForAnalysis(asset, { start, end, stripVideo: audioOnly });- const sessionUrl = await startResumableSession(uploadUrl, contentType);- const uploadPromise = uploadResumableStream(transcoder.readable, sessionUrl);- await transcoder.run?.();- await uploadPromise;- }-- const { analysis } = await trpc.analyze.mutate({ media: fileRef, prompt });- return { result: analysis, start, end };+ throw new DapiError("unsupported", "Audio analysis is not available in this version of the app.");};diff --git a/apps/web/src/dapi/handlers/media-transcribe.ts b/apps/web/src/dapi/handlers/media-transcribe.tsindex 234c2632..0b9e33a3 100644--- a/apps/web/src/dapi/handlers/media-transcribe.ts+++ b/apps/web/src/dapi/handlers/media-transcribe.ts@@ -2,36 +2,15 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { transcodeForTranscription } from "@diffusionstudio/runtime";-import { trpc } from "@/lib/trpc";-import { uploadBlob } from "@/lib/uploads";+import { DapiError } from "@diffusionstudio/dapi";import { requireAssetType, resolveAsset } from "../lib/assets";-import type { TranscriptSegment } from "@diffusionstudio/dapi";import type { ToolHandler } from "../handler";-// Transcripts are remembered per asset for the app's lifetime: the audio-// does not change, and the transcription is the expensive part.-const transcripts = new Map<string, TranscriptSegment[]>();-+// Placeholder: transcription ran on the backend, which is being rebuilt. The+// tool keeps its place in the catalog so callers get a clear answer.export const mediaTranscribe: ToolHandler<"media_transcribe"> = async ({ path }, ctx) => {const asset = await resolveAsset(ctx, path);requireAssetType(asset, ["AUDIO", "VIDEO"], "a video or audio asset");-- let transcript = transcripts.get(asset.id);- if (!transcript) {- const uploadId = crypto.randomUUID();- const audioFile = await transcodeForTranscription(asset);- const fileRef = await uploadBlob(audioFile, uploadId);- if (!fileRef) throw new Error(`Failed to upload asset ${asset.id} for transcription.`);-- ({ results: transcript } = await trpc.transcribe.mutate({ audio: fileRef }));- if (!transcript.length || transcript.every((s) => s.words.length === 0)) {- throw new Error("No speech detected. The audio does not appear to contain recognizable speech.");- }-- transcripts.set(asset.id, transcript);- }-- return { segments: transcript };+ throw new DapiError("unsupported", "Transcription is not available in this version of the app.");};diff --git a/apps/web/src/dapi/handlers/models.ts b/apps/web/src/dapi/handlers/models.tsdeleted file mode 100644index 5209511a..00000000--- a/apps/web/src/dapi/handlers/models.ts+++ /dev/null@@ -1,35 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import {- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,-} from "@/components/genai/config";--import type { ModelInfo } from "@diffusionstudio/dapi";-import type { ToolHandler } from "../handler";--export const models: ToolHandler<"models"> = async ({ type }) => {- const out: ModelInfo[] = [];- if (!type || type === "image") {- for (const { id, name } of PROMPT_INPUT_IMAGE_MODEL_OPTIONS) out.push({ type: "image", id, name });- }- if (!type || type === "video") {- for (const option of PROMPT_INPUT_VIDEO_MODEL_OPTIONS) {- out.push({- type: "video",- id: option.id,- name: option.name,- durations: option.durations,- aspectRatios: option.aspectRatios,- features: option.features,- });- }- }- if (!type || type === "audio") {- for (const { id, name } of PROMPT_INPUT_AUDIO_MODEL_OPTIONS) out.push({ type: "audio", id, name });- }- return { models: out };-};diff --git a/apps/web/src/dapi/handlers/voices.ts b/apps/web/src/dapi/handlers/voices.tsdeleted file mode 100644index 65fd8982..00000000--- a/apps/web/src/dapi/handlers/voices.ts+++ /dev/null@@ -1,11 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { PROMPT_INPUT_VOICE_OPTIONS } from "@/components/genai/config";--import type { ToolHandler } from "../handler";--export const voices: ToolHandler<"voices"> = async () => ({- voices: PROMPT_INPUT_VOICE_OPTIONS.map((v) => ({ id: v.value, label: v.label, description: v.description })),-});diff --git a/apps/web/src/engine/capture.ts b/apps/web/src/engine/capture.tsindex 52ef6180..bd810837 100644--- a/apps/web/src/engine/capture.ts+++ b/apps/web/src/engine/capture.ts@@ -22,8 +22,8 @@import { mount } from '@diffusionstudio/reconciler';import {- Ai, ChildOf, FramePromises, FrameRate, Fonts, Hidden, Library, Mode, Project,- RenderSurface, Root, Source, TranscriptionRequest,+ ChildOf, FramePromises, FrameRate, Fonts, Hidden, Library, Mode, Project,+ RenderSurface, Root, Source,createRuntimeWorld, disposeDecoders, getParentNode, removeChild, resetCamera,} from '@diffusionstudio/runtime';@@ -112,14 +112,13 @@ export async function createCapture(source: World, node: Entity, options: Capturcanvas.style.cssText = 'position:fixed;left:0;top:0;z-index:-9999;opacity:0;will-change:opacity;pointer-events:none;';document.body.appendChild(canvas);- // Shares the source's assets and generation service — a `src` names the- // same file in both — and its fonts, which are loaded into the document+ // Shares the source's assets — a `src` names the same file in both — and+ // its fonts, which are loaded into the document// already, so copying the list only tracks them as loaded rather than// having this world add them again.const world = createRuntimeWorld(source.get(Project)?.id ?? '');world.set(Mode, { value: mode });world.set(Library, source.get(Library) ?? null);- world.set(Ai, source.get(Ai) ?? null);world.set(Fonts, { list: [...(source.get(Fonts)?.list ?? [])] });world.set(RenderSurface, { canvas, ctx: canvas.getContext('2d'), resolution: 1 });world.set(FrameRate, { value: frameRate });@@ -175,15 +174,5 @@ export async function createCapture(source: World, node: Entity, options: Captur// encode draws with is the plain one.resetCamera(world);- // A `<captions>` with no src transcribes the scene it is in, and a- // transcription encodes that scene's audio — through a capture world of- // its own, whose captions would ask for the transcript that is still being- // made. Nothing audible comes from a caption anyway.- if (mode === 'offline-audio') {- for (const entity of world.query(TranscriptionRequest)) {- entity.remove(TranscriptionRequest);- }- }-return { world, node: rendered, dispose: dispose(mounted) };}diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.tsindex 17316791..3481d2f6 100644--- a/apps/web/src/engine/editor.ts+++ b/apps/web/src/engine/editor.ts@@ -11,7 +11,7 @@*/import { Active, Background, Chars, colorToHex, Computed, DEFAULT_BACKGROUND, FrameRate, framesToSeconds, getActiveEntity, getEntityChildren, getEntityTree, getIntrinsicPaint, getParentEntity, getTimelineOrigin, isText, Loop, PaintType, Selected, Sequential, setActive, Size, Source, Stage } from '@diffusionstudio/runtime';-import { isAssetRef, isPropValue, serializeAssetRef, SOURCE_ATTR } from '@diffusionstudio/jsx';+import { isPropValue, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createRoot } from 'solid-js';import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode, renderAuthored, withDocument } from '@diffusionstudio/reconciler';@@ -20,16 +20,12 @@ import { findInspectEntry } from './inspect';import { AssetSelection } from './traits';import type { SceneNode } from '@diffusionstudio/runtime';-import type { AssetRef, InspectValue, PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';+import type { InspectValue, PropValue } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { AuthoredTree, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';-/**- * A value an edit can carry to the file: what a source spells as a literal,- * or a `generate.*` declaration in its wire form (see `SerializedAssetRef`),- * which the writer spells as the call that reproduces it.- */-export type EditValue = PropValue | SerializedAssetRef;+/** A value an edit can carry to the file: what a source spells as a literal. */+export type EditValue = PropValue;/*** A property the editor changed, in the vocabulary of the JSX rather than of@@ -109,8 +105,7 @@ export interface MoveEdit {/*** A deleted subtree as it stood the moment before it went, one node per* authored element: the source it was stamped with, what a project would- * author it as (live values, `AssetRef`s included — this never crosses a- * wire), and its children in order. What an undo needs to put the subtree+ * author it as (live values included — this never crosses a wire), and its children in order. What an undo needs to put the subtree* back, and to pair the old sources with the new ones the reinsert mints.*/export interface CapturedNode {@@ -200,15 +195,13 @@ interface Recorded {/*** A prop as an edit can carry it (see `EditValue`): a value a file spells as- * it is, a declaration in its wire form, or undefined for anything else — an- * element is written with whatever of it the file can say, and the rest is- * the project's to keep, not the writer's to guess at. The one place that- * decides this, so a new kind of value is taught here once.+ * it is, or undefined for anything else — an element is written with whatever+ * of it the file can say, and the rest is the project's to keep, not the+ * writer's to guess at. The one place that decides this, so a new kind of+ * value is taught here once.*/function wireValue(value: unknown): EditValue | undefined {- if (isPropValue(value)) return value;- if (isAssetRef(value)) return serializeAssetRef(value);- return undefined;+ return isPropValue(value) ? value : undefined;}/** `wireValue` over a whole authored element, dropping what will not travel. */@@ -325,7 +318,7 @@ export class DocumentEditor {}/** Writes a prop to the document and reports it. */- public editProperty(entity: Entity, name: string, value: PropValue | AssetRef): void {+ public editProperty(entity: Entity, name: string, value: PropValue): void {const node = this.document.node(entity);let previous = node.props[name];// The stage keeps no authored record (see RuntimeDocument.setProperty),diff --git a/apps/web/src/engine/generations.ts b/apps/web/src/engine/generations.tsdeleted file mode 100644index 4b033aa3..00000000--- a/apps/web/src/engine/generations.ts+++ /dev/null@@ -1,44 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--/**- * Asking again for a generation that failed. A failed generation stands in- * the library as a partial document in error, and a declaration whose key- * stands there resolves to the failure rather than to another run — in- * this session and the next. Removing the record is what asks again; this- * is that removal, plus making the elements still waiting on the answer ask- * right away rather than on the next mount.- */--import { isAssetRef } from '@diffusionstudio/jsx';-import { authoredElement } from '@diffusionstudio/reconciler';-import {- Caption, Generating, GenerationRequest, PendingSource, SourceError, TranscriptionRequest,-} from '@diffusionstudio/runtime';--import type { AssetLibrary, PartialAsset } from '@diffusionstudio/assets';-import type { World } from 'koota';--/** Forgets the failure `partial` records and re-asks for every failed source. */-export async function retryGeneration(world: World, library: AssetLibrary, partial: PartialAsset): Promise<void> {- await library.remove([partial]);-- for (const entity of world.query(SourceError)) {- if (!entity.get(SourceError)!.generated) continue;- const authored = authoredElement(entity);-- if (!authored) continue;-- const src = authored.props.src;- entity.remove(SourceError, PendingSource, Generating);-- if (isAssetRef(src)) {- entity.add(GenerationRequest);- entity.set(GenerationRequest, { ref: src });- } else if (entity.has(Caption)) {- entity.add(TranscriptionRequest);- entity.set(TranscriptionRequest, { seed: Number(authored.props.seed) || 0 });- }- }-}diff --git a/apps/web/src/engine/hooks/use-asset-selection.ts b/apps/web/src/engine/hooks/use-asset-selection.tsindex bc31a2a0..3362d59b 100644--- a/apps/web/src/engine/hooks/use-asset-selection.ts+++ b/apps/web/src/engine/hooks/use-asset-selection.ts@@ -8,12 +8,11 @@ import { createMemo } from 'solid-js';import { AssetSelection } from '../traits';import { useLibrary } from '../library';-import type { Asset, PartialAsset } from '@diffusionstudio/assets';+import type { Asset } from '@diffusionstudio/assets';/**- * Reactive read and write of the AssetSelection trait: the one library entry- * the assets panel has picked and the inspector describes — an asset, or the- * partial document of a generation without bytes. Shared between the two+ * Reactive read and write of the AssetSelection trait: the one asset the+ * assets panel has picked and the inspector describes. Shared between the two* sidebars, so it lives on the world rather than in either panel.*/export function useAssetSelection() {@@ -32,17 +31,9 @@ export function useAssetSelection() {return lib.list().find((entry) => entry.id === current);});- /** The picked partial document, when the pick is one; drops out as its bytes land. */- const partial = createMemo(() => {- const lib = library();- const current = id();- if (!lib || !current) return undefined;- return lib.partials().find((entry) => entry.id === current);- });-- const select = (next: Asset | PartialAsset | string | null) => {+ const select = (next: Asset | string | null) => {world.set(AssetSelection, { id: typeof next === 'string' ? next : (next?.id ?? null) });};- return { id, asset, partial, select };+ return { id, asset, select };}diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.tsindex 06635b73..600ac39f 100644--- a/apps/web/src/engine/hud/hud-system.ts+++ b/apps/web/src/engine/hud/hud-system.ts@@ -8,7 +8,7 @@ import {HitRegions, Hovering, Name, Playback, RenderSurface, Root, Selected,Sequential, ChildOf,entityQuad, entityWorldMat, getMaskSelection, getSelectionMask, getSourceFailure,- invert2D, isGenerating,+ invert2D,multiply2D, rectToQuad, rotate2D, scale2D, store, transformPoint,translate2D,} from '@diffusionstudio/runtime';@@ -110,7 +110,7 @@ function drawHoverOutlines(world: World, ctx: Ctx2D, resolution: number): void {*/function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number): void {const failure = getSourceFailure(entity);- const label = failure || (isGenerating(entity) ? 'Generating...' : entity.get(Name)?.value);+ const label = failure || entity.get(Name)?.value;if (!label) return;const header = getHeaderLayout(world, entity, resolution);diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.tsindex a7e2fcbd..87eea159 100644--- a/apps/web/src/engine/library.ts+++ b/apps/web/src/engine/library.ts@@ -10,14 +10,12 @@ import { AssetId, Library } from '@diffusionstudio/runtime';import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';import { useTrait, useWorld } from '@diffusionstudio/koota-solid';import { authoredElement } from '@diffusionstudio/reconciler';-import { isAssetRef, mapAssetInputs } from '@diffusionstudio/jsx';import type { Accessor } from 'solid-js';import { createProjectFS } from '@/projects/fs';import { getDocumentEditor } from './editor';import type { Asset } from '@diffusionstudio/assets';-import type { AssetInput } from '@diffusionstudio/jsx';import type { World } from 'koota';/**@@ -42,22 +40,13 @@ export function isLibraryFile(path: string): boolean {return path === MANIFEST_FILE || path === ASSETS_DIR || path.startsWith(`${ASSETS_DIR}/`);}-/**- * Rewrites `src` on every element that named an asset by its old path —- * directly, or as an input somewhere inside a declaration: an element- * showing `transform.upscale("old.png")` is bound to what the transform made,- * and its `src` still has to follow the rename.- */+/** Rewrites `src` on every element that named an asset by its old path. */function followRename(world: World, asset: Asset, from: string): void {const editor = getDocumentEditor(world);- const renamed = (input: AssetInput): AssetInput =>- isAssetRef(input) ? mapAssetInputs(input, renamed) : input === from ? asset.path : input;for (const entity of world.query(AssetId)) {const src = authoredElement(entity)?.props.src;- if (typeof src !== 'string' && !isAssetRef(src)) continue;- const next = renamed(src);- if (next !== src) editor.editProperty(entity, 'src', next);+ if (src === from) editor.editProperty(entity, 'src', asset.path);}}diff --git a/apps/web/src/engine/placement.ts b/apps/web/src/engine/placement.tsdeleted file mode 100644index 3ab7d8fd..00000000--- a/apps/web/src/engine/placement.ts+++ /dev/null@@ -1,102 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// Where something new goes on the infinite canvas: beside the work rather-// than over it. Carried over from the engine the editor is migrating off,-// where this is what generations have always been placed by.--import {- aabbsIntersect, getEntityBounds, getEntityChildren, getViewport, Hidden, Root, screenToWorld,-} from '@diffusionstudio/runtime';--import type { AABB, Point } from '@diffusionstudio/runtime';-import type { World } from 'koota';--/** The visible part of the canvas in document space, or null before a surface is mounted. */-function viewportBounds(world: World): AABB | null {- const viewport = getViewport(world);- if (!viewport) return null;-- // All four corners, so the box still holds if the camera is ever rotated.- const corners = [- screenToWorld(world, 0, 0),- screenToWorld(world, viewport.width, 0),- screenToWorld(world, viewport.width, viewport.height),- screenToWorld(world, 0, viewport.height),- ];-- return {- minX: Math.min(...corners.map((corner) => corner.x)),- minY: Math.min(...corners.map((corner) => corner.y)),- maxX: Math.max(...corners.map((corner) => corner.x)),- maxY: Math.max(...corners.map((corner) => corner.y)),- };-}--/**- * What stands on the canvas: the bounds of every visible root-level node.- * A node the transform system has not measured yet reports nothing and is- * left out — it has no place on the canvas to be avoided at.- */-function occupied(world: World): AABB[] {- const boxes: AABB[] = [];-- for (const entity of getEntityChildren(world, world.get(Root)!)) {- if (entity.has(Hidden)) continue;- const rect = getEntityBounds(world, [entity]);- if (!rect) continue;- boxes.push({ minX: rect.x, minY: rect.y, maxX: rect.x + rect.width, maxY: rect.y + rect.height });- }-- return boxes;-}--const centerOf = (box: AABB): Point => ({ x: (box.minX + box.maxX) / 2, y: (box.minY + box.maxY) / 2 });--const distance = (a: Point, b: Point): number => Math.hypot(a.x - b.x, a.y - b.y);--/**- * Where the middle of a `width`×`height` block goes on the canvas: beside- * something already standing there, level with it — to its right, or to its- * left when the right is taken — leaving `gap` between the two. Neighbours are- * tried nearest the middle of the view first, and those in view before those- * out of it, so a block lands next to what the user is looking at.- *- * The middle of the view when the canvas is empty, or when nothing has a free- * side left; the origin when no surface is mounted yet.- */-export function findEmptyPlacement(world: World, width: number, height: number, gap: number): Point {- const viewport = viewportBounds(world);- const center = viewport ? centerOf(viewport) : { x: 0, y: 0 };-- const boxes = occupied(world);- if (boxes.length === 0) return center;-- const inView = viewport ? boxes.filter((box) => aabbsIntersect(box, viewport)) : [];- const ordered = (inView.length > 0 ? inView : boxes)- .slice()- .sort((a, b) => distance(centerOf(a), center) - distance(centerOf(b), center));-- const free = (rect: AABB): boolean => !boxes.some((box) => aabbsIntersect(rect, box));-- for (const box of ordered) {- // Level with the neighbour: the block keeps its middle, whatever the- // two are sized like.- const y = (box.minY + box.maxY) / 2;- const minY = y - height / 2;- const maxY = y + height / 2;-- const right = box.maxX + gap;- if (free({ minX: right, minY, maxX: right + width, maxY })) {- return { x: right + width / 2, y };- }-- const left = box.minX - gap;- if (free({ minX: left - width, minY, maxX: left, maxY })) {- return { x: left - width / 2, y };- }- }-- return center;-}diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.tsindex 6e9e2c02..20b929b2 100644--- a/apps/web/src/engine/timeline/render/clip.ts+++ b/apps/web/src/engine/timeline/render/clip.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getGeneratingColor, getSourceFailure, isCaption, isGenerating, isGroup, isText, store } from '@diffusionstudio/runtime';+import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Hidden, Name, Selected, TrimDragOrigin, fitsChildren, getSourceFailure, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';import { getDocumentEditor } from '../../editor';import {@@ -67,16 +67,14 @@ export function renderClip(left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);}- const generating = isGenerating(entity);-ctx.save();ctx.beginPath();ctx.roundRect(left, 0, width, row.height, CLIP_CORNER_RADIUS);- ctx.fillStyle = generating ? getGeneratingColor(world) : style.background;+ ctx.fillStyle = style.background;ctx.fill();ctx.restore();- if (!generating && !error) {+ if (!error) {renderContent(world, scene, surface, entity, asset, style, row);}@@ -88,7 +86,6 @@ export function renderClip(if (isCaption(entity)) label = label || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;else if (isText(entity)) label = entity.get(Chars)?.value ?? label ?? '';if (!label) label = getClipFallbackName(world, entity);- if (generating) label = 'Generating...';if (error) label = error;const [viewportLeft] = getViewport(world, scene, surface.layout.width);diff --git a/apps/web/src/lib/backend.ts b/apps/web/src/lib/backend.tsnew file mode 100644index 00000000..76512498--- /dev/null+++ b/apps/web/src/lib/backend.ts@@ -0,0 +1,122 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * Stand-ins for the Diffusion Studio backend, which is being rebuilt. Every+ * call the app used to make to it lands here: reads answer with empty mock+ * data, and actions that need the server fail with `BackendUnavailableError`,+ * so the screens that show them keep working until the new API is wired in.+ * The Supabase database and auth are not part of this; they stay as they are.+ */++export type BillingPeriod = "month" | "year";++export type SubscriptionCredits = "2_500" | "5_000" | "15_000" | "30_000" | "45_000";++export type TopupCredits = "1_000" | "2_000" | "5_000" | "10_000" | "25_000";++/** Credits a free account gets each month. */+export const FREE_CREDITS_QUOTA = 50;++/** Row shape of the Supabase `user_data` table. */+export type UserData = {+ id: string;+ lifetime_credit_balance: number;+ monthly_credit_balance: number;+ monthly_credit_quota: number;+ access_level: number;+ stripe_customer_id: string | null;+ last_credit_reset_at: string | null;+ product_updates_enabled: boolean;+ marketing_announcements_enabled: boolean;+};++export type EmailPreferenceColumn = "product_updates_enabled" | "marketing_announcements_enabled";++export type SubscriptionSummary = {+ amount: number;+ currency: string;+ billingPeriod: BillingPeriod;+ currentPeriodEnd: number;+};++export type BillingInfo = {+ email: string;+ address: {+ legalName: string;+ line1: string;+ line2: string;+ city: string;+ state: string;+ postalCode: string;+ country: string;+ } | null;+ taxId: string | null;+ paymentMethod: {+ brand: string;+ last4: string;+ expMonth: number;+ expYear: number;+ } | null;+ billingPeriod: BillingPeriod | null;+};++export type Invoice = {+ id: string;+ status: "draft" | "open" | "paid" | "uncollectible" | "void" | null;+ total: number;+ currency: string;+ dueDate: number;+ hostedUrl: string | null;+};++export class BackendUnavailableError extends Error {+ constructor(action: string) {+ super(`${action} is not available yet.`);+ this.name = "BackendUnavailableError";+ }+}++export const backend = {+ async getSubscriptionSummary(): Promise<SubscriptionSummary | null> {+ return null;+ },++ async getBillingInfo(): Promise<BillingInfo | null> {+ return null;+ },++ async listInvoices(): Promise<Invoice[]> {+ return [];+ },++ async createSubscription(_input: {+ creditQuantity: SubscriptionCredits;+ billingPeriod: BillingPeriod;+ successUrl: string;+ cancelUrl: string;+ }): Promise<{ url: string }> {+ throw new BackendUnavailableError("Checkout");+ },++ async createTopup(_input: {+ creditQuantity: TopupCredits;+ successUrl: string;+ cancelUrl: string;+ }): Promise<{ url: string }> {+ throw new BackendUnavailableError("Checkout");+ },++ async createBillingPortal(): Promise<{ url: string }> {+ throw new BackendUnavailableError("The billing portal");+ },++ async updateEmailPreference(_input: { column: EmailPreferenceColumn; value: boolean }): Promise<{ success: boolean }> {+ throw new BackendUnavailableError("Changing email preferences");+ },++ async deleteAccount(): Promise<{ success: boolean }> {+ throw new BackendUnavailableError("Deleting your account");+ },+};diff --git a/apps/web/src/lib/checkout.ts b/apps/web/src/lib/checkout.tsindex 6410369a..91d56285 100644--- a/apps/web/src/lib/checkout.ts+++ b/apps/web/src/lib/checkout.ts@@ -2,24 +2,15 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { TRPCClientError } from "@trpc/client";import { toast } from "somoto";-import type {- BillingPeriod,- SubscriptionCredits,- TopupCredits,-} from "@diffusionstudio/api-contract";-import { trpc } from "./trpc";+import { backend } from "./backend";import { mainBridge } from "./ipc";import { MAIN_CHANNELS } from "@desktop/main-channels";+import type { BillingPeriod, SubscriptionCredits, TopupCredits } from "./backend";+function toastMessage(err: unknown, fallback: string): string {- // Empty / non-JSON 500 bodies (e.g. upstream crashed before tRPC could- // serialize the error) surface as a SyntaxError — not useful to display.- if (err instanceof TRPCClientError && err.cause instanceof SyntaxError) {- return fallback;- }return err instanceof Error ? err.message : fallback;}@@ -103,7 +94,7 @@ export async function startSubscriptionCheckout(input: {billingPeriod: BillingPeriod;}): Promise<void> {try {- const { url } = await trpc.createSubscription.mutate({+ const { url } = await backend.createSubscription({...input,...successAndCancelUrls(),});@@ -117,7 +108,7 @@ export async function startTopupCheckout(creditQuantity: TopupCredits,): Promise<void> {try {- const { url } = await trpc.createTopup.mutate({+ const { url } = await backend.createTopup({creditQuantity,...successAndCancelUrls(),});@@ -130,7 +121,7 @@ export async function startTopupCheckout(export async function openBillingPortal(): Promise<void> {if (window.desktop) {try {- const { url } = await trpc.createBillingPortal.mutate();+ const { url } = await backend.createBillingPortal();await mainBridge.call(MAIN_CHANNELS.APP_OPEN_EXTERNAL, { url });} catch (err) {toast.error(toastMessage(err, "Failed to open billing portal"));@@ -144,7 +135,7 @@ export async function openBillingPortal(): Promise<void> {if (!tab) return;try {- const { url } = await trpc.createBillingPortal.mutate();+ const { url } = await backend.createBillingPortal();tab.location.href = url;} catch (err) {tab.close();diff --git a/apps/web/src/lib/trpc.ts b/apps/web/src/lib/trpc.tsdeleted file mode 100644index efb662a9..00000000--- a/apps/web/src/lib/trpc.ts+++ /dev/null@@ -1,39 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { createTRPCClient, httpBatchLink, TRPCClientError, type TRPCLink } from "@trpc/client";-import { observable } from "@trpc/server/observable";-import { supabase } from "./supabase";-import { showUpgradeDialog } from "@/components/upgrade-dialog";--import type { AppRouter } from "@diffusionstudio/api-contract";--const paymentRequiredLink: TRPCLink<AppRouter> = () => ({ next, op }) =>- observable((observer) => {- const sub = next(op).subscribe({- next: (value) => observer.next(value),- error: (err) => {- if (err instanceof TRPCClientError && err.data?.code === "PAYMENT_REQUIRED") {- showUpgradeDialog();- }- observer.error(err);- },- complete: () => observer.complete(),- });- return () => sub.unsubscribe();- });--export const trpc = createTRPCClient<AppRouter>({- links: [- paymentRequiredLink,- httpBatchLink({- url: `${import.meta.env.VITE_API_URL ?? ""}/api/trpc`,- async headers() {- const session = await supabase?.auth.getSession();- const token = session?.data.session?.access_token;- return token ? { Authorization: `Bearer ${token}` } : {};- },- }),- ],-});diff --git a/apps/web/src/lib/uploads.ts b/apps/web/src/lib/uploads.tsdeleted file mode 100644index 4a004561..00000000--- a/apps/web/src/lib/uploads.ts+++ /dev/null@@ -1,127 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--/**- * Getting bytes into the cloud bucket the API resolves a `FileRef` against.- * A whole Blob goes in one signed PUT; anything produced as it is encoded- * (a transcode feeding a generation) goes through a GCS resumable session so- * the payload is never held in memory in full.- */--import { trpc } from "@/lib/trpc";--/**- * Upload a Blob to the cloud bucket via a signed URL under the asset id.- * Returns the FileRef the API can resolve.- */-export async function uploadBlob(blob: Blob, id: string) {- const contentType = blob.type || "application/octet-stream";- const { uploadUrl, fileRef } = await trpc.getUploadUrl.mutate({ id, contentType });-- if (uploadUrl) {- const res = await fetch(uploadUrl, {- method: "PUT",- headers: { "Content-Type": contentType },- body: blob,- });-- if (!res.ok) {- console.error(`Failed to upload asset ${id}: ${res.status}`);- return null;- }- }-- return fileRef;-}--/** GCS requires every resumable chunk except the last to be a multiple of 256 KiB. */-const RESUMABLE_CHUNK_SIZE = 8 * 1024 * 1024;--/**- * Open a GCS resumable upload session against a signed "resumable" URL and- * return the session URI that the bytes are streamed to.- */-export async function startResumableSession(uploadUrl: string, mimeType: string): Promise<string> {- const res = await fetch(uploadUrl, {- method: "POST",- headers: { "x-goog-resumable": "start", "Content-Type": mimeType },- });- if (!res.ok) {- throw new Error(`Failed to start resumable upload session: ${res.status}`);- }- const sessionUrl = res.headers.get("Location");- if (!sessionUrl) {- throw new Error("Resumable upload session response is missing the Location header.");- }- return sessionUrl;-}--/**- * Stream a ReadableStream into an open GCS resumable session, PUTting- * fixed-size chunks as bytes arrive so the full payload is never buffered.- */-export async function uploadResumableStream(- readable: ReadableStream<Uint8Array<ArrayBuffer>>,- sessionUrl: string,-): Promise<void> {- const reader = readable.getReader();- let pending: Uint8Array<ArrayBuffer> = new Uint8Array(0);- let uploaded = 0;- let done = false;-- while (!done) {- const { value, done: streamDone } = await reader.read();- if (streamDone) {- done = true;- } else if (value) {- pending = concatBytes(pending, value);- }-- while (pending.length >= RESUMABLE_CHUNK_SIZE) {- await putResumableChunk(sessionUrl, pending.subarray(0, RESUMABLE_CHUNK_SIZE), uploaded, null);- uploaded += RESUMABLE_CHUNK_SIZE;- pending = pending.subarray(RESUMABLE_CHUNK_SIZE);- }- }-- await putResumableChunk(sessionUrl, pending, uploaded, uploaded + pending.length);-}--async function putResumableChunk(- sessionUrl: string,- chunk: Uint8Array<ArrayBuffer>,- offset: number,- total: number | null,-): Promise<void> {- const last = offset + chunk.length - 1;- const contentRange =- total === null- ? `bytes ${offset}-${last}/*`- : chunk.length === 0- ? `bytes */${total}`- : `bytes ${offset}-${last}/${total}`;-- const res = await fetch(sessionUrl, {- method: "PUT",- headers: { "Content-Range": contentRange },- body: chunk,- });-- // 308 "Resume Incomplete" is expected while more chunks remain; the final- // chunk finalizes the object and returns 200/201.- if (total === null) {- if (res.status !== 308) {- throw new Error(`Unexpected status during resumable upload: ${res.status}`);- }- } else if (!res.ok) {- throw new Error(`Failed to finalize resumable upload: ${res.status}`);- }-}--function concatBytes(a: Uint8Array<ArrayBuffer>, b: Uint8Array<ArrayBuffer>): Uint8Array<ArrayBuffer> {- const out = new Uint8Array(a.length + b.length);- out.set(a);- out.set(b, a.length);- return out;-}diff --git a/apps/web/src/pages/editor.tsx b/apps/web/src/pages/editor.tsxindex 28a6b752..ed279bf3 100644--- a/apps/web/src/pages/editor.tsx+++ b/apps/web/src/pages/editor.tsx@@ -19,7 +19,6 @@ import { getDocumentEditor } from '@/engine/editor';import { getEditHistory } from '@/engine/history';import { setInspectEntries } from '@/engine/inspect';import { attachLibrary, isLibraryFile } from '@/engine/library';-import { attachAi } from '@/utils/gen-ai';import { attachProjectConfig, isProjectConfigFile } from '@/engine/project-config';import { loadProjectBundle, rememberProjectBundle } from '@/lib/db';import { isCacheFile } from '@diffusionstudio/assets';@@ -59,8 +58,6 @@ export function EditorPage() {// The library first: a mounted project's `src` values name its assets.const library = attachLibrary(world, dir);- // The generation service over it: what `generate.*` sources resolve through.- attachAi(world, library, dir);// The project's own settings (package.json `diffusion`), next to the scene.const config = attachProjectConfig(world, dir);diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsdeleted file mode 100644index 7a73fd81..00000000--- a/apps/web/src/utils/gen-ai.ts+++ /dev/null@@ -1,599 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { getAssetSpec, isAssetRef, isTransformSpec, isTransformType, parseSource } from "@diffusionstudio/jsx";-import {- Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,- Paint, PaintType, Project, Source,-} from "@diffusionstudio/runtime";-import { createEncoder } from "@diffusionstudio/encoder";-import { createCapture } from "@/engine/capture";-import { assetName, GENERATED_DIR, isPartialAsset } from "@diffusionstudio/assets";-import {- PROMPT_INPUT_AUDIO_MODEL_OPTIONS,- PROMPT_INPUT_IMAGE_MODEL_OPTIONS,- PROMPT_INPUT_VIDEO_MODEL_OPTIONS,- PROMPT_INPUT_VOICE_MODEL,- PROMPT_INPUT_VOICE_OPTIONS,-} from "@/components/genai/config";-import { assert, mimeTypeToExtension } from "@/utils";-import { uploadBlob } from "@/lib/uploads";-import { track } from "@/lib/analytics";-import { trpc } from "@/lib/trpc";-import { toast } from "somoto";--import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput, GenerateSpec, TransformType } from "@diffusionstudio/jsx";-import type { Asset, AssetLibrary, AssetType, PartialAsset, ReserveOptions } from "@diffusionstudio/assets";-import type { FileRef } from "@diffusionstudio/api-contract";-import type { ExportResult } from "@diffusionstudio/encoder";-import type { Entity, World } from "koota";--/** What a failed generation is called where the user reads about it. */-const FAILURE_TITLES: Record<GenerateSpec["type"] | "transcript", string> = {- image: "Image generation failed",- video: "Video generation failed",- voice: "Voice generation failed",- audio: "Audio generation failed",- transcript: "Caption generation failed",-};--/** What each generation is to become, for the partial that stands for it. */-const ASSET_TYPES: Record<GenerateSpec["type"], AssetType> = {- image: "IMAGE",- video: "VIDEO",- voice: "AUDIO",- audio: "AUDIO",-};--/** What each transform's failure is called where the user reads about it, and what names its result. */-const TRANSFORMS: Record<TransformType, { title: string; suffix: string }> = {- removeBackground: { title: "Background removal failed", suffix: "Background removed" },- upscale: { title: "Upscale failed", suffix: "Upscaled" },- addAudio: { title: "Adding audio failed", suffix: "With audio" },-};--/**- * A spec with defaults applied and every `AssetInput` reduced to an asset id.- * Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.- * A transform's key is its type and input alone: the endpoints take no model- * and no factor, so every call over the same asset is the same call.- */-type ResolvedGeneration =- | { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }- | { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }- | { type: "voice"; model: string; prompt: string; voice: string; seed?: number }- | { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };-type ResolvedTransform = { type: TransformType; inputId: string };-type ResolvedSpec = ResolvedGeneration | ResolvedTransform;--const isResolvedTransform = (spec: ResolvedSpec): spec is ResolvedTransform => isTransformType(spec.type);--/** What the partial standing for a run is called and is to become, and what its failure is called. */-type RunDescription = Omit<ReserveOptions, "key" | "folder"> & { title: string };--/**- * A failure the user has already been told of — as a toast when it happened,- * and in the library's record from then on. It goes up the dependency graph- * as it is: a declaration whose input failed is not a second thing gone- * wrong.- */-class ReportedError extends Error {}--/** Creates the project's GenAi over `library` and attaches it as the world's Ai. */-export function attachAi(world: World, library: AssetLibrary, dir?: string): EditorGenAi {- const ai = new EditorGenAi(library, world.get(Project)?.id ?? "project", dir);- world.set(Ai, ai);- return ai;-}--export class EditorGenAi extends GenAi {- private readonly library: AssetLibrary;- /** Prefixes upload keys, so referenced assets land project-unique in the bucket. */- private readonly projectId: string;- /** The project's folder, so a transcription's capture compiles the sources as they are now. */- private readonly dir?: string;-- /**- * Declarations already resolved, keyed by ref identity — a ref consumed by- * several elements resolves (and validates) once. A failed one is- * forgotten, so that asking again — once its record is gone from the- * library — asks the library again rather than this map.- */- private readonly memo = new Map<AssetRef, Promise<Asset>>();- /** Runs in flight, keyed by generation key. */- private readonly inflight = new Map<string, Promise<Asset>>();-- public constructor(library: AssetLibrary, projectId: string, dir?: string) {- super();- this.library = library;- this.projectId = projectId;- this.dir = dir;- }-- /** Identical concurrent declarations collapse to one request. */- public resolve(ref: AssetRef): Promise<Asset> {- assert(isAssetRef(ref), "Not a generate.* declaration");-- let promise = this.memo.get(ref);- if (!promise) {- promise = this.generateFromRef(ref);- promise.catch(() => this.memo.delete(ref));- this.memo.set(ref, promise);- }- return promise;- }-- /**- * Transcribes the scene's audible mix for a `<captions>` element (see the- * runtime's asset system). Keyed by scene id + seed: the same pair is the- * same transcript asset across sessions, and a new seed transcribes the- * scene again.- */- public transcribe(world: World, scene: Entity, seed: number): Promise<Asset> {- const key = transcriptKey(scene, seed);- return this.generated(- key,- () => ({ type: "TRANSCRIPT", name: `${this.nextCaptionsName()}.json`, title: FAILURE_TITLES.transcript }),- (partial) => this.runTranscription(world, scene, key, partial),- );- }-- /**- * The library's answer for `key`, or the run that produces one. An asset- * the key landed as is returned; a key standing in error rejects with the- * recorded reason — answered, not run or paid for again, and not toasted- * again either; a run in flight is joined. Otherwise the run starts, with- * a partial document reserved for it first (a pending one the last session- * never finished is taken over), and ends either as the asset that- * replaces the partial or as the partial's recorded failure.- */- private async generated(- key: string,- describe: () => RunDescription,- run: (partial: PartialAsset) => Promise<Asset>,- ): Promise<Asset> {- const known = this.library.generated(key);- if (known && !isPartialAsset(known)) return known;- if (known?.state === "error") throw new ReportedError(known.error || "Generation failed");-- const running = this.inflight.get(key);- if (running) return await running;-- const { title, ...partial } = describe();- const promise = this.library.reserve({ key, folder: GENERATED_DIR, ...partial }).then(async (reserved) => {- try {- return await run(reserved);- } catch (error) {- const failure = reportFailure(error, title);- this.library.fail(reserved, failure.message);- throw failure;- }- });- this.inflight.set(key, promise);- try {- return await promise;- } finally {- this.inflight.delete(key);- }- }-- private async generateFromRef(ref: AssetRef): Promise<Asset> {- const spec = getAssetSpec(ref);- const title = isTransformSpec(spec) ? TRANSFORMS[spec.type].title : FAILURE_TITLES[spec.type];-- // Inputs first: their ids are part of the key. An input that failed has- // said so already; anything else wrong with them is said here.- let resolved: ResolvedSpec;- try {- resolved = await this.resolveSpec(spec);- } catch (error) {- throw error instanceof ReportedError ? error : reportFailure(error, title);- }-- return this.generated(- JSON.stringify(resolved),- () => ({ ...this.describe(resolved), title }),- (partial) => isResolvedTransform(resolved)- ? this.runTransform(resolved, partial)- : this.runGeneration(resolved, partial),- );- }-- /** What the partial standing for a run is called, and what it is to become. */- private describe(spec: ResolvedSpec): { type: AssetType; name: string } {- if (!isResolvedTransform(spec)) return { type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt) };- const input = this.library.get(spec.inputId);- assert(input, `Input asset ${spec.inputId} not found`);- return { type: input.type, name: `${stem(input)} (${TRANSFORMS[spec.type].suffix})` };- }-- private resolveInput(input: AssetInput): Promise<Asset> {- return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);- }-- private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {- if (isTransformSpec(spec)) {- const input = await this.resolveInput(spec.input);- return { type: spec.type, inputId: input.id };- }-- switch (spec.type) {- case "image": {- const refs = await Promise.all((spec.refs ?? []).map((ref) => this.resolveInput(ref)));- return {- type: "image",- model: spec.model ?? PROMPT_INPUT_IMAGE_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- seed: spec.seed,- refIds: refs.map((asset) => asset.id),- };- }- case "video": {- const [startFrame, endFrame] = await Promise.all([- spec.startFrame !== undefined ? this.resolveInput(spec.startFrame) : undefined,- spec.endFrame !== undefined ? this.resolveInput(spec.endFrame) : undefined,- ]);- return {- type: "video",- model: spec.model ?? PROMPT_INPUT_VIDEO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio ?? "16:9",- duration: spec.duration ?? 5,- audio: spec.audio ?? false,- seed: spec.seed,- startFrameId: startFrame?.id,- endFrameId: endFrame?.id,- };- }- case "voice": {- return {- type: "voice",- model: PROMPT_INPUT_VOICE_MODEL,- prompt: spec.prompt,- voice: spec.voice ?? PROMPT_INPUT_VOICE_OPTIONS[0].value,- seed: spec.seed,- };- }- case "audio": {- return {- type: "audio",- model: spec.model ?? PROMPT_INPUT_AUDIO_MODEL_OPTIONS[0].id,- prompt: spec.prompt,- duration: spec.duration,- seed: spec.seed,- };- }- }- }-- /**- * Runs a generation and stores its first result in place of `partial`. A- * spec a model cannot take fails here rather than before the partial is- * reserved, so the refusal is recorded like any other and not asked again.- */- private async runGeneration(spec: ResolvedGeneration, partial: PartialAsset): Promise<Asset> {- if (spec.type === "video") checkVideoConstraints(spec);-- const startedAt = performance.now();- track("generation_started", {- mode: spec.type,- model: spec.model,- prompt_length: spec.prompt.length,- ...("aspectRatio" in spec ? { aspect_ratio: spec.aspectRatio } : {}),- ...(spec.type === "image" ? { reference_count: spec.refIds.length } : {}),- });-- try {- const { name, results, generationId } = await this.requestGeneration(spec);- assert(results.length > 0, "No results returned from the model");-- const asset = await this.store(results[0].url, name, { key: partial.generation.key, id: generationId });- track("generation_completed", {- mode: spec.type,- model: spec.model,- duration_ms: Math.round(performance.now() - startedAt),- });- return asset;- } catch (err) {- track("generation_failed", {- mode: spec.type,- model: spec.model,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : "unknown",- });- throw err;- }- }-- /** Encodes the scene's audio, transcribes it, and stores the transcript in place of `partial`. */- private async runTranscription(world: World, scene: Entity, key: string, partial: PartialAsset): Promise<Asset> {- assert(sceneHasAudio(world, scene), "No audio found. Add an audio or video clip to the scene to generate captions.");-- // The scene's own capture world: the project rendered again, reduced to- // this scene, with nothing drawn — see `createCapture`.- const capture = await createCapture(world, scene, { mode: "offline-audio", dir: this.dir });- let result: ExportResult;- try {- const encoder = await createEncoder(capture.world, {- format: "ogg",- video: { enabled: false },- audio: { enabled: true, codec: "opus", sampleRate: 24000 },- });- result = await encoder.render();- } finally {- capture.dispose();- }- assert(result.type === "success" && result.data !== undefined, "Failed to encode the scene audio");-- const uploadId = crypto.randomUUID();- const audioFile = new File([result.data], `${uploadId}.ogg`, { type: "audio/ogg" });- const fileRef = await uploadBlob(audioFile, uploadId);- assert(fileRef, "Failed to upload the scene audio for transcription");-- const { results: transcript } = await trpc.transcribe.mutate({ audio: fileRef });- assert(- transcript.length > 0 && transcript.some((segment) => segment.words.length > 0),- "No speech detected. The audio does not appear to contain recognizable speech.",- );-- const blob = new Blob([JSON.stringify(transcript)], { type: "application/json" });- return this.library.store(blob, {- name: assetName(partial),- folder: GENERATED_DIR,- generation: { key },- });- }-- /** The next free `Captions N`, counting the takes still in flight. */- private nextCaptionsName(): string {- let max = 0;- for (const entry of [...this.library.list(), ...this.library.partials()]) {- const match = assetName(entry).match(/^Captions (\d+)\.json$/);- if (match) max = Math.max(max, Number(match[1]));- }- return `Captions ${max + 1}`;- }-- /**- * Uploads the transform's input, runs the call, and stores the result in- * place of `partial`. A transform over the wrong kind of asset fails here,- * like a spec a model cannot take: recorded, and not asked again.- */- private async runTransform(spec: ResolvedTransform, partial: PartialAsset): Promise<Asset> {- const asset = this.library.get(spec.inputId);- assert(asset, `Input asset ${spec.inputId} not found`);- checkTransformInput(spec.type, asset);-- const startedAt = performance.now();- track("generation_started", { mode: spec.type });-- try {- const input = await this.uploadInput(asset.id);- const { url, generationId } = await this.requestTransform(spec.type, asset, input);- const stored = await this.store(url, assetName(partial), { key: partial.generation.key, id: generationId });-- track("generation_completed", {- mode: spec.type,- duration_ms: Math.round(performance.now() - startedAt),- });- return stored;- } catch (err) {- track("generation_failed", {- mode: spec.type,- duration_ms: Math.round(performance.now() - startedAt),- error: err instanceof Error ? err.message.slice(0, 200) : "unknown",- });- throw err;- }- }-- private requestTransform(type: TransformType, asset: Asset, input: FileRef) {- switch (type) {- case "removeBackground":- return trpc.removeBackground.mutate({ image: input });- case "upscale":- return isMoving(asset.type)- ? trpc.upscaleVideo.mutate({ video: input })- : trpc.upscaleImage.mutate({ image: input });- case "addAudio":- return trpc.addAudioToVideo.mutate({ video: input });- }- }-- /**- * Downloads what a model returned and files it in the library. The name is- * the model's, the extension the result's — what came back decides what the- * file is called, not what was asked for.- */- private async store(url: string, name: string, generation: { key: string; id?: string | null }): Promise<Asset> {- const response = await fetch(url);- assert(response.ok, `Failed to fetch the generated asset: ${response.status}`);- const blob = await response.blob();-- return this.library.store(blob, {- name: name + resultExtension(url, blob),- folder: GENERATED_DIR,- generation,- });- }-- private requestGeneration(spec: ResolvedGeneration) {- switch (spec.type) {- case "image": {- return (async () => {- const images = spec.refIds.length > 0- ? await Promise.all(spec.refIds.map((id) => this.uploadInput(id)))- : undefined;-- return await trpc.generateImage.mutate({- model: spec.model,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio,- count: 1,- seed: spec.seed,- images,- });- })();- }- case "video": {- return (async () => {- const [startFrame, endFrame] = await Promise.all([- spec.startFrameId ? this.uploadInput(spec.startFrameId) : undefined,- spec.endFrameId ? this.uploadInput(spec.endFrameId) : undefined,- ]);-- return await trpc.generateVideo.mutate({- model: spec.model,- prompt: spec.prompt,- aspectRatio: spec.aspectRatio,- duration: spec.duration,- generateAudio: spec.audio,- seed: spec.seed,- startFrame,- endFrame,- });- })();- }- case "voice": {- return trpc.textToSpeech.mutate({- model: spec.model,- prompt: spec.prompt,- voice: spec.voice,- seed: spec.seed,- });- }- case "audio": {- return trpc.generateSound.mutate({- model: spec.model,- prompt: spec.prompt,- duration: spec.duration,- seed: spec.seed,- });- }- }- }-- /** Uploads a referenced asset for the model to read; the bucket key is project-unique. */- private async uploadInput(assetId: string) {- const asset = this.library.get(assetId);- assert(asset, `Referenced asset ${assetId} not found`);- const uploaded = await uploadBlob(await getAssetFile(asset), `${this.projectId}-${assetId}`);- assert(uploaded, `Failed to upload referenced asset ${assetId}`);- return uploaded;- }-}--/** Whether an asset type is footage, for the calls that only take footage. */-const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";--/** An asset's name without its extension: what a transform's result is named after. */-const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, "");--/** What a transform can be put over; known only once the input has resolved. */-function checkTransformInput(type: TransformType, input: Asset): void {- switch (type) {- case "removeBackground":- assert(input.type === "IMAGE", "Only a picture has a background to remove");- return;- case "upscale":- assert(input.type === "IMAGE" || isMoving(input.type), "Only a picture or footage can be upscaled");- return;- case "addAudio":- assert(isMoving(input.type), "Only footage can be scored");- return;- }-}--/** How much of a prompt names the partial standing for its generation. */-const PROVISIONAL_NAME_LENGTH = 48;--/**- * What a generation is called before its result names it: the head of its- * prompt, cut at a word. The model's own name takes over when the bytes- * land (see `store`).- */-function provisionalName(prompt: string): string {- const text = prompt.trim().replace(/\s+/g, " ").replace(/\//g, "-");- if (text.length <= PROVISIONAL_NAME_LENGTH) return text || "Generation";- const cut = text.lastIndexOf(" ", PROVISIONAL_NAME_LENGTH);- return `${text.slice(0, cut > 0 ? cut : PROVISIONAL_NAME_LENGTH)}…`;-}--/**- * What to call a generated result: the extension its type implies, falling- * back to the one its URL carries. Neither is guaranteed — some models hand- * back a plain slug (`.../files/young-man-city-skyline`), some servers say- * only `application/octet-stream` — so this can come back empty, and the- * library reads the bytes instead. The name is what the file is identified- * by once on disk, so getting it right saves that guess.- */-function resultExtension(url: string, blob: Blob): string {- const fromType = blob.type ? mimeTypeToExtension(blob.type) : ".bin";- if (fromType !== ".bin") return fromType;-- const fileName = url.split(/[?#]/)[0]!.split("/").pop() ?? "";- const dot = fileName.lastIndexOf(".");- const fromUrl = dot > 0 ? fileName.slice(dot + 1) : "";- return fromUrl && fromUrl.length <= 5 ? `.${fromUrl}` : "";-}--/**- * Says what a generation failed with, and hands the failure on as one- * already reported. The message is what the library records on the partial- * and what the element carries (the runtime's `SourceError`), so the- * sentence in the asset panel, the one on the canvas and the one in the- * toast are the same. The toast is keyed by it: variants that failed the- * same way are one thing gone wrong rather than four, and a render that ran- * into the same wall does not stack up.- */-function reportFailure(error: unknown, title: string): ReportedError {- const message = error instanceof Error ? error.message : String(error);-- console.error(`[gen-ai] ${title}:`, error);- toast.error(title, {- id: `gen-ai:${title}:${message}`,- description: message,- });-- return new ReportedError(message);-}--/**- * The transcript cache key: scene id + seed. The scene's durable name is the- * id in its source stamp (`<file>:<id>`, stamped once by the compiler — the- * same identity the project config keys by); a scene without one falls back- * to its entity id, which only holds within the session.- */-function transcriptKey(scene: Entity, seed: number): string {- const source = scene.get(Source)?.value;- const locator = source ? parseSource(source)?.locator : undefined;- const sceneId = typeof locator === "string" ? locator : source ?? String(scene.id());- return `transcript:v1:${sceneId}:${seed}`;-}--/**- * Whether anything in the scene contributes to its audible mix: an unmuted,- * unhidden audio clip, video, or video paint with its asset bound.- */-function sceneHasAudio(world: World, scene: Entity): boolean {- for (const entity of getEntityTree(world, scene)) {- if (entity.has(Hidden) || entity.has(Muted) || !entity.has(AssetId)) continue;- if (entity.has(Audio) || entity.get(Paint)?.value === PaintType.VIDEO) return true;- }- return false;-}--/**- * Per-model constraints (`diffusion models video`); unknown models are left to the server.- */-function checkVideoConstraints(spec: Extract<ResolvedGeneration, { type: "video" }>): void {- const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);- if (!model) return;-- assert(model.aspectRatios.includes(spec.aspectRatio), `${spec.model} does not support aspect ratio ${spec.aspectRatio}`);- assert(model.durations.includes(`${spec.duration}s`), `${spec.model} does not support a duration of ${spec.duration}s`);- assert(!spec.audio || model.features.includes("audio"), `${spec.model} does not support audio generation`);- assert(spec.endFrameId === undefined || model.features.includes("end-frame"), `${spec.model} does not support an end frame`);-}diff --git a/docs/INSTRUCTIONS.md b/docs/INSTRUCTIONS.mdindex 75039307..6c66453a 100644--- a/docs/INSTRUCTIONS.md+++ b/docs/INSTRUCTIONS.md@@ -2,5 +2,5 @@ Diffusion Studio, a video editor, is running on this machine and you are connectBefore starting on the work itself, read the skill page that covers it, and follow it for the rest of the session:-- `skills/editor.md` (the default) — Understand, generate, and edit footage with Diffusion Studio: analyze video/audio/images, generate them with AI, and compose video compositions. Use for any media analysis, media generation, or video editing task.+- `skills/editor.md` (the default) — Understand and edit footage with Diffusion Studio: analyze video/audio/images and compose video compositions. Use for any media analysis or video editing task.- `skills/watch.md` — Watch and understand footage with Diffusion Studio: answer questions about a video or audio file, summarize it, find scenes and moments, pull quotes, and describe what happens and when. Use whenever the user asks what's in a piece of footage, wants a summary or recap, wants to locate a moment, or needs a claim about a video or audio file checked.diff --git a/docs/examples/02-genai.tsx b/docs/examples/02-genai.tsxdeleted file mode 100644index 9b93e18e..00000000--- a/docs/examples/02-genai.tsx+++ /dev/null@@ -1,74 +0,0 @@-/* Multi-stage generation: text-to-image keyframes, image-to-video motion,- * a text-to-speech voiceover, generated ambience, and captions on top.- *- * cp examples/02-genai.tsx ~/Projects/genai/index.tsx- * diffusion open ~/Projects/genai- *- * `generate.*` declarations are pure values; nothing generates until commit.- * Refs form a dependency graph (keyframe -> matching second keyframe -> two- * motion clips) that generates in topological order, blocking the command- * until every asset lands. <captions> reads the finished scene audio after- * generation, so caption timing matches the voiceover. Consumes credits;- * results are cached in the project's library, so re-mounting unchanged- * specs is free.- * Discover models and voices with `diffusion models <type>` and `diffusion voices`;- * omitted here, so each stage uses the default model.- */--import { generate } from "@diffusionstudio/jsx";--const SEED = 7;--const keyframe = generate.image({- prompt: "Lighthouse on a basalt cliff at dawn, low fog rolling in, cinematic wide shot",- aspectRatio: "16:9",- seed: SEED,-});--// second keyframe references the first, keeping the location consistent-const counterShot = generate.image({- prompt: "The same lighthouse seen from the open sea, waves in the foreground, dawn light",- aspectRatio: "16:9",- refs: [keyframe],- seed: SEED,-});--const shot1 = generate.video({- prompt: "slow aerial push-in toward the lighthouse, fog drifting",- startFrame: keyframe,- duration: 5,-});--const shot2 = generate.video({- prompt: "gentle drift across the waves, lighthouse holding in the distance",- startFrame: counterShot,- duration: 5,-});--const voiceover = generate.voice({- prompt:- "Long before radar, a single beam of light was the only promise sailors had that land was near.",-});--const ambience = generate.audio({- prompt: "distant ocean waves, low wind, calm coastal ambience",- duration: 10,-});--export default function GenAi() {- return (- <stage camera={[0.3, 0, 0, 0.3, 85, 150]}>- <scene name="GenAI" width={1920} height={1080} fill="black" active>- <sequence name="Generated shots">- <video src={shot1} width={1920} height={1080} start={0} end={5} transition={{}} />- <video src={shot2} width={1920} height={1080} start={5} end={10} />- </sequence>-- <audio name="Voiceover" src={voiceover} start={1} />- <audio name="Ambience" src={ambience} start={0} end={10} volume={-18} />-- <captions preset="spotlight" verticalAlign="bottom" />- </scene>- </stage>- );-}diff --git a/docs/examples/README.md b/docs/examples/README.mdindex cf9c6f19..6d20e3e6 100644--- a/docs/examples/README.md+++ b/docs/examples/README.md@@ -13,7 +13,6 @@ diffusion open ~/Projects/basics| Example | Shows || --- | --- || [01-basics.tsx](01-basics.tsx) | `<stage>` / `<scene>`, `<sequence>` with a dissolve, `<video>`, `<audio>`, `<image>`, `<text>` titles from data via `<For>`, `<animation>` children |-| [02-genai.tsx](02-genai.tsx) | multi-stage generation: `generate.image` refs feeding `generate.video`, TTS voiceover, generated ambience, `<captions>` || [03-ticker.tsx](03-ticker.tsx) | declarative animation: `useTicker` + `createMemo` derived values driving props || [04-html-in-canvas.tsx](04-html-in-canvas.tsx) | `<htmlPaint>`: an AI prompt box as real DOM, typed out from the playhead || [05-anime-timeline.tsx](05-anime-timeline.tsx) | anime.js timeline seeked from `useTicker`, driving an ECS node and `<html>` content in lockstep |@@ -24,8 +23,7 @@ diffusion open ~/Projects/basics| [10-typegpu.tsx](10-typegpu.tsx) | TypeGPU on a `<surface>`: shaders written in TypeScript (`'use gpu'`), compiled through the project's own [babel config](../reference/jsx/module.md#compile-time-plugins-babel-config) || [11-redraw.tsx](11-redraw.tsx) | [Redraw](https://redraw.dev) on a `<surface>`: the docs' Hello World write-on stroke, a vendored-tarball package driven by composition time; the surface spans one animation cycle, `@inspect` variables tune it, and `useResolution` keeps it sharp at any export size |-Requirements: `02-genai.tsx` consumes generation credits (results are cached per session);-`01-basics.tsx`, `06-three.tsx`, and `08-shader-paint.tsx` fetch remote media.+Requirements: `01-basics.tsx`, `06-three.tsx`, and `08-shader-paint.tsx` fetch remote media.`10-typegpu.tsx` and `11-redraw.tsx` need packages installed in the project folderand a babel config next to the entry — the commands are in their header comments.`11-redraw.tsx` additionally needs the `redraw` tarball vendored into the projectdiff --git a/docs/guides/walkthroughs/podcast-clip.md b/docs/guides/walkthroughs/podcast-clip.mdindex 213ada01..6c499dca 100644--- a/docs/guides/walkthroughs/podcast-clip.md+++ b/docs/guides/walkthroughs/podcast-clip.md@@ -104,16 +104,16 @@ const speakerX = () =>## 8. Captions-Add captions last, after the trim and framing are verified. Use the **`classic`** preset centred — it is the first choice for vertical content.+Add captions last, after the trim and framing are verified. `src` names the episode's subtitle file (`.srt`, `.vtt`, or a transcript `.json`). Use the **`classic`** preset centred — it is the first choice for vertical content.```tsx-<captions preset="classic" verticalAlign="center" />+<captions src="episode.srt" preset="classic" verticalAlign="center" />```If the caption block lands on the speakers' faces, push it off with `offsetY` rather than changing the framing you just verified:```tsx-<captions preset="classic" verticalAlign="center" offsetY={420} />+<captions src="episode.srt" preset="classic" verticalAlign="center" offsetY={420} />```Capture a frame per caption line with `capture` and check readability at delivery size — a caption over a mouth is worse than no caption.diff --git a/docs/guides/walkthroughs/talking-head.md b/docs/guides/walkthroughs/talking-head.mdindex 436c5e7b..f1e30648 100644--- a/docs/guides/walkthroughs/talking-head.md+++ b/docs/guides/walkthroughs/talking-head.md@@ -97,9 +97,9 @@ One line of copy is a job for the native `<text>` tag:## 7. Captions (optional)-Add captions **last**, after every cut and overlay is verified, so they transcribe the final audio at its final placement. Use the **`whisper`** preset aligned to the bottom, and start it where the hook ends — set both `start` and `sourceIn` to that timestamp so the transcript stays aligned (see `reference/jsx/captions.md` in the project's docs). Keep captions off overlays and off the important part of the A-roll.+Add captions **last**, after every cut and overlay is verified. `src` names the recording's subtitle file (`.srt`, `.vtt`, or a transcript `.json`). Use the **`whisper`** preset aligned to the bottom, and start it where the hook ends — set both `start` and `sourceIn` to that timestamp so the transcript stays aligned (see `reference/jsx/captions.md` in the project's docs). Keep captions off overlays and off the important part of the A-roll.```tsx{/* Hook holds until 00:03; captions begin there. */}-<captions preset="whisper" verticalAlign="bottom" start={3} sourceIn={3} />+<captions src="talk.srt" preset="whisper" verticalAlign="bottom" start={3} sourceIn={3} />```diff --git a/docs/reference/jsx/README.md b/docs/reference/jsx/README.mdindex 464bcce4..4ad9799e 100644--- a/docs/reference/jsx/README.md+++ b/docs/reference/jsx/README.md@@ -40,7 +40,7 @@ export default function Project() {| [html.md](./html.md) | `<html>`: reactive HTML children drawn into the box || [surface-paint.md](./surface-paint.md) | `<surface>`: a ref-provided canvas you draw into, sampled every frame || [shader-paint.md](./shader-paint.md) | `<shaderPaint>`: a WGSL fragment shader over the media paint below it |-| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`), image sequences, transforms, adding an asset to the library |+| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids), image sequences, adding an asset to the library || [timing.md](./timing.md) | `start` / `end` / `sourceIn` / `sourceOut` / `playbackRate`, time formats || [keyframes.md](./keyframes.md) | Keyframe animation and easing || [animations.md](./animations.md) | `<animation>`: preset in/out animations |@@ -49,7 +49,6 @@ export default function Project() {| [adjustment-layer.md](./adjustment-layer.md) | `<adjustmentLayer>`: a transform over the clip below it || [audio-sync.md](./audio-sync.md) | `syncTo` audio alignment || [captions.md](./captions.md) | `<captions>` and style presets |-| [generate.md](./generate.md) | Declarative AI asset generation (`generate.*`) || [variables.md](./variables.md) | `@inspect` variables: annotated consts as live inspector controls (number, color, text, font, boolean, select) || [lifecycle.md](./lifecycle.md) | Mount lifecycle: always live, persisted + re-executed, `useTicker`, `useResolution` || [errors.md](./errors.md) | Where each pipeline stage fails and with what effect |@@ -62,7 +61,7 @@ Everything below runs in the app, on open and again on every save of a project f2. **Compile**: the entry file bundles with esbuild + `babel-preset-solid` in `universal` mode, so JSX compiles against the editor's renderer runtime instead of the DOM. `solid-js`, `solid-js/store` and `@diffusionstudio/jsx` are external — the app supplies its own instances; everything else is bundled. A compile error is reported and the canvas keeps the last good render.3. **Evaluate**: the app imports the module. Top-level code runs to completion. The module's **default export** is the project component.4. **Mount**: the component tree renders **directly into the world**. There is no staging tree: each element materializes as an entity as it is created, and the root `<stage>` is the one already there. Mounting is synchronous, and a throw part-way through leaves nothing behind — the document is disposed and the previous render stays on the canvas.-5. **Resolve**: every `src` resolves in the background — a path or asset id is loaded, a `generate.*` declaration is generated in dependency order, a `<captions>` without a `src` transcribes its scene. Each element renders a generating state until its asset lands, and an element that fails carries the reason (see [generate.md](./generate.md), [errors.md](./errors.md)). Non-blocking: the composition is on the canvas and editable while this runs, and [`context`](../tools/context.md) reports where each generation stands.+5. **Resolve**: every `src` — a library path, asset id, URL, or absolute path — loads in the background, and an element whose source fails to load carries the reason (see [errors.md](./errors.md#failed-sources)). Non-blocking: the composition is on the canvas and editable while this runs.6. **Live**: the reactive graph keeps running for as long as the project is open — signals, effects, `useTicker` (see [lifecycle.md](./lifecycle.md)). Export, capture and the next open re-execute the same module.A save replaces the whole run: the old mount is disposed and the new one takes the stage. Scenes are rebuilt rather than accumulated, and the entities the previous render owned go with it.diff --git a/docs/reference/jsx/audio-sync.md b/docs/reference/jsx/audio-sync.mdindex 8605fc22..905c279d 100644--- a/docs/reference/jsx/audio-sync.md+++ b/docs/reference/jsx/audio-sync.md@@ -19,4 +19,4 @@## Execution-Alignment runs at the resolve stage of the [pipeline](./README.md#pipeline), after both sides' assets have landed (either side may be generated) and before captions read the scene. It is local and consumes no credits. A correlation too weak to trust is reported rather than guessed at (see [errors.md](./errors.md)), and the node keeps its default placement. Offsets are **cached** by the pair of source contents, so re-mounting an unchanged project re-measures nothing.+Alignment runs at the resolve stage of the [pipeline](./README.md#pipeline), after both sides' assets have loaded. It is local and consumes no credits. A correlation too weak to trust is reported rather than guessed at (see [errors.md](./errors.md)), and the node keeps its default placement. Offsets are **cached** by the pair of source contents, so re-mounting an unchanged project re-measures nothing.diff --git a/docs/reference/jsx/audio.md b/docs/reference/jsx/audio.mdindex 09f33cce..0efe845b 100644--- a/docs/reference/jsx/audio.md+++ b/docs/reference/jsx/audio.md@@ -12,7 +12,7 @@ It draws nothing inside a scene, but on the canvas it is still something to poin| Prop | Type | Default | Meaning || ---- | ---- | ------- | ------- |-| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |+| `src` | `string` | **required** | See [media.md](./media.md). || `id`, `name` | `string` | see [elements.md](./elements.md#common-props) | Address and label. || `x`, `y` | `number` | `0` | Where the waveform box sits on the canvas. No meaning inside a scene. || `width`, `height` | `number` | `500`, `150` | Size of that box. |diff --git a/docs/reference/jsx/captions.md b/docs/reference/jsx/captions.mdindex 9e700294..7cefd2a8 100644--- a/docs/reference/jsx/captions.md+++ b/docs/reference/jsx/captions.md@@ -1,33 +1,24 @@# Captions-`<captions />` inside a scene transcribes that scene's audio into a caption node: a styled, timed transcript.+`<captions src>` inside a scene mounts a transcript as a caption node: a styled, timed transcript.```tsx<scene name="Talk" width={1920} height={1080}><video src="a-roll/take-3.mp4" width={1920} height={1080} />- <captions preset="whisper" />+ <captions src="transcripts/take-3.srt" preset="whisper" /></scene>```-Transcription is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once ready. Because it reads the scene's audible mix, it waits until **every other source in the scene has landed** — a generated `voice` or `audio` track is transcribed at its final placement, not at the placeholder. The scene must contain an unmuted, unhidden audio or video source; without one the node carries an error saying so, recorded in the library like any failed generation (see [errors.md](./errors.md#failed-sources)).--## Bringing your own transcript--Give `src` a transcript file — `.srt`, `.vtt`, or a transcript `.json` — and it is mounted as it is; no transcription runs and no credits are spent. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL), except that `generate.*` is not accepted.--```tsx-<captions src="transcripts/interview.srt" preset="classic" />-```+`src` is **required**: a transcript file — `.srt`, `.vtt`, or a transcript `.json` — or an audio or video asset that carries a transcript. It resolves like any other [`src`](./media.md) (library path, asset id, path, URL) and is mounted as it is; `<captions>` does not transcribe its scene. Loading is **asynchronous and non-blocking**: the caption node is on the canvas from the moment the project mounts and its transcript attaches once loaded. A source that fails to load leaves the node carrying the reason (see [errors.md](./errors.md#failed-sources)).## Props| Prop | Type | Default | Meaning || ---- | ---- | ------- | ------- |-| `src` | `string` | none | A transcript file to mount instead of transcribing the scene. |+| `src` | `string` | **required** | The transcript to mount: a transcript file, or an audio or video asset that carries one. || `preset` | see below | `"classic"` | Caption style preset. || `colors` | `string[]` | preset defaults | Fills the preset's color slots in order; any CSS color, alpha ignored. Ignored by presets without slots. || `verticalAlign` | `"top" \| "center" \| "bottom"` | preset default | Vertical placement of the caption block: anchored to the top or bottom safe margin, or centered. Horizontal placement stays with the preset. |-| `seed` | `number` | `0` | Joins the transcript cache key — see [Caching](#caching). Ignored when `src` is set. || `offsetX`, `offsetY` | `number` | `0` | Render-time nudge in px on top of the preset placement; subpixel values are kept. A slide animation drives the same channel and wins while it plays. || `start`, `end`, `sourceIn`, `sourceOut` | `Time` | full transcript | Trim which stretch of the transcript is captioned, using the same [timing](./timing.md) semantics as media nodes. See [Trimming](#trimming). || `id`, `name` | `string` | see [elements.md](./elements.md#common-props) | Address and label. |@@ -36,25 +27,15 @@ Give `src` a transcript file — `.srt`, `.vtt`, or a transcript `.json` — andThe preset positions the caption block; `verticalAlign` overrides only its vertical anchor (`whisper` and `cascade` default to `bottom`, all other presets to `center`), and `offsetX`/`offsetY` nudge the drawn result from there.-## Caching--A transcript is cached under **the scene's id and the `seed`**, and the cached asset is reused whenever that pair comes up again — so reopening a project transcribes nothing and consumes no credits.--The audio itself is not part of the key. **Recutting a scene does not re-transcribe it**: to pick up changed audio, bump `seed`. A value used before replays that take from cache; a value that has not been used for this scene transcribes it again, which costs credits.--A transcription that failed is recorded under the same key, as a library entry in error, and is not run again until that entry is removed — see [errors.md](./errors.md#failed-sources).-## TrimmingCaptions carry the same [timing](./timing.md) props as media nodes, and the transcript is source content that must stay aligned to the audio — so advance `start` and `sourceIn` together (and `end`/`sourceOut`), never `start` alone. To show only from 15 s onward, set both `sourceIn={15}` and `start={15}`. One node can't skip a gap, so to blank captions out for a middle stretch — e.g. under an overlay — use two nodes that meet at the gap:```tsx-<captions start={0} end={15} sourceIn={0} sourceOut={15} /> {/* before the overlay */}-<captions start={20} sourceIn={20} /> {/* after the overlay */}+<captions src="transcripts/take-3.srt" start={0} end={15} sourceIn={0} sourceOut={15} /> {/* before the overlay */}+<captions src="transcripts/take-3.srt" start={20} sourceIn={20} /> {/* after the overlay */}```-Both read the same cached transcript: two `<captions>` in one scene do not transcribe it twice.-## Presets`preset` selects the caption style: the same presets as the editor's caption inspector. Some presets expose **color slots**, filled in order by the `colors` prop; a missing or omitted entry falls back to the slot's default.diff --git a/docs/reference/jsx/elements.md b/docs/reference/jsx/elements.mdindex fae1961e..2996151d 100644--- a/docs/reference/jsx/elements.md+++ b/docs/reference/jsx/elements.md@@ -21,7 +21,7 @@ camelCase composition elements map 1:1 onto entities. Lowercase DOM vocabulary i| [`<video>`](./video.md) | A video clip: a rect whose intrinsic paint is the media `src` names. || [`<image>`](./image.md) | A picture: a rect whose intrinsic paint is the media `src` names. || [`<audio>`](./audio.md) | A clip with a sound and no picture; on the canvas, a waveform box. |-| [`<captions>`](./captions.md) | A styled, timed transcript of the enclosing scene (or of a transcript file). |+| [`<captions>`](./captions.md) | A styled, timed transcript, mounted from a transcript file or an asset that carries one. || [`<adjustmentLayer>`](./adjustment-layer.md) | Draws nothing; its transform composes onto the clip directly below it, for as long as its own clip lasts. || [`<html>`](./html.md) | A rect whose paint is real, reactive HTML drawn into the box by the browser. || [`<surface>`](./surface-paint.md) | A rect whose paint is a canvas your `ref` draws into, sampled every frame. |@@ -74,7 +74,7 @@ User-defined components are ordinary Solid components; they compose the elements- A `<text>` given neither `width` nor `height` sizes itself to what it says; giving it either fixes the box and wraps into it.- `<group>` and `<sequence>` never take a size: theirs is the union of their children's.-- How media pixels map into the box is controlled by `objectFit` (default `"cover"`), never by the box itself. A generated asset's placeholder therefore always has a definite size, even before the asset exists.+- How media pixels map into the box is controlled by `objectFit` (default `"cover"`), never by the box itself. A media element therefore always has a definite size, even before its asset has loaded.## Common propsdiff --git a/docs/reference/jsx/errors.md b/docs/reference/jsx/errors.mdindex 44afc16c..59b56b24 100644--- a/docs/reference/jsx/errors.md+++ b/docs/reference/jsx/errors.md@@ -6,29 +6,13 @@ Where each [pipeline](./README.md#pipeline) stage fails, and with what effect:| ----- | ----------------- | ------ || **Compile** (syntax, an unresolved import, a PascalCase composition tag, a control-flow component with no import) | A "Project failed to compile" toast, with the compiler's message; also on the app console ([`logs`](../tools/logs.md)) | Nothing is remounted. The canvas keeps the last good render, so a project in the middle of an edit is never blanked. || **Evaluate / mount** (a throw at module scope or during render, a root that is not `<stage>`, a tag the host does not know, an element parented into its own subtree) | A "Project failed to render" toast with the thrown message | The half-built document is disposed — **nothing is left behind** — and the previous render stays on the canvas. |-| **Source resolution** (a path that does not exist, an unreachable URL, a per-model constraint on `aspectRatio` / `duration` / a feature flag, a caption with no audible audio in its scene) | The element on the canvas, and `generations` in [`context`](../tools/context.md) | Per element, not per mount: everything else stays mounted and playable. The element stops showing its generating state and is left without a paint, carrying the reason — see below. |+| **Source resolution** (a path that does not exist, an unreachable URL, an unsupported file) | The element on the canvas, and `source-error` in [`check`](../tools/check.md) | Per element, not per mount: everything else stays mounted and playable. The element is left without its media, carrying the reason — see below. |Runtime errors are reported against the compiled module. Since types are stripped rather than checked, run `npx tsc --noEmit` in the project folder to catch what the compile will not.## Failed sources-A failed **generation** is recorded in the library, never in your file. The library holds a partial document for every generation the project has asked for: `pending` while the model runs, replaced by the asset when it lands, and kept as `error` — with the reason — when it does not. In `assets.yml`:--```yaml-- id: 3f9c1a7e2b8d4c05- path: generated/a red fox- type: IMAGE- generation:- key: '{"type":"image","model":"…","prompt":"a red fox",…}'- state: error- error: Model refused the prompt-```--That record is what keeps a refused or impossible generation from being run — or paid for — again by every reopen of the project: a declaration whose key stands in error resolves to that error, and its element carries the message without anything being requested. The source that declared it is not written to.--**Removing the record is what asks for the run again**, and it is the only thing that does — not another take, not another prompt. Delete the entry in the asset panel (or its record in `assets.yml`), or use *Retry* on it, which deletes the record and re-asks at once for every element still waiting on the answer. Changing the declaration is a new key, and so a new generation; the old record stays until removed.--Only generations are recorded this way: a load that failed is cheap to try again, and an asset that has since been put back should simply load.+A source fails only to load: a path that does not exist, a URL that cannot be reached, a file the app cannot decode. The element carries the reason and is drawn with a still dark-red fill in place of its media; [`check`](../tools/check.md) reports it as `source-error` with the message. Nothing is recorded in the library and nothing is written to your file: fix the path or put the asset back, and the next save or open loads it.## Blank or partial `<html>` content in capturesdiff --git a/docs/reference/jsx/generate.md b/docs/reference/jsx/generate.mddeleted file mode 100644index 5406309d..00000000--- a/docs/reference/jsx/generate.md+++ /dev/null@@ -1,107 +0,0 @@-# Generated assets--Assets that don't exist yet are **declared as values** with the `generate` namespace from `@diffusionstudio/jsx`. A declaration returns an **`AssetRef`** that is passed wherever a source is expected (`src`, `startFrame`, `endFrame`, `refs`). This makes generative content declarative: the project describes the asset it wants and the app produces it on mount.--```tsx-import { generate } from "@diffusionstudio/jsx";--const hero = generate.image({- prompt: "A neon city at night, cinematic",- model: "flux-2-turbo",- aspectRatio: "16:9",- seed: 42,-});--// one generated asset can feed another-const heroMotion = generate.video({- prompt: "slow camera push-in",- model: "kling-3-pro",- startFrame: hero,- duration: 5,-});--export default function Project() {- return (- <stage background="#161616" camera={[0.3, 0, 0, 0.3, 85, 150]}>- <scene name="Intro" width={1920} height={1080} active>- <video src={heroMotion} width={1920} height={1080} start={0} end={5} />- <image src={hero} x={40} y={40} width={200} height={112} />- </scene>- </stage>- );-}-```--Declarations are **pure**: calling `generate.*` validates its options and returns a ref; nothing is requested until an element carrying it mounts. A ref that is never used by a mounted element (directly or as an input to another asset) is never generated. Declarations may live at module scope or inside components.--Generation is **asynchronous and non-blocking**: the element is on the canvas immediately, showing a generating state, and its paint attaches when the asset lands. [`context`](../tools/context.md) reports where each one stands — generating, failed with the reason, or done with the library path it landed as — and a declaration that fails leaves its element carrying the reason, recorded in the library (see [errors.md](./errors.md#failed-sources)).--## Declaration options--[`models`](../tools/models.md) lists valid `model` ids and per-model constraints; [`voices`](../tools/voices.md) lists voices.--```ts-type AssetInput = string | AssetRef; // path, URL, asset id, or another declaration--generate.image(opts: {- prompt: string; // required- model?: string; // default: the first image model from `models`- aspectRatio?: "1:1" | "4:3" | "3:4" | "16:9" | "9:16"; // default "16:9"- refs?: AssetInput[]; // image references- seed?: number; // reproducible generation-}): AssetRef;--generate.video(opts: {- prompt: string; // required- model?: string; // default: the first video model from `models`- aspectRatio?: "1:1" | "4:3" | "3:4" | "16:9" | "9:16"; // default "16:9"- duration?: number; // whole seconds; default 5- audio?: boolean; // generate audio alongside; models with the `audio` feature only- startFrame?: AssetInput; // image used as the first frame- endFrame?: AssetInput; // image used as the last frame; `end-frame` feature only- seed?: number;-}): AssetRef;--generate.voice(opts: {- prompt: string; // required: the text to speak- voice?: string; // default: the first voice from `voices`- seed?: number;-}): AssetRef;--generate.audio(opts: {- prompt: string; // required- model?: string; // default: the first audio model from `models`- duration?: number; // seconds; default 30 for music, the model's own default for sfx- seed?: number;-}): AssetRef;-```--## Transforms--`transform.*` declares an asset made from another. Each takes what it works on as its argument — a source string or another declaration — and returns an `AssetRef` like `generate.*` does, usable anywhere a source is:--```ts-transform.upscale(input: AssetInput): AssetRef; // twice the pixels; a picture or footage — enlarges the source, not the box-transform.removeBackground(input: AssetInput): AssetRef; // the subject cut out, the rest transparent; pictures only-transform.addAudio(input: AssetInput): AssetRef; // footage scored with a generated soundtrack; independent of `volume` and `muted`-```--A chain reads inside out and runs in that order:--```tsx-<image src={transform.upscale(transform.removeBackground(generate.image({ prompt: "a red fox" })))} width={800} height={450} />-```--generates, cuts out, then enlarges. Each step is cached by step and input, so wrapping a further transform around a chain does not re-run what is inside it, and a transform over a library asset (`transform.upscale("footage/fox.png")`) leaves that asset as it is. A transform over the wrong kind of asset — a background removal over footage — fails like a spec a model cannot take (see [errors.md](./errors.md#failed-sources)).--## Dependency order--`startFrame`, `endFrame`, `refs`, and a transform's input accept other `AssetRef`s. The dependency graph is built from these values and generated in topological order, so referenced assets exist before the assets that consume them. Because dependencies are **values, not string ids**, a declaration can only reference refs that already exist; reference cycles are impossible by construction. Assets referenced only as inputs generate too, but produce no node of their own.--## Caching and idempotency--Generation is long-running and consumes credits, so results are **cached by content**: a key derived from the fully-resolved spec — `type`, `model`, `prompt`, the resolved ids of any references, `seed`, and the rest of the options, with the defaults filled in. Re-mounting an unchanged project reuses cached assets instead of regenerating, two declarations with identical specs collapse to a single asset, and identical concurrent declarations share one request; changing any option produces a new asset. Set `seed` to make a spec reproducible.--Finished generations are stored in the project's library under `generated/` with their spec key, so the cache survives app restarts — and a save that only touched an unrelated part of the file regenerates nothing. Deleting the asset from the library regenerates it on the next mount.--Where a generation stands is recorded there too: a run in flight is a `pending` entry of the library, and one that failed stays as an `error` entry carrying its reason, which answers the declaration — instead of another run — until the entry is removed (see [errors.md](./errors.md#failed-sources)).diff --git a/docs/reference/jsx/html.md b/docs/reference/jsx/html.mdindex 112d31a3..7fd6bc98 100644--- a/docs/reference/jsx/html.md+++ b/docs/reference/jsx/html.md@@ -61,7 +61,7 @@ setInterval(() => setCount((c) => c + 1), 1000);## Images-`<img>` takes the same sources as a composition [`src`](./media.md): a path, an asset id, a URL, or a [`generate.*`](./generate.md) ref. The host resolves them exactly as it does for [`<image>`](./image.md)+`<img>` takes the same sources as a composition [`src`](./media.md): a path, an asset id, or a URL. The host resolves them exactly as it does for [`<image>`](./image.md)## Propsdiff --git a/docs/reference/jsx/image.md b/docs/reference/jsx/image.mdindex 56819c75..4521cfd2 100644--- a/docs/reference/jsx/image.md+++ b/docs/reference/jsx/image.md@@ -12,7 +12,7 @@ All [common props](./elements.md#common-props), plus:| Prop | Type | Default | Meaning || ---- | ---- | ------- | ------- |-| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |+| `src` | `string` | **required** | See [media.md](./media.md). || `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. || `frameRate` | `number` | `30` | Frames per second for a `src` naming a directory of numbered frames, which plays on `<image>` as footage does (see [media.md](./media.md#image-sequences)). Nothing for a still to read. |diff --git a/docs/reference/jsx/lifecycle.md b/docs/reference/jsx/lifecycle.mdindex 0c59828b..568db028 100644--- a/docs/reference/jsx/lifecycle.md+++ b/docs/reference/jsx/lifecycle.md@@ -2,7 +2,7 @@## A mount stays live-Opening a project compiles the entry file and renders it into the document. After that the reactive graph **keeps running**: signals, effects, timers, and [`useTicker`](#useticker) keep driving the mounted entities for as long as the project is open. Updates land in the document immediately — prop writes, conditional inserts and removals (`<Show>`, `<For>`), text, and reactive `src` swaps including `generate.*`. The materialized nodes are ordinary editable entities; asset resolution is owned by the engine.+Opening a project compiles the entry file and renders it into the document. After that the reactive graph **keeps running**: signals, effects, timers, and [`useTicker`](#useticker) keep driving the mounted entities for as long as the project is open. Updates land in the document immediately — prop writes, conditional inserts and removals (`<Show>`, `<For>`), text, and reactive `src` swaps. The materialized nodes are ordinary editable entities; asset resolution is owned by the engine.A save re-runs the pipeline. The old mount is disposed and the new render takes the stage, the way reloading a page rebuilds it — scenes are rebuilt rather than accumulated, and the entities the previous render owned go with it. A compile error changes nothing: the last good render stays on the canvas and the failure is reported.diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.mdindex 264978f0..c537ae29 100644--- a/docs/reference/jsx/media.md+++ b/docs/reference/jsx/media.md@@ -6,9 +6,8 @@- **Asset id**: e.g. `"9f3a2c1d7e4b8a01"`, the content hash of a library asset (`assets.yml` lists them).- **Global path**: e.g. `"/Movies/video.mp4"`, resolved against the user's OS. Not added to the library.- **Remote URL**: e.g. `"https://my.videoarchive.com/audio/clip.wav"`, fetched on mount. Not added to the library.-- **`AssetRef`**: the value returned by a `generate.*` declaration (see [generate.md](./generate.md)). The node is mounted immediately as a placeholder and its paint is attached once the asset has generated; the result is stored under the library's `generated/` folder.-Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts, showing a generating state until its source lands. A source that never lands leaves the element carrying the reason (see [errors.md](./errors.md#failed-sources)).+Resolution is **asynchronous and non-blocking**: the element is on the canvas as soon as the project mounts and its paint is attached once its source has loaded. A source that fails to load leaves the element carrying the reason (see [errors.md](./errors.md#failed-sources)).An `<img>` inside [`<html>`](./html.md) additionally takes a `data:` or `blob:` URL, which goes to the browser as it is.@@ -24,24 +23,10 @@ A folder of pictures has a count, not a duration, so `frameRate` is what says ho`frameRate` is not [`playbackRate`](./timing.md#playback-rate), which retimes a source against the timeline whatever its natural speed is; this is what that natural speed *is*. It is unrelated to the composition's own frame rate, which the export sets.-## Transforms--A source can be put through a model before the element shows it, by declaring the result as the `src` — see [generate.md](./generate.md#transforms):--```tsx-import { transform } from "@diffusionstudio/jsx";--<image src={transform.upscale(transform.removeBackground("footage/fox.png"))} width={800} height={450} />-```--The inner source is untouched — nothing is overwritten and nothing is lost — and the chain runs inside out, in the order it is written. Results are cached by step and input like any [generated asset](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and wrapping a further transform around a chain does not re-run what is inside it.-## The libraryA project's assets are recorded in `assets.yml` next to its entry file: for each asset, its library `path`, where its bytes are (`source`: the absolute path of a file imported from disk — imports never move or copy files — or a project-relative path under `assets/`, whether the app wrote the bytes there or a symlink points at them), and what it was found to be. Folders are listed too, so an empty one survives a reload. Renaming or moving an asset in the panel rewrites the `src` props that named it.-Generations are listed from the moment they are asked for: a partial record (`state: pending`) with no `source` while the model runs, the asset once it lands, and a record in `state: error` carrying the reason when it fails, which stands until removed (see [errors.md](./errors.md#failed-sources)).-## Adding an assetSymlink the file into the project's `assets/` folder and it is in the library. The app is watching, so it lands while the project is open; a project that is closed takes it in on the next open. Its library path is where the link sits under `assets/` — folder and name, so `assets/b-roll/drone.mp4` is `"b-roll/drone.mp4"` and `assets/logo.png` is `"logo.png"`.@@ -54,7 +39,7 @@ mkdir -p assets/b-roll && ln -s ~/Movies/drone.mp4 assets/b-roll/<video src="b-roll/drone.mp4" width={1920} height={1080} />```-Link rather than copy: the bytes stay where they are, the project holds nothing but a name for them, and the library still gets a portable path to reach them by. Give `ln -s` an absolute target — a relative link is read against the folder the link sits in, so it breaks as soon as the project moves. A directory of numbered frames is taken in whole, as one [image sequence](#image-sequences), rather than as a file each, and a link to that directory does the same. A download becomes an asset the same way: point [yt-dlp](https://github.com/yt-dlp/yt-dlp) at `assets/b-roll/` with `-o`; `generate.*` results arrive under `assets/generated/` on their own.+Link rather than copy: the bytes stay where they are, the project holds nothing but a name for them, and the library still gets a portable path to reach them by. Give `ln -s` an absolute target — a relative link is read against the folder the link sits in, so it breaks as soon as the project moves. A directory of numbered frames is taken in whole, as one [image sequence](#image-sequences), rather than as a file each, and a link to that directory does the same. A download becomes an asset the same way: point [yt-dlp](https://github.com/yt-dlp/yt-dlp) at `assets/b-roll/` with `-o`.A link is the project's, and what it points at is not. Deleting a linked asset in the panel removes the link and stops there.diff --git a/docs/reference/jsx/module.md b/docs/reference/jsx/module.mdindex f887472b..6021a504 100644--- a/docs/reference/jsx/module.md+++ b/docs/reference/jsx/module.md@@ -26,7 +26,7 @@ The component receives no props. [`open`](../tools/open.md) opens the folder —| `package.json` | The project record: `projectId` (its identity, kept across renames), `displayName`, `main`, and the diffusion commands as scripts. || `tsconfig.json` | Types for the composition tags, through `jsxImportSource`. || `assets.yml` | The asset library (see [media.md](./media.md#the-library)). |-| `assets/` | The library's files: symlinks to media brought in from elsewhere, plus what the app produced itself, generations under `assets/generated/`. Media imported through the app is linked where it lies, never copied. |+| `assets/` | The library's files: symlinks to media brought in from elsewhere, plus what the app produced itself. Media imported through the app is linked where it lies, never copied. || `cache/` | Derived data (thumbnails, waveforms). Disposable. || `README.md` | What the folder holds and how to work in it — including, for agents, where to find this reference. |diff --git a/docs/reference/jsx/stage.md b/docs/reference/jsx/stage.mdindex d5d9f9f9..886626f7 100644--- a/docs/reference/jsx/stage.md+++ b/docs/reference/jsx/stage.md@@ -13,7 +13,7 @@ export default function Project() {}```-A node at root level is material rather than composition: a generated clip that has not been cut into a scene yet, a reference still lying about. It draws on the canvas and can be dragged, selected and inspected, but it has no scene's clock to be placed against, so it plays nothing and exports nowhere — that is what moving it into a scene is for. It is where the editor puts what it generates.+A node at root level is material rather than composition: a clip that has not been cut into a scene yet, a reference still lying about. It draws on the canvas and can be dragged, selected and inspected, but it has no scene's clock to be placed against, so it plays nothing and exports nowhere — that is what moving it into a scene is for.The stage is a **singleton**: it is the canvas already on screen rather than something the project creates, so a project spelling `<stage>` addresses the one that is there. One mount per world — a save disposes the running mount before the new one takes the stage, and the scenes it owned go with it.diff --git a/docs/reference/jsx/video.md b/docs/reference/jsx/video.mdindex 499c8f0e..cafb5e04 100644--- a/docs/reference/jsx/video.md+++ b/docs/reference/jsx/video.md@@ -12,7 +12,7 @@ All [common props](./elements.md#common-props), plus:| Prop | Type | Default | Meaning || ---- | ---- | ------- | ------- |-| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |+| `src` | `string` | **required** | See [media.md](./media.md). || `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. || `frameRate` | `number` | `30` | Frames per second for a `src` naming a directory of numbered frames — the only thing that says how long such a clip runs. Nothing for encoded video to read: a file carries its own rate (see [media.md](./media.md#image-sequences)). || `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. |diff --git a/docs/reference/tools/README.md b/docs/reference/tools/README.mdindex 30dc7378..c623dcb5 100644--- a/docs/reference/tools/README.md+++ b/docs/reference/tools/README.md@@ -44,8 +44,6 @@ Every tool runs inside the app, so the app has to be running. Over MCP that is a| [`media_waveform`](./media/waveform.md) | `diffusion media waveform` | Waveform preview || [`media_listen`](./media/listen.md) | `diffusion media listen` | Listen to audio || [`media_segment`](./media/segment.md) | `diffusion media segment` | Segment object |-| [`models`](./models.md) | `diffusion models` | Generation models |-| [`voices`](./voices.md) | `diffusion voices` | Speech voices || [`logs`](./logs.md) | `diffusion logs` | App logs || [`screenshot`](./screenshot.md) | `diffusion screenshot` | Window screenshot || [`window`](./window.md) | `diffusion window` | App window |@@ -57,7 +55,7 @@ How the surface is divided:- **The project loop.** [`open`](./open.md) a folder, edit its JSX, [`context`](./context.md) for what the source cannot say, [`capture`](./capture.md) and [`check`](./check.md) to verify, [`export`](./export.md) when asked.- **Media inspection** (`media_*`): a file by path, without adding it to the project. Absolute paths and URLs work with or without an open project; library paths (`b-roll/clip.mp4`) need one.- **Masks.** [`media_segment`](./media/segment.md) segments and tracks an object in footage and writes the mask file a `<mask src>` names.-- **What a declaration may name.** [`models`](./models.md), [`voices`](./voices.md), [`fonts`](./fonts.md). Generation itself is declared in the project module (`generate.*`, see [jsx/generate.md](../jsx/generate.md)); no tool generates.+- **What a declaration may name.** [`fonts`](./fonts.md).- **The app and the machine.** [`logs`](./logs.md), [`screenshot`](./screenshot.md), [`window`](./window.md), [`report`](./report.md).## Downloading footagediff --git a/docs/reference/tools/check.md b/docs/reference/tools/check.mdindex 2a020fb3..03513c17 100644--- a/docs/reference/tools/check.md+++ b/docs/reference/tools/check.md@@ -1,6 +1,6 @@# check-Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load or generate — plus subtree stats (node count by kind, nesting depth, played duration). Times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock capture uses. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with capture.+Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load — plus subtree stats (node count by kind, nesting depth, played duration). Times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock capture uses. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with capture.| | || --- | --- |@@ -52,7 +52,7 @@ One JSON object:| `never-visible` | warning | A node scheduled entirely outside the window its ancestors play || `zero-duration` | warning | A node that spans no frames || `transparent` | warning | A node with static opacity 0 (nodes with keyframes are given the benefit of the doubt) |-| `source-error` | error | An asset that failed to load or generate, with the failure message |+| `source-error` | error | An asset that failed to load, with the failure message |## Severity and the shelldiff --git a/docs/reference/tools/context.md b/docs/reference/tools/context.mdindex 6b055ca9..589e5c1a 100644--- a/docs/reference/tools/context.md+++ b/docs/reference/tools/context.md@@ -1,6 +1,6 @@# context-Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, where its `generate.*` declarations stand, and the progress of `media_segment` tracks running in the background. Poll it to wait for generations and tracks without blocking.+Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and the progress of `media_segment` tracks running in the background. Poll it to wait for tracks without blocking.| | || --- | --- |@@ -26,13 +26,6 @@ One JSON object:projectDir: string | null; // absolute open project folder — where the JSX being edited lives; null when none is opencurrentTime: number | null; // playhead in the active scene, in seconds; null if no scene is activefontFamilies: string[]; // families registered in the running world, valid as `fontFamily`- generations: { // every generated source in the project, and where it stands- element: string | null; // the element's source stamp, `<file>:<id>`; null for an entity no element produced- name: string | null; // the element's `name`, when it has one- state: "generating" | "failed" | "done";- error?: string; // what it failed with, on `failed` rows- asset?: string; // the library path it landed as, on `done` rows- }[];masks: { // media_segment tracks started while this project is open, oldest firstid: string; // the track's id, the same across pollssrc: string; // the mask's library path, for <mask src>; the file is there once done@@ -47,7 +40,7 @@ One JSON object:}```-With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }` with empty lists: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).+With no project open (the app sits at the dashboard) the report is just `{ rootDir, projectDir: null }` with an empty `masks` list: there is no playhead, no world, and no fonts to speak of. Open one with [`open`](./open.md).`rootDir` is reported whether or not a project is open — it is where a caller with nothing open goes to create or find one.@@ -57,6 +50,4 @@ With no project open (the app sits at the dashboard) the report is just `{ rootD`fontFamilies` is what text can be drawn with right now — loaded into the world, not merely named in the source — and always includes the editor default. For every family a text can name, see [`fonts`](./fonts.md).-`generations` is how a caller waits for `generate.*` declarations without blocking: generation is asynchronous, so poll this until nothing is `generating`. A `done` row's `asset` is a library path, ready for [`media_probe`](./media/probe.md) and its siblings; a `failed` row's `error` is the message the library recorded for the generation, which is what keeps it from being generated again (see [jsx/errors.md](../jsx/errors.md#failed-sources)).-`masks` is how a caller waits for a [`media_segment`](./media/segment.md) track that went into the library: the call returns as soon as it starts, so poll this until the row whose `src` it returned is `done` or `failed`. `loading` covers the model loading (its download, the first time, is `progress`) and waiting behind the editor's object mask tool or another track. A `done` row carries the same findings `media_segment` returns for a finished track, and its contact sheet as `image` — over MCP it also arrives inline, once, with the first poll that reports the row done. Rows last as long as the project stays open; closing it stops its tracks.diff --git a/docs/reference/tools/logs.md b/docs/reference/tools/logs.mdindex 5177cd44..5ffc904f 100644--- a/docs/reference/tools/logs.md+++ b/docs/reference/tools/logs.md@@ -16,7 +16,7 @@ Recent console output from the running app (what the devtools console shows: pag| `since` | `number` | `--since <ms>` | only entries logged after this unix time in milliseconds; pass the last entry's `ts` to get only new ones || `contains` | `string` | `-c, --contains <text>` | only entries whose message contains this (case-insensitive) |-The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations — an export's percentage, a generation landing — shows up here, so polling `logs` is how a caller follows work it started; pass `since` with the last `ts` seen so each poll returns only what is new.+The buffer lives in the app's main process, so the log survives page reloads and project switches. Progress of long operations, such as an export's percentage, shows up here, so polling `logs` is how a caller follows work it started; pass `since` with the last `ts` seen so each poll returns only what is new.`level`, `since`, and `contains` filter first, then `tail` keeps the last entries of what is left. A message longer than 4000 characters is cut, ending in `… (N more chars)`.diff --git a/docs/reference/tools/media/listen.md b/docs/reference/tools/media/listen.mdindex 73f97951..4799a1f4 100644--- a/docs/reference/tools/media/listen.md+++ b/docs/reference/tools/media/listen.md@@ -1,6 +1,8 @@# media_listen-Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed. Needs a signed-in account.+Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed.++**Temporarily unavailable in this version:** every call fails with `unsupported`.| | || --- | --- |@@ -16,7 +18,7 @@ Prompt a multimodal model for a semantic analysis of an audio track and return i| `start` | `Time` | `-s, --start <time>` | start of the segment to analyze (default: 0); timestamps in the analysis are relative to this point || `end` | `Time` | `-e, --end <time>` | end of the segment to analyze (default: media duration) |-With no `prompt` it returns a general description of what is heard; with one it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). See [the prompt guide](../../../guides/prompts/media-listen.md) for prompts that work. Costs credits, and needs a signed-in account; signed out, it fails with `sign-in-required`.+With no `prompt` it returns a general description of what is heard; with one it answers that question about the audio (e.g. "who is speaking?", "what music is playing?", "summarize what is said"). See [the prompt guide](../../../guides/prompts/media-listen.md) for prompts that work.## Output@@ -28,4 +30,4 @@ One JSON object, the model's answer. `start`/`end` echo the analyzed window (in## Errors-Fails when the path can't be resolved, the asset isn't a video or audio asset, `start`/`end` cross (`start` >= `end`), or no account is signed in.+Fails when the path can't be resolved, the asset isn't a video or audio asset, or `start`/`end` cross (`start` >= `end`). In this version every call fails with `unsupported`.diff --git a/docs/reference/tools/media/transcribe.md b/docs/reference/tools/media/transcribe.mdindex e5c38e34..816bcbf8 100644--- a/docs/reference/tools/media/transcribe.md+++ b/docs/reference/tools/media/transcribe.md@@ -2,6 +2,8 @@Transcribe the speech in a video or audio file and write the timed transcript to a JSON file, with word-level start/end times in seconds; returns the file's path and its segment and word counts. Commonly useful for footage with speakers (talking head, interview), where the word times let you cut on a line. A transcript marks only speech; the gaps are not necessarily silent (music, score, applause).+**Temporarily unavailable in this version:** every call fails with `unsupported`.+| | || --- | --- || MCP tool | `media_transcribe` |@@ -41,4 +43,4 @@ The file holds the transcript as indented JSON, so it searches line by line: a g## Errors-Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`).+Fails when the path can't be resolved, the asset is not a video/audio asset, or no speech is detected in the audio at all (`No speech detected`). In this version every call fails with `unsupported`.diff --git a/docs/reference/tools/models.md b/docs/reference/tools/models.mddeleted file mode 100644index 5a3254b9..00000000--- a/docs/reference/tools/models.md+++ /dev/null@@ -1,33 +0,0 @@-# models--List available AI generation models and their per-model constraints (durations, aspect ratios, features), for `generate.*` asset declarations in a project module.--| | |-| --- | --- |-| MCP tool | `models` |-| CLI | `diffusion models [type]` |--## Input--| Field | Type | CLI | Description |-| --- | --- | --- | --- |-| `type` | `"image" \| "video" \| "audio"` | `[type]` | filter to one kind of model, image, video, or audio (default: all three) |--Use it to discover valid model ids and the per-model constraints (durations, aspect ratios, features) to set on an asset declaration (see [jsx/generate.md](../jsx/generate.md)). No tool generates: asset generation is declared in the project module and produced on mount.--## Output--One JSON object:--```ts-{- models: Array<{- type: "image" | "video" | "audio";- id: string; // the model id to set on a generate.* declaration- name: string;- durations?: string[]; // video only, e.g. ["5s","10s"]- aspectRatios?: string[]; // video only- features?: Array<"start-frame" | "end-frame" | "audio">; // video only- }>;-}-```diff --git a/docs/reference/tools/report.md b/docs/reference/tools/report.mdindex 892a627c..de1ee92e 100644--- a/docs/reference/tools/report.md+++ b/docs/reference/tools/report.md@@ -21,7 +21,7 @@ For a tool that errors, contradicts this reference, or returns something it shouThe issue is submitted immediately, with no review step: the call returns once the issue exists. Filing goes through the [`gh`](https://cli.github.com) CLI, which must be installed and authenticated (`gh auth login`) on this machine; without it nothing is filed. It runs in the app's main process, so no project needs to be open.-This is for defects in the tooling, not for problems inside a project: a composition that looks wrong, a node in the wrong place, or a generation that missed the prompt are editing problems, not reported here.+This is for defects in the tooling, not for problems inside a project: a composition that looks wrong or a node in the wrong place are editing problems, not reported here.## Outputdiff --git a/docs/reference/tools/voices.md b/docs/reference/tools/voices.mddeleted file mode 100644index 51c64b5e..00000000--- a/docs/reference/tools/voices.md+++ /dev/null@@ -1,22 +0,0 @@-# voices--List the speech voices available for `generate.voice` declarations in a project module.--| | |-| --- | --- |-| MCP tool | `voices` |-| CLI | `diffusion voices` |--## Input--None.--See [jsx/generate.md](../jsx/generate.md) for the declaration a voice id goes on.--## Output--One JSON object:--```ts-{ voices: Array<{ id: string; label: string; description: string }> }-```diff --git a/docs/skills/editor.md b/docs/skills/editor.mdindex 9d47b6f5..a19426f4 100644--- a/docs/skills/editor.md+++ b/docs/skills/editor.md@@ -4,8 +4,8 @@ How to understand source material before editing it. Inspect only the modalities- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.-- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.+- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.+- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.# The editing loop@@ -14,7 +14,7 @@ How to understand source material before editing it. Inspect only the modalities- Lay down the A-roll. Assemble the primary footage as JSX and save. Get the spine of the edit right before anything else.- Layer the rest on top. Once the A-roll holds, add B-roll and secondary assets (sound effects, captions, overlays) in the same source.- Symlink media the project uses into its `assets/` folder and name it by library path (`assets/b-roll/drone.mp4` is `"b-roll/drone.mp4"`); local and remote paths work too but stay outside the library.-- `context` reports which folder the app actually has open, where the playhead sits, and where every `generate.*` declaration stands — poll it to wait for generations without blocking.+- `context` reports which folder the app actually has open and where the playhead sits.**The source is the document.** A project is a folder of JSX; `open` a folder once, then write the files and save. Saving recompiles and re-renders the canvas.@@ -31,7 +31,7 @@ How to understand source material before editing it. Inspect only the modalitiesHow to confirm a change actually produced what you intended. A clean save does not guarantee a correct-looking composition.-- Run `check` on the scene first. It catches black frames, clips that never become visible, zero-duration or fully transparent nodes, and assets that failed to load or generate.+- Run `check` on the scene first. It catches black frames, clips that never become visible, zero-duration or fully transparent nodes, and assets that failed to load.- Use `capture` on the scene to see what the viewer actually gets.- Reconcile captured frames with the brief, and the brief with these guidelines.- Verify after every stage, not only at the end — build the composition incrementally so a problem is caught next to the change that caused it.@@ -49,7 +49,7 @@ How to confirm a change actually produced what you intended. A clean save does n- To cut an object out of footage, or confine an effect to it, track it with `media_segment` (preview the prompt first) and name the mask file in a `<mask>` under the clip's `<effect>`. A track into the library runs in the background: keep working, and poll `context` until its row in `masks` is done before naming it.- For motion graphics, overlays and UI-heavy graphics, the `<html>` tag driven by a paused [anime.js](https://animejs.com) timeline- Before animating anything, read the [easings reference](../guides/motion/easings.md) and choose easings deliberately — default or linear easing is what makes motion read as a slideshow.-- Add auto captions last, after everything else is assembled, so they transcribe the finished audio at its final placement.+- Add captions last, after everything else is assembled. `<captions>` mounts the transcript its `src` names; it does not transcribe the scene.- For tasks that don't need an editing UI, keep the app in the background: from a shell, `diffusion open -b <dir>` launches it that way. Over MCP the app is already running, and `open` only opens the folder.- Only render (export) the result when prompted.- Start with a fresh project.@@ -58,7 +58,7 @@ How to confirm a change actually produced what you intended. A clean save does nThe authoring reference is served by the app for the installed version: read it there and trust it over memory.-- [JSX reference — elements, timing, paints, generation, captions](../reference/jsx/README.md)+- [JSX reference — elements, timing, paints, captions](../reference/jsx/README.md)- [Tool reference — every tool, its fields, its CLI spelling and its output](../reference/tools/README.md)- [Examples — complete compositions, basics through shaders](../examples/README.md)- [Easings: which cubic-bezier to use and when](../guides/motion/easings.md)diff --git a/docs/skills/watch.md b/docs/skills/watch.mdindex c8761daf..8769a870 100644--- a/docs/skills/watch.md+++ b/docs/skills/watch.md@@ -4,20 +4,20 @@ How to understand source material. Inspect only the modalities the question turn- **Always probe first.** `media_probe` reports the container and its tracks, telling you up front whether the file has a video track, an audio track, or both. Everything after branches on that.- **Get the lay of the land.** Render a `media_waveform` (audio) and a `media_filmstrip` (video) for a fast, cheap overview of where the loud and quiet stretches fall, and where the visual scene changes are. A filmstrip shows coarse structure and scene state, not crop, framing, readability, or an exact cut frame.-- **Listen to the audio.** Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.-- **Transcribe speech.** For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.+- **Listen to the audio.** (`media_listen` is temporarily unavailable in this version.) Run `media_listen` with a prompt tailored to the context (what you actually need to know), and explicitly ask the model to include timestamps in its answer. See [media-listen.md](../guides/prompts/media-listen.md) for prompt patterns.+- **Transcribe speech.** (`media_transcribe` is temporarily unavailable in this version.) For speech, `media_transcribe` writes the full transcript with word-level start/end times to a JSON file and returns its path — search the file for the line you need and read its times from there, rather than loading it whole.- **Sample the video against the audio.** Use `media_grab` to pull frames. When the audio has already pointed you at specific moments, feed those timestamps straight in from the transcript or listen output as `times` (e.g. `00:32`, `00:45`). When you need a visual pass without such cues, reach for `auto`: it scans the footage and keeps only the frames where the picture settles into a new visual state, dropping near-duplicates.The `media_*` tools take a path or a direct media URL. Footage behind a page URL (YouTube, TikTok, Instagram, …) is downloaded first with `yt-dlp` from a shell — audio only (`yt-dlp -x --audio-format m4a -o talk.m4a <url>`) unless the question needs frames; see the [tool reference](https://github.com/yt-dlp/yt-dlp).# Matching depth to the question-Read only as much of the footage as the answer requires — each pass costs time, and `media_listen` costs credits.+Read only as much of the footage as the answer requires — each pass costs time.- A duration or format question ends at `media_probe`.- "Where is the quiet part" or "how is it paced" is usually answered by the waveform and filmstrip alone.-- A question about what was said resolves fastest through `media_transcribe`; search the transcript file for the passage and quote it with its times.-- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for.+- A question about what was said resolves fastest through `media_transcribe` (temporarily unavailable in this version); search the transcript file for the passage and quote it with its times.+- Questions about non-speech audio (music, tone, sound events, speaker identity) are what `media_listen` is for (temporarily unavailable in this version).- Only questions about what is *seen* need frames — and the audio pass usually tells you which moments to grab, so grab those instead of scanning blind.- For open-ended questions ("summarize this", "what happens here"), combine passes: structure from waveform + filmstrip, content from transcript or listen, then frames at the salient moments to confirm what the picture shows.diff --git a/package-lock.json b/package-lock.jsonindex befe3f7e..23229b13 100644--- a/package-lock.json+++ b/package-lock.json@@ -92,7 +92,6 @@"license": "MPL-2.0","dependencies": {"@diffusionstudio/agent-chat": "*",- "@diffusionstudio/api-contract": "^0.1.0","@diffusionstudio/assets": "*","@diffusionstudio/dapi": "*","@diffusionstudio/encoder": "*",@@ -108,8 +107,6 @@"@solid-primitives/resize-observer": "^2.1.3","@solidjs/router": "^0.15.4","@supabase/supabase-js": "^2.99.2",- "@trpc/client": "^11.10.0",- "@trpc/server": "^11.10.0","animejs": "^4.4.1","cmdk-solid": "^1.1.2","colord": "^2.9.3",@@ -773,16 +770,6 @@"resolved": "packages/agent-chat","link": true},- "node_modules/@diffusionstudio/api-contract": {- "version": "0.1.0",- "resolved": "https://registry.npmjs.org/@diffusionstudio/api-contract/-/api-contract-0.1.0.tgz",- "integrity": "sha512-oufWqwI9XZDV1hRexeRALTyW0c3xR2KShEz3YRh3Ejr9GIV2t58dZiMHUbPgbARafjEg7YsyLwfRqchgOuHKEA==",- "license": "MIT",- "dependencies": {- "@trpc/server": "^11.10.0",- "zod": "^4.3.6"- }- },"node_modules/@diffusionstudio/assets": {"resolved": "packages/assets","link": true@@ -4279,37 +4266,6 @@"node": ">= 10"}},- "node_modules/@trpc/client": {- "version": "11.18.0",- "resolved": "https://registry.npmjs.org/@trpc/client/-/client-11.18.0.tgz",- "integrity": "sha512-wOqeg3Fvl25V1ZisQhUD3K8G60ZJDlSGJNSyeXrLH24xAo5w6GSR2Kzb1cSNY9Y+IQ2YZvYGZstBU+V/ulo/ow==",- "funding": [- "https://trpc.io/sponsor"- ],- "license": "MIT",- "bin": {- "intent": "bin/intent.js"- },- "peerDependencies": {- "@trpc/server": "11.18.0",- "typescript": ">=5.7.2"- }- },- "node_modules/@trpc/server": {- "version": "11.18.0",- "resolved": "https://registry.npmjs.org/@trpc/server/-/server-11.18.0.tgz",- "integrity": "sha512-JAvXOuNTxgXjIDfQaOvDq1j66LMNfDJUH1IU7Slfn8EvRv2EkH6ehu3A7zpYhjO0syHHiYg77v2lG2JFJgvw7Q==",- "funding": [- "https://trpc.io/sponsor"- ],- "license": "MIT",- "bin": {- "intent": "bin/intent.js"- },- "peerDependencies": {- "typescript": ">=5.7.2"- }- },"node_modules/@ts-morph/common": {"version": "0.29.0","resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.29.0.tgz",@@ -13172,6 +13128,7 @@"version": "5.9.3","resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz","integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",+ "devOptional": true,"license": "Apache-2.0","bin": {"tsc": "bin/tsc",diff --git a/packages/assets/src/hash.ts b/packages/assets/src/hash.tsindex 7abe7ea3..13b05267 100644--- a/packages/assets/src/hash.ts+++ b/packages/assets/src/hash.ts@@ -42,16 +42,6 @@ export async function hashSequence(frames: { name: string; size: number }[]): Prreturn hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);}-/**- * The id of a partial document: the hash of its generation key, since there- * are no bytes yet. Stable for the key, so the same generation reserved twice- * is one record, and unlike any content hash the bytes could come to.- */-export async function hashKey(key: string): Promise<string> {- const bytes = new TextEncoder().encode(`key\n${key}`);- return hexOf(await crypto.subtle.digest('SHA-256', bytes)).slice(0, ID_LENGTH);-}-/** Whether a string could be an asset id (and not, say, a library path). */export const looksLikeId = (value: string): boolean =>value.length === ID_LENGTH && /^[0-9a-f]+$/.test(value);diff --git a/packages/assets/src/library.ts b/packages/assets/src/library.tsindex c4211a68..7668efb8 100644--- a/packages/assets/src/library.ts+++ b/packages/assets/src/library.ts@@ -3,8 +3,8 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */// The asset library of a project: the manifest, loaded with handles attached,-// plus everything that changes it — imports, renames, folders, generated-// output — and the one place a `src` is turned into an asset. Files the user+// plus everything that changes it — imports, renames, folders, bytes the app+// produced — and the one place a `src` is turned into an asset. Files the user// imports stay where they are (the manifest links to them by absolute path);// only bytes the app itself produces are written into the project's// `assets/` directory, and anything found there that the manifest does not@@ -14,29 +14,22 @@// Knows nothing of the host: files come and go through a `ProjectFS`, and// what it cannot do itself (follow a rename into whatever names assets by// path, rebind whatever is bound to a relinked asset's old id) it asks of-// `LibraryOptions`. Generation is not its business either — a generated-// file is stored like any other bytes the app produced, with its-// `generation` record along — but where a generation stands is: a run that-// has started is a partial document here (`reserve`), one that failed stays-// as one with its reason (`fail`), and one that landed replaces its partial-// (`store`). Its state is solid signals (`assets()`, `partials()`,-// `folders()`), so readers inside a tracking scope follow it.+// `LibraryOptions`. Its state is solid signals (`assets()`, `folders()`), so+// readers inside a tracking scope follow it.import { createSignal, type Accessor } from 'solid-js';import { AssetCache } from './cache';-import { hashBlob, hashKey, hashSequence } from './hash';+import { hashBlob, hashSequence } from './hash';import {- ASSETS_DIR, isAbsoluteSource, isPartialRecord, isProjectSource, isUrlSource, normalizeManifest, toRecord,+ ASSETS_DIR, isAbsoluteSource, isProjectSource, isUrlSource, normalizeManifest, toRecord,} from './manifest';import { detectMimeType, DEFAULT_SEQUENCE_FPS, isSequenceListing, probeMedia, sortFrames } from './probe';-import { assetFolder, assetName, basename, dirname, isPartialAsset, joinPath, normalizePath } from './types';+import { assetFolder, assetName, basename, dirname, joinPath, normalizePath } from './types';import type { FsEntry, ProjectFS } from './fs';import type { AssetRecord, Manifest } from './manifest';-import type {- Asset, AssetDirectoryHandle, AssetEntry, AssetFileHandle, AssetGeneration, AssetType, PartialAsset, SequenceAsset,-} from './types';+import type { Asset, AssetDirectoryHandle, AssetFileHandle, SequenceAsset } from './types';/** How long changes pile up before the manifest is written. */const SAVE_DEBOUNCE = 200;@@ -66,18 +59,6 @@ export interface ImportOptions {folder?: string;/** Library name; the source's file name by default. */name?: string;- generation?: AssetGeneration;-}--export interface ReserveOptions {- /** The generation key the partial stands for (see `AssetGeneration`). */- key: string;- /** What the generation is to become. */- type: AssetType;- /** A provisional library name; the bytes, once they land, are named by whoever stores them. */- name: string;- /** Library folder to place it in; root by default. */- folder?: string;}export class AssetLibrary {@@ -86,21 +67,15 @@ export class AssetLibrary {public readonly cache: AssetCache;/** Library assets, newest first; transient ones excluded. Reactive. */public readonly assets: Accessor<Asset[]>;- /** Partial documents — generations pending or failed — newest first. Reactive. */- public readonly partials: Accessor<PartialAsset[]>;/** Every folder: declared ones, those implied by asset paths, and their ancestors. Reactive. */public readonly folders: Accessor<ReadonlySet<string>>;private readonly setAssets: (assets: Asset[]) => void;- private readonly setPartials: (partials: PartialAsset[]) => void;private readonly setFolders: (folders: ReadonlySet<string>) => void;/** Folders declared in the manifest (the empty ones need declaring). */private declared = new Set<string>();- /**- * Every entry by id: the library's assets, transient ones resolved from- * outside it, and the partial documents of generations without bytes.- */- private readonly map = new Map<string, AssetEntry>();+ /** Every asset by id: the library's, and transient ones resolved from outside it. */+ private readonly map = new Map<string, Asset>();private readonly onRename: LibraryOptions['onRename'];private readonly onRelink: LibraryOptions['onRelink'];private inflight = new Map<string, Promise<Asset>>();@@ -115,7 +90,6 @@ export class AssetLibrary {this.onRename = options.onRename;this.onRelink = options.onRelink;[this.assets, this.setAssets] = createSignal<Asset[]>([]);- [this.partials, this.setPartials] = createSignal<PartialAsset[]>([]);[this.folders, this.setFolders] = createSignal<ReadonlySet<string>>(new Set());}@@ -131,25 +105,15 @@ export class AssetLibrary {private listNow(): Asset[] {const assets: Asset[] = [];for (const entry of this.map.values()) {- if (!isPartialAsset(entry) && !entry.transient) assets.push(entry);+ if (!entry.transient) assets.push(entry);}return assets;}- /** The partial documents as of now, straight from the map. */- private partialsNow(): PartialAsset[] {- return Array.from(this.map.values()).filter(isPartialAsset);- }-- /** Everything at a library path as of now: assets and partials, transient ones excluded. */- private entriesNow(): AssetEntry[] {- return Array.from(this.map.values()).filter((entry) => isPartialAsset(entry) || !entry.transient);- }-/** The asset with `id`, or at library path `path`; undefined otherwise. */public get(pathOrId: string): Asset | undefined {const byId = this.map.get(pathOrId);- if (byId && !isPartialAsset(byId)) return byId;+ if (byId) return byId;const path = normalizePath(pathOrId);for (const asset of this.listNow()) {if (asset.path === path) return asset;@@ -157,12 +121,6 @@ export class AssetLibrary {return undefined;}- /** The partial document with `id`; undefined when there is none, or the id is an asset's. */- public getPartial(id: string): PartialAsset | undefined {- const entry = this.map.get(id);- return entry && isPartialAsset(entry) ? entry : undefined;- }-/** The asset whose bytes are `source`, if the library links to it. */public bySource(source: string): Asset | undefined {for (const asset of this.listNow()) {@@ -171,21 +129,8 @@ export class AssetLibrary {return undefined;}- /**- * What the library holds for a generation key: the asset the generation- * landed as, or the partial document standing for it while it runs or- * after it failed. Undefined when the generation was never asked for- * here — or its record was removed, which is how one is asked for again.- */- public generated(key: string): AssetEntry | undefined {- for (const entry of this.entriesNow()) {- if (entry.generation?.key === key) return entry;- }- return undefined;- }-- /** Direct children of a folder ('' for the root): its subfolders, assets and partials. Reactive. */- public childrenOf(folder: string): { folders: string[]; assets: Asset[]; partials: PartialAsset[] } {+ /** Direct children of a folder ('' for the root): its subfolders and assets. Reactive. */+ public childrenOf(folder: string): { folders: string[]; assets: Asset[] } {const prefix = folder ? `${folder}/` : '';const folders = new Set<string>();for (const path of this.folders()) {@@ -194,25 +139,23 @@ export class AssetLibrary {}}const assets = this.assets().filter((asset) => assetFolder(asset) === folder);- const partials = this.partials().filter((partial) => assetFolder(partial) === folder);- return { folders: [...folders].sort(), assets, partials };+ return { folders: [...folders].sort(), assets };}- /** Every folder as of now: declared, implied by entry paths, and all their ancestors. */+ /** Every folder as of now: declared, implied by asset paths, and all their ancestors. */private foldersNow(): Set<string> {const folders = new Set<string>();const declare = (path: string): void => {for (let folder = path; folder && !folders.has(folder); folder = dirname(folder)) folders.add(folder);};for (const folder of this.declared) declare(folder);- for (const entry of this.entriesNow()) declare(assetFolder(entry));+ for (const asset of this.listNow()) declare(assetFolder(asset));return folders;}/** Publishes the map's state to the signals. */private publish(): void {this.setAssets(this.listNow());- this.setPartials(this.partialsNow());this.setFolders(this.foldersNow());}@@ -230,13 +173,8 @@ export class AssetLibrary {const manifest = normalizeManifest(await this.fs.readManifest());this.declared = new Set(manifest.folders);- const next = new Map<string, AssetEntry>();+ const next = new Map<string, Asset>();await Promise.all(manifest.assets.map(async (record) => {- // A partial has no bytes to attach or re-examine; it is its record.- if (isPartialRecord(record)) {- next.set(record.id, { ...record });- return;- }const asset = await this.revive(record).catch((error: unknown) => {console.warn(`[assets] could not load ${record.path}:`, error);return this.attach(record);@@ -246,7 +184,7 @@ export class AssetLibrary {// Keep transient assets and the same-id instances entities already hold.for (const [id, entry] of this.map) {- if (!isPartialAsset(entry) && entry.transient && !next.has(id)) next.set(id, entry);+ if (entry.transient && !next.has(id)) next.set(id, entry);}this.map.clear();for (const [id, asset] of next) {@@ -274,11 +212,7 @@ export class AssetLibrary {if (!stat || (record.stat && stat.size === record.stat.size && stat.mtime === record.stat.mtime)) {return this.attach(record);}- return this.describeFile(record.source, {- path: record.path,- createdAt: record.createdAt,- generation: record.generation,- });+ return this.describeFile(record.source, { path: record.path, createdAt: record.createdAt });}/**@@ -335,9 +269,7 @@ export class AssetLibrary {/*** The asset a source string names: a library path or id, or an absolute- * path or URL (resolved on the fly, kept in memory only). Anything the- * library cannot name — a `generate.*` declaration — is not a source; it- * is the host's Ai's to resolve.+ * path or URL (resolved on the fly, kept in memory only).*/public async resolve(input: string): Promise<Asset> {const value = input.trim();@@ -368,7 +300,7 @@ export class AssetLibrary {asset.transient = true;// A transient asset that turns out to be one the library has is that one.const owned = this.map.get(asset.id);- if (owned && !isPartialAsset(owned) && !owned.transient) return owned;+ if (owned && !owned.transient) return owned;this.map.set(asset.id, asset);return asset;});@@ -376,7 +308,7 @@ export class AssetLibrary {private transientBySource(source: string): Asset | undefined {for (const entry of this.map.values()) {- if (!isPartialAsset(entry) && entry.transient && entry.source === source) return entry;+ if (entry.transient && entry.source === source) return entry;}return undefined;}@@ -437,98 +369,27 @@ export class AssetLibrary {return this.once(`link:${source}`, async () => {const name = options.name ?? basename(source);const path = this.uniquePath(joinPath(options.folder ?? '', name));- const asset = await this.describeSource(source, { path, generation: options.generation });+ const asset = await this.describeSource(source, { path });return this.add(asset);});}/*** Writes bytes the app produced into `assets/<folder>/<name>` and takes- * the file into the library at the same library path. Stored under a- * generation key, the asset takes the place of the partial document- * reserved for that key, and answers for the key from here on — even- * when the bytes turn out to be ones the library already had under- * another key, as a re-take of unchanged speech does.+ * the file into the library at the same library path.*/public async store(blob: Blob, options: ImportOptions & { name: string }): Promise<Asset> {- const key = options.generation?.key;- const reserved = key === undefined ? undefined : this.partialFor(key);- const path = this.uniquePath(joinPath(options.folder ?? '', options.name), reserved);+ const path = this.uniquePath(joinPath(options.folder ?? '', options.name));const source = joinPath(ASSETS_DIR, path);await this.fs.write(source, blob);- const asset = await this.describeFile(source, { path, generation: options.generation });- if (key !== undefined) this.dropPartial(key);- const stored = this.add(asset);- if (options.generation && stored.generation?.key !== key) {- this.update(stored, { generation: options.generation });- }- return stored;- }-- /**- * Puts a partial document in the library for a generation about to run:- * `pending`, at a path of its own under `folder`. A partial already- * standing for the key — a run the last session never finished, or one- * that failed and is being asked for again — is set pending again rather- * than doubled. An asset the key already landed as is left alone: this- * is not the place to ask whether to run (see `generated`).- */- public async reserve(options: ReserveOptions): Promise<PartialAsset> {- const id = await hashKey(options.key);- const existing = this.getPartial(id);- if (existing) {- existing.state = 'pending';- delete existing.error;- this.changed();- return existing;- }-- const partial: PartialAsset = {- id,- path: this.uniquePath(joinPath(options.folder ?? '', options.name)),- type: options.type,- createdAt: new Date().toISOString(),- generation: { key: options.key },- state: 'pending',- };- this.reorder(partial);- this.changed();- return partial;- }-- /**- * Records that the generation `partial` stands for failed, and why. The- * record stays: it is what answers for the key from here on, until it- * is removed. A partial no longer in the library (removed while the run- * went on) is not brought back — that removal was the answer.- */- public fail(partial: PartialAsset, error: string): void {- const current = this.getPartial(partial.id);- if (!current) return;- current.state = 'error';- current.error = error;- this.changed();- }-- /** The partial document standing for `key`, if one is in the library. */- private partialFor(key: string): PartialAsset | undefined {- return this.partialsNow().find((partial) => partial.generation.key === key);- }-- /** Takes the partial standing for `key` out, its bytes having landed. */- private dropPartial(key: string): void {- for (const partial of this.partialsNow()) {- if (partial.generation.key === key) this.map.delete(partial.id);- }+ const asset = await this.describeFile(source, { path });+ return this.add(asset);}/** Puts an asset into the library, deduplicating by content. */private add(asset: Asset): Asset {const existing = this.map.get(asset.id);- if (existing && isPartialAsset(existing)) {- // A content hash colliding with a key hash: the bytes win the id.- this.map.delete(existing.id);- } else if (existing) {+ if (existing) {if (!existing.transient) return existing;// A transient asset the library now takes in becomes a real one; the// instance entities hold stays valid, so mutate rather than replace.@@ -543,11 +404,11 @@ export class AssetLibrary {return asset;}- /** Puts `entry` at the front of the map (newest first). */- private reorder(entry: AssetEntry): void {- const rest = Array.from(this.map.values()).filter((other) => other.id !== entry.id);+ /** Puts `asset` at the front of the map (newest first). */+ private reorder(asset: Asset): void {+ const rest = Array.from(this.map.values()).filter((other) => other.id !== asset.id);this.map.clear();- this.map.set(entry.id, entry);+ this.map.set(asset.id, asset);for (const other of rest) this.map.set(other.id, other);}@@ -567,11 +428,7 @@ export class AssetLibrary {* is the host's to move over (`onRelink`).*/public async relink(asset: Asset, source: string): Promise<Asset> {- const next = await this.describeSource(source, {- path: asset.path,- createdAt: asset.createdAt,- generation: asset.generation,- });+ const next = await this.describeSource(source, { path: asset.path, createdAt: asset.createdAt });const from = asset.id;if (next.id === from) {Object.assign(asset, next);@@ -586,45 +443,39 @@ export class AssetLibrary {return next;}- /** Renames an entry within its folder; an asset's source file is untouched. */- public rename(entry: AssetEntry, name: string): void {+ /** Renames an asset within its folder; its source file is untouched. */+ public rename(asset: Asset, name: string): void {const clean = normalizePath(name).replace(/\//g, '-');- if (!clean || clean === assetName(entry)) return;- this.setPath(entry, this.uniquePath(joinPath(assetFolder(entry), clean), entry));+ if (!clean || clean === assetName(asset)) return;+ this.setPath(asset, this.uniquePath(joinPath(assetFolder(asset), clean), asset));}- /** Moves entries into a folder ('' for the root). */- public move(entries: AssetEntry[], folder: string): void {+ /** Moves assets into a folder ('' for the root). */+ public move(assets: Asset[], folder: string): void {const target = normalizePath(folder);if (target) this.declared.add(target);- for (const entry of entries) {- if (assetFolder(entry) === target) continue;- this.setPath(entry, this.uniquePath(joinPath(target, assetName(entry)), entry));+ for (const asset of assets) {+ if (assetFolder(asset) === target) continue;+ this.setPath(asset, this.uniquePath(joinPath(target, assetName(asset)), asset));}}- private setPath(entry: AssetEntry, path: string): void {- const from = entry.path;+ private setPath(asset: Asset, path: string): void {+ const from = asset.path;if (from === path) return;- entry.path = path;+ asset.path = path;this.changed();- // Nothing names a partial by path: it is not a source yet.- if (!isPartialAsset(entry)) this.onRename?.(entry, from);+ this.onRename?.(asset, from);}- /**- * Removes entries from the library. Bytes the app wrote into `assets/`- * go too. Removing a partial forgets where its generation stood — for a- * failed one, that is what asks for the run again.- */- public async remove(entries: AssetEntry[]): Promise<void> {- for (const entry of entries) {- this.map.delete(entry.id);- if (isPartialAsset(entry)) continue;- if (isProjectSource(entry.source) && entry.source.startsWith(`${ASSETS_DIR}/`)) {- await this.fs.remove(entry.source).catch(() => { });+ /** Removes assets from the library. Bytes the app wrote into `assets/` go too. */+ public async remove(assets: Asset[]): Promise<void> {+ for (const asset of assets) {+ this.map.delete(asset.id);+ if (isProjectSource(asset.source) && asset.source.startsWith(`${ASSETS_DIR}/`)) {+ await this.fs.remove(asset.source).catch(() => { });}- this.cache.remove(entry);+ this.cache.remove(asset);}this.changed();}@@ -675,20 +526,20 @@ export class AssetLibrary {}}this.declared.add(to);- for (const entry of this.entriesNow()) {- if (entry.path.startsWith(prefix)) this.setPath(entry, to + entry.path.slice(from.length));+ for (const asset of this.listNow()) {+ if (asset.path.startsWith(prefix)) this.setPath(asset, to + asset.path.slice(from.length));}this.changed();return to;}- /** Deletes a folder and everything in it. Returns the removed entries. */- public async deleteFolder(path: string): Promise<AssetEntry[]> {+ /** Deletes a folder and everything in it. Returns the removed assets. */+ public async deleteFolder(path: string): Promise<Asset[]> {const prefix = `${path}/`;for (const folder of [...this.declared]) {if (folder === path || folder.startsWith(prefix)) this.declared.delete(folder);}- const doomed = this.entriesNow().filter((entry) => entry.path.startsWith(prefix));+ const doomed = this.listNow().filter((asset) => asset.path.startsWith(prefix));await this.remove(doomed);return doomed;}@@ -709,7 +560,7 @@ export class AssetLibrary {return {version: 1,folders: [...this.declared].sort(),- assets: this.entriesNow().map(toRecord),+ assets: this.listNow().map(toRecord),};}@@ -734,9 +585,9 @@ export class AssetLibrary {// -----------------------------------------------------------------------// Describing- /** A library path not taken by any other entry: `name`, `name 2`, … */- private uniquePath(path: string, except?: AssetEntry): string {- const taken = new Set(this.entriesNow().filter((entry) => entry !== except).map((entry) => entry.path));+ /** A library path not taken by any other asset: `name`, `name 2`, … */+ private uniquePath(path: string, except?: Asset): string {+ const taken = new Set(this.listNow().filter((asset) => asset !== except).map((asset) => asset.path));if (!taken.has(path)) return path;const folder = dirname(path);const name = basename(path);@@ -783,7 +634,6 @@ export class AssetLibrary {createdAt: meta.createdAt ?? new Date().toISOString(),mimeType,stat: { size: file.size, mtime: file.lastModified },- ...(meta.generation ? { generation: meta.generation } : {}),handle,...probe,};@@ -885,5 +735,4 @@ export class AssetLibrary {interface DescribeMeta {path: string;createdAt?: string;- generation?: AssetGeneration;}diff --git a/packages/assets/src/manifest.ts b/packages/assets/src/manifest.tsindex f6ac2422..4cd6dc3e 100644--- a/packages/assets/src/manifest.ts+++ b/packages/assets/src/manifest.ts@@ -5,7 +5,7 @@import { normalizePath } from './types';-import type { Asset, PartialAsset } from './types';+import type { Asset } from './types';export const MANIFEST_VERSION = 1;@@ -15,19 +15,11 @@ export const MANIFEST_FILE = 'assets.yml';/** The project directory the app writes its own asset bytes into. */export const ASSETS_DIR = 'assets';-/** Where generated assets are stored inside `ASSETS_DIR`. */-export const GENERATED_DIR = 'generated';-type DistributiveOmit<T, K extends PropertyKey> = T extends unknown ? Omit<T, K> : never;/** An asset as written to the manifest: the model minus its runtime handles. */export type AssetRecord = DistributiveOmit<Asset, 'handle' | 'directoryHandle' | 'transient'>;-/** A partial document as written to the manifest: it has no handles to lose. */-export type PartialRecord = PartialAsset;--export type ManifestRecord = AssetRecord | PartialRecord;-export interface Manifest {version: number;/**@@ -35,20 +27,20 @@ export interface Manifest {* in it is also implied by their paths). `/`-separated, no leading slash.*/folders: string[];- /** Assets and the partial records of generations without bytes, newest first. */- assets: ManifestRecord[];+ /** Assets, newest first. */+ assets: AssetRecord[];}const RECORD_KEYS = new Set(['handle', 'directoryHandle', 'transient']);-/** The manifest record of a library entry: its data, none of its handles. */-export function toRecord(entry: Asset | PartialAsset): ManifestRecord {+/** The manifest record of an asset: its data, none of its handles. */+export function toRecord(asset: Asset): AssetRecord {const record: Record<string, unknown> = {};- for (const [key, value] of Object.entries(entry)) {+ for (const [key, value] of Object.entries(asset)) {if (RECORD_KEYS.has(key) || value === undefined) continue;record[key] = value;}- return record as ManifestRecord;+ return record as AssetRecord;}/** Whether `record` has what every asset needs; unrecognized fields are kept. */@@ -64,21 +56,6 @@ export function isAssetRecord(record: unknown): record is AssetRecord {);}-/** Whether `record` is a partial document: a generation's place and state, no bytes. */-export function isPartialRecord(record: unknown): record is PartialRecord {- if (!record || typeof record !== 'object') return false;- const r = record as Record<string, unknown>;- const generation = r.generation as Record<string, unknown> | undefined;- return (- typeof r.id === 'string' && r.id.length > 0 &&- typeof r.path === 'string' && r.path.length > 0 &&- typeof r.type === 'string' &&- (r.state === 'pending' || r.state === 'error') &&- !!generation && typeof generation === 'object' && typeof generation.key === 'string' &&- (r.error === undefined || typeof r.error === 'string')- );-}-/*** Reads whatever the host handed back as a manifest into a well-formed one.* Tolerant: a missing or malformed file is an empty library, and a broken@@ -100,7 +77,7 @@ export function normalizeManifest(input: unknown): Manifest {if (Array.isArray(raw.assets)) {const seen = new Set<string>();for (const record of raw.assets) {- if (!isAssetRecord(record) && !isPartialRecord(record)) continue;+ if (!isAssetRecord(record)) continue;const path = normalizePath(record.path);if (!path || seen.has(record.id)) continue;seen.add(record.id);diff --git a/packages/assets/src/types.ts b/packages/assets/src/types.tsindex e139256f..65da731a 100644--- a/packages/assets/src/types.ts+++ b/packages/assets/src/types.ts@@ -29,13 +29,6 @@ export interface AssetStat {mtime: number;}-/** How a generated asset came to be: the content key of the fully-resolved- * `generate.*` spec (dedup across runs) and the backend generation id. */-export interface AssetGeneration {- key: string;- id?: string | null;-}-interface AssetBase {/** Short content hash; identity of the asset. */id: string;@@ -48,13 +41,12 @@ interface AssetBase {/*** Where the bytes are: an absolute OS path (a linked file, left where the* user had it), an `http(s)://` URL, or a project-relative path (`assets/…`,- * bytes the app itself produced: generations, downloads, transcodes).+ * bytes the app itself produced: downloads, transcodes).*/source: string;createdAt: string;mimeType: string;stat?: AssetStat;- generation?: AssetGeneration;/*** Resolved on the fly for a `src` that names a path or URL outside the* library; lives in memory only and is never written to the manifest.@@ -133,45 +125,11 @@ export type Asset =export type AssetType = Asset['type'];-/** Where a generation stands while the library has no bytes for it. */-export type PartialAssetState = 'pending' | 'error';--/**- * A partial document: the library's record of a generation before — or- * instead of — its bytes. Written `pending` when a run starts, so the- * generation is in the library from the moment it is asked for; replaced by- * the asset when the run lands; kept as `error`, with what went wrong, when- * it does not. The error record is what keeps a refused or impossible- * generation from being run — or paid for — again on every reopen: a- * declaration whose key stands in error resolves to that error, and removing- * the record is what asks for the run again. Never bound to an entity;- * elements see the state through their resolution.- */-export interface PartialAsset {- /** Hash of the generation key: there are no bytes to hash. */- id: string;- /** Library path, like any asset's; provisional until the bytes name themselves. */- path: string;- /** What the generation is to become. */- type: AssetType;- createdAt: string;- generation: AssetGeneration;- state: PartialAssetState;- /** What the run failed with, on an `error` record. */- error?: string;-}--/** Anything the library holds at a path: an asset, or a partial standing for one. */-export type AssetEntry = Asset | PartialAsset;--/** Whether a library entry is a partial document rather than an asset with bytes. */-export const isPartialAsset = (entry: AssetEntry): entry is PartialAsset => 'state' in entry;-/** The file name of an asset: the last segment of its library path. */-export const assetName = (asset: Pick<AssetEntry, 'path'>): string => basename(asset.path);+export const assetName = (asset: Pick<Asset, 'path'>): string => basename(asset.path);/** The folder of an asset: its library path without the name, '' at root. */-export const assetFolder = (asset: Pick<AssetEntry, 'path'>): string => dirname(asset.path);+export const assetFolder = (asset: Pick<Asset, 'path'>): string => dirname(asset.path);export function basename(path: string): string {const trimmed = path.replace(/[/\\]+$/, '');diff --git a/packages/dapi/src/catalog.ts b/packages/dapi/src/catalog.tsindex 7821d416..081cc01e 100644--- a/packages/dapi/src/catalog.ts+++ b/packages/dapi/src/catalog.ts@@ -10,8 +10,6 @@ import { context } from "./tools/context";import { capture } from "./tools/capture";import { check } from "./tools/check";import { exportScene } from "./tools/export";-import { models } from "./tools/models";-import { voices } from "./tools/voices";import { logs } from "./tools/logs";import { screenshot } from "./tools/screenshot";import { mediaProbe } from "./tools/media-probe";@@ -43,8 +41,6 @@ export const catalog = [mediaWaveform,mediaListen,mediaSegment,- models,- voices,logs,screenshot,appWindow,diff --git a/packages/dapi/src/index.ts b/packages/dapi/src/index.tsindex 98f4fc45..aa447f78 100644--- a/packages/dapi/src/index.ts+++ b/packages/dapi/src/index.ts@@ -33,11 +33,9 @@ export { FONT_LIMIT } from "./tools/fonts";// Named request and result types, for handlers that spell out their// signature. Each is the parsed (output) side of the tool's schema.import type { ImageRef as ImageRefSchema, LogEntry as LogEntrySchema, LogLevel as LogLevelSchema, TimecodedImage as TimecodedImageSchema } from "./schemas";-import type { GenerationRow as GenerationRowType, MaskRow as MaskRowType } from "./tools/context";+import type { MaskRow as MaskRowType } from "./tools/context";import type { CheckIssue as CheckIssueSchema, CheckIssueCode as CheckIssueCodeSchema } from "./tools/check";import type { ExportFormat as ExportFormatSchema, ExportSettings as ExportSettingsSchema } from "./tools/export";-import type { ModelInfo as ModelInfoSchema } from "./tools/models";-import type { VoiceInfo as VoiceInfoSchema } from "./tools/voices";import type { FrameQuality as FrameQualitySchema } from "./tools/media-grab";import type { TranscriptSegment as TranscriptSegmentSchema, TranscriptWord as TranscriptWordSchema } from "./tools/media-transcribe";import type { FontFamily as FontFamilySchema } from "./tools/fonts";@@ -47,14 +45,11 @@ export type LogLevel = z.output<typeof LogLevelSchema>;export type LogEntry = z.output<typeof LogEntrySchema>;export type TimecodedImage = z.output<typeof TimecodedImageSchema>;export type ImageRef = z.output<typeof ImageRefSchema>;-export type GenerationRow = GenerationRowType;export type MaskRow = MaskRowType;export type CheckIssueCode = z.output<typeof CheckIssueCodeSchema>;export type CheckIssue = z.output<typeof CheckIssueSchema>;export type ExportFormat = z.output<typeof ExportFormatSchema>;export type ExportSettings = z.output<typeof ExportSettingsSchema>;-export type ModelInfo = z.output<typeof ModelInfoSchema>;-export type VoiceInfo = z.output<typeof VoiceInfoSchema>;export type FrameQuality = z.output<typeof FrameQualitySchema>;export type TranscriptWord = z.output<typeof TranscriptWordSchema>;export type TranscriptSegment = z.output<typeof TranscriptSegmentSchema>;@@ -69,7 +64,6 @@ export type CheckRequest = ToolArgs<"check">;export type CheckResult = ToolResult<"check">;export type ExportRequest = ToolArgs<"export">;export type ExportResult = ToolResult<"export">;-export type ModelsRequest = ToolArgs<"models">;export type LogsRequest = ToolArgs<"logs">;export type ScreenshotResult = ToolResult<"screenshot">;export type ScreenshotOutput = ToolOutput<"screenshot">;diff --git a/packages/dapi/src/tools/check.ts b/packages/dapi/src/tools/check.tsindex 18996035..2dc3df7a 100644--- a/packages/dapi/src/tools/check.ts+++ b/packages/dapi/src/tools/check.ts@@ -34,7 +34,7 @@ export const check = defineTool({name: "check",title: "Check structure",description:- "Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load or generate — plus subtree stats (node count by kind, nesting depth, played duration). Times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock capture uses. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with capture.",+ "Check a node's subtree for obvious structural mistakes, without rendering (local analysis, no credits): spans where no visual is scheduled (likely black frames), children that never become visible, zero-duration or fully transparent nodes, and assets that failed to load — plus subtree stats (node count by kind, nesting depth, played duration). Times in issue ranges are seconds relative to the node's start — for a scene whose workarea starts at 0, the same clock capture uses. Structural only: a scheduled clip can still render black (dark footage, content smaller than the canvas), so confirm suspicious spans visually with capture.",input: z.object({ id: NodeId }),output: z.object({stats: z.object({diff --git a/packages/dapi/src/tools/context.ts b/packages/dapi/src/tools/context.tsindex dd923c89..87399f38 100644--- a/packages/dapi/src/tools/context.ts+++ b/packages/dapi/src/tools/context.ts@@ -7,20 +7,6 @@ import { defineTool } from "../tool";import { Bytes } from "../schemas";import { segmentFound, segmentSpan } from "./media-segment";-const GenerationRow = z.object({- element: z- .string()- .nullable()- .describe("the element's source stamp, `<file>:<key or position>`; null for an entity no element produced"),- name: z.string().nullable(),- state: z.enum(["generating", "failed", "done"]),- error: z.string().optional().describe("what the generation failed with, on failed rows"),- asset: z- .string()- .optional()- .describe("the library path the generation landed as, on done rows — ready for media_probe and its siblings"),-});-const maskRow = {id: z.string().describe("the track's id, the same across polls"),src: z.string().describe("the mask's library path, for `<mask src>`: the file is there once the row is done"),@@ -45,14 +31,14 @@ const MaskRow = z.object({/*** What the project's source cannot say: the JSX already holds the scenes,* the selection, and the work area, so the report is only the folders, the- * playhead, the fonts actually registered, and where generations and- * background mask tracks stand.+ * playhead, the fonts actually registered, and where background mask tracks+ * stand.*/export const context = defineTool({name: "context",title: "App context",description:- "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, where its `generate.*` declarations stand, and the progress of `media_segment` tracks running in the background. Poll it to wait for generations and tracks without blocking.",+ "Report the current app context: the folder new projects are created in (always reported), the folder of the project the app has open (null when none is), where its playhead sits in seconds, the registered font families, and the progress of `media_segment` tracks running in the background. Poll it to wait for tracks without blocking.",input: z.object({}),output: z.object({rootDir: z.string().nullable().describe("folder new projects are created in; null until one has been chosen"),@@ -64,7 +50,6 @@ export const context = defineTool({fontFamilies: z.array(z.string()).describe("families registered in the world drawing the project; the editor default is always among them"),- generations: z.array(GenerationRow),masks: z.array(MaskRow).describe("media_segment tracks started while this project is open, oldest first"),}),result: z.object({@@ -72,12 +57,10 @@ export const context = defineTool({projectDir: z.string().nullable(),currentTime: z.number().nullable(),fontFamilies: z.array(z.string()),- generations: z.array(GenerationRow),/** A done row's contact sheet comes as bytes, which the server writes to a file once per track. */masks: z.array(z.object({ ...maskRow, png: Bytes.optional() })),}),environment: "renderer",});-export type GenerationRow = z.output<typeof GenerationRow>;export type MaskRow = z.output<typeof MaskRow>;diff --git a/packages/dapi/src/tools/media-listen.ts b/packages/dapi/src/tools/media-listen.tsindex 1c56b097..c333777b 100644--- a/packages/dapi/src/tools/media-listen.ts+++ b/packages/dapi/src/tools/media-listen.ts@@ -10,7 +10,7 @@ export const mediaListen = defineTool({name: "media_listen",title: "Listen to audio",description:- "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed. Needs a signed-in account.",+ "Prompt a multimodal model for a semantic analysis of an audio track and return its answer. Shines on audio semantics (the name of the music playing, who is speaking, the spoken content with second-granularity timestamps). Accepts an audio file or a video; of a video only the audio track is analyzed.",input: z.object({path: AssetPath,diff --git a/packages/dapi/src/tools/models.ts b/packages/dapi/src/tools/models.tsdeleted file mode 100644index 91ac7253..00000000--- a/packages/dapi/src/tools/models.ts+++ /dev/null@@ -1,29 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { z } from "zod";-import { defineTool } from "../tool";--const ModelType = z.enum(["image", "video", "audio"]);--export const ModelInfo = z.object({- type: ModelType,- id: z.string(),- name: z.string(),- durations: z.array(z.string()).optional(),- aspectRatios: z.array(z.string()).optional(),- features: z.array(z.enum(["start-frame", "end-frame", "audio"])).optional(),-});--export const models = defineTool({- name: "models",- title: "Generation models",- description:- "List available AI generation models and their per-model constraints (durations, aspect ratios, features), for `generate.*` asset declarations in a project module.",- input: z.object({- type: ModelType.optional().describe("filter to one kind of model, image, video, or audio (default: all three)"),- }),- output: z.object({ models: z.array(ModelInfo) }),- environment: "renderer",-});diff --git a/packages/dapi/src/tools/tools.test.ts b/packages/dapi/src/tools/tools.test.tsindex 5a9e4aa5..4540b95c 100644--- a/packages/dapi/src/tools/tools.test.ts+++ b/packages/dapi/src/tools/tools.test.ts@@ -100,7 +100,7 @@ describe("logs and export", () => {describe("context", () => {it("reports the same shape with and without an open project", () => {expect(- context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], generations: [], masks: [] }).success,+ context.output.safeParse({ rootDir: "/p", projectDir: null, currentTime: null, fontFamilies: [], masks: [] }).success,).toBe(true);expect(context.output.safeParse({@@ -108,7 +108,6 @@ describe("context", () => {projectDir: "/p/a",currentTime: null,fontFamilies: ["Inter"],- generations: [{ element: "index.tsx:3", name: null, state: "done", asset: "gen/a.mp4" }],masks: [],}).success,).toBe(true);@@ -119,7 +118,7 @@ describe("context", () => {const span = { model: "tiny", frameRate: 30, start: 0, end: 2, frames: 60 };const running = { id: "t1", src: "masks/a/Tracking 1.mask", video: "a.mp4", state: "tracking", progress: 0.4, ...span };const found = { bbox: null, area: 0, score: 1, iou: 0.9, lost: [], weak: [] };- const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [], generations: [] };+ const base = { rootDir: "/p", projectDir: "/p/a", currentTime: null, fontFamilies: [] };expect(context.output.safeParse({ ...base, masks: [running] }).success).toBe(true);expect(context.result!.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, png: new Uint8Array(3), ...found }] }).success).toBe(true);expect(context.output.safeParse({ ...base, masks: [{ ...running, state: "done", progress: 1, image: "/tmp/s.png", ...found }] }).success).toBe(true);diff --git a/packages/dapi/src/tools/voices.ts b/packages/dapi/src/tools/voices.tsdeleted file mode 100644index cb052805..00000000--- a/packages/dapi/src/tools/voices.ts+++ /dev/null@@ -1,21 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { z } from "zod";-import { defineTool } from "../tool";--export const VoiceInfo = z.object({- id: z.string(),- label: z.string(),- description: z.string(),-});--export const voices = defineTool({- name: "voices",- title: "Speech voices",- description: "List the speech voices available for `generate.voice` declarations in a project module.",- input: z.object({}),- output: z.object({ voices: z.array(VoiceInfo) }),- environment: "renderer",-});diff --git a/packages/jsx/README.md b/packages/jsx/README.mdindex c1ca5a29..646eb0e8 100644--- a/packages/jsx/README.md+++ b/packages/jsx/README.md@@ -4,7 +4,7 @@ The editor supplies the runtime when a project is mounted, so this package isneeded for **types and tooling** — IntelliSense and `tsc --noEmit`. It carriesno renderer: `useTicker` is a declaration that throws outside a mount, andelements only become a composition once the editor renders them. The pure-helpers (`generate.*`, `parseTime`, the source-stamp constants) are real here;+helpers (`parseTime`, the source-stamp constants) are real here;everything else is a type.See [reference/jsx](https://github.com/diffusionstudio/editor/blob/main/reference/jsx/README.md)diff --git a/packages/jsx/package.json b/packages/jsx/package.jsonindex 75e304b5..602e8855 100644--- a/packages/jsx/package.json+++ b/packages/jsx/package.json@@ -1,7 +1,7 @@{"name": "@diffusionstudio/jsx","version": "0.207.0",- "description": "Types and authoring API for Diffusion Studio compositions: the JSX element vocabulary, declarative generated assets (`generate.*`), and the hooks the editor implements when it mounts a project.",+ "description": "Types and authoring API for Diffusion Studio compositions: the JSX element vocabulary and the hooks the editor implements when it mounts a project.","license": "MPL-2.0","type": "module","sideEffects": false,@@ -27,7 +27,6 @@"diffusion-studio","solid-js","jsx",- "generative-ai","video-editing"],"dependencies": {diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.tsdeleted file mode 100644index 9a44bc0a..00000000--- a/packages/jsx/src/generate.ts+++ /dev/null@@ -1,309 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--/**- * Declarative assets. A `generate.*` call makes an asset from a prompt; a- * `transform.*` call makes one from an asset. Both are pure: they validate- * their arguments and return an `AssetRef` value that can be passed wherever- * a source is expected (`src`, `startFrame`, `endFrame`, `refs`, a- * transform's input). Nothing runs until the mounted tree commits; refs- * never used by a mounted element are dropped. Because dependencies are- * values (not string ids), reference cycles are impossible by construction,- * and the order of a chain is its nesting:- * `transform.upscale(transform.removeBackground(generate.image({ prompt })))`.- */--export type AspectRatio = "1:1" | "4:3" | "3:4" | "16:9" | "9:16";--/** A path, URL, asset id, or another declaration. */-export type AssetInput = string | AssetRef;--export type GenerateImageOptions = {- prompt: string;- model?: string;- aspectRatio?: AspectRatio;- refs?: AssetInput[];- seed?: number;-};--export type GenerateVideoOptions = {- prompt: string;- model?: string;- aspectRatio?: AspectRatio;- /** Whole seconds; default 5. */- duration?: number;- /** Generate audio alongside; models with the `audio` feature only. */- audio?: boolean;- /** Image used as the first frame. */- startFrame?: AssetInput;- /** Image used as the last frame; `end-frame` feature only. */- endFrame?: AssetInput;- seed?: number;-};--export type GenerateVoiceOptions = {- /** The text to speak. */- prompt: string;- /** Voice id; default the first voice from `dapi voices`. */- voice?: string;- seed?: number;-};--export type GenerateAudioOptions = {- prompt: string;- model?: string;- duration?: number;- seed?: number;-};--/** What a `generate.*` call declares: an asset made from a prompt. */-export type GenerateSpec =- | ({ type: "image" } & GenerateImageOptions)- | ({ type: "video" } & GenerateVideoOptions)- | ({ type: "voice" } & GenerateVoiceOptions)- | ({ type: "audio" } & GenerateAudioOptions);--/** What a `transform.*` call declares: an asset made from another. */-export type TransformSpec =- /** The input at twice the pixels; a picture or footage. */- | { type: "upscale"; input: AssetInput }- /** The subject of a picture cut out, the rest transparent. */- | { type: "removeBackground"; input: AssetInput }- /** Footage scored with a generated soundtrack. */- | { type: "addAudio"; input: AssetInput };--export type TransformType = TransformSpec["type"];--export type AssetSpecInput = GenerateSpec | TransformSpec;--const GENERATE_TYPES: readonly string[] = ["image", "video", "voice", "audio"];-const TRANSFORM_TYPES: readonly string[] = ["upscale", "removeBackground", "addAudio"];--/** Whether a spec type is a transform's, as opposed to a generation's. */-export function isTransformType(type: string): type is TransformType {- return TRANSFORM_TYPES.includes(type);-}--export function isTransformSpec(spec: AssetSpecInput): spec is TransformSpec {- return isTransformType(spec.type);-}--/**- * Opaque handle to a declared (not-yet-made) asset. Obtained from- * `generate.*` or `transform.*`; consumed by `src`, `startFrame`, `endFrame`,- * `refs`, and a transform's input.- */-export class AssetRef {- /** @internal — read via `getAssetSpec`. */- readonly spec: AssetSpecInput;-- /** @internal — declare refs with `generate.*` / `transform.*`, not `new AssetRef()`. */- constructor(spec: AssetSpecInput) {- this.spec = spec;- }-}--export function isAssetRef(value: unknown): value is AssetRef {- return value instanceof AssetRef;-}--/** Host-side accessor for a declaration's spec. */-export function getAssetSpec(ref: AssetRef): AssetSpecInput {- return ref.spec;-}--/** The inputs a spec names, in option order: what the declaration depends on. */-export function getAssetInputs(spec: AssetSpecInput): AssetInput[] {- switch (spec.type) {- case "image":- return spec.refs ?? [];- case "video":- return [spec.startFrame, spec.endFrame].filter((input): input is AssetInput => input !== undefined);- case "voice":- case "audio":- return [];- default:- return [spec.input];- }-}--/**- * `ref` with each of its direct inputs replaced by what `map` makes of it.- * Direct only: a caller that wants the whole chain recurses through the refs- * `map` is handed. A ref none of whose inputs change is returned as it is.- */-export function mapAssetInputs(ref: AssetRef, map: (input: AssetInput) => AssetInput): AssetRef {- const spec = ref.spec;- let changed = false;- const mapped = (input: AssetInput): AssetInput => {- const next = map(input);- if (next !== input) changed = true;- return next;- };-- let next: AssetSpecInput;- switch (spec.type) {- case "image":- next = { ...spec, refs: spec.refs?.map(mapped) };- break;- case "video":- next = {- ...spec,- startFrame: spec.startFrame === undefined ? undefined : mapped(spec.startFrame),- endFrame: spec.endFrame === undefined ? undefined : mapped(spec.endFrame),- };- break;- case "voice":- case "audio":- return ref;- default:- next = { ...spec, input: mapped(spec.input) };- }- return changed ? new AssetRef(next) : ref;-}--// ----------------------------------------------------------------------------// The wire form--/** An input as data: a source string, or a declaration serialized in turn. */-export type SerializedAssetInput = string | SerializedAssetRef;--/** A spec as data: the same fields, with every declared input serialized. */-export type SerializedAssetSpec =- | ({ type: "image"; refs?: SerializedAssetInput[] } & Omit<GenerateImageOptions, "refs">)- | ({ type: "video"; startFrame?: SerializedAssetInput; endFrame?: SerializedAssetInput } & Omit<GenerateVideoOptions, "startFrame" | "endFrame">)- | ({ type: "voice" } & GenerateVoiceOptions)- | ({ type: "audio" } & GenerateAudioOptions)- | { type: TransformType; input: SerializedAssetInput };--/**- * A declaration as data, for the edit protocol. An `AssetRef` itself cannot- * travel as a prop value (see `isPropValue`: an object literal would not read- * back as one), so an edit carries the spec in this tagged shape instead, and- * the source writer spells it as the call that reproduces it — inputs- * included, so a transform over a declaration travels as the nested call it- * is.- */-export interface SerializedAssetRef {- $asset: SerializedAssetSpec;-}--export function serializeAssetRef(ref: AssetRef): SerializedAssetRef {- const serializeInput = (input: AssetInput): SerializedAssetInput =>- isAssetRef(input) ? serializeAssetRef(input) : input;-- const spec = ref.spec;- const serialized: Record<string, unknown> = {};- // Without the undefined entries: they are not part of the declaration, and- // the writer spells every entry it is handed.- for (const [key, value] of Object.entries(spec)) {- if (value === undefined) continue;- serialized[key] = key === "refs"- ? (value as AssetInput[]).map(serializeInput)- : key === "startFrame" || key === "endFrame" || key === "input"- ? serializeInput(value as AssetInput)- : value;- }- return { $asset: serialized as SerializedAssetSpec };-}--export function isSerializedAssetRef(value: unknown): value is SerializedAssetRef {- if (typeof value !== "object" || value === null || !("$asset" in value)) return false;- return isSerializedSpec((value as SerializedAssetRef).$asset);-}--function isSerializedInput(value: unknown): value is SerializedAssetInput {- return typeof value === "string" || isSerializedAssetRef(value);-}--function isSerializedSpec(spec: unknown): spec is SerializedAssetSpec {- if (typeof spec !== "object" || spec === null) return false;- const record = spec as Record<string, unknown>;- const type = record.type;- if (typeof type !== "string") return false;-- if (isTransformType(type)) return isSerializedInput(record.input);- if (!GENERATE_TYPES.includes(type) || typeof record.prompt !== "string") return false;- if (record.refs !== undefined && !(Array.isArray(record.refs) && record.refs.every(isSerializedInput))) return false;- if (record.startFrame !== undefined && !isSerializedInput(record.startFrame)) return false;- if (record.endFrame !== undefined && !isSerializedInput(record.endFrame)) return false;- return true;-}--// ----------------------------------------------------------------------------// The declarations--function requirePrompt(value: unknown, call: string): string {- if (typeof value !== "string" || value.trim().length === 0) {- throw new Error(`${call} requires a non-empty string prompt`);- }- return value;-}--function checkAssetInput(value: unknown, call: string, option: string): void {- if (value === undefined) return;- if (typeof value === "string" && value.trim().length > 0) return;- if (isAssetRef(value)) return;- throw new Error(`${call}: ${option} must be a path, URL, asset id, or an AssetRef`);-}--function requireAssetInput(value: unknown, call: string): AssetInput {- if (value === undefined) throw new Error(`${call} requires an input: a path, URL, asset id, or an AssetRef`);- checkAssetInput(value, call, "input");- return value as AssetInput;-}--export const generate = {- image(opts: GenerateImageOptions): AssetRef {- requirePrompt(opts.prompt, "generate.image");- for (const ref of opts.refs ?? []) checkAssetInput(ref, "generate.image", "refs");- return new AssetRef({ type: "image", ...opts });- },-- video(opts: GenerateVideoOptions): AssetRef {- requirePrompt(opts.prompt, "generate.video");- checkAssetInput(opts.startFrame, "generate.video", "startFrame");- checkAssetInput(opts.endFrame, "generate.video", "endFrame");- if (opts.duration !== undefined && (!Number.isInteger(opts.duration) || opts.duration <= 0)) {- throw new Error("generate.video: duration must be a positive whole number of seconds");- }- return new AssetRef({ type: "video", ...opts });- },-- voice(opts: GenerateVoiceOptions): AssetRef {- requirePrompt(opts.prompt, "generate.voice");- return new AssetRef({ type: "voice", ...opts });- },-- audio(opts: GenerateAudioOptions): AssetRef {- requirePrompt(opts.prompt, "generate.audio");- if (opts.duration !== undefined && (!Number.isFinite(opts.duration) || opts.duration <= 0)) {- throw new Error("generate.audio: duration must be a positive number of seconds");- }- return new AssetRef({ type: "audio", ...opts });- },-};--/**- * Assets made from assets. Each takes what it works on as its argument — a- * source string or another declaration — so a chain reads inside out, and- * runs in that order. The result is cached by step and input like any- * generation: the same call on the same asset is the same asset.- */-export const transform = {- /** `input` at twice the pixels. A picture or footage; the box it is shown in is unchanged. */- upscale(input: AssetInput): AssetRef {- return new AssetRef({ type: "upscale", input: requireAssetInput(input, "transform.upscale") });- },-- /** The subject of the picture `input` cut out, the rest transparent. */- removeBackground(input: AssetInput): AssetRef {- return new AssetRef({ type: "removeBackground", input: requireAssetInput(input, "transform.removeBackground") });- },-- /** The footage `input` scored with a generated soundtrack. */- addAudio(input: AssetInput): AssetRef {- return new AssetRef({ type: "addAudio", input: requireAssetInput(input, "transform.addAudio") });- },-};diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 4aebee5d..a0ec6f0c 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -10,34 +10,6 @@* @diffusionstudio/reconciler), so nothing in this package touches a host.*/-export {- generate,- transform,- AssetRef,- isAssetRef,- getAssetSpec,- getAssetInputs,- mapAssetInputs,- isTransformSpec,- isTransformType,- serializeAssetRef,- isSerializedAssetRef,-} from "./generate";-export type {- AspectRatio,- AssetInput,- AssetSpecInput,- GenerateSpec,- TransformSpec,- TransformType,- SerializedAssetInput,- SerializedAssetRef,- SerializedAssetSpec,- GenerateAudioOptions,- GenerateImageOptions,- GenerateVideoOptions,- GenerateVoiceOptions,-} from "./generate";export { parseTime, TIME_FPS } from "./time";export {COMPOSITION_TAGS,diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.tsindex 413ec625..358be176 100644--- a/packages/jsx/src/jsx-runtime.ts+++ b/packages/jsx/src/jsx-runtime.ts@@ -19,7 +19,6 @@*/import type { JSX as SolidJSX } from "solid-js";-import type { AssetInput } from "./generate";import type {AdjustmentLayerProps,AnimationProps,@@ -60,9 +59,9 @@ import type {type HtmlElementTags = Omit<SolidJSX.HTMLElementTags, "canvas" | "audio" | "video" | "html" | "img">;// `<img>` keeps every DOM attribute but takes the composition `src` inputs:-// a path, an asset id, a URL, or a `generate.*` ref all resolve through the-// host, and `data:`/`blob:` sources pass through to the browser untouched.-type ImgTag = Omit<SolidJSX.HTMLElementTags["img"], "src"> & { src?: AssetInput };+// a path, an asset id, or a URL all resolve through the host, and+// `data:`/`blob:` sources pass through to the browser untouched.+type ImgTag = Omit<SolidJSX.HTMLElementTags["img"], "src"> & { src?: string };// The shared names are re-declared below as unions with the composition props.type SvgElementTags = Omit<SolidJSX.SVGElementTags, "rect" | "text" | "image">;diff --git a/packages/jsx/src/source.ts b/packages/jsx/src/source.tsindex 5610f831..c9ccce5a 100644--- a/packages/jsx/src/source.ts+++ b/packages/jsx/src/source.ts@@ -60,7 +60,7 @@ export type PropValue = number | string | boolean | null | PropValue[] | { [key:/*** Whether a value is one a source file could spell — a `PropValue`. An- * instance of a class (an `AssetRef`, say) is not, however plain its fields:+ * instance of a class (a `Date`, say) is not, however plain its fields:* spelled as an object literal it would read back as something else.*/export function isPropValue(value: unknown): value is PropValue {diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 9188c69a..14f7347f 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -4,7 +4,6 @@import type { JSX as SolidJSX } from "solid-js";import type { Entity } from "koota";-import type { AssetRef } from "./generate";/*** Composition-relative time: seconds (number), frames ("30f"), or a@@ -312,14 +311,13 @@ type FillProps = {type MediaProps = {/**- * Path, URL, asset id, or a `generate.*` / `transform.*` declaration. A path naming a- * directory of numbered frames (`shot_001.png`, `shot_002.png`, ...) is an- * image sequence, and plays on `<Video>` or `<Image>` as footage does — see- * `frameRate` for how long it lasts. On `<Captions>` a transcript source- * (.srt, .vtt, or transcript .json) mounted instead of transcribing the- * scene; `generate.*` is not accepted there.+ * Path, URL, or asset id. A path naming a directory of numbered frames+ * (`shot_001.png`, `shot_002.png`, ...) is an image sequence, and plays on+ * `<Video>` or `<Image>` as footage does — see `frameRate` for how long it+ * lasts. On `<Captions>` a transcript source: .srt, .vtt, transcript .json,+ * or an audio or video asset carrying a transcript.*/- src: string | AssetRef;+ src: string;};type FitProps = {@@ -852,7 +850,7 @@ export type SequenceProps = Pick<IdentityProps, "name"> & {children?: SolidJSX.Element;};-export type CaptionsProps = IdentityProps & TimingProps & OffsetProps & Partial<MediaProps> & {+export type CaptionsProps = IdentityProps & TimingProps & OffsetProps & MediaProps & {/** Caption style preset. Default "classic". */preset?: CaptionPreset;/** Fills the caption preset's color slots in order; any CSS color, alpha is ignored. */@@ -863,12 +861,6 @@ export type CaptionsProps = IdentityProps & TimingProps & OffsetProps & Partial<* placement. Defaults to the preset's own alignment.*/verticalAlign?: "top" | "center" | "bottom";- /**- * Transcription seed. Part of the transcript cache key (scene id + seed),- * so a new value bypasses the cached transcript and transcribes the scene- * again; reusing a value replays that take from cache. Default 0.- */- seed?: number;/** `<Animation>` children. */children?: SolidJSX.Element;};diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex cc3d37e9..50da72d8 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, Constraint, ConstraintCache, ConstraintType, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Mask, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsClipPath, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setPlayhead, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, Constraint, ConstraintCache, ConstraintType, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, getActiveEntity, Loop, LoadRequest, Mask, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsClipPath, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setPlayhead, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';import { DEFAULT_MASK_SMOOTHING } from '@diffusionstudio/assets';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';@@ -11,7 +11,7 @@ import { SVGElements } from 'solid-js/web';import { IsExcluded } from 'koota';import type { CameraMatrix, PropertyPath, SceneNode, TimelineView } from '@diffusionstudio/runtime';-import type { AnimatableProperty, AssetRef } from '@diffusionstudio/jsx';+import type { AnimatableProperty } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { ProjectDocument, ProjectTick } from './host';@@ -501,7 +501,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {entity.set(Position, { x: 0, y: 0 });entity.add(Chars);entity.add(Caption);- entity.add(TranscriptionRequest);entity.add(TextStyle);entity.set(TextStyle, CAPTION_PRESET_STYLES[CaptionType.CLASSIC]);entity.add(Color);@@ -1189,28 +1188,13 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {return;}case 'src': {- // Generating goes with the wait it belongs to: the resolution- // running for the old src will not clear it, having been- // superseded by whatever this one starts.- entity.remove(GenerationRequest, LoadRequest, PendingSource, Generating);+ // The resolution running for the old src is superseded by+ // whatever this one starts.+ entity.remove(LoadRequest, PendingSource);// `false` is how an editor unsets a prop (the writer drops the attribute).- if (value === undefined || value === null || value === false || value === '') {- entity.remove(AssetId);- // A `<captions>` without a src transcribes its scene instead.- if (entity.has(Caption)) {- entity.add(TranscriptionRequest);- entity.set(TranscriptionRequest, { seed: toNumber(node.props.seed) ?? 0 });- }- return;- }-- entity.remove(TranscriptionRequest);-- if (typeof value !== 'string') {+ if (typeof value !== 'string' || value === '') {entity.remove(AssetId);- entity.add(GenerationRequest);- entity.set(GenerationRequest, { ref: value as AssetRef });return;}@@ -1394,19 +1378,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {});return;}- case 'seed': {- if (!entity.has(Caption)) return;- // An authored src mounts a transcript directly; there is no- // transcription for the seed to key.- if (node.props.src !== undefined) return;- if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;-- // A resolution running for another seed must not bind late- entity.remove(PendingSource, Generating);- entity.add(TranscriptionRequest);- entity.set(TranscriptionRequest, { seed: toNumber(value) ?? 0 });- return;- }case 'workarea': {// Two times or none: a half-authored range says nothing, and// `false` is what the editor writes to take the brackets off.diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex 662716a9..cef51b6e 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -6,12 +6,11 @@// goes through the AssetLibrary (@diffusionstudio/assets); these are the// lookups the decoders and hosts share.-import { Ai, AssetId, Library, Paint, SourceFrameRate } from '../traits';+import { AssetId, Library, Paint, SourceFrameRate } from '../traits';import { PaintType } from '../constants';import type { Entity, World } from 'koota';import type { Asset, AssetLibrary, MaskAsset, SequenceAsset } from '@diffusionstudio/assets';-import type { GenAi } from '../ai';/** The world's asset library; throws when the host attached none. */export function getLibrary(world: World): AssetLibrary {@@ -20,13 +19,6 @@ export function getLibrary(world: World): AssetLibrary {return library;}-/** The world's generation service; throws when the host attached none. */-export function getAi(world: World): GenAi {- const ai = world.get(Ai);- if (!ai) throw new Error('This world cannot generate assets (no Ai attached)');- return ai;-}-/*** Binds an entity to an asset: stamps its AssetId, and follows the asset with* the paint. A media paint is whichever of the two the asset can be drawndiff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsdeleted file mode 100644index 21c81db8..00000000--- a/packages/runtime/src/ai.ts+++ /dev/null@@ -1,64 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--import { generate, transform } from '@diffusionstudio/jsx';--import type { Entity, World } from 'koota';-import type { Asset } from '@diffusionstudio/assets';-import type {- AssetInput,- AssetRef,- GenerateAudioOptions,- GenerateImageOptions,- GenerateVideoOptions,- GenerateVoiceOptions,-} from '@diffusionstudio/jsx';--/**- * The generation service a host attaches to its world. Where a generation- * stands is the library's to hold, as partial documents (see- * `PartialAsset` in @diffusionstudio/assets): a run that starts is- * `pending` there, one that lands is the asset, one that fails stays as- * `error` with its reason — and a key standing in error resolves to that- * error rather than to another run, in this session and the next. Nothing- * here writes to the source that declared the generation.- */-export abstract class GenAi {- /**- * The declaration, made real: an asset whose bytes the model produced —- * from a prompt (`generate.*`) or from another asset (`transform.*`), its- * inputs resolved first. Content-addressed — the fully-resolved spec's- * hash is the asset's `generation.key`, so the same spec is the same- * asset in this session and the next, and identical concurrent- * declarations share one run. Rejects with the recorded reason when the- * key stands in error.- */- public abstract resolve(ref: AssetRef): Promise<Asset>;-- /**- * The transcript of `scene`'s audible mix, as a TRANSCRIPT asset of the- * project's library. Cached by scene id + seed: the same pair resolves to- * the same asset, in this session and the next, and a new seed transcribes- * the scene again. Identical concurrent requests share one run.- */- public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;-- /**- * The imperative surface: `ai.generate.image({...})` declares and resolves- * in one call, returning the finished asset.- */- public readonly generate = {- image: (options: GenerateImageOptions): Promise<Asset> => this.resolve(generate.image(options)),- video: (options: GenerateVideoOptions): Promise<Asset> => this.resolve(generate.video(options)),- voice: (options: GenerateVoiceOptions): Promise<Asset> => this.resolve(generate.voice(options)),- audio: (options: GenerateAudioOptions): Promise<Asset> => this.resolve(generate.audio(options)),- };-- /** The same for `transform.*`: `ai.transform.upscale(input)` is the upscaled asset. */- public readonly transform = {- upscale: (input: AssetInput): Promise<Asset> => this.resolve(transform.upscale(input)),- removeBackground: (input: AssetInput): Promise<Asset> => this.resolve(transform.removeBackground(input)),- addAudio: (input: AssetInput): Promise<Asset> => this.resolve(transform.addAudio(input)),- };-}diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.tsindex f777bd2a..2a3827d2 100644--- a/packages/runtime/src/index.ts+++ b/packages/runtime/src/index.ts@@ -1,5 +1,4 @@export * from './constants';-export * from './ai';export * from './traits';export * from './world/create-world';export * from './world/observers';diff --git a/packages/runtime/src/media/transcode.ts b/packages/runtime/src/media/transcode.tsindex cce17f00..b68af806 100644--- a/packages/runtime/src/media/transcode.ts+++ b/packages/runtime/src/media/transcode.ts@@ -2,88 +2,20 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Asset transcoding and preview rendering (was engine/utils/transcode.ts;-// the timecode formatters moved to utils/time.ts). Filmstrips and waveforms+// Asset preview rendering (was engine/utils/transcode.ts; the timecode+// formatters moved to utils/time.ts). Filmstrips and waveforms// draw into an OffscreenCanvas, so everything here is worker-capable.-import { ALL_FORMATS, AudioSampleSink, BlobSource, BufferTarget, CanvasSink, Conversion, Input, InputAudioTrack, Mp4OutputFormat, OggOutputFormat, Output, StreamTarget } from 'mediabunny';+import { ALL_FORMATS, AudioSampleSink, BlobSource, CanvasSink, Input, InputAudioTrack } from 'mediabunny';import { assert } from '../utils/assert';import { formatTimestamp } from '../utils/time';import { getAssetFile } from '../actions/assets';-import type { StreamTargetChunk } from 'mediabunny';import type { Asset, VideoAsset, AudioAsset } from '@diffusionstudio/assets';type Ctx2D = CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;-export async function transcodeForTranscription(asset: Asset): Promise<File> {- const blob = await getAssetFile(asset);- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });- try {- const output = new Output({ format: new OggOutputFormat(), target: new BufferTarget() });- const conversion = await Conversion.init({- input,- output,- video: { discard: true },- audio: { codec: "opus", numberOfChannels: 1, sampleRate: 16000 },- });- if (!conversion.isValid || conversion.utilizedTracks.length === 0) {- throw new Error("No audio found. The asset has no audio track to transcribe.");- }- await conversion.execute();-- const buffer = output.target.buffer;- assert(buffer, "Transcoding produced no output.");- return new File([buffer], `${crypto.randomUUID()}.ogg`, { type: "audio/ogg" });- } finally {- input.dispose();- }-}--export async function transcodeForAnalysis(asset: VideoAsset | AudioAsset, window?: TimeWindow & { stripVideo?: boolean }) {- const blob = await getAssetFile(asset);- const hasWindow = window?.start !== undefined || window?.end !== undefined;-- const isVideo = asset.type === "VIDEO" && !window?.stripVideo;- const trim = hasWindow ? resolveWindow(asset.duration, window) : undefined;-- const input = new Input({ formats: ALL_FORMATS, source: new BlobSource(blob) });-- const { writable, readable } = new TransformStream<StreamTargetChunk, Uint8Array<ArrayBuffer>>({- transform: (chunk, controller) => controller.enqueue(chunk.data),- });-- const target = new StreamTarget(writable);- const format = isVideo- ? new Mp4OutputFormat({ fastStart: "fragmented" })- : new OggOutputFormat();- const output = new Output({ format, target });-- const conversion = await Conversion.init({- input,- output,- trim: (trim && { start: trim.start, end: trim.end }),- video: { width: 640, frameRate: 24, bitrate: 500_000, discard: !isVideo },- audio: { codec: isVideo ? undefined : "opus", numberOfChannels: 1, sampleRate: 16_000 },- });-- if (!conversion.isValid || conversion.utilizedTracks.length === 0) {- input.dispose();- throw new Error("Nothing to analyze. The asset has no usable video or audio track.");- }-- const run = async () => {- try {- await conversion.execute();- } finally {- input.dispose();- }- };-- return { readable, run };-}-const PATCH_SIZE = 28;const MAX_WIDTH_IN_TOKENS = 64; // 92const MAX_HEIGHT_IN_TOKENS = 52;diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex a0e2e647..f14a5b92 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -4,9 +4,9 @@import { parseSource } from '@diffusionstudio/jsx';-import { Ai, Cache, Computed, Delay, FrameRate, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, TranscriptionRequest, Trim } from '../traits';+import { Cache, Computed, Delay, FrameRate, FramePromises, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, Trim } from '../traits';import { bindAsset, getAssetFile } from '../actions/assets';-import { getEntityTree, getParentNode, getSceneAncestor } from '../queries/hierarchy';+import { getParentNode } from '../queries/hierarchy';import { findAssetDuration, findGeometryAsset } from '../utils/time';import { computeAudioSyncOffsetCached } from '../media/audio-sync';import { store } from '../world/store';@@ -34,26 +34,6 @@ export function assetSystem(world: World): void {}}- const ai = world.get(Ai);- if (ai) {- for (const entity of world.query(GenerationRequest)) {- const ref = entity.get(GenerationRequest)!.ref;- entity.remove(GenerationRequest);- if (ref === null) continue;- if (isDomImage(entity)) pointDomImageAt(entity, null);- resolve(world, entity, ai.resolve(ref), true);- }-- for (const entity of world.query(TranscriptionRequest)) {- const scene = getSceneAncestor(entity);- if (!scene || hasPendingSources(world, scene, entity)) continue;-- const seed = entity.get(TranscriptionRequest)!.seed;- entity.remove(TranscriptionRequest);- resolve(world, entity, ai.transcribe(world, scene, seed), true);- }- }-// Handle syncTo requestsfor (const entity of world.query(SyncRequest)) {if (waitingOnSources(entity)) continue;@@ -200,14 +180,14 @@ function waitingOnSources(entity: Entity): boolean {}function pendingSource(entity: Entity): boolean {- return entity.has(LoadRequest) || entity.has(GenerationRequest) || entity.has(PendingSource);+ return entity.has(LoadRequest) || entity.has(PendingSource);}/** Takes the sync off and leaves why on the element, `resolve`-style. */function fail(entity: Entity, message: string): void {entity.remove(SyncRequest, PendingSync);entity.add(SourceError);- entity.set(SourceError, { value: message, generated: false });+ entity.set(SourceError, { value: message });console.error('[runtime] could not sync:', message);}@@ -215,23 +195,6 @@ function currentSync(entity: Entity, token: object): boolean {return entity.isAlive() && entity.get(PendingSync)?.value === token;}-/**- * Whether anything in `scene`'s subtree (besides `except`, the requesting- * element itself) is still waiting on a source: a request this system has- * not consumed, or a resolution it started that has not landed. A pending- * `syncTo` counts — the clip has a source but not yet the placement a- * transcription would read it at.- */-function hasPendingSources(world: World, scene: Entity, except: Entity): boolean {- for (const entity of getEntityTree(world, scene)) {- if (entity === except) continue;- if (pendingSource(entity) || entity.has(SyncRequest) || entity.has(PendingSync)) {- return true;- }- }- return false;-}-/*** Tracks one started resolution: the entity remembers which one it is waiting* on (PendingSource), so of overlapping resolutions only the latest binds, and@@ -239,23 +202,22 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean* the identity rather than what was asked for — the later question is always* the one being asked.*/-function resolve(world: World, entity: Entity, promise: Promise<Asset>, generating = false): void {+function resolve(world: World, entity: Entity, promise: Promise<Asset>): void {const token = {};entity.remove(SourceError);entity.add(PendingSource);entity.set(PendingSource, { value: token });- if (generating) entity.add(Generating);const done = promise.then(async (asset) => {if (!current(entity, token)) return;bindAsset(entity, asset);await bindDomImage(entity, asset, token);- if (current(entity, token)) entity.remove(PendingSource, Generating);+ if (current(entity, token)) entity.remove(PendingSource);}).catch((error: unknown) => {if (!current(entity, token)) return;- entity.remove(PendingSource, Generating);+ entity.remove(PendingSource);entity.add(SourceError);- entity.set(SourceError, { value: errorMessage(error), generated: generating });+ entity.set(SourceError, { value: errorMessage(error) });console.error('[runtime] could not resolve src:', error);});diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex f25ca488..b21a8c55 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -29,7 +29,7 @@ import {import { getParentNode } from '../queries/hierarchy';import { getViewMatrix } from '../queries/camera';import { colorToHex } from '../utils/color';-import { FAILED_COLOR, getGeneratingColor, getSourceFailure, isGenerating } from '../utils/generating';+import { FAILED_COLOR, getSourceFailure } from '../utils/source-failure';import { applyStrokeStyle } from '../utils/stroke';import { renderText } from '../utils/text';import { getTransitionWindow } from '../utils/transition';@@ -589,14 +589,13 @@ function renderStrokes(world: World, entity: Entity): void {}/**- * The pulse a node waiting on a generation is filled with+ * The still fill a node whose source failed is left with*/-function renderGenerating(world: World, entity: Entity): void {- const errored = getSourceFailure(entity) !== undefined;- if (!errored && !isGenerating(entity)) return;+function renderSourceFailure(world: World, entity: Entity): void {+ if (getSourceFailure(entity) === undefined) return;const ctx = getCtx(world);- ctx.fillStyle = errored ? FAILED_COLOR : getGeneratingColor(world);+ ctx.fillStyle = FAILED_COLOR;ctx.fill();}@@ -667,7 +666,7 @@ function renderShapeNode(world: World, entity: Entity): void {renderShadows(world, entity);renderIntrinsicFill(world, entity);renderFills(world, entity);- renderGenerating(world, entity);+ renderSourceFailure(world, entity);renderStrokes(world, entity);}diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex b4ae1e72..d7455338 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -6,7 +6,6 @@ import { trait } from 'koota';import { GeometryType, PaintType, CaptionType, CaptionAlign } from '../constants';-import type { AssetRef } from '@diffusionstudio/jsx';// Geometric primitive: RECT or TEXT (see GeometryType). Other node-like roles// (group, audio, scene, caption) are layered on top via tag traits.@@ -48,9 +47,6 @@ export const Hidden = trait();export const ClipsContent = trait();-// Tag for entities whose content is still being generated.-export const Generating = trait();-export const Name = trait({ value: '' });// Stable identity for entities.@@ -78,26 +74,15 @@ export const AssetId = trait({ value: '' });// and stamps AssetId when the asset lands. Never serialized: a request is// this world's business, re-derived from the src on any re-render.-// The `generate.*` declaration a src names, to run through the world's Ai.-export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }));-// A source outside the library — a path or URL — to load into memory// through the library.export const LoadRequest = trait({ value: '' });-// On a `<captions>` element without a src: transcribe the scene it sits-// under, through the world's Ai. The seed keys the take — the transcript is-// cached by scene id + seed, so re-running with a seed replays that take and-// bumping the seed transcribes the scene again. Seed 0 is the default take.-export const TranscriptionRequest = trait({ seed: 0 });-// On an element authored with `syncTo`: the id of the element to align// against by listening instead of arithmetic. The asset system waits for both// sides' assets to land, cross-correlates the two recordings, and derives-// Delay/Trim so they coincide on the timeline. Stands (with PendingSync) as a-// pending source, so a transcription of the scene waits for the clip to land-// where it will play. Never serialized, for the same reason the requests-// above are not.+// Delay/Trim so they coincide on the timeline. Never serialized, for the same+// reason the requests above are not.export const SyncRequest = trait({ value: '' });// The syncTo measurement inflight, kept while the asset system correlates, so@@ -113,14 +98,8 @@ export const PendingSource = trait(() => ({ value: undefined as unknown }));// Why the entity's src never became an asset: the message of the rejection// the asset system saw. Session state, like the requests above: it stands// until a resolution for the src starts again, and is never written back to-// the source. What outlives the session is the library's: a generation that-// failed stands there as a partial document carrying this same message, and-// answers the next request for it with the failure rather than another run.-//-// `generated` tells a failed generation from a failed load, for whoever-// reports them: the one is a model refusing or a spec a model cannot take,-// the other a path that is not there.-export const SourceError = trait({ value: '', generated: false });+// the source.+export const SourceError = trait({ value: '' });// Sibling order under a ChildOf parent.export const ItemIndex = trait({ value: 0 });diff --git a/packages/runtime/src/traits/world.ts b/packages/runtime/src/traits/world.tsindex e5523e6f..7af2e9ab 100644--- a/packages/runtime/src/traits/world.ts+++ b/packages/runtime/src/traits/world.ts@@ -6,7 +6,6 @@ import { trait, type Entity, type World } from 'koota';import type { FontSource } from '../fonts/types';import type { AssetLibrary } from '@diffusionstudio/assets';-import type { GenAi } from '../ai';import type { Quad } from '../math/aabb';@@ -70,12 +69,6 @@ export const Fonts = trait({ list: () => [] as FontSource[] });// and transient alike. Null in a world without a project.export const Library = trait(() => null as AssetLibrary | null);-// The generation service, once the host has attached one (it needs a-// backend): what the asset system runs `generate.*` sources through, and-// what `getAi(world).generate.image(...)` calls. Optional — in a world-// without one, such sources stay pending and never resolve.-export const Ai = trait(() => null as GenAi | null);-// Per-tick callbacks. A mounted project adds one to feed its `useTicker`// clock; the playback system fires the set once per tick, after playheads// advance. Held on the world so anything that runs the systems (the editordiff --git a/packages/runtime/src/utils/generating.ts b/packages/runtime/src/utils/generating.tsdeleted file mode 100644index cf67ecd7..00000000--- a/packages/runtime/src/utils/generating.ts+++ /dev/null@@ -1,96 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--/**- * Pulse animation for entities with the Generating tag, and the still fill- * that takes its place when the wait ends in an error. Everything that draws- * one reads it from here — the stage and the timeline both — so a node- * waiting on a generation pulses in step wherever it is shown.- *- * Easing: cubic-bezier(0.52, 0.18, 0.56, 0.88)- * Duration: 0.6s per half-cycle (alternating)- * Delay: 0.2s before the first transition- * Full cycle: 0.2 delay + 0.6 forward + 0.6 reverse = 1.4s- * repeats ∞ for the entire gen phase (~4.5s)- *- * Pulses between --background (#1c1c1c) and --secondary (#292929).- */--import { cubicBezier } from 'animejs';--import { Cache, Generating, SourceError, Time } from '../traits';--import type { Entity, World } from 'koota';--const generatingEase = cubicBezier(0.52, 0.18, 0.56, 0.88);--const GEN_DELAY = 200; // ms-const GEN_DURATION = 600; // ms per half-cycle-const GEN_CYCLE = GEN_DURATION * 2; // full forward+reverse cycle--// Colors: #1c1c1c → rgb(28,28,28) and #292929 → rgb(41,41,41)-const GEN_FROM_R = 28, GEN_FROM_G = 28, GEN_FROM_B = 28;-const GEN_TO_R = 41, GEN_TO_G = 41, GEN_TO_B = 41;--/**- * Whether the node is waiting on a generation. The wait sits wherever the- * src does: on the node itself where the paint is intrinsic (`<image>`,- * `<video>`, `<audio>`, `<captions>`), or on one of its paints where it is- * authored as a child (`<rect><imagePaint src /></rect>`).- */-export function isGenerating(entity: Entity): boolean {- if (entity.has(Generating)) return true;- for (const fill of entity.get(Cache)?.fills ?? []) {- if (fill.has(Generating)) return true;- }- return false;-}--/**- * The reason the node's source failed, from wherever the src sits — the- * counterpart of `isGenerating`, and what a still red fill is drawn for.- */-export function getSourceFailure(entity: Entity): string | undefined {- const own = entity.get(SourceError)?.value;- if (own) return own;- for (const fill of entity.get(Cache)?.fills ?? []) {- const failure = fill.get(SourceError)?.value;- if (failure) return failure;- }- return undefined;-}--/** Where the pulse stands on the world's clock: 0 at rest, 1 fully lit. */-function generatingFactor(world: World): number {- const t = world.get(Time)?.now ?? 0;-- // Before initial delay → show base color- const elapsed = t % (GEN_DELAY + GEN_CYCLE);- if (elapsed < GEN_DELAY) return 0;-- const cycleT = elapsed - GEN_DELAY;- const half = cycleT % GEN_DURATION;- const eased = generatingEase(half / GEN_DURATION);-- // Alternate direction: first half forward, second half reverse- return cycleT >= GEN_DURATION ? 1 - eased : eased;-}--/**- * The fill a node whose source failed is left with: the pulse gone still and- * turned toward red, so a node that will never fill itself in is not mistaken- * for one that still might.- */-export const FAILED_COLOR = '#2e1d1d';--/** The fill a node in generation is painted with this frame. */-export function getGeneratingColor(world: World): string {- const factor = generatingFactor(world);-- const r = Math.round(GEN_FROM_R + (GEN_TO_R - GEN_FROM_R) * factor);- const g = Math.round(GEN_FROM_G + (GEN_TO_G - GEN_FROM_G) * factor);- const b = Math.round(GEN_FROM_B + (GEN_TO_B - GEN_FROM_B) * factor);-- return `rgb(${r},${g},${b})`;-}diff --git a/packages/runtime/src/utils/index.ts b/packages/runtime/src/utils/index.tsindex ccd788a5..6e61532f 100644--- a/packages/runtime/src/utils/index.ts+++ b/packages/runtime/src/utils/index.ts@@ -2,7 +2,7 @@export * from './assert';export * from './async';export * from './color';-export * from './generating';+export * from './source-failure';export * from './text';export * from './sort';export * from './stroke';diff --git a/packages/runtime/src/utils/source-failure.ts b/packages/runtime/src/utils/source-failure.tsnew file mode 100644index 00000000..5a3ac077--- /dev/null+++ b/packages/runtime/src/utils/source-failure.ts@@ -0,0 +1,35 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++/**+ * The still fill a node is left with when its source failed. Everything that+ * draws one reads it from here — the stage and the timeline both — so a+ * broken node looks the same wherever it is shown.+ */++import { Cache, SourceError } from '../traits';++import type { Entity } from 'koota';++/**+ * The reason the node's source failed, from wherever the src sits: on the+ * node itself where the paint is intrinsic (`<image>`, `<video>`, `<audio>`,+ * `<captions>`), or on one of its paints where it is authored as a child+ * (`<rect><imagePaint src /></rect>`).+ */+export function getSourceFailure(entity: Entity): string | undefined {+ const own = entity.get(SourceError)?.value;+ if (own) return own;+ for (const fill of entity.get(Cache)?.fills ?? []) {+ const failure = fill.get(SourceError)?.value;+ if (failure) return failure;+ }+ return undefined;+}++/**+ * The fill a node whose source failed is left with: a dark red, so a node+ * that will never fill itself in is told apart from an empty one.+ */+export const FAILED_COLOR = '#2e1d1d';diff --git a/packages/runtime/src/world/create-world.ts b/packages/runtime/src/world/create-world.tsindex 425ad6b4..273bd3eb 100644--- a/packages/runtime/src/world/create-world.ts+++ b/packages/runtime/src/world/create-world.ts@@ -18,7 +18,6 @@ import {AudioEngine,Fonts,Library,- Ai,Tickers,FramePromises,HitRegions,@@ -41,7 +40,6 @@ export function createRuntimeWorld(projectId: string) {AudioEngine,Fonts,Library,- Ai,Tickers,FramePromises,HitRegions,