The VS Code of video editing, built for agents.
TypeScript
Refactor and enhance transform functionality in the asset management system
konstantin-paulus1bbe99234578
diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.tsindex f0f7dd70..16a830cf 100644--- a/apps/desktop/src/edit.ts+++ b/apps/desktop/src/edit.ts@@ -9,7 +9,7 @@ import { join } from "node:path";import { IndentationText, Project, SyntaxKind } from "ts-morph";-import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, parseSource } from "@diffusionstudio/jsx";+import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, isTransformType, parseSource } from "@diffusionstudio/jsx";import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";import type {@@ -119,7 +119,8 @@ const round = (value: number): number => Math.round(value * 100) / 100;const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;/** A prop value as the JavaScript that would have produced it. */-function literalText(value: PropValue): string {+function literalText(value: PropValue | SerializedAssetRef): string {+ if (isSerializedAssetRef(value)) return declarationText(value);if (typeof value === "string") return JSON.stringify(value);if (typeof value === "number") return String(round(value));if (typeof value === "boolean" || value === null) return String(value);@@ -149,24 +150,41 @@ const initializerText = (value: PropValue): string =>? `"${value}"`: `{${literalText(value)}}`;-/** A declaration's wire form as the `generate.*` call that reproduces it. */-function generateCallText(ref: SerializedAssetRef): string {- const { type, ...options } = ref.$generate;+/**+ * A declaration's wire form as the call that reproduces it: `generate.<type>`+ * over its options, or `transform.<type>` over its input, with declared+ * inputs spelled as the nested calls they are.+ */+function declarationText(ref: SerializedAssetRef): string {+ const { type, ...options } = ref.$asset;+ if (isTransformType(type)) return `transform.${type}(${literalText((options as { input: EditValue }).input)})`;return `generate.${type}(${literalText(options as Record<string, PropValue>)})`;}+/** The namespaces a declaration is spelled with: what its file has to import. */+function declarationImports(ref: SerializedAssetRef, names = new Set<string>()): Set<string> {+ const { type, ...options } = ref.$asset;+ names.add(isTransformType(type) ? "transform" : "generate");+ for (const value of Object.values(options)) {+ for (const input of Array.isArray(value) ? value : [value]) {+ if (isSerializedAssetRef(input)) declarationImports(input, names);+ }+ }+ return names;+}+/*** Writes a prop onto a tag as one would write it: `muted`, not `muted={true}`,* and no attribute at all rather than `muted={false}`, since absence is what a- * boolean prop's false reads as. A declaration is spelled as its `generate.*`- * call — the caller makes sure `generate` is imported (see- * `ensureGenerateImport`).+ * boolean prop's false reads as. A declaration is spelled as the call that+ * reproduces it — the caller makes sure its namespaces are imported (see+ * `ensureDeclarationImports`).*/function setProp(tag: JsxTag, name: string, value: EditValue): void {const attribute = attributeOf(tag, name);if (isSerializedAssetRef(value)) {- const initializer = `{${generateCallText(value)}}`;+ const initializer = `{${declarationText(value)}}`;if (attribute) attribute.setInitializer(initializer);else tag.addAttribute({ name, initializer });return;@@ -187,32 +205,37 @@ function setProp(tag: JsxTag, name: string, value: EditValue): void {else tag.addAttribute({ name, initializer: initializerText(value) });}-/** Where `generate` comes from — the module every project authors against. */-const GENERATE_MODULE = "@diffusionstudio/jsx";+/** Where `generate` and `transform` come from — the module every project authors against. */+const DECLARATION_MODULE = "@diffusionstudio/jsx";/**- * Makes sure `generate` is in scope once a declaration has been spelled into- * the file: added to the module's existing import, or as an import of its own- * after the last one — or above the first statement, past the file's header- * comments, when there are no imports at all. Idempotent.+ * Makes sure the namespaces a declaration is spelled with are in scope once+ * it has been written into the file: added to the module's existing import,+ * or as an import of their own after the last one — or above the first+ * statement, past the file's header comments, when there are no imports at+ * all. Idempotent.*/-function ensureGenerateImport(sourceFile: SourceFile): void {+function ensureDeclarationImports(sourceFile: SourceFile, ref: SerializedAssetRef): void {+ for (const name of declarationImports(ref)) ensureNamedImport(sourceFile, name);+}++function ensureNamedImport(sourceFile: SourceFile, name: string): void {const declarations = sourceFile.getImportDeclarations();const importable = declarations.find((declaration) =>- declaration.getModuleSpecifierValue() === GENERATE_MODULE &&+ declaration.getModuleSpecifierValue() === DECLARATION_MODULE &&!declaration.isTypeOnly() &&!declaration.getNamespaceImport(),);if (importable) {const named = importable.getNamedImports();- if (named.some((specifier) => specifier.getName() === "generate" && !specifier.getAliasNode())) return;- importable.addNamedImport("generate");+ if (named.some((specifier) => specifier.getName() === name && !specifier.getAliasNode())) return;+ importable.addNamedImport(name);return;}- const statement = `import { generate } from "${GENERATE_MODULE}";`;+ const statement = `import { ${name} } from "${DECLARATION_MODULE}";`;const last = declarations.at(-1);if (last) sourceFile.insertText(last.getEnd(), `\n${statement}`);else {@@ -423,8 +446,14 @@ function reindent(text: string, from: string, to: string): string {.join("\n");}-/** Whether an expression is a value rather than a way of computing one. */+/**+ * Whether an expression is a value rather than a way of computing one. A+ * declaration call over values is one too: it is what the writer spells a+ * declaration as, so it is an initializer the writer may also overwrite.+ */function isLiteral(node: Node): boolean {+ if (isDeclarationCall(node)) return true;+if (node.isKind(SyntaxKind.StringLiteral) ||node.isKind(SyntaxKind.NumericLiteral) ||@@ -458,16 +487,19 @@ function isLiteral(node: Node): boolean {return false;}+/** The namespaces a declaration call is made through. */+const DECLARATION_NAMESPACES = new Set(["generate", "transform"]);+/**- * A `generate.*` call over a literal spec: what the writer spells a- * declaration as, and so an initializer it may also overwrite. One whose- * argument computes anything is authored reactivity, like any expression.+ * A `generate.*` or `transform.*` call over literals — a nested declaration+ * among them. One whose argument computes anything is authored reactivity,+ * like any expression.*/-function isGenerateCall(node: Node): boolean {+function isDeclarationCall(node: Node): boolean {const call = node.asKind(SyntaxKind.CallExpression);const callee = call?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);if (!call || !callee || !callee.getExpression().isKind(SyntaxKind.Identifier)) return false;- if (callee.getExpression().getText() !== "generate") return false;+ if (!DECLARATION_NAMESPACES.has(callee.getExpression().getText())) return false;const args = call.getArguments();return args.length === 1 && isLiteral(args[0]!);@@ -486,7 +518,7 @@ function isWritable(attribute: JsxAttribute): boolean {if (!initializer.isKind(SyntaxKind.JsxExpression)) return false;const expression = initializer.getExpression();- return expression !== undefined && (isLiteral(expression) || isGenerateCall(expression));+ return expression !== undefined && isLiteral(expression);}// ---------------------------------------------------------------------------@@ -801,7 +833,7 @@ class SourceWriter {}setProp(tag, name, value);- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);wrote = true;}@@ -898,7 +930,7 @@ class SourceWriter {insertChild(sourceFile, parent, child, before);for (const [name, value] of Object.entries(edit.props)) {setProp(findTag(sourceFile, id)!, name, value);- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);}ids[edit.source] = formatSource(file, id);return true;@@ -1138,7 +1170,7 @@ class SourceWriter {}for (const [name, value] of Object.entries(record.props)) {setProp(found(), name, value);- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);}const element = elementOf(found());diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsxindex bae122e1..edb70d2b 100644--- a/apps/web/src/components/genai/action-bar.tsx+++ b/apps/web/src/components/genai/action-bar.tsx@@ -21,7 +21,7 @@ import { useGenerateVoice } from "./use-generate-voice";import { useGenerateAudio } from "./use-generate-audio";import { useAutoCaptions } from "./use-auto-captions";import { useMediaSelection } from "./selection";-import { useModifiers } from "./use-modifiers";+import { useTransforms } from "./use-transforms";import { createDefaultConfig } from "./prompt-input";import { toast } from "somoto";@@ -33,7 +33,7 @@ interface ActionBarProps {export function ActionBar(props: ActionBarProps) {const { imageNodes, videoNodes } = useMediaSelection();- const { isOn, toggle } = useModifiers();+ const { isOn, toggle } = useTransforms();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsxindex 239fc8af..8c606030 100644--- a/apps/web/src/components/genai/prompt-input-actions.tsx+++ b/apps/web/src/components/genai/prompt-input-actions.tsx@@ -23,14 +23,14 @@ import { useGenerateVideo } from "./use-generate-video";import { useGenerateVoice } from "./use-generate-voice";import { useGenerateAudio } from "./use-generate-audio";import { useMediaSelection } from "./selection";-import { useModifiers } from "./use-modifiers";+import { useTransforms } from "./use-transforms";-import type { ModifierName } from "./use-modifiers";+import type { TransformType } from "@diffusionstudio/jsx";export function PromptInputActions() {const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();const { imageNodes, videoNodes } = useMediaSelection();- const { isOn, toggle } = useModifiers();+ const { isOn, toggle } = useTransforms();const { generate: generateImage } = useGenerateImage();const { generate: generateVideo } = useGenerateVideo();@@ -121,13 +121,13 @@ export function PromptInputActions() {</Show><DropdownMenuGroup><Show when={hasImageSelection() || hasVideoSelection()}>- <ModifierItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />+ <TransformItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} /></Show><Show when={hasImageSelection()}>- <ModifierItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />+ <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} /></Show><Show when={hasVideoSelection()}>- <ModifierItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />+ <TransformItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} /></Show></DropdownMenuGroup></DropdownMenuContent>@@ -138,16 +138,16 @@ export function PromptInputActions() {);}-type ModifierItemProps = {- name: ModifierName;+type TransformItemProps = {+ name: TransformType;icon: string;label: string;- isOn(name: ModifierName): boolean;- toggle(name: ModifierName): void;+ isOn(name: TransformType): boolean;+ toggle(name: TransformType): void;};-/** A modifier as a menu row: a check when the selection is asking for it. */-function ModifierItem(props: ModifierItemProps) {+/** A transform as a menu row: a check when the selection is asking for it. */+function TransformItem(props: TransformItemProps) {return (<DropdownMenuItem onSelect={() => props.toggle(props.name)}><Icon name={props.icon} class="mr-2 text-foreground" />diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.tsindex 23a28d77..1cc4fac6 100644--- a/apps/web/src/components/genai/selection.ts+++ b/apps/web/src/components/genai/selection.ts@@ -6,7 +6,7 @@// two different questions://// `imageNodes`/`videoNodes` are what the element *is*, which is what a-// source modifier applies to — a picture that is still generating is still a+// transform applies to — a picture that is still generating is still a// picture, and `removeBackground` can be asked of it before it exists.//// `images`/`videos` are what the element currently *shows*, which is what adiff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.tsindex a6fb4822..2b91461e 100644--- a/apps/web/src/components/genai/use-generation-records.ts+++ b/apps/web/src/components/genai/use-generation-records.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */import { createMemo, createResource } from "solid-js";-import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";+import { getAssetSpec, isAssetRef, isTransformSpec } from "@diffusionstudio/jsx";import { authoredElement } from "@diffusionstudio/reconciler";import { useSelection } from "@/engine/hooks";import { useLibrary } from "@/engine/library";@@ -18,7 +18,7 @@ import {} from "./config";import { generationConfigSchema, type GenerationConfig } from "./schemas";-import type { AssetInput, AssetSpecInput } from "@diffusionstudio/jsx";+import type { AssetInput, AssetRef, GenerateSpec } from "@diffusionstudio/jsx";import type { AssetLibrary } from "@diffusionstudio/assets";/**@@ -63,7 +63,7 @@ export function toClientConfig(stored: unknown): GenerationConfig | undefined {* "Rerun" and "Reuse" work from. Inputs are named by library path in the* file and by asset id in the prompt box, so they are looked up on the way.*/-function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): GenerationConfig {+function toPromptConfig(spec: GenerateSpec, library: AssetLibrary): GenerationConfig {const idOf = (input: AssetInput | undefined): string | undefined =>typeof input === "string" ? library.get(input)?.id : undefined;@@ -104,6 +104,13 @@ function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): Generation}}+/** The generation a declaration rests on: itself, or what its transforms were put over. */+function generationUnder(ref: AssetRef): GenerateSpec | undefined {+ const spec = getAssetSpec(ref);+ if (!isTransformSpec(spec)) return spec;+ return isAssetRef(spec.input) ? generationUnder(spec.input) : undefined;+}+export function useGenerationRecords() {const library = useLibrary();const { nodes } = useSelection();@@ -111,12 +118,14 @@ export function useGenerationRecords() {// What the selected elements declare their source to be. An element made by// the prompt box carries the whole spec, so this answers before the asset- // exists — and without asking the server what it was asked for.+ // exists — and without asking the server what it was asked for. Transforms+ // are looked through: the generation under them is what the box made.const declarations = createMemo(() =>nodes().map((entity) => authoredElement(entity)?.props.src).filter(isAssetRef)- .map(getAssetSpec),+ .map(generationUnder)+ .filter((spec): spec is GenerateSpec => spec !== undefined),);// Credits are the server's to know, and it knows them per generation.diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.tsdeleted file mode 100644index 829b6cc1..00000000--- a/apps/web/src/components/genai/use-modifiers.ts+++ /dev/null@@ -1,66 +0,0 @@-/* This Source Code Form is subject to the terms of the Mozilla Public- * License, v. 2.0. If a copy of the MPL was not distributed with this- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */--// Upscaling, taking a background out, scoring footage: the editor's side of-// the source modifiers (see the runtime's `SourceModifiers` and-// reference/jsx/media.md). The action bar sets a prop and stops there — the-// asset system is what notices, calls the model and binds the result — so-// these are toggles rather than commands, and turning one off gives the-// original back rather than running anything.--import { authoredElement } from "@diffusionstudio/reconciler";-import { useEditor } from "@/engine/hooks";-import { useMediaSelection } from "./selection";--import type { Entity } from "koota";--/** The props an element carries to say what its source is put through. */-export type ModifierName = "removeBackground" | "upscale" | "addAudio";--/**- * What `upscale` is set to when it is switched on. The endpoint takes no- * factor today, so this is the one the language says and the number every- * upscale is; see the cache key in `EditorGenAi.transform`.- */-const UPSCALE_FACTOR = 2;--/** Whether `entity` currently asks for `name`, as its element says it. */-function isSet(entity: Entity, name: ModifierName): boolean {- const value = authoredElement(entity)?.props[name];- return name === "upscale" ? typeof value === "number" && value > 1 : value === true;-}--export function useModifiers() {- const editor = useEditor();- const { imageNodes, videoNodes } = useMediaSelection();-- /** The selected elements a modifier can be asked of. */- const targets = (name: ModifierName): Entity[] => {- if (name === "removeBackground") return imageNodes();- if (name === "addAudio") return videoNodes();- return [...imageNodes(), ...videoNodes()];- };-- /** On when every element it applies to asks for it, so the toggle turns the odd one on. */- const isOn = (name: ModifierName): boolean => {- const selected = targets(name);- return selected.length > 0 && selected.every((entity) => isSet(entity, name));- };-- /**- * Turns `name` on for the selection, or off when they all have it. Off is- * written as `false`, which the source writer spells as the prop's absence- * — an element says what it asks for, and nothing about what it does not.- */- const toggle = (name: ModifierName): void => {- const next = !isOn(name);- const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;-- for (const entity of targets(name)) {- editor.editProperty(entity, name, value);- }- };-- return { isOn, toggle };-}diff --git a/apps/web/src/components/genai/use-transforms.ts b/apps/web/src/components/genai/use-transforms.tsnew file mode 100644index 00000000..e06b87de--- /dev/null+++ b/apps/web/src/components/genai/use-transforms.ts@@ -0,0 +1,86 @@+/* This Source Code Form is subject to the terms of the Mozilla Public+ * License, v. 2.0. If a copy of the MPL was not distributed with this+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */++// Upscaling, taking a background out, scoring footage: the editor's side of+// `transform.*` (see reference/jsx/generate.md). A toggle wraps the+// element's `src` in the transform — `transform.upscale(<what it was>)` —+// or takes that layer out again, and stops there: the asset system is what+// notices the new declaration, runs it and binds the result. Turning one+// off gives the inner declaration back rather than running anything.++import { getAssetSpec, isAssetRef, isTransformSpec, mapAssetInputs, transform } from "@diffusionstudio/jsx";+import { authoredElement } from "@diffusionstudio/reconciler";+import { useEditor } from "@/engine/hooks";+import { useMediaSelection } from "./selection";++import type { AssetInput, TransformType } from "@diffusionstudio/jsx";+import type { Entity } from "koota";++/** Whether `input` is, somewhere down its chain of transforms, put through `type`. */+function hasTransform(input: AssetInput, type: TransformType): boolean {+ if (!isAssetRef(input)) return false;+ const spec = getAssetSpec(input);+ if (!isTransformSpec(spec)) return false;+ return spec.type === type || hasTransform(spec.input, type);+}++/**+ * `input` with every `type` layer taken out of its chain of transforms. A+ * generation under the chain is left as it is: its own inputs are its+ * business, not the element's.+ */+function withoutTransform(input: AssetInput, type: TransformType): AssetInput {+ if (!isAssetRef(input)) return input;+ const spec = getAssetSpec(input);+ if (!isTransformSpec(spec)) return input;+ if (spec.type === type) return withoutTransform(spec.input, type);+ return mapAssetInputs(input, (inner) => withoutTransform(inner, type));+}++/** The element's `src` as authored, when it is something a transform can be put over. */+function sourceOf(entity: Entity): AssetInput | undefined {+ const src = authoredElement(entity)?.props.src;+ return typeof src === "string" && src !== "" ? src : isAssetRef(src) ? src : undefined;+}++export function useTransforms() {+ const editor = useEditor();+ const { imageNodes, videoNodes } = useMediaSelection();++ /** The selected elements a transform can be asked of. */+ const targets = (type: TransformType): Entity[] => {+ if (type === "removeBackground") return imageNodes();+ if (type === "addAudio") return videoNodes();+ return [...imageNodes(), ...videoNodes()];+ };++ /** On when every element it applies to asks for it, so the toggle turns the odd one on. */+ const isOn = (type: TransformType): boolean => {+ const selected = targets(type);+ return selected.length > 0 && selected.every((entity) => {+ const src = sourceOf(entity);+ return src !== undefined && hasTransform(src, type);+ });+ };++ /**+ * Puts `type` over the selection's sources, or takes it off when they all+ * have it. On goes outermost — the last thing done to the source — and off+ * takes every layer of it out, wherever it sits in the chain.+ */+ const toggle = (type: TransformType): void => {+ const next = !isOn(type);++ for (const entity of targets(type)) {+ const src = sourceOf(entity);+ if (src === undefined) continue;+ const declared = next+ ? (hasTransform(src, type) ? src : transform[type](src))+ : withoutTransform(src, type);+ if (declared !== src) editor.editProperty(entity, "src", declared);+ }+ };++ return { isOn, toggle };+}diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.tsindex 85d396ad..b6960968 100644--- a/apps/web/src/engine/editor.ts+++ b/apps/web/src/engine/editor.ts@@ -19,7 +19,7 @@ import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode,import { findInspectEntry } from './inspect';import type { SceneNode } from '@diffusionstudio/runtime';-import type { InspectValue, PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';+import type { AssetRef, InspectValue, PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { AuthoredTree, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';@@ -266,11 +266,10 @@ function nextSourceAfter(world: World, entity: Entity): string | undefined {* The props of a `<video>`/`<image>` that only mean something while the node* plays media, dropped when the intrinsic paint is removed (see* `removeIntrinsicPaint`): the source, what qualifies it (its fit, its- * window, a frames directory's rate, the modifiers put on it), and the audio- * mix a rect has no track to apply to. `start`/`end` stay: they place the- * clip, media or not.+ * window, a frames directory's rate), and the audio mix a rect has no track+ * to apply to. `start`/`end` stay: they place the clip, media or not.*/-const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;+const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'volume', 'muted', 'syncTo'] as const;/*** What `copy` took: the subtrees as authored, and the source of the parent@@ -325,7 +324,7 @@ export class DocumentEditor {}/** Writes a prop to the document and reports it. */- public editProperty(entity: Entity, name: string, value: PropValue): void {+ public editProperty(entity: Entity, name: string, value: PropValue | AssetRef): void {const node = this.document.node(entity);let previous = node.props[name];// The stage keeps no authored record (see RuntimeDocument.setProperty),@@ -336,7 +335,7 @@ export class DocumentEditor {if (background !== undefined && background !== DEFAULT_BACKGROUND) previous = colorToHex(background);}this.document.setProperty(node, name, value);- this.reportEdit(entity, name, value, previous ?? false);+ this.reportEdit(entity, name, wireValue(value)!, wireValue(previous) ?? false);}/**@@ -380,7 +379,7 @@ export class DocumentEditor {* (see `PropEdit`); a caller that cannot say what the value was leaves it* out, and the edit is not undoable.*/- public reportEdit(entity: Entity, name: string, value: PropValue, previous?: unknown): void {+ public reportEdit(entity: Entity, name: string, value: EditValue, previous?: unknown): void {this.settle(entity);const source = entity.get(Source)?.value;diff --git a/apps/web/src/engine/generations.ts b/apps/web/src/engine/generations.tsindex 95dd97f4..4b033aa3 100644--- a/apps/web/src/engine/generations.ts+++ b/apps/web/src/engine/generations.ts@@ -14,7 +14,7 @@import { isAssetRef } from '@diffusionstudio/jsx';import { authoredElement } from '@diffusionstudio/reconciler';import {- Caption, Generating, GenerationRequest, LoadRequest, PendingSource, SourceError, TranscriptionRequest,+ Caption, Generating, GenerationRequest, PendingSource, SourceError, TranscriptionRequest,} from '@diffusionstudio/runtime';import type { AssetLibrary, PartialAsset } from '@diffusionstudio/assets';@@ -36,10 +36,6 @@ export async function retryGeneration(world: World, library: AssetLibrary, partiif (isAssetRef(src)) {entity.add(GenerationRequest);entity.set(GenerationRequest, { ref: src });- } else if (typeof src === 'string' && src !== '') {- // A source put through modifiers: the load is cheap, the derivation is what failed.- entity.add(LoadRequest);- entity.set(LoadRequest, { value: src });} else if (entity.has(Caption)) {entity.add(TranscriptionRequest);entity.set(TranscriptionRequest, { seed: Number(authored.props.seed) || 0 });diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.tsindex 58139f16..a7e2fcbd 100644--- a/apps/web/src/engine/library.ts+++ b/apps/web/src/engine/library.ts@@ -10,12 +10,14 @@ import { AssetId, Library } from '@diffusionstudio/runtime';import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';import { useTrait, useWorld } from '@diffusionstudio/koota-solid';import { authoredElement } from '@diffusionstudio/reconciler';+import { isAssetRef, mapAssetInputs } from '@diffusionstudio/jsx';import type { Accessor } from 'solid-js';import { createProjectFS } from '@/projects/fs';import { getDocumentEditor } from './editor';import type { Asset } from '@diffusionstudio/assets';+import type { AssetInput } from '@diffusionstudio/jsx';import type { World } from 'koota';/**@@ -41,18 +43,21 @@ export function isLibraryFile(path: string): boolean {}/**- * Rewrites `src` on every element that named an asset by its old path. Matched- * on the path alone, not on what the element is bound to: an element showing a- * modified source (`removeBackground`, `upscale`) is bound to what the- * modifiers made of the asset rather than to the asset itself, and its `src`- * still has to follow the rename.+ * Rewrites `src` on every element that named an asset by its old path —+ * directly, or as an input somewhere inside a declaration: an element+ * showing `transform.upscale("old.png")` is bound to what the transform made,+ * and its `src` still has to follow the rename.*/function followRename(world: World, asset: Asset, from: string): void {const editor = getDocumentEditor(world);+ const renamed = (input: AssetInput): AssetInput =>+ isAssetRef(input) ? mapAssetInputs(input, renamed) : input === from ? asset.path : input;+for (const entity of world.query(AssetId)) {const src = authoredElement(entity)?.props.src;- if (src !== from) continue;- editor.editProperty(entity, 'src', asset.path);+ if (typeof src !== 'string' && !isAssetRef(src)) continue;+ const next = renamed(src);+ if (next !== src) editor.editProperty(entity, 'src', next);}}diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.tsindex 71e72bf6..523f5a21 100644--- a/apps/web/src/utils/gen-ai.ts+++ b/apps/web/src/utils/gen-ai.ts@@ -2,12 +2,11 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { getAssetSpec, isAssetRef, parseSource } from "@diffusionstudio/jsx";+import { getAssetSpec, isAssetRef, isTransformSpec, isTransformType, parseSource } from "@diffusionstudio/jsx";import {Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,Paint, PaintType, Project, Source,} from "@diffusionstudio/runtime";-import type { SourceModifierValues } from "@diffusionstudio/runtime";import { createEncoder } from "@diffusionstudio/encoder";import { createCapture } from "@/engine/capture";import { assetName, GENERATED_DIR, isPartialAsset } from "@diffusionstudio/assets";@@ -24,14 +23,14 @@ import { track } from "@/lib/analytics";import { trpc } from "@/lib/trpc";import { toast } from "somoto";-import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";+import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput, GenerateSpec, TransformType } from "@diffusionstudio/jsx";import type { Asset, AssetLibrary, AssetType, PartialAsset, ReserveOptions } from "@diffusionstudio/assets";import type { FileRef } from "@diffusionstudio/api-contract";import type { ExportResult } from "@diffusionstudio/encoder";import type { Entity, World } from "koota";-/** What a failure is called where the user reads about it. */-const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {+/** What a failed generation is called where the user reads about it. */+const FAILURE_TITLES: Record<GenerateSpec["type"] | "transcript", string> = {image: "Image generation failed",video: "Video generation failed",voice: "Voice generation failed",@@ -39,48 +38,36 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {transcript: "Caption generation failed",};-/** What each declaration is to become, for the partial that stands for it. */-const ASSET_TYPES: Record<AssetSpecInput["type"], AssetType> = {+/** What each generation is to become, for the partial that stands for it. */+const ASSET_TYPES: Record<GenerateSpec["type"], AssetType> = {image: "IMAGE",video: "VIDEO",voice: "AUDIO",audio: "AUDIO",};-/**- * One step of a source modifier (see the runtime's `SourceModifiers`): a- * model call that makes a new asset out of one the library already holds.- * An element asks for these by prop and goes on naming what it was made- * from, so each step is stored and cached on its own, and a set of modifiers- * is these run in order (see `derive`).- */-export type TransformKind = "remove-background" | "upscale" | "add-audio";--/** What each step is called where the user reads about it, and names its result. */-const TRANSFORMS: Record<TransformKind, { title: string; suffix: string }> = {- "remove-background": { title: "Background removal failed", suffix: "Background removed" },- "upscale": { title: "Upscale failed", suffix: "Upscaled" },- "add-audio": { title: "Adding audio failed", suffix: "With audio" },+/** What each transform's failure is called where the user reads about it, and what names its result. */+const TRANSFORMS: Record<TransformType, { title: string; suffix: string }> = {+ removeBackground: { title: "Background removal failed", suffix: "Background removed" },+ upscale: { title: "Upscale failed", suffix: "Upscaled" },+ addAudio: { title: "Adding audio failed", suffix: "With audio" },};-/** The steps a set of modifiers comes to, in the order they are applied. */-function steps(modifiers: SourceModifierValues): TransformKind[] {- const kinds: TransformKind[] = [];- if (modifiers.removeBackground) kinds.push("remove-background");- if (modifiers.upscale > 1) kinds.push("upscale");- if (modifiers.addAudio) kinds.push("add-audio");- return kinds;-}-/*** A spec with defaults applied and every `AssetInput` reduced to an asset id.* Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.+ * A transform's key is its type and input alone: the endpoints take no model+ * and no factor, so every call over the same asset is the same call.*/-type ResolvedSpec =+type ResolvedGeneration =| { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }| { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }| { type: "voice"; model: string; prompt: string; voice: string; seed?: number }| { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };+type ResolvedTransform = { type: TransformType; inputId: string };+type ResolvedSpec = ResolvedGeneration | ResolvedTransform;++const isResolvedTransform = (spec: ResolvedSpec): spec is ResolvedTransform => isTransformType(spec.type);/** What the partial standing for a run is called and is to become, and what its failure is called. */type RunDescription = Omit<ReserveOptions, "key" | "folder"> & { title: string };@@ -152,39 +139,6 @@ export class EditorGenAi extends GenAi {);}- /**- * `asset` through every modifier the element asked for, one step at a time- * and in a fixed order. Each step is cached in its own right, so turning- * one on leaves what the others already made alone — adding `upscale` to a- * cut-out picture pays for the enlarging, not for the matte again.- */- public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {- let derived = asset;- for (const kind of steps(modifiers)) {- derived = await this.transform(kind, derived);- }- return derived;- }-- /**- * Runs one step over `asset` and returns what it produced, stored under- * `generated/` like any other model output. Keyed by step and input, so the- * same call on the same asset is the same result in this session and the- * next: upscaling a picture twice costs what upscaling it once did.- */- public transform(kind: TransformKind, asset: Asset): Promise<Asset> {- // No upscale factor in the key: the endpoint takes none, so every factor- // is the same call and would otherwise be billed once per number asked- // for. It belongs here the moment the API can be told one.- const key = `transform:v1:${kind}:${asset.id}`;- const base = assetName(asset).replace(/\.[^.]+$/, "");- return this.generated(- key,- () => ({ type: asset.type, name: `${base} (${TRANSFORMS[kind].suffix})`, title: TRANSFORMS[kind].title }),- () => this.runTransform(kind, asset, key),- );- }-/*** The library's answer for `key`, or the run that produces one. An asset* the key landed as is returned; a key standing in error rejects with the@@ -226,6 +180,7 @@ export class EditorGenAi extends GenAi {private async generateFromRef(ref: AssetRef): Promise<Asset> {const spec = getAssetSpec(ref);+ const title = isTransformSpec(spec) ? TRANSFORMS[spec.type].title : FAILURE_TITLES[spec.type];// Inputs first: their ids are part of the key. An input that failed has// said so already; anything else wrong with them is said here.@@ -233,21 +188,36 @@ export class EditorGenAi extends GenAi {try {resolved = await this.resolveSpec(spec);} catch (error) {- throw error instanceof ReportedError ? error : reportFailure(error, FAILURE_TITLES[spec.type]);+ throw error instanceof ReportedError ? error : reportFailure(error, title);}return this.generated(JSON.stringify(resolved),- () => ({ type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt), title: FAILURE_TITLES[spec.type] }),- (partial) => this.runGeneration(resolved, partial),+ () => ({ ...this.describe(resolved), title }),+ (partial) => isResolvedTransform(resolved)+ ? this.runTransform(resolved, partial)+ : this.runGeneration(resolved, partial),);}+ /** What the partial standing for a run is called, and what it is to become. */+ private describe(spec: ResolvedSpec): { type: AssetType; name: string } {+ if (!isResolvedTransform(spec)) return { type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt) };+ const input = this.library.get(spec.inputId);+ assert(input, `Input asset ${spec.inputId} not found`);+ return { type: input.type, name: `${stem(input)} (${TRANSFORMS[spec.type].suffix})` };+ }+private resolveInput(input: AssetInput): Promise<Asset> {return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);}private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {+ if (isTransformSpec(spec)) {+ const input = await this.resolveInput(spec.input);+ return { type: spec.type, inputId: input.id };+ }+switch (spec.type) {case "image": {const refs = await Promise.all((spec.refs ?? []).map((ref) => this.resolveInput(ref)));@@ -303,7 +273,7 @@ export class EditorGenAi extends GenAi {* spec a model cannot take fails here rather than before the partial is* reserved, so the refusal is recorded like any other and not asked again.*/- private async runGeneration(spec: ResolvedSpec, partial: PartialAsset): Promise<Asset> {+ private async runGeneration(spec: ResolvedGeneration, partial: PartialAsset): Promise<Asset> {if (spec.type === "video") checkVideoConstraints(spec);const startedAt = performance.now();@@ -389,27 +359,33 @@ export class EditorGenAi extends GenAi {return `Captions ${max + 1}`;}- /** Uploads the input, runs the call, and stores the result beside the generations. */- private async runTransform(kind: TransformKind, asset: Asset, key: string): Promise<Asset> {+ /**+ * Uploads the transform's input, runs the call, and stores the result in+ * place of `partial`. A transform over the wrong kind of asset fails here,+ * like a spec a model cannot take: recorded, and not asked again.+ */+ private async runTransform(spec: ResolvedTransform, partial: PartialAsset): Promise<Asset> {+ const asset = this.library.get(spec.inputId);+ assert(asset, `Input asset ${spec.inputId} not found`);+ checkTransformInput(spec.type, asset);+const startedAt = performance.now();- track("generation_started", { mode: kind });+ track("generation_started", { mode: spec.type });try {- console.log(`[gen-ai] running ${kind} on ${asset.path}`);+ console.log(`[gen-ai] running ${spec.type} on ${asset.path}`);const input = await this.uploadInput(asset.id);- const { url, generationId } = await this.requestTransform(kind, asset, input);-- const base = assetName(asset).replace(/\.[^.]+$/, "");- const stored = await this.store(url, `${base} (${TRANSFORMS[kind].suffix})`, { key, id: generationId });+ const { url, generationId } = await this.requestTransform(spec.type, asset, input);+ const stored = await this.store(url, assetName(partial), { key: partial.generation.key, id: generationId });track("generation_completed", {- mode: kind,+ mode: spec.type,duration_ms: Math.round(performance.now() - startedAt),});return stored;} catch (err) {track("generation_failed", {- mode: kind,+ mode: spec.type,duration_ms: Math.round(performance.now() - startedAt),error: err instanceof Error ? err.message.slice(0, 200) : "unknown",});@@ -417,17 +393,15 @@ export class EditorGenAi extends GenAi {}}- private requestTransform(kind: TransformKind, asset: Asset, input: FileRef) {- switch (kind) {- case "remove-background":- assert(asset.type === "IMAGE", "Only a picture has a background to remove");+ private requestTransform(type: TransformType, asset: Asset, input: FileRef) {+ switch (type) {+ case "removeBackground":return trpc.removeBackground.mutate({ image: input });case "upscale":return isMoving(asset.type)? trpc.upscaleVideo.mutate({ video: input }): trpc.upscaleImage.mutate({ image: input });- case "add-audio":- assert(isMoving(asset.type), "Only footage can be scored");+ case "addAudio":return trpc.addAudioToVideo.mutate({ video: input });}}@@ -449,7 +423,7 @@ export class EditorGenAi extends GenAi {});}- private requestGeneration(spec: ResolvedSpec) {+ private requestGeneration(spec: ResolvedGeneration) {switch (spec.type) {case "image": {return (async () => {@@ -518,6 +492,24 @@ export class EditorGenAi extends GenAi {/** Whether an asset type is footage, for the calls that only take footage. */const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";+/** An asset's name without its extension: what a transform's result is named after. */+const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, "");++/** What a transform can be put over; known only once the input has resolved. */+function checkTransformInput(type: TransformType, input: Asset): void {+ switch (type) {+ case "removeBackground":+ assert(input.type === "IMAGE", "Only a picture has a background to remove");+ return;+ case "upscale":+ assert(input.type === "IMAGE" || isMoving(input.type), "Only a picture or footage can be upscaled");+ return;+ case "addAudio":+ assert(isMoving(input.type), "Only footage can be scored");+ return;+ }+}+/** How much of a prompt names the partial standing for its generation. */const PROVISIONAL_NAME_LENGTH = 48;@@ -600,7 +592,7 @@ function sceneHasAudio(world: World, scene: Entity): boolean {/*** Per-model constraints (`dapi models video`); unknown models are left to the server.*/-function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {+function checkVideoConstraints(spec: Extract<ResolvedGeneration, { type: "video" }>): void {const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);if (!model) return;diff --git a/docs/reference/jsx/README.md b/docs/reference/jsx/README.mdindex 6608dee3..464bcce4 100644--- a/docs/reference/jsx/README.md+++ b/docs/reference/jsx/README.md@@ -40,7 +40,7 @@ export default function Project() {| [html.md](./html.md) | `<html>`: reactive HTML children drawn into the box || [surface-paint.md](./surface-paint.md) | `<surface>`: a ref-provided canvas you draw into, sampled every frame || [shader-paint.md](./shader-paint.md) | `<shaderPaint>`: a WGSL fragment shader over the media paint below it |-| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`), image sequences, modifiers, adding an asset to the library |+| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`), image sequences, transforms, adding an asset to the library || [timing.md](./timing.md) | `start` / `end` / `sourceIn` / `sourceOut` / `playbackRate`, time formats || [keyframes.md](./keyframes.md) | Keyframe animation and easing || [animations.md](./animations.md) | `<animation>`: preset in/out animations |diff --git a/docs/reference/jsx/audio.md b/docs/reference/jsx/audio.mdindex fc52982e..09f33cce 100644--- a/docs/reference/jsx/audio.md+++ b/docs/reference/jsx/audio.md@@ -21,7 +21,6 @@ It draws nothing inside a scene, but on the canvas it is still something to poin| `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. || `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. || `syncTo` | `string` | none | `id` of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |An audio clip has no picture, so it takes no transform beyond its canvas box and no paints. Its children are [`<keyframeTrack>`](./keyframes.md) (a `volume` track) and [`<animation>`](./animations.md), of which only `"gain"` is audible:diff --git a/docs/reference/jsx/generate.md b/docs/reference/jsx/generate.mdindex fe0326da..5406309d 100644--- a/docs/reference/jsx/generate.md+++ b/docs/reference/jsx/generate.md@@ -76,9 +76,27 @@ generate.audio(opts: {}): AssetRef;```+## Transforms++`transform.*` declares an asset made from another. Each takes what it works on as its argument — a source string or another declaration — and returns an `AssetRef` like `generate.*` does, usable anywhere a source is:++```ts+transform.upscale(input: AssetInput): AssetRef; // twice the pixels; a picture or footage — enlarges the source, not the box+transform.removeBackground(input: AssetInput): AssetRef; // the subject cut out, the rest transparent; pictures only+transform.addAudio(input: AssetInput): AssetRef; // footage scored with a generated soundtrack; independent of `volume` and `muted`+```++A chain reads inside out and runs in that order:++```tsx+<image src={transform.upscale(transform.removeBackground(generate.image({ prompt: "a red fox" })))} width={800} height={450} />+```++generates, cuts out, then enlarges. Each step is cached by step and input, so wrapping a further transform around a chain does not re-run what is inside it, and a transform over a library asset (`transform.upscale("footage/fox.png")`) leaves that asset as it is. A transform over the wrong kind of asset — a background removal over footage — fails like a spec a model cannot take (see [errors.md](./errors.md#failed-sources)).+## Dependency order-`startFrame`, `endFrame`, and `refs` accept other `AssetRef`s. The dependency graph is built from these values and generated in topological order, so referenced assets exist before the assets that consume them. Because dependencies are **values, not string ids**, a declaration can only reference refs that already exist; reference cycles are impossible by construction. Assets referenced only as inputs generate too, but produce no node of their own.+`startFrame`, `endFrame`, `refs`, and a transform's input accept other `AssetRef`s. The dependency graph is built from these values and generated in topological order, so referenced assets exist before the assets that consume them. Because dependencies are **values, not string ids**, a declaration can only reference refs that already exist; reference cycles are impossible by construction. Assets referenced only as inputs generate too, but produce no node of their own.## Caching and idempotencydiff --git a/docs/reference/jsx/image.md b/docs/reference/jsx/image.mdindex 8008a2a7..56819c75 100644--- a/docs/reference/jsx/image.md+++ b/docs/reference/jsx/image.md@@ -15,9 +15,6 @@ All [common props](./elements.md#common-props), plus:| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). || `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. || `frameRate` | `number` | `30` | Frames per second for a `src` naming a directory of numbered frames, which plays on `<image>` as footage does (see [media.md](./media.md#image-sequences)). Nothing for a still to read. |-| `removeBackground` | `boolean` | `false` | Cuts the subject out, leaving the rest transparent (see [media.md](./media.md#source-modifiers)). |-| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |Without `width`/`height` the box is 1920×1080. A still has no duration to fit, so with no `end` it takes the 16-second default (see [timing.md](./timing.md)).diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.mdindex 641ac896..f9604e70 100644--- a/docs/reference/jsx/media.md+++ b/docs/reference/jsx/media.md@@ -24,23 +24,17 @@ A folder of pictures has a count, not a duration, so `frameRate` is what says ho`frameRate` is not [`playbackRate`](./timing.md#playback-rate), which retimes a source against the timeline whatever its natural speed is; this is what that natural speed *is*. It is unrelated to the composition's own frame rate, which the export sets.-## Source modifiers+## Transforms-`<image>` and `<video>` take props that put the source through a model before the element shows it:--| Prop | On | Meaning |-| ---- | -- | ------- |-| `removeBackground` | `<image>` | Cuts the subject out, leaving the rest transparent. |-| `upscale` | both | Resolution multiplier — `upscale={2}` asks for twice the pixels. Enlarges the source, not the box. |-| `addAudio` | `<video>` | Scores footage that has no sound. Independent of `volume` and `muted`, which mix whatever track the clip ends up with. |+A source can be put through a model before the element shows it, by declaring the result as the `src` — see [generate.md](./generate.md#transforms):```tsx-<image src="footage/fox.png" removeBackground upscale={2} width={800} height={450} />-```+import { transform } from "@diffusionstudio/jsx";-The `src` goes on naming what the picture was made from, so taking a modifier off gives the original back — nothing is overwritten and nothing is lost. Results are cached by source and modifiers, like [generated assets](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and adding a second modifier does not re-run the first. They are applied in the order of the table above, whatever order they are written in.+<image src={transform.upscale(transform.removeBackground("footage/fox.png"))} width={800} height={450} />+```-Modifiers compose with declarations: `<image src={generate.image({ prompt: "a red fox" })} removeBackground />` generates, then cuts out.+The inner source is untouched — nothing is overwritten and nothing is lost — and the chain runs inside out, in the order it is written. Results are cached by step and input like any [generated asset](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and wrapping a further transform around a chain does not re-run what is inside it.## The librarydiff --git a/docs/reference/jsx/video.md b/docs/reference/jsx/video.mdindex 794fa368..499c8f0e 100644--- a/docs/reference/jsx/video.md+++ b/docs/reference/jsx/video.md@@ -18,9 +18,6 @@ All [common props](./elements.md#common-props), plus:| `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. || `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. || `syncTo` | `string` | none | `id` of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |-| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |-| `addAudio` | `boolean` | `false` | Scores footage that has no sound (see [media.md](./media.md#source-modifiers)). |-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |Without `width`/`height` the box is 1920×1080.diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.tsindex 08a5be99..9a44bc0a 100644--- a/packages/jsx/src/generate.ts+++ b/packages/jsx/src/generate.ts@@ -3,17 +3,20 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. *//**- * Declarative generated assets. A `generate.*` call is pure: it validates its- * options and returns an `AssetRef` value that can be passed wherever a source- * is expected (`src`, `startFrame`, `endFrame`, `refs`). Nothing generates- * until the mounted tree commits; refs never used by a mounted element are- * dropped. Because dependencies are values (not string ids), reference cycles- * are impossible by construction.+ * Declarative assets. A `generate.*` call makes an asset from a prompt; a+ * `transform.*` call makes one from an asset. Both are pure: they validate+ * their arguments and return an `AssetRef` value that can be passed wherever+ * a source is expected (`src`, `startFrame`, `endFrame`, `refs`, a+ * transform's input). Nothing runs until the mounted tree commits; refs+ * never used by a mounted element are dropped. Because dependencies are+ * values (not string ids), reference cycles are impossible by construction,+ * and the order of a chain is its nesting:+ * `transform.upscale(transform.removeBackground(generate.image({ prompt })))`.*/export type AspectRatio = "1:1" | "4:3" | "3:4" | "16:9" | "9:16";-/** A path, URL, asset id, or another `generate.*` declaration. */+/** A path, URL, asset id, or another declaration. */export type AssetInput = string | AssetRef;export type GenerateImageOptions = {@@ -54,21 +57,48 @@ export type GenerateAudioOptions = {seed?: number;};-export type AssetSpecInput =+/** What a `generate.*` call declares: an asset made from a prompt. */+export type GenerateSpec =| ({ type: "image" } & GenerateImageOptions)| ({ type: "video" } & GenerateVideoOptions)| ({ type: "voice" } & GenerateVoiceOptions)| ({ type: "audio" } & GenerateAudioOptions);+/** What a `transform.*` call declares: an asset made from another. */+export type TransformSpec =+ /** The input at twice the pixels; a picture or footage. */+ | { type: "upscale"; input: AssetInput }+ /** The subject of a picture cut out, the rest transparent. */+ | { type: "removeBackground"; input: AssetInput }+ /** Footage scored with a generated soundtrack. */+ | { type: "addAudio"; input: AssetInput };++export type TransformType = TransformSpec["type"];++export type AssetSpecInput = GenerateSpec | TransformSpec;++const GENERATE_TYPES: readonly string[] = ["image", "video", "voice", "audio"];+const TRANSFORM_TYPES: readonly string[] = ["upscale", "removeBackground", "addAudio"];++/** Whether a spec type is a transform's, as opposed to a generation's. */+export function isTransformType(type: string): type is TransformType {+ return TRANSFORM_TYPES.includes(type);+}++export function isTransformSpec(spec: AssetSpecInput): spec is TransformSpec {+ return isTransformType(spec.type);+}+/**- * Opaque handle to a declared (not-yet-generated) asset. Obtained from- * `generate.*`; consumed by `src` / `startFrame` / `endFrame` / `refs`.+ * Opaque handle to a declared (not-yet-made) asset. Obtained from+ * `generate.*` or `transform.*`; consumed by `src`, `startFrame`, `endFrame`,+ * `refs`, and a transform's input.*/export class AssetRef {/** @internal — read via `getAssetSpec`. */readonly spec: AssetSpecInput;- /** @internal — declare refs with `generate.*`, not `new AssetRef()`. */+ /** @internal — declare refs with `generate.*` / `transform.*`, not `new AssetRef()`. */constructor(spec: AssetSpecInput) {this.spec = spec;}@@ -78,81 +108,163 @@ export function isAssetRef(value: unknown): value is AssetRef {return value instanceof AssetRef;}-/** An `AssetSpecInput` whose inputs are all strings — no nested declarations. */-export type FlatAssetSpec = AssetSpecInput & {- refs?: string[];- startFrame?: string;- endFrame?: string;-};+/** Host-side accessor for a declaration's spec. */+export function getAssetSpec(ref: AssetRef): AssetSpecInput {+ return ref.spec;+}++/** The inputs a spec names, in option order: what the declaration depends on. */+export function getAssetInputs(spec: AssetSpecInput): AssetInput[] {+ switch (spec.type) {+ case "image":+ return spec.refs ?? [];+ case "video":+ return [spec.startFrame, spec.endFrame].filter((input): input is AssetInput => input !== undefined);+ case "voice":+ case "audio":+ return [];+ default:+ return [spec.input];+ }+}++/**+ * `ref` with each of its direct inputs replaced by what `map` makes of it.+ * Direct only: a caller that wants the whole chain recurses through the refs+ * `map` is handed. A ref none of whose inputs change is returned as it is.+ */+export function mapAssetInputs(ref: AssetRef, map: (input: AssetInput) => AssetInput): AssetRef {+ const spec = ref.spec;+ let changed = false;+ const mapped = (input: AssetInput): AssetInput => {+ const next = map(input);+ if (next !== input) changed = true;+ return next;+ };++ let next: AssetSpecInput;+ switch (spec.type) {+ case "image":+ next = { ...spec, refs: spec.refs?.map(mapped) };+ break;+ case "video":+ next = {+ ...spec,+ startFrame: spec.startFrame === undefined ? undefined : mapped(spec.startFrame),+ endFrame: spec.endFrame === undefined ? undefined : mapped(spec.endFrame),+ };+ break;+ case "voice":+ case "audio":+ return ref;+ default:+ next = { ...spec, input: mapped(spec.input) };+ }+ return changed ? new AssetRef(next) : ref;+}++// ---------------------------------------------------------------------------+// The wire form++/** An input as data: a source string, or a declaration serialized in turn. */+export type SerializedAssetInput = string | SerializedAssetRef;++/** A spec as data: the same fields, with every declared input serialized. */+export type SerializedAssetSpec =+ | ({ type: "image"; refs?: SerializedAssetInput[] } & Omit<GenerateImageOptions, "refs">)+ | ({ type: "video"; startFrame?: SerializedAssetInput; endFrame?: SerializedAssetInput } & Omit<GenerateVideoOptions, "startFrame" | "endFrame">)+ | ({ type: "voice" } & GenerateVoiceOptions)+ | ({ type: "audio" } & GenerateAudioOptions)+ | { type: TransformType; input: SerializedAssetInput };/*** A declaration as data, for the edit protocol. An `AssetRef` itself cannot* travel as a prop value (see `isPropValue`: an object literal would not read* back as one), so an edit carries the spec in this tagged shape instead, and- * the source writer spells it as the `generate.*` call that reproduces it.+ * the source writer spells it as the call that reproduces it — inputs+ * included, so a transform over a declaration travels as the nested call it+ * is.*/-export type SerializedAssetRef = { $generate: FlatAssetSpec };+export interface SerializedAssetRef {+ $asset: SerializedAssetSpec;+}-/**- * The wire form of a declaration, or undefined when the spec references- * other declarations: nested refs only exist in authored code, where the- * call spelling them already is — the editor declares over library assets.- */-export function serializeAssetRef(ref: AssetRef): SerializedAssetRef | undefined {- const spec = ref.spec;- const inputs = [- ...(spec.type === "image" ? (spec.refs ?? []) : []),- ...(spec.type === "video" ? [spec.startFrame, spec.endFrame] : []),- ];- if (inputs.some((input) => isAssetRef(input))) return undefined;+export function serializeAssetRef(ref: AssetRef): SerializedAssetRef {+ const serializeInput = (input: AssetInput): SerializedAssetInput =>+ isAssetRef(input) ? serializeAssetRef(input) : input;+ const spec = ref.spec;+ const serialized: Record<string, unknown> = {};// Without the undefined entries: they are not part of the declaration, and// the writer spells every entry it is handed.- const entries = Object.entries(spec).filter(([, value]) => value !== undefined);- return { $generate: Object.fromEntries(entries) as FlatAssetSpec };+ for (const [key, value] of Object.entries(spec)) {+ if (value === undefined) continue;+ serialized[key] = key === "refs"+ ? (value as AssetInput[]).map(serializeInput)+ : key === "startFrame" || key === "endFrame" || key === "input"+ ? serializeInput(value as AssetInput)+ : value;+ }+ return { $asset: serialized as SerializedAssetSpec };}-const SPEC_TYPES = ["image", "video", "voice", "audio"];-export function isSerializedAssetRef(value: unknown): value is SerializedAssetRef {- if (typeof value !== "object" || value === null || !("$generate" in value)) return false;- const spec = (value as SerializedAssetRef).$generate;- return (- typeof spec === "object" && spec !== null &&- SPEC_TYPES.includes(spec.type) && typeof spec.prompt === "string"- );+ if (typeof value !== "object" || value === null || !("$asset" in value)) return false;+ return isSerializedSpec((value as SerializedAssetRef).$asset);}-/** Host-side accessor for a declaration's spec. */-export function getAssetSpec(ref: AssetRef): AssetSpecInput {- return ref.spec;+function isSerializedInput(value: unknown): value is SerializedAssetInput {+ return typeof value === "string" || isSerializedAssetRef(value);+}++function isSerializedSpec(spec: unknown): spec is SerializedAssetSpec {+ if (typeof spec !== "object" || spec === null) return false;+ const record = spec as Record<string, unknown>;+ const type = record.type;+ if (typeof type !== "string") return false;++ if (isTransformType(type)) return isSerializedInput(record.input);+ if (!GENERATE_TYPES.includes(type) || typeof record.prompt !== "string") return false;+ if (record.refs !== undefined && !(Array.isArray(record.refs) && record.refs.every(isSerializedInput))) return false;+ if (record.startFrame !== undefined && !isSerializedInput(record.startFrame)) return false;+ if (record.endFrame !== undefined && !isSerializedInput(record.endFrame)) return false;+ return true;}-function requirePrompt(value: unknown, kind: string): string {+// ---------------------------------------------------------------------------+// The declarations++function requirePrompt(value: unknown, call: string): string {if (typeof value !== "string" || value.trim().length === 0) {- throw new Error(`generate.${kind} requires a non-empty string prompt`);+ throw new Error(`${call} requires a non-empty string prompt`);}return value;}-function checkAssetInput(value: unknown, kind: string, option: string): void {+function checkAssetInput(value: unknown, call: string, option: string): void {if (value === undefined) return;if (typeof value === "string" && value.trim().length > 0) return;if (isAssetRef(value)) return;- throw new Error(`generate.${kind}: ${option} must be a path, URL, asset id, or an AssetRef`);+ throw new Error(`${call}: ${option} must be a path, URL, asset id, or an AssetRef`);+}++function requireAssetInput(value: unknown, call: string): AssetInput {+ if (value === undefined) throw new Error(`${call} requires an input: a path, URL, asset id, or an AssetRef`);+ checkAssetInput(value, call, "input");+ return value as AssetInput;}export const generate = {image(opts: GenerateImageOptions): AssetRef {- requirePrompt(opts.prompt, "image");- for (const ref of opts.refs ?? []) checkAssetInput(ref, "image", "refs");+ requirePrompt(opts.prompt, "generate.image");+ for (const ref of opts.refs ?? []) checkAssetInput(ref, "generate.image", "refs");return new AssetRef({ type: "image", ...opts });},video(opts: GenerateVideoOptions): AssetRef {- requirePrompt(opts.prompt, "video");- checkAssetInput(opts.startFrame, "video", "startFrame");- checkAssetInput(opts.endFrame, "video", "endFrame");+ requirePrompt(opts.prompt, "generate.video");+ checkAssetInput(opts.startFrame, "generate.video", "startFrame");+ checkAssetInput(opts.endFrame, "generate.video", "endFrame");if (opts.duration !== undefined && (!Number.isInteger(opts.duration) || opts.duration <= 0)) {throw new Error("generate.video: duration must be a positive whole number of seconds");}@@ -160,15 +272,38 @@ export const generate = {},voice(opts: GenerateVoiceOptions): AssetRef {- requirePrompt(opts.prompt, "voice");+ requirePrompt(opts.prompt, "generate.voice");return new AssetRef({ type: "voice", ...opts });},audio(opts: GenerateAudioOptions): AssetRef {- requirePrompt(opts.prompt, "audio");+ requirePrompt(opts.prompt, "generate.audio");if (opts.duration !== undefined && (!Number.isFinite(opts.duration) || opts.duration <= 0)) {throw new Error("generate.audio: duration must be a positive number of seconds");}return new AssetRef({ type: "audio", ...opts });},};++/**+ * Assets made from assets. Each takes what it works on as its argument — a+ * source string or another declaration — so a chain reads inside out, and+ * runs in that order. The result is cached by step and input like any+ * generation: the same call on the same asset is the same asset.+ */+export const transform = {+ /** `input` at twice the pixels. A picture or footage; the box it is shown in is unchanged. */+ upscale(input: AssetInput): AssetRef {+ return new AssetRef({ type: "upscale", input: requireAssetInput(input, "transform.upscale") });+ },++ /** The subject of the picture `input` cut out, the rest transparent. */+ removeBackground(input: AssetInput): AssetRef {+ return new AssetRef({ type: "removeBackground", input: requireAssetInput(input, "transform.removeBackground") });+ },++ /** The footage `input` scored with a generated soundtrack. */+ addAudio(input: AssetInput): AssetRef {+ return new AssetRef({ type: "addAudio", input: requireAssetInput(input, "transform.addAudio") });+ },+};diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex 8e2a15d3..dc964fca 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -12,9 +12,14 @@export {generate,+ transform,AssetRef,isAssetRef,getAssetSpec,+ getAssetInputs,+ mapAssetInputs,+ isTransformSpec,+ isTransformType,serializeAssetRef,isSerializedAssetRef,} from "./generate";@@ -22,8 +27,12 @@ export type {AspectRatio,AssetInput,AssetSpecInput,- FlatAssetSpec,+ GenerateSpec,+ TransformSpec,+ TransformType,+ SerializedAssetInput,SerializedAssetRef,+ SerializedAssetSpec,GenerateAudioOptions,GenerateImageOptions,GenerateVideoOptions,diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 9acc1ec7..3b8374af 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -286,7 +286,7 @@ type FillProps = {type MediaProps = {/**- * Path, URL, asset id, or a `generate.*` declaration. A path naming a+ * Path, URL, asset id, or a `generate.*` / `transform.*` declaration. A path naming a* directory of numbered frames (`shot_001.png`, `shot_002.png`, ...) is an* image sequence, and plays on `<Video>` or `<Image>` as footage does — see* `frameRate` for how long it lasts. On `<Captions>` a transcript source@@ -296,22 +296,6 @@ type MediaProps = {src: string | AssetRef;};-/**- * Model calls the source is put through before the element shows it. The- * `src` goes on naming what it was made from, so taking a modifier off gives- * the original back; what they made is cached by source and modifiers, so it- * is made once however many elements ask for it, and adding a second- * modifier does not re-run the first. Applied in the order below.- */-type UpscaleProps = {- /**- * Resolution multiplier: 2 asks for twice the pixels. Enlarges the source,- * not the box — the element keeps the width and height it was given, and- * renders sharper. Default 1, the source as it is.- */- upscale?: number;-};-type FitProps = {/** How the source maps into the box. Default "cover" on `<Video>`, "contain" on `<Image>`. */objectFit?: Fit;@@ -655,24 +639,12 @@ export type ColorStopProps = ColorProps & OpacityProps & TrackChildren & {*/export type MediaPaintProps = PaintProps & MediaProps & FitProps & FrameRateProps & TrackChildren;-export type VideoProps = CommonProps & MediaProps & FitProps & FrameRateProps & AudioTrackProps & UpscaleProps & {- /**- * Scores the footage: a generated soundtrack for a clip that has none. See- * `UpscaleProps` for what a modifier is; applied last, after `upscale`, so- * a re-encode cannot drop the track. Independent of `volume` and `muted`,- * which mix whatever track the clip ends up with.- */- addAudio?: boolean;+export type VideoProps = CommonProps & MediaProps & FitProps & FrameRateProps & AudioTrackProps & {/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */children?: SolidJSX.Element;};-export type ImageProps = CommonProps & MediaProps & FitProps & FrameRateProps & UpscaleProps & {- /**- * Cuts the subject out, leaving the rest of the picture transparent. See- * `UpscaleProps` for what a modifier is; applied before `upscale`.- */- removeBackground?: boolean;+export type ImageProps = CommonProps & MediaProps & FitProps & FrameRateProps & {/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */children?: SolidJSX.Element;};diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 6375ae42..51c7a934 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import { createSignal } from 'solid-js';import { SVGElements } from 'solid-js/web';@@ -26,15 +26,6 @@ const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, '*/const HOLD_TIMEOUT_MS = 30_000;-/** What an element with no `SourceModifiers` trait is asking for: nothing. */-const NO_MODIFIERS = { removeBackground: false, upscale: 1, addAudio: false };--/** `upscale` as a factor; anything that is not one above 1 is natural size. */-function upscaleFactor(value: unknown): number {- const factor = typeof value === 'number' ? value : Number(value);- return Number.isFinite(factor) && factor > 1 ? factor : 1;-}-export interface AuthoredElement {/** The camelCase tag the project used. */tag: string;@@ -1325,47 +1316,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {});return;}- case 'removeBackground':- case 'addAudio':- case 'upscale': {- const current = entity.get(SourceModifiers) ?? NO_MODIFIERS;- const next = {- ...current,- [name]: name === 'upscale' ? upscaleFactor(value) : value === true,- };-- if (- next.removeBackground === current.removeBackground- && next.upscale === current.upscale- && next.addAudio === current.addAudio- ) return;-- if (hasModifier(next)) {- entity.add(SourceModifiers);- entity.set(SourceModifiers, next);- } else {- entity.remove(SourceModifiers);- }-- if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;-- const src = node.props.src;- if (src === undefined || src === null || src === '') return;-- // A resolution running for the old modifiers must not bind late.- entity.remove(PendingSource, Generating);-- if (typeof src === 'string') {- entity.add(LoadRequest);- entity.set(LoadRequest, { value: src });- return;- }-- entity.add(GenerationRequest);- entity.set(GenerationRequest, { ref: src as AssetRef });-- return;- }case 'seed': {if (!entity.has(Caption)) return;// An authored src mounts a transcript directly; there is nodiff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex 14c73ef5..e0a88cc5 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -6,7 +6,7 @@// goes through the AssetLibrary (@diffusionstudio/assets); these are the// lookups the decoders and hosts share.-import { Ai, AssetId, Library, Paint, SourceFrameRate, SourceModifiers } from '../traits';+import { Ai, AssetId, Library, Paint, SourceFrameRate } from '../traits';import { PaintType } from '../constants';import type { Entity, World } from 'koota';@@ -27,28 +27,6 @@ export function getAi(world: World): GenAi {return ai;}-/** What a source is put through after it resolves (see `SourceModifiers`). */-export interface SourceModifierValues {- removeBackground: boolean;- /** Factor, 1 = natural size. */- upscale: number;- addAudio: boolean;-}--/** Whether a set of modifiers asks for anything at all. */-export const hasModifier = (modifiers: SourceModifierValues): boolean =>- modifiers.removeBackground || modifiers.upscale > 1 || modifiers.addAudio;--/**- * The modifiers `entity` asks its source to be put through, or undefined- * when it asks for none — which is what the trait's absence means, so this- * is also the question "is this element showing a derived source".- */-export function getModifiers(entity: Entity): SourceModifierValues | undefined {- const modifiers = entity.get(SourceModifiers);- return modifiers && hasModifier(modifiers) ? modifiers : undefined;-}-/*** Binds an entity to an asset: stamps its AssetId, and follows the asset with* the paint. A media paint is whichever of the two the asset can be drawndiff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.tsindex f0a2ae2c..21c81db8 100644--- a/packages/runtime/src/ai.ts+++ b/packages/runtime/src/ai.ts@@ -2,12 +2,12 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { generate } from '@diffusionstudio/jsx';+import { generate, transform } from '@diffusionstudio/jsx';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';-import type { SourceModifierValues } from './actions/assets';import type {+ AssetInput,AssetRef,GenerateAudioOptions,GenerateImageOptions,@@ -26,11 +26,13 @@ import type {*/export abstract class GenAi {/**- * The declaration, made real: an asset whose bytes the model produced.- * Content-addressed — the fully-resolved spec's hash is the asset's- * `generation.key`, so the same spec is the same asset in this session- * and the next, and identical concurrent declarations share one run.- * Rejects with the recorded reason when the key stands in error.+ * The declaration, made real: an asset whose bytes the model produced —+ * from a prompt (`generate.*`) or from another asset (`transform.*`), its+ * inputs resolved first. Content-addressed — the fully-resolved spec's+ * hash is the asset's `generation.key`, so the same spec is the same+ * asset in this session and the next, and identical concurrent+ * declarations share one run. Rejects with the recorded reason when the+ * key stands in error.*/public abstract resolve(ref: AssetRef): Promise<Asset>;@@ -42,14 +44,6 @@ export abstract class GenAi {*/public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;- /**- * `asset` put through the modifiers an element asked of its source (see- * `SourceModifiers`): the same asset and the same modifiers are the same- * result, in this session and the next, and every step is cached on its- * own, so adding one to an element does not re-run the others.- */- public abstract derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset>;-/*** The imperative surface: `ai.generate.image({...})` declares and resolves* in one call, returning the finished asset.@@ -60,4 +54,11 @@ export abstract class GenAi {voice: (options: GenerateVoiceOptions): Promise<Asset> => this.resolve(generate.voice(options)),audio: (options: GenerateAudioOptions): Promise<Asset> => this.resolve(generate.audio(options)),};++ /** The same for `transform.*`: `ai.transform.upscale(input)` is the upscaled asset. */+ public readonly transform = {+ upscale: (input: AssetInput): Promise<Asset> => this.resolve(transform.upscale(input)),+ removeBackground: (input: AssetInput): Promise<Asset> => this.resolve(transform.removeBackground(input)),+ addAudio: (input: AssetInput): Promise<Asset> => this.resolve(transform.addAudio(input)),+ };}diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.tsindex cc81aa7f..a0e2e647 100644--- a/packages/runtime/src/systems/assets.ts+++ b/packages/runtime/src/systems/assets.ts@@ -5,7 +5,7 @@import { parseSource } from '@diffusionstudio/jsx';import { Ai, Cache, Computed, Delay, FrameRate, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, TranscriptionRequest, Trim } from '../traits';-import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';+import { bindAsset, getAssetFile } from '../actions/assets';import { getEntityTree, getParentNode, getSceneAncestor } from '../queries/hierarchy';import { findAssetDuration, findGeometryAsset } from '../utils/time';import { computeAudioSyncOffsetCached } from '../media/audio-sync';@@ -30,7 +30,7 @@ export function assetSystem(world: World): void {for (const entity of world.query(LoadRequest)) {const source = entity.get(LoadRequest)!.value;entity.remove(LoadRequest);- start(world, entity, library.resolve(source));+ resolve(world, entity, library.resolve(source));}}@@ -41,7 +41,7 @@ export function assetSystem(world: World): void {entity.remove(GenerationRequest);if (ref === null) continue;if (isDomImage(entity)) pointDomImageAt(entity, null);- start(world, entity, ai.resolve(ref), true);+ resolve(world, entity, ai.resolve(ref), true);}for (const entity of world.query(TranscriptionRequest)) {@@ -215,24 +215,6 @@ function currentSync(entity: Entity, token: object): boolean {return entity.isAlive() && entity.get(PendingSync)?.value === token;}-/**- * Starts a resolution for whatever the element's src named, putting it- * through the modifiers the element asks of it (see `SourceModifiers`). The- * two are one wait: the element binds the asset it is going to show, not- * first the one it was made from.- */-function start(world: World, entity: Entity, base: Promise<Asset>, generating = false): void {- const modifiers = getModifiers(entity);- const ai = world.get(Ai);-- if (!modifiers || !ai) {- resolve(world, entity, base, generating);- return;- }-- resolve(world, entity, base.then((asset) => ai.derive(asset, modifiers)), true);-}-/*** Whether anything in `scene`'s subtree (besides `except`, the requesting* element itself) is still waiting on a source: a request this system has@@ -254,8 +236,7 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean* Tracks one started resolution: the entity remembers which one it is waiting* on (PendingSource), so of overlapping resolutions only the latest binds, and* one that outlives its element (or its src) is dropped. The wait itself is- * the identity rather than what was asked for — the same src put through- * different modifiers is a different answer, and the later question is always+ * the identity rather than what was asked for — the later question is always* the one being asked.*/function resolve(world: World, entity: Entity, promise: Promise<Asset>, generating = false): void {diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.tsindex 7c90436d..5960b674 100644--- a/packages/runtime/src/traits/node.ts+++ b/packages/runtime/src/traits/node.ts@@ -85,16 +85,6 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))// through the library.export const LoadRequest = trait({ value: '' });-// Model calls the src is put through once it resolves: the element shows-// what they made of what it named, and the src goes on naming the original,-// so taking a modifier off gives it back. Every field's default is off —-// `upscale` 1 is natural size — and the trait is only present while one is-// not. Applied in the order declared here: the matte before the enlarging-// (cheaper, and the model wants a normally-sized picture), the audio last so-// a re-encode cannot drop it. Each step is cached on its own, so turning one-// on does not re-pay for the ones already applied.-export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addAudio: false });-// On a `<captions>` element without a src: transcribe the scene it sits// under, through the world's Ai. The seed keys the take — the transcript is// cached by scene id + seed, so re-running with a seed replays that take and