Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor and enhance transform functionality in the asset management system

konstantin-paulus1bbe99234578
diff --git a/apps/desktop/src/edit.ts b/apps/desktop/src/edit.ts
index f0f7dd70..16a830cf 100644
--- a/apps/desktop/src/edit.ts
+++ b/apps/desktop/src/edit.ts
@@ -9,7 +9,7 @@ import { join } from "node:path";
import { IndentationText, Project, SyntaxKind } from "ts-morph";
-import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, parseSource } from "@diffusionstudio/jsx";
+import { ID_ATTR, INSPECT_TAG, formatSource, isCompositionTag, isLoopTag, isSerializedAssetRef, isTransformType, parseSource } from "@diffusionstudio/jsx";
import type { PropValue, SerializedAssetRef } from "@diffusionstudio/jsx";
import type {
@@ -119,7 +119,8 @@ const round = (value: number): number => Math.round(value * 100) / 100;
const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
/** A prop value as the JavaScript that would have produced it. */
-function literalText(value: PropValue): string {
+function literalText(value: PropValue | SerializedAssetRef): string {
+ if (isSerializedAssetRef(value)) return declarationText(value);
if (typeof value === "string") return JSON.stringify(value);
if (typeof value === "number") return String(round(value));
if (typeof value === "boolean" || value === null) return String(value);
@@ -149,24 +150,41 @@ const initializerText = (value: PropValue): string =>
? `"${value}"`
: `{${literalText(value)}}`;
-/** A declaration's wire form as the `generate.*` call that reproduces it. */
-function generateCallText(ref: SerializedAssetRef): string {
- const { type, ...options } = ref.$generate;
+/**
+ * A declaration's wire form as the call that reproduces it: `generate.<type>`
+ * over its options, or `transform.<type>` over its input, with declared
+ * inputs spelled as the nested calls they are.
+ */
+function declarationText(ref: SerializedAssetRef): string {
+ const { type, ...options } = ref.$asset;
+ if (isTransformType(type)) return `transform.${type}(${literalText((options as { input: EditValue }).input)})`;
return `generate.${type}(${literalText(options as Record<string, PropValue>)})`;
}
+/** The namespaces a declaration is spelled with: what its file has to import. */
+function declarationImports(ref: SerializedAssetRef, names = new Set<string>()): Set<string> {
+ const { type, ...options } = ref.$asset;
+ names.add(isTransformType(type) ? "transform" : "generate");
+ for (const value of Object.values(options)) {
+ for (const input of Array.isArray(value) ? value : [value]) {
+ if (isSerializedAssetRef(input)) declarationImports(input, names);
+ }
+ }
+ return names;
+}
+
/**
* Writes a prop onto a tag as one would write it: `muted`, not `muted={true}`,
* and no attribute at all rather than `muted={false}`, since absence is what a
- * boolean prop's false reads as. A declaration is spelled as its `generate.*`
- * call — the caller makes sure `generate` is imported (see
- * `ensureGenerateImport`).
+ * boolean prop's false reads as. A declaration is spelled as the call that
+ * reproduces it — the caller makes sure its namespaces are imported (see
+ * `ensureDeclarationImports`).
*/
function setProp(tag: JsxTag, name: string, value: EditValue): void {
const attribute = attributeOf(tag, name);
if (isSerializedAssetRef(value)) {
- const initializer = `{${generateCallText(value)}}`;
+ const initializer = `{${declarationText(value)}}`;
if (attribute) attribute.setInitializer(initializer);
else tag.addAttribute({ name, initializer });
return;
@@ -187,32 +205,37 @@ function setProp(tag: JsxTag, name: string, value: EditValue): void {
else tag.addAttribute({ name, initializer: initializerText(value) });
}
-/** Where `generate` comes from — the module every project authors against. */
-const GENERATE_MODULE = "@diffusionstudio/jsx";
+/** Where `generate` and `transform` come from — the module every project authors against. */
+const DECLARATION_MODULE = "@diffusionstudio/jsx";
/**
- * Makes sure `generate` is in scope once a declaration has been spelled into
- * the file: added to the module's existing import, or as an import of its own
- * after the last one — or above the first statement, past the file's header
- * comments, when there are no imports at all. Idempotent.
+ * Makes sure the namespaces a declaration is spelled with are in scope once
+ * it has been written into the file: added to the module's existing import,
+ * or as an import of their own after the last one — or above the first
+ * statement, past the file's header comments, when there are no imports at
+ * all. Idempotent.
*/
-function ensureGenerateImport(sourceFile: SourceFile): void {
+function ensureDeclarationImports(sourceFile: SourceFile, ref: SerializedAssetRef): void {
+ for (const name of declarationImports(ref)) ensureNamedImport(sourceFile, name);
+}
+
+function ensureNamedImport(sourceFile: SourceFile, name: string): void {
const declarations = sourceFile.getImportDeclarations();
const importable = declarations.find(
(declaration) =>
- declaration.getModuleSpecifierValue() === GENERATE_MODULE &&
+ declaration.getModuleSpecifierValue() === DECLARATION_MODULE &&
!declaration.isTypeOnly() &&
!declaration.getNamespaceImport(),
);
if (importable) {
const named = importable.getNamedImports();
- if (named.some((specifier) => specifier.getName() === "generate" && !specifier.getAliasNode())) return;
- importable.addNamedImport("generate");
+ if (named.some((specifier) => specifier.getName() === name && !specifier.getAliasNode())) return;
+ importable.addNamedImport(name);
return;
}
- const statement = `import { generate } from "${GENERATE_MODULE}";`;
+ const statement = `import { ${name} } from "${DECLARATION_MODULE}";`;
const last = declarations.at(-1);
if (last) sourceFile.insertText(last.getEnd(), `\n${statement}`);
else {
@@ -423,8 +446,14 @@ function reindent(text: string, from: string, to: string): string {
.join("\n");
}
-/** Whether an expression is a value rather than a way of computing one. */
+/**
+ * Whether an expression is a value rather than a way of computing one. A
+ * declaration call over values is one too: it is what the writer spells a
+ * declaration as, so it is an initializer the writer may also overwrite.
+ */
function isLiteral(node: Node): boolean {
+ if (isDeclarationCall(node)) return true;
+
if (
node.isKind(SyntaxKind.StringLiteral) ||
node.isKind(SyntaxKind.NumericLiteral) ||
@@ -458,16 +487,19 @@ function isLiteral(node: Node): boolean {
return false;
}
+/** The namespaces a declaration call is made through. */
+const DECLARATION_NAMESPACES = new Set(["generate", "transform"]);
+
/**
- * A `generate.*` call over a literal spec: what the writer spells a
- * declaration as, and so an initializer it may also overwrite. One whose
- * argument computes anything is authored reactivity, like any expression.
+ * A `generate.*` or `transform.*` call over literals — a nested declaration
+ * among them. One whose argument computes anything is authored reactivity,
+ * like any expression.
*/
-function isGenerateCall(node: Node): boolean {
+function isDeclarationCall(node: Node): boolean {
const call = node.asKind(SyntaxKind.CallExpression);
const callee = call?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);
if (!call || !callee || !callee.getExpression().isKind(SyntaxKind.Identifier)) return false;
- if (callee.getExpression().getText() !== "generate") return false;
+ if (!DECLARATION_NAMESPACES.has(callee.getExpression().getText())) return false;
const args = call.getArguments();
return args.length === 1 && isLiteral(args[0]!);
@@ -486,7 +518,7 @@ function isWritable(attribute: JsxAttribute): boolean {
if (!initializer.isKind(SyntaxKind.JsxExpression)) return false;
const expression = initializer.getExpression();
- return expression !== undefined && (isLiteral(expression) || isGenerateCall(expression));
+ return expression !== undefined && isLiteral(expression);
}
// ---------------------------------------------------------------------------
@@ -801,7 +833,7 @@ class SourceWriter {
}
setProp(tag, name, value);
- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);
wrote = true;
}
@@ -898,7 +930,7 @@ class SourceWriter {
insertChild(sourceFile, parent, child, before);
for (const [name, value] of Object.entries(edit.props)) {
setProp(findTag(sourceFile, id)!, name, value);
- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);
}
ids[edit.source] = formatSource(file, id);
return true;
@@ -1138,7 +1170,7 @@ class SourceWriter {
}
for (const [name, value] of Object.entries(record.props)) {
setProp(found(), name, value);
- if (isSerializedAssetRef(value)) ensureGenerateImport(sourceFile);
+ if (isSerializedAssetRef(value)) ensureDeclarationImports(sourceFile, value);
}
const element = elementOf(found());
diff --git a/apps/web/src/components/genai/action-bar.tsx b/apps/web/src/components/genai/action-bar.tsx
index bae122e1..edb70d2b 100644
--- a/apps/web/src/components/genai/action-bar.tsx
+++ b/apps/web/src/components/genai/action-bar.tsx
@@ -21,7 +21,7 @@ import { useGenerateVoice } from "./use-generate-voice";
import { useGenerateAudio } from "./use-generate-audio";
import { useAutoCaptions } from "./use-auto-captions";
import { useMediaSelection } from "./selection";
-import { useModifiers } from "./use-modifiers";
+import { useTransforms } from "./use-transforms";
import { createDefaultConfig } from "./prompt-input";
import { toast } from "somoto";
@@ -33,7 +33,7 @@ interface ActionBarProps {
export function ActionBar(props: ActionBarProps) {
const { imageNodes, videoNodes } = useMediaSelection();
- const { isOn, toggle } = useModifiers();
+ const { isOn, toggle } = useTransforms();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
diff --git a/apps/web/src/components/genai/prompt-input-actions.tsx b/apps/web/src/components/genai/prompt-input-actions.tsx
index 239fc8af..8c606030 100644
--- a/apps/web/src/components/genai/prompt-input-actions.tsx
+++ b/apps/web/src/components/genai/prompt-input-actions.tsx
@@ -23,14 +23,14 @@ import { useGenerateVideo } from "./use-generate-video";
import { useGenerateVoice } from "./use-generate-voice";
import { useGenerateAudio } from "./use-generate-audio";
import { useMediaSelection } from "./selection";
-import { useModifiers } from "./use-modifiers";
+import { useTransforms } from "./use-transforms";
-import type { ModifierName } from "./use-modifiers";
+import type { TransformType } from "@diffusionstudio/jsx";
export function PromptInputActions() {
const { isGenerated, totalCredits, firstConfig } = useGenerationRecords();
const { imageNodes, videoNodes } = useMediaSelection();
- const { isOn, toggle } = useModifiers();
+ const { isOn, toggle } = useTransforms();
const { generate: generateImage } = useGenerateImage();
const { generate: generateVideo } = useGenerateVideo();
@@ -121,13 +121,13 @@ export function PromptInputActions() {
</Show>
<DropdownMenuGroup>
<Show when={hasImageSelection() || hasVideoSelection()}>
- <ModifierItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />
+ <TransformItem name="upscale" icon="arrow-scale" label="Upscale" isOn={isOn} toggle={toggle} />
</Show>
<Show when={hasImageSelection()}>
- <ModifierItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />
+ <TransformItem name="removeBackground" icon="ai-generate" label="Remove background" isOn={isOn} toggle={toggle} />
</Show>
<Show when={hasVideoSelection()}>
- <ModifierItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />
+ <TransformItem name="addAudio" icon="audio-on" label="Add audio" isOn={isOn} toggle={toggle} />
</Show>
</DropdownMenuGroup>
</DropdownMenuContent>
@@ -138,16 +138,16 @@ export function PromptInputActions() {
);
}
-type ModifierItemProps = {
- name: ModifierName;
+type TransformItemProps = {
+ name: TransformType;
icon: string;
label: string;
- isOn(name: ModifierName): boolean;
- toggle(name: ModifierName): void;
+ isOn(name: TransformType): boolean;
+ toggle(name: TransformType): void;
};
-/** A modifier as a menu row: a check when the selection is asking for it. */
-function ModifierItem(props: ModifierItemProps) {
+/** A transform as a menu row: a check when the selection is asking for it. */
+function TransformItem(props: TransformItemProps) {
return (
<DropdownMenuItem onSelect={() => props.toggle(props.name)}>
<Icon name={props.icon} class="mr-2 text-foreground" />
diff --git a/apps/web/src/components/genai/selection.ts b/apps/web/src/components/genai/selection.ts
index 23a28d77..1cc4fac6 100644
--- a/apps/web/src/components/genai/selection.ts
+++ b/apps/web/src/components/genai/selection.ts
@@ -6,7 +6,7 @@
// two different questions:
//
// `imageNodes`/`videoNodes` are what the element *is*, which is what a
-// source modifier applies to — a picture that is still generating is still a
+// transform applies to — a picture that is still generating is still a
// picture, and `removeBackground` can be asked of it before it exists.
//
// `images`/`videos` are what the element currently *shows*, which is what a
diff --git a/apps/web/src/components/genai/use-generation-records.ts b/apps/web/src/components/genai/use-generation-records.ts
index a6fb4822..2b91461e 100644
--- a/apps/web/src/components/genai/use-generation-records.ts
+++ b/apps/web/src/components/genai/use-generation-records.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import { createMemo, createResource } from "solid-js";
-import { getAssetSpec, isAssetRef } from "@diffusionstudio/jsx";
+import { getAssetSpec, isAssetRef, isTransformSpec } from "@diffusionstudio/jsx";
import { authoredElement } from "@diffusionstudio/reconciler";
import { useSelection } from "@/engine/hooks";
import { useLibrary } from "@/engine/library";
@@ -18,7 +18,7 @@ import {
} from "./config";
import { generationConfigSchema, type GenerationConfig } from "./schemas";
-import type { AssetInput, AssetSpecInput } from "@diffusionstudio/jsx";
+import type { AssetInput, AssetRef, GenerateSpec } from "@diffusionstudio/jsx";
import type { AssetLibrary } from "@diffusionstudio/assets";
/**
@@ -63,7 +63,7 @@ export function toClientConfig(stored: unknown): GenerationConfig | undefined {
* "Rerun" and "Reuse" work from. Inputs are named by library path in the
* file and by asset id in the prompt box, so they are looked up on the way.
*/
-function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): GenerationConfig {
+function toPromptConfig(spec: GenerateSpec, library: AssetLibrary): GenerationConfig {
const idOf = (input: AssetInput | undefined): string | undefined =>
typeof input === "string" ? library.get(input)?.id : undefined;
@@ -104,6 +104,13 @@ function toPromptConfig(spec: AssetSpecInput, library: AssetLibrary): Generation
}
}
+/** The generation a declaration rests on: itself, or what its transforms were put over. */
+function generationUnder(ref: AssetRef): GenerateSpec | undefined {
+ const spec = getAssetSpec(ref);
+ if (!isTransformSpec(spec)) return spec;
+ return isAssetRef(spec.input) ? generationUnder(spec.input) : undefined;
+}
+
export function useGenerationRecords() {
const library = useLibrary();
const { nodes } = useSelection();
@@ -111,12 +118,14 @@ export function useGenerationRecords() {
// What the selected elements declare their source to be. An element made by
// the prompt box carries the whole spec, so this answers before the asset
- // exists — and without asking the server what it was asked for.
+ // exists — and without asking the server what it was asked for. Transforms
+ // are looked through: the generation under them is what the box made.
const declarations = createMemo(() =>
nodes()
.map((entity) => authoredElement(entity)?.props.src)
.filter(isAssetRef)
- .map(getAssetSpec),
+ .map(generationUnder)
+ .filter((spec): spec is GenerateSpec => spec !== undefined),
);
// Credits are the server's to know, and it knows them per generation.
diff --git a/apps/web/src/components/genai/use-modifiers.ts b/apps/web/src/components/genai/use-modifiers.ts
deleted file mode 100644
index 829b6cc1..00000000
--- a/apps/web/src/components/genai/use-modifiers.ts
+++ /dev/null
@@ -1,66 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Upscaling, taking a background out, scoring footage: the editor's side of
-// the source modifiers (see the runtime's `SourceModifiers` and
-// reference/jsx/media.md). The action bar sets a prop and stops there — the
-// asset system is what notices, calls the model and binds the result — so
-// these are toggles rather than commands, and turning one off gives the
-// original back rather than running anything.
-
-import { authoredElement } from "@diffusionstudio/reconciler";
-import { useEditor } from "@/engine/hooks";
-import { useMediaSelection } from "./selection";
-
-import type { Entity } from "koota";
-
-/** The props an element carries to say what its source is put through. */
-export type ModifierName = "removeBackground" | "upscale" | "addAudio";
-
-/**
- * What `upscale` is set to when it is switched on. The endpoint takes no
- * factor today, so this is the one the language says and the number every
- * upscale is; see the cache key in `EditorGenAi.transform`.
- */
-const UPSCALE_FACTOR = 2;
-
-/** Whether `entity` currently asks for `name`, as its element says it. */
-function isSet(entity: Entity, name: ModifierName): boolean {
- const value = authoredElement(entity)?.props[name];
- return name === "upscale" ? typeof value === "number" && value > 1 : value === true;
-}
-
-export function useModifiers() {
- const editor = useEditor();
- const { imageNodes, videoNodes } = useMediaSelection();
-
- /** The selected elements a modifier can be asked of. */
- const targets = (name: ModifierName): Entity[] => {
- if (name === "removeBackground") return imageNodes();
- if (name === "addAudio") return videoNodes();
- return [...imageNodes(), ...videoNodes()];
- };
-
- /** On when every element it applies to asks for it, so the toggle turns the odd one on. */
- const isOn = (name: ModifierName): boolean => {
- const selected = targets(name);
- return selected.length > 0 && selected.every((entity) => isSet(entity, name));
- };
-
- /**
- * Turns `name` on for the selection, or off when they all have it. Off is
- * written as `false`, which the source writer spells as the prop's absence
- * — an element says what it asks for, and nothing about what it does not.
- */
- const toggle = (name: ModifierName): void => {
- const next = !isOn(name);
- const value = next ? (name === "upscale" ? UPSCALE_FACTOR : true) : false;
-
- for (const entity of targets(name)) {
- editor.editProperty(entity, name, value);
- }
- };
-
- return { isOn, toggle };
-}
diff --git a/apps/web/src/components/genai/use-transforms.ts b/apps/web/src/components/genai/use-transforms.ts
new file mode 100644
index 00000000..e06b87de
--- /dev/null
+++ b/apps/web/src/components/genai/use-transforms.ts
@@ -0,0 +1,86 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Upscaling, taking a background out, scoring footage: the editor's side of
+// `transform.*` (see reference/jsx/generate.md). A toggle wraps the
+// element's `src` in the transform — `transform.upscale(<what it was>)` —
+// or takes that layer out again, and stops there: the asset system is what
+// notices the new declaration, runs it and binds the result. Turning one
+// off gives the inner declaration back rather than running anything.
+
+import { getAssetSpec, isAssetRef, isTransformSpec, mapAssetInputs, transform } from "@diffusionstudio/jsx";
+import { authoredElement } from "@diffusionstudio/reconciler";
+import { useEditor } from "@/engine/hooks";
+import { useMediaSelection } from "./selection";
+
+import type { AssetInput, TransformType } from "@diffusionstudio/jsx";
+import type { Entity } from "koota";
+
+/** Whether `input` is, somewhere down its chain of transforms, put through `type`. */
+function hasTransform(input: AssetInput, type: TransformType): boolean {
+ if (!isAssetRef(input)) return false;
+ const spec = getAssetSpec(input);
+ if (!isTransformSpec(spec)) return false;
+ return spec.type === type || hasTransform(spec.input, type);
+}
+
+/**
+ * `input` with every `type` layer taken out of its chain of transforms. A
+ * generation under the chain is left as it is: its own inputs are its
+ * business, not the element's.
+ */
+function withoutTransform(input: AssetInput, type: TransformType): AssetInput {
+ if (!isAssetRef(input)) return input;
+ const spec = getAssetSpec(input);
+ if (!isTransformSpec(spec)) return input;
+ if (spec.type === type) return withoutTransform(spec.input, type);
+ return mapAssetInputs(input, (inner) => withoutTransform(inner, type));
+}
+
+/** The element's `src` as authored, when it is something a transform can be put over. */
+function sourceOf(entity: Entity): AssetInput | undefined {
+ const src = authoredElement(entity)?.props.src;
+ return typeof src === "string" && src !== "" ? src : isAssetRef(src) ? src : undefined;
+}
+
+export function useTransforms() {
+ const editor = useEditor();
+ const { imageNodes, videoNodes } = useMediaSelection();
+
+ /** The selected elements a transform can be asked of. */
+ const targets = (type: TransformType): Entity[] => {
+ if (type === "removeBackground") return imageNodes();
+ if (type === "addAudio") return videoNodes();
+ return [...imageNodes(), ...videoNodes()];
+ };
+
+ /** On when every element it applies to asks for it, so the toggle turns the odd one on. */
+ const isOn = (type: TransformType): boolean => {
+ const selected = targets(type);
+ return selected.length > 0 && selected.every((entity) => {
+ const src = sourceOf(entity);
+ return src !== undefined && hasTransform(src, type);
+ });
+ };
+
+ /**
+ * Puts `type` over the selection's sources, or takes it off when they all
+ * have it. On goes outermost — the last thing done to the source — and off
+ * takes every layer of it out, wherever it sits in the chain.
+ */
+ const toggle = (type: TransformType): void => {
+ const next = !isOn(type);
+
+ for (const entity of targets(type)) {
+ const src = sourceOf(entity);
+ if (src === undefined) continue;
+ const declared = next
+ ? (hasTransform(src, type) ? src : transform[type](src))
+ : withoutTransform(src, type);
+ if (declared !== src) editor.editProperty(entity, "src", declared);
+ }
+ };
+
+ return { isOn, toggle };
+}
diff --git a/apps/web/src/engine/editor.ts b/apps/web/src/engine/editor.ts
index 85d396ad..b6960968 100644
--- a/apps/web/src/engine/editor.ts
+++ b/apps/web/src/engine/editor.ts
@@ -19,7 +19,7 @@ import { authoredElement, authoredTree, getRuntimeDocument, insert, isSceneNode,
import { findInspectEntry } from './inspect';
import type { SceneNode } from '@diffusionstudio/runtime';
-import type { InspectValue, PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';
+import type { AssetRef, InspectValue, PropValue, SerializedAssetRef } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
import type { AuthoredTree, ProjectDocument, RuntimeDocument } from '@diffusionstudio/reconciler';
@@ -266,11 +266,10 @@ function nextSourceAfter(world: World, entity: Entity): string | undefined {
* The props of a `<video>`/`<image>` that only mean something while the node
* plays media, dropped when the intrinsic paint is removed (see
* `removeIntrinsicPaint`): the source, what qualifies it (its fit, its
- * window, a frames directory's rate, the modifiers put on it), and the audio
- * mix a rect has no track to apply to. `start`/`end` stay: they place the
- * clip, media or not.
+ * window, a frames directory's rate), and the audio mix a rect has no track
+ * to apply to. `start`/`end` stay: they place the clip, media or not.
*/
-const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'upscale', 'removeBackground', 'addAudio', 'volume', 'muted', 'syncTo'] as const;
+const MEDIA_PROPS = ['src', 'objectFit', 'frameRate', 'sourceIn', 'sourceOut', 'volume', 'muted', 'syncTo'] as const;
/**
* What `copy` took: the subtrees as authored, and the source of the parent
@@ -325,7 +324,7 @@ export class DocumentEditor {
}
/** Writes a prop to the document and reports it. */
- public editProperty(entity: Entity, name: string, value: PropValue): void {
+ public editProperty(entity: Entity, name: string, value: PropValue | AssetRef): void {
const node = this.document.node(entity);
let previous = node.props[name];
// The stage keeps no authored record (see RuntimeDocument.setProperty),
@@ -336,7 +335,7 @@ export class DocumentEditor {
if (background !== undefined && background !== DEFAULT_BACKGROUND) previous = colorToHex(background);
}
this.document.setProperty(node, name, value);
- this.reportEdit(entity, name, value, previous ?? false);
+ this.reportEdit(entity, name, wireValue(value)!, wireValue(previous) ?? false);
}
/**
@@ -380,7 +379,7 @@ export class DocumentEditor {
* (see `PropEdit`); a caller that cannot say what the value was leaves it
* out, and the edit is not undoable.
*/
- public reportEdit(entity: Entity, name: string, value: PropValue, previous?: unknown): void {
+ public reportEdit(entity: Entity, name: string, value: EditValue, previous?: unknown): void {
this.settle(entity);
const source = entity.get(Source)?.value;
diff --git a/apps/web/src/engine/generations.ts b/apps/web/src/engine/generations.ts
index 95dd97f4..4b033aa3 100644
--- a/apps/web/src/engine/generations.ts
+++ b/apps/web/src/engine/generations.ts
@@ -14,7 +14,7 @@
import { isAssetRef } from '@diffusionstudio/jsx';
import { authoredElement } from '@diffusionstudio/reconciler';
import {
- Caption, Generating, GenerationRequest, LoadRequest, PendingSource, SourceError, TranscriptionRequest,
+ Caption, Generating, GenerationRequest, PendingSource, SourceError, TranscriptionRequest,
} from '@diffusionstudio/runtime';
import type { AssetLibrary, PartialAsset } from '@diffusionstudio/assets';
@@ -36,10 +36,6 @@ export async function retryGeneration(world: World, library: AssetLibrary, parti
if (isAssetRef(src)) {
entity.add(GenerationRequest);
entity.set(GenerationRequest, { ref: src });
- } else if (typeof src === 'string' && src !== '') {
- // A source put through modifiers: the load is cheap, the derivation is what failed.
- entity.add(LoadRequest);
- entity.set(LoadRequest, { value: src });
} else if (entity.has(Caption)) {
entity.add(TranscriptionRequest);
entity.set(TranscriptionRequest, { seed: Number(authored.props.seed) || 0 });
diff --git a/apps/web/src/engine/library.ts b/apps/web/src/engine/library.ts
index 58139f16..a7e2fcbd 100644
--- a/apps/web/src/engine/library.ts
+++ b/apps/web/src/engine/library.ts
@@ -10,12 +10,14 @@ import { AssetId, Library } from '@diffusionstudio/runtime';
import { AssetLibrary, MANIFEST_FILE, ASSETS_DIR } from '@diffusionstudio/assets';
import { useTrait, useWorld } from '@diffusionstudio/koota-solid';
import { authoredElement } from '@diffusionstudio/reconciler';
+import { isAssetRef, mapAssetInputs } from '@diffusionstudio/jsx';
import type { Accessor } from 'solid-js';
import { createProjectFS } from '@/projects/fs';
import { getDocumentEditor } from './editor';
import type { Asset } from '@diffusionstudio/assets';
+import type { AssetInput } from '@diffusionstudio/jsx';
import type { World } from 'koota';
/**
@@ -41,18 +43,21 @@ export function isLibraryFile(path: string): boolean {
}
/**
- * Rewrites `src` on every element that named an asset by its old path. Matched
- * on the path alone, not on what the element is bound to: an element showing a
- * modified source (`removeBackground`, `upscale`) is bound to what the
- * modifiers made of the asset rather than to the asset itself, and its `src`
- * still has to follow the rename.
+ * Rewrites `src` on every element that named an asset by its old path —
+ * directly, or as an input somewhere inside a declaration: an element
+ * showing `transform.upscale("old.png")` is bound to what the transform made,
+ * and its `src` still has to follow the rename.
*/
function followRename(world: World, asset: Asset, from: string): void {
const editor = getDocumentEditor(world);
+ const renamed = (input: AssetInput): AssetInput =>
+ isAssetRef(input) ? mapAssetInputs(input, renamed) : input === from ? asset.path : input;
+
for (const entity of world.query(AssetId)) {
const src = authoredElement(entity)?.props.src;
- if (src !== from) continue;
- editor.editProperty(entity, 'src', asset.path);
+ if (typeof src !== 'string' && !isAssetRef(src)) continue;
+ const next = renamed(src);
+ if (next !== src) editor.editProperty(entity, 'src', next);
}
}
diff --git a/apps/web/src/utils/gen-ai.ts b/apps/web/src/utils/gen-ai.ts
index 71e72bf6..523f5a21 100644
--- a/apps/web/src/utils/gen-ai.ts
+++ b/apps/web/src/utils/gen-ai.ts
@@ -2,12 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { getAssetSpec, isAssetRef, parseSource } from "@diffusionstudio/jsx";
+import { getAssetSpec, isAssetRef, isTransformSpec, isTransformType, parseSource } from "@diffusionstudio/jsx";
import {
Ai, AssetId, Audio, GenAi, getAssetFile, getEntityTree, Hidden, Muted,
Paint, PaintType, Project, Source,
} from "@diffusionstudio/runtime";
-import type { SourceModifierValues } from "@diffusionstudio/runtime";
import { createEncoder } from "@diffusionstudio/encoder";
import { createCapture } from "@/engine/capture";
import { assetName, GENERATED_DIR, isPartialAsset } from "@diffusionstudio/assets";
@@ -24,14 +23,14 @@ import { track } from "@/lib/analytics";
import { trpc } from "@/lib/trpc";
import { toast } from "somoto";
-import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput } from "@diffusionstudio/jsx";
+import type { AspectRatio, AssetInput, AssetRef, AssetSpecInput, GenerateSpec, TransformType } from "@diffusionstudio/jsx";
import type { Asset, AssetLibrary, AssetType, PartialAsset, ReserveOptions } from "@diffusionstudio/assets";
import type { FileRef } from "@diffusionstudio/api-contract";
import type { ExportResult } from "@diffusionstudio/encoder";
import type { Entity, World } from "koota";
-/** What a failure is called where the user reads about it. */
-const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
+/** What a failed generation is called where the user reads about it. */
+const FAILURE_TITLES: Record<GenerateSpec["type"] | "transcript", string> = {
image: "Image generation failed",
video: "Video generation failed",
voice: "Voice generation failed",
@@ -39,48 +38,36 @@ const FAILURE_TITLES: Record<AssetSpecInput["type"] | "transcript", string> = {
transcript: "Caption generation failed",
};
-/** What each declaration is to become, for the partial that stands for it. */
-const ASSET_TYPES: Record<AssetSpecInput["type"], AssetType> = {
+/** What each generation is to become, for the partial that stands for it. */
+const ASSET_TYPES: Record<GenerateSpec["type"], AssetType> = {
image: "IMAGE",
video: "VIDEO",
voice: "AUDIO",
audio: "AUDIO",
};
-/**
- * One step of a source modifier (see the runtime's `SourceModifiers`): a
- * model call that makes a new asset out of one the library already holds.
- * An element asks for these by prop and goes on naming what it was made
- * from, so each step is stored and cached on its own, and a set of modifiers
- * is these run in order (see `derive`).
- */
-export type TransformKind = "remove-background" | "upscale" | "add-audio";
-
-/** What each step is called where the user reads about it, and names its result. */
-const TRANSFORMS: Record<TransformKind, { title: string; suffix: string }> = {
- "remove-background": { title: "Background removal failed", suffix: "Background removed" },
- "upscale": { title: "Upscale failed", suffix: "Upscaled" },
- "add-audio": { title: "Adding audio failed", suffix: "With audio" },
+/** What each transform's failure is called where the user reads about it, and what names its result. */
+const TRANSFORMS: Record<TransformType, { title: string; suffix: string }> = {
+ removeBackground: { title: "Background removal failed", suffix: "Background removed" },
+ upscale: { title: "Upscale failed", suffix: "Upscaled" },
+ addAudio: { title: "Adding audio failed", suffix: "With audio" },
};
-/** The steps a set of modifiers comes to, in the order they are applied. */
-function steps(modifiers: SourceModifierValues): TransformKind[] {
- const kinds: TransformKind[] = [];
- if (modifiers.removeBackground) kinds.push("remove-background");
- if (modifiers.upscale > 1) kinds.push("upscale");
- if (modifiers.addAudio) kinds.push("add-audio");
- return kinds;
-}
-
/**
* A spec with defaults applied and every `AssetInput` reduced to an asset id.
* Field order is fixed, so `JSON.stringify` of it is a stable `generationKey`.
+ * A transform's key is its type and input alone: the endpoints take no model
+ * and no factor, so every call over the same asset is the same call.
*/
-type ResolvedSpec =
+type ResolvedGeneration =
| { type: "image"; model: string; prompt: string; aspectRatio: AspectRatio; seed?: number; refIds: string[] }
| { type: "video"; model: string; prompt: string; aspectRatio: AspectRatio; duration: number; audio: boolean; seed?: number; startFrameId?: string; endFrameId?: string }
| { type: "voice"; model: string; prompt: string; voice: string; seed?: number }
| { type: "audio"; model: string; prompt: string; duration?: number; seed?: number };
+type ResolvedTransform = { type: TransformType; inputId: string };
+type ResolvedSpec = ResolvedGeneration | ResolvedTransform;
+
+const isResolvedTransform = (spec: ResolvedSpec): spec is ResolvedTransform => isTransformType(spec.type);
/** What the partial standing for a run is called and is to become, and what its failure is called. */
type RunDescription = Omit<ReserveOptions, "key" | "folder"> & { title: string };
@@ -152,39 +139,6 @@ export class EditorGenAi extends GenAi {
);
}
- /**
- * `asset` through every modifier the element asked for, one step at a time
- * and in a fixed order. Each step is cached in its own right, so turning
- * one on leaves what the others already made alone — adding `upscale` to a
- * cut-out picture pays for the enlarging, not for the matte again.
- */
- public async derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset> {
- let derived = asset;
- for (const kind of steps(modifiers)) {
- derived = await this.transform(kind, derived);
- }
- return derived;
- }
-
- /**
- * Runs one step over `asset` and returns what it produced, stored under
- * `generated/` like any other model output. Keyed by step and input, so the
- * same call on the same asset is the same result in this session and the
- * next: upscaling a picture twice costs what upscaling it once did.
- */
- public transform(kind: TransformKind, asset: Asset): Promise<Asset> {
- // No upscale factor in the key: the endpoint takes none, so every factor
- // is the same call and would otherwise be billed once per number asked
- // for. It belongs here the moment the API can be told one.
- const key = `transform:v1:${kind}:${asset.id}`;
- const base = assetName(asset).replace(/\.[^.]+$/, "");
- return this.generated(
- key,
- () => ({ type: asset.type, name: `${base} (${TRANSFORMS[kind].suffix})`, title: TRANSFORMS[kind].title }),
- () => this.runTransform(kind, asset, key),
- );
- }
-
/**
* The library's answer for `key`, or the run that produces one. An asset
* the key landed as is returned; a key standing in error rejects with the
@@ -226,6 +180,7 @@ export class EditorGenAi extends GenAi {
private async generateFromRef(ref: AssetRef): Promise<Asset> {
const spec = getAssetSpec(ref);
+ const title = isTransformSpec(spec) ? TRANSFORMS[spec.type].title : FAILURE_TITLES[spec.type];
// Inputs first: their ids are part of the key. An input that failed has
// said so already; anything else wrong with them is said here.
@@ -233,21 +188,36 @@ export class EditorGenAi extends GenAi {
try {
resolved = await this.resolveSpec(spec);
} catch (error) {
- throw error instanceof ReportedError ? error : reportFailure(error, FAILURE_TITLES[spec.type]);
+ throw error instanceof ReportedError ? error : reportFailure(error, title);
}
return this.generated(
JSON.stringify(resolved),
- () => ({ type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt), title: FAILURE_TITLES[spec.type] }),
- (partial) => this.runGeneration(resolved, partial),
+ () => ({ ...this.describe(resolved), title }),
+ (partial) => isResolvedTransform(resolved)
+ ? this.runTransform(resolved, partial)
+ : this.runGeneration(resolved, partial),
);
}
+ /** What the partial standing for a run is called, and what it is to become. */
+ private describe(spec: ResolvedSpec): { type: AssetType; name: string } {
+ if (!isResolvedTransform(spec)) return { type: ASSET_TYPES[spec.type], name: provisionalName(spec.prompt) };
+ const input = this.library.get(spec.inputId);
+ assert(input, `Input asset ${spec.inputId} not found`);
+ return { type: input.type, name: `${stem(input)} (${TRANSFORMS[spec.type].suffix})` };
+ }
+
private resolveInput(input: AssetInput): Promise<Asset> {
return isAssetRef(input) ? this.resolve(input) : this.library.resolve(input);
}
private async resolveSpec(spec: AssetSpecInput): Promise<ResolvedSpec> {
+ if (isTransformSpec(spec)) {
+ const input = await this.resolveInput(spec.input);
+ return { type: spec.type, inputId: input.id };
+ }
+
switch (spec.type) {
case "image": {
const refs = await Promise.all((spec.refs ?? []).map((ref) => this.resolveInput(ref)));
@@ -303,7 +273,7 @@ export class EditorGenAi extends GenAi {
* spec a model cannot take fails here rather than before the partial is
* reserved, so the refusal is recorded like any other and not asked again.
*/
- private async runGeneration(spec: ResolvedSpec, partial: PartialAsset): Promise<Asset> {
+ private async runGeneration(spec: ResolvedGeneration, partial: PartialAsset): Promise<Asset> {
if (spec.type === "video") checkVideoConstraints(spec);
const startedAt = performance.now();
@@ -389,27 +359,33 @@ export class EditorGenAi extends GenAi {
return `Captions ${max + 1}`;
}
- /** Uploads the input, runs the call, and stores the result beside the generations. */
- private async runTransform(kind: TransformKind, asset: Asset, key: string): Promise<Asset> {
+ /**
+ * Uploads the transform's input, runs the call, and stores the result in
+ * place of `partial`. A transform over the wrong kind of asset fails here,
+ * like a spec a model cannot take: recorded, and not asked again.
+ */
+ private async runTransform(spec: ResolvedTransform, partial: PartialAsset): Promise<Asset> {
+ const asset = this.library.get(spec.inputId);
+ assert(asset, `Input asset ${spec.inputId} not found`);
+ checkTransformInput(spec.type, asset);
+
const startedAt = performance.now();
- track("generation_started", { mode: kind });
+ track("generation_started", { mode: spec.type });
try {
- console.log(`[gen-ai] running ${kind} on ${asset.path}`);
+ console.log(`[gen-ai] running ${spec.type} on ${asset.path}`);
const input = await this.uploadInput(asset.id);
- const { url, generationId } = await this.requestTransform(kind, asset, input);
-
- const base = assetName(asset).replace(/\.[^.]+$/, "");
- const stored = await this.store(url, `${base} (${TRANSFORMS[kind].suffix})`, { key, id: generationId });
+ const { url, generationId } = await this.requestTransform(spec.type, asset, input);
+ const stored = await this.store(url, assetName(partial), { key: partial.generation.key, id: generationId });
track("generation_completed", {
- mode: kind,
+ mode: spec.type,
duration_ms: Math.round(performance.now() - startedAt),
});
return stored;
} catch (err) {
track("generation_failed", {
- mode: kind,
+ mode: spec.type,
duration_ms: Math.round(performance.now() - startedAt),
error: err instanceof Error ? err.message.slice(0, 200) : "unknown",
});
@@ -417,17 +393,15 @@ export class EditorGenAi extends GenAi {
}
}
- private requestTransform(kind: TransformKind, asset: Asset, input: FileRef) {
- switch (kind) {
- case "remove-background":
- assert(asset.type === "IMAGE", "Only a picture has a background to remove");
+ private requestTransform(type: TransformType, asset: Asset, input: FileRef) {
+ switch (type) {
+ case "removeBackground":
return trpc.removeBackground.mutate({ image: input });
case "upscale":
return isMoving(asset.type)
? trpc.upscaleVideo.mutate({ video: input })
: trpc.upscaleImage.mutate({ image: input });
- case "add-audio":
- assert(isMoving(asset.type), "Only footage can be scored");
+ case "addAudio":
return trpc.addAudioToVideo.mutate({ video: input });
}
}
@@ -449,7 +423,7 @@ export class EditorGenAi extends GenAi {
});
}
- private requestGeneration(spec: ResolvedSpec) {
+ private requestGeneration(spec: ResolvedGeneration) {
switch (spec.type) {
case "image": {
return (async () => {
@@ -518,6 +492,24 @@ export class EditorGenAi extends GenAi {
/** Whether an asset type is footage, for the calls that only take footage. */
const isMoving = (type: AssetType): boolean => type === "VIDEO" || type === "SEQUENCE";
+/** An asset's name without its extension: what a transform's result is named after. */
+const stem = (asset: Asset): string => assetName(asset).replace(/\.[^.]+$/, "");
+
+/** What a transform can be put over; known only once the input has resolved. */
+function checkTransformInput(type: TransformType, input: Asset): void {
+ switch (type) {
+ case "removeBackground":
+ assert(input.type === "IMAGE", "Only a picture has a background to remove");
+ return;
+ case "upscale":
+ assert(input.type === "IMAGE" || isMoving(input.type), "Only a picture or footage can be upscaled");
+ return;
+ case "addAudio":
+ assert(isMoving(input.type), "Only footage can be scored");
+ return;
+ }
+}
+
/** How much of a prompt names the partial standing for its generation. */
const PROVISIONAL_NAME_LENGTH = 48;
@@ -600,7 +592,7 @@ function sceneHasAudio(world: World, scene: Entity): boolean {
/**
* Per-model constraints (`dapi models video`); unknown models are left to the server.
*/
-function checkVideoConstraints(spec: Extract<ResolvedSpec, { type: "video" }>): void {
+function checkVideoConstraints(spec: Extract<ResolvedGeneration, { type: "video" }>): void {
const model = PROMPT_INPUT_VIDEO_MODEL_OPTIONS.find((option) => option.id === spec.model);
if (!model) return;
diff --git a/docs/reference/jsx/README.md b/docs/reference/jsx/README.md
index 6608dee3..464bcce4 100644
--- a/docs/reference/jsx/README.md
+++ b/docs/reference/jsx/README.md
@@ -40,7 +40,7 @@ export default function Project() {
| [html.md](./html.md) | `<html>`: reactive HTML children drawn into the box |
| [surface-paint.md](./surface-paint.md) | `<surface>`: a ref-provided canvas you draw into, sampled every frame |
| [shader-paint.md](./shader-paint.md) | `<shaderPaint>`: a WGSL fragment shader over the media paint below it |
-| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`), image sequences, modifiers, adding an asset to the library |
+| [media.md](./media.md) | `src` resolution (paths, URLs, asset ids, `AssetRef`), image sequences, transforms, adding an asset to the library |
| [timing.md](./timing.md) | `start` / `end` / `sourceIn` / `sourceOut` / `playbackRate`, time formats |
| [keyframes.md](./keyframes.md) | Keyframe animation and easing |
| [animations.md](./animations.md) | `<animation>`: preset in/out animations |
diff --git a/docs/reference/jsx/audio.md b/docs/reference/jsx/audio.md
index fc52982e..09f33cce 100644
--- a/docs/reference/jsx/audio.md
+++ b/docs/reference/jsx/audio.md
@@ -21,7 +21,6 @@ It draws nothing inside a scene, but on the canvas it is still something to poin
| `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. |
| `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |
| `syncTo` | `string` | none | `id` of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |
-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |
An audio clip has no picture, so it takes no transform beyond its canvas box and no paints. Its children are [`<keyframeTrack>`](./keyframes.md) (a `volume` track) and [`<animation>`](./animations.md), of which only `"gain"` is audible:
diff --git a/docs/reference/jsx/generate.md b/docs/reference/jsx/generate.md
index fe0326da..5406309d 100644
--- a/docs/reference/jsx/generate.md
+++ b/docs/reference/jsx/generate.md
@@ -76,9 +76,27 @@ generate.audio(opts: {
}): AssetRef;
```
+## Transforms
+
+`transform.*` declares an asset made from another. Each takes what it works on as its argument — a source string or another declaration — and returns an `AssetRef` like `generate.*` does, usable anywhere a source is:
+
+```ts
+transform.upscale(input: AssetInput): AssetRef; // twice the pixels; a picture or footage — enlarges the source, not the box
+transform.removeBackground(input: AssetInput): AssetRef; // the subject cut out, the rest transparent; pictures only
+transform.addAudio(input: AssetInput): AssetRef; // footage scored with a generated soundtrack; independent of `volume` and `muted`
+```
+
+A chain reads inside out and runs in that order:
+
+```tsx
+<image src={transform.upscale(transform.removeBackground(generate.image({ prompt: "a red fox" })))} width={800} height={450} />
+```
+
+generates, cuts out, then enlarges. Each step is cached by step and input, so wrapping a further transform around a chain does not re-run what is inside it, and a transform over a library asset (`transform.upscale("footage/fox.png")`) leaves that asset as it is. A transform over the wrong kind of asset — a background removal over footage — fails like a spec a model cannot take (see [errors.md](./errors.md#failed-sources)).
+
## Dependency order
-`startFrame`, `endFrame`, and `refs` accept other `AssetRef`s. The dependency graph is built from these values and generated in topological order, so referenced assets exist before the assets that consume them. Because dependencies are **values, not string ids**, a declaration can only reference refs that already exist; reference cycles are impossible by construction. Assets referenced only as inputs generate too, but produce no node of their own.
+`startFrame`, `endFrame`, `refs`, and a transform's input accept other `AssetRef`s. The dependency graph is built from these values and generated in topological order, so referenced assets exist before the assets that consume them. Because dependencies are **values, not string ids**, a declaration can only reference refs that already exist; reference cycles are impossible by construction. Assets referenced only as inputs generate too, but produce no node of their own.
## Caching and idempotency
diff --git a/docs/reference/jsx/image.md b/docs/reference/jsx/image.md
index 8008a2a7..56819c75 100644
--- a/docs/reference/jsx/image.md
+++ b/docs/reference/jsx/image.md
@@ -15,9 +15,6 @@ All [common props](./elements.md#common-props), plus:
| `src` | `string \| AssetRef` | **required** | See [media.md](./media.md). |
| `objectFit` | `"cover" \| "contain" \| "fill"` | `"cover"` | How the source maps into the box. |
| `frameRate` | `number` | `30` | Frames per second for a `src` naming a directory of numbered frames, which plays on `<image>` as footage does (see [media.md](./media.md#image-sequences)). Nothing for a still to read. |
-| `removeBackground` | `boolean` | `false` | Cuts the subject out, leaving the rest transparent (see [media.md](./media.md#source-modifiers)). |
-| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |
-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |
Without `width`/`height` the box is 1920×1080. A still has no duration to fit, so with no `end` it takes the 16-second default (see [timing.md](./timing.md)).
diff --git a/docs/reference/jsx/media.md b/docs/reference/jsx/media.md
index 641ac896..f9604e70 100644
--- a/docs/reference/jsx/media.md
+++ b/docs/reference/jsx/media.md
@@ -24,23 +24,17 @@ A folder of pictures has a count, not a duration, so `frameRate` is what says ho
`frameRate` is not [`playbackRate`](./timing.md#playback-rate), which retimes a source against the timeline whatever its natural speed is; this is what that natural speed *is*. It is unrelated to the composition's own frame rate, which the export sets.
-## Source modifiers
+## Transforms
-`<image>` and `<video>` take props that put the source through a model before the element shows it:
-
-| Prop | On | Meaning |
-| ---- | -- | ------- |
-| `removeBackground` | `<image>` | Cuts the subject out, leaving the rest transparent. |
-| `upscale` | both | Resolution multiplier — `upscale={2}` asks for twice the pixels. Enlarges the source, not the box. |
-| `addAudio` | `<video>` | Scores footage that has no sound. Independent of `volume` and `muted`, which mix whatever track the clip ends up with. |
+A source can be put through a model before the element shows it, by declaring the result as the `src` — see [generate.md](./generate.md#transforms):
```tsx
-<image src="footage/fox.png" removeBackground upscale={2} width={800} height={450} />
-```
+import { transform } from "@diffusionstudio/jsx";
-The `src` goes on naming what the picture was made from, so taking a modifier off gives the original back — nothing is overwritten and nothing is lost. Results are cached by source and modifiers, like [generated assets](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and adding a second modifier does not re-run the first. They are applied in the order of the table above, whatever order they are written in.
+<image src={transform.upscale(transform.removeBackground("footage/fox.png"))} width={800} height={450} />
+```
-Modifiers compose with declarations: `<image src={generate.image({ prompt: "a red fox" })} removeBackground />` generates, then cuts out.
+The inner source is untouched — nothing is overwritten and nothing is lost — and the chain runs inside out, in the order it is written. Results are cached by step and input like any [generated asset](./generate.md#caching-and-idempotency) and stored in the same `generated/` folder, so one is made however many elements ask for it, and wrapping a further transform around a chain does not re-run what is inside it.
## The library
diff --git a/docs/reference/jsx/video.md b/docs/reference/jsx/video.md
index 794fa368..499c8f0e 100644
--- a/docs/reference/jsx/video.md
+++ b/docs/reference/jsx/video.md
@@ -18,9 +18,6 @@ All [common props](./elements.md#common-props), plus:
| `volume` | `number` | `0` | Decibels: `0` = unity, negative attenuates (`-6` ≈ half as loud), `-Infinity` = silence. Not linear. |
| `muted` | `boolean` | `false` | Excludes the node's audio from the mix; independent of `volume`. |
| `syncTo` | `string` | none | `id` of another element carrying audio; derives `start` by audio alignment (see [audio-sync.md](./audio-sync.md)). Mutually exclusive with `start`. |
-| `upscale` | `number` | `1` | Resolution multiplier; enlarges the source, not the box (see [media.md](./media.md#source-modifiers)). |
-| `addAudio` | `boolean` | `false` | Scores footage that has no sound (see [media.md](./media.md#source-modifiers)). |
-| `error` | `string` | none | Why this element's source never became an asset (see [errors.md](./errors.md#failed-sources)). |
Without `width`/`height` the box is 1920×1080.
diff --git a/packages/jsx/src/generate.ts b/packages/jsx/src/generate.ts
index 08a5be99..9a44bc0a 100644
--- a/packages/jsx/src/generate.ts
+++ b/packages/jsx/src/generate.ts
@@ -3,17 +3,20 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
- * Declarative generated assets. A `generate.*` call is pure: it validates its
- * options and returns an `AssetRef` value that can be passed wherever a source
- * is expected (`src`, `startFrame`, `endFrame`, `refs`). Nothing generates
- * until the mounted tree commits; refs never used by a mounted element are
- * dropped. Because dependencies are values (not string ids), reference cycles
- * are impossible by construction.
+ * Declarative assets. A `generate.*` call makes an asset from a prompt; a
+ * `transform.*` call makes one from an asset. Both are pure: they validate
+ * their arguments and return an `AssetRef` value that can be passed wherever
+ * a source is expected (`src`, `startFrame`, `endFrame`, `refs`, a
+ * transform's input). Nothing runs until the mounted tree commits; refs
+ * never used by a mounted element are dropped. Because dependencies are
+ * values (not string ids), reference cycles are impossible by construction,
+ * and the order of a chain is its nesting:
+ * `transform.upscale(transform.removeBackground(generate.image({ prompt })))`.
*/
export type AspectRatio = "1:1" | "4:3" | "3:4" | "16:9" | "9:16";
-/** A path, URL, asset id, or another `generate.*` declaration. */
+/** A path, URL, asset id, or another declaration. */
export type AssetInput = string | AssetRef;
export type GenerateImageOptions = {
@@ -54,21 +57,48 @@ export type GenerateAudioOptions = {
seed?: number;
};
-export type AssetSpecInput =
+/** What a `generate.*` call declares: an asset made from a prompt. */
+export type GenerateSpec =
| ({ type: "image" } & GenerateImageOptions)
| ({ type: "video" } & GenerateVideoOptions)
| ({ type: "voice" } & GenerateVoiceOptions)
| ({ type: "audio" } & GenerateAudioOptions);
+/** What a `transform.*` call declares: an asset made from another. */
+export type TransformSpec =
+ /** The input at twice the pixels; a picture or footage. */
+ | { type: "upscale"; input: AssetInput }
+ /** The subject of a picture cut out, the rest transparent. */
+ | { type: "removeBackground"; input: AssetInput }
+ /** Footage scored with a generated soundtrack. */
+ | { type: "addAudio"; input: AssetInput };
+
+export type TransformType = TransformSpec["type"];
+
+export type AssetSpecInput = GenerateSpec | TransformSpec;
+
+const GENERATE_TYPES: readonly string[] = ["image", "video", "voice", "audio"];
+const TRANSFORM_TYPES: readonly string[] = ["upscale", "removeBackground", "addAudio"];
+
+/** Whether a spec type is a transform's, as opposed to a generation's. */
+export function isTransformType(type: string): type is TransformType {
+ return TRANSFORM_TYPES.includes(type);
+}
+
+export function isTransformSpec(spec: AssetSpecInput): spec is TransformSpec {
+ return isTransformType(spec.type);
+}
+
/**
- * Opaque handle to a declared (not-yet-generated) asset. Obtained from
- * `generate.*`; consumed by `src` / `startFrame` / `endFrame` / `refs`.
+ * Opaque handle to a declared (not-yet-made) asset. Obtained from
+ * `generate.*` or `transform.*`; consumed by `src`, `startFrame`, `endFrame`,
+ * `refs`, and a transform's input.
*/
export class AssetRef {
/** @internal — read via `getAssetSpec`. */
readonly spec: AssetSpecInput;
- /** @internal — declare refs with `generate.*`, not `new AssetRef()`. */
+ /** @internal — declare refs with `generate.*` / `transform.*`, not `new AssetRef()`. */
constructor(spec: AssetSpecInput) {
this.spec = spec;
}
@@ -78,81 +108,163 @@ export function isAssetRef(value: unknown): value is AssetRef {
return value instanceof AssetRef;
}
-/** An `AssetSpecInput` whose inputs are all strings — no nested declarations. */
-export type FlatAssetSpec = AssetSpecInput & {
- refs?: string[];
- startFrame?: string;
- endFrame?: string;
-};
+/** Host-side accessor for a declaration's spec. */
+export function getAssetSpec(ref: AssetRef): AssetSpecInput {
+ return ref.spec;
+}
+
+/** The inputs a spec names, in option order: what the declaration depends on. */
+export function getAssetInputs(spec: AssetSpecInput): AssetInput[] {
+ switch (spec.type) {
+ case "image":
+ return spec.refs ?? [];
+ case "video":
+ return [spec.startFrame, spec.endFrame].filter((input): input is AssetInput => input !== undefined);
+ case "voice":
+ case "audio":
+ return [];
+ default:
+ return [spec.input];
+ }
+}
+
+/**
+ * `ref` with each of its direct inputs replaced by what `map` makes of it.
+ * Direct only: a caller that wants the whole chain recurses through the refs
+ * `map` is handed. A ref none of whose inputs change is returned as it is.
+ */
+export function mapAssetInputs(ref: AssetRef, map: (input: AssetInput) => AssetInput): AssetRef {
+ const spec = ref.spec;
+ let changed = false;
+ const mapped = (input: AssetInput): AssetInput => {
+ const next = map(input);
+ if (next !== input) changed = true;
+ return next;
+ };
+
+ let next: AssetSpecInput;
+ switch (spec.type) {
+ case "image":
+ next = { ...spec, refs: spec.refs?.map(mapped) };
+ break;
+ case "video":
+ next = {
+ ...spec,
+ startFrame: spec.startFrame === undefined ? undefined : mapped(spec.startFrame),
+ endFrame: spec.endFrame === undefined ? undefined : mapped(spec.endFrame),
+ };
+ break;
+ case "voice":
+ case "audio":
+ return ref;
+ default:
+ next = { ...spec, input: mapped(spec.input) };
+ }
+ return changed ? new AssetRef(next) : ref;
+}
+
+// ---------------------------------------------------------------------------
+// The wire form
+
+/** An input as data: a source string, or a declaration serialized in turn. */
+export type SerializedAssetInput = string | SerializedAssetRef;
+
+/** A spec as data: the same fields, with every declared input serialized. */
+export type SerializedAssetSpec =
+ | ({ type: "image"; refs?: SerializedAssetInput[] } & Omit<GenerateImageOptions, "refs">)
+ | ({ type: "video"; startFrame?: SerializedAssetInput; endFrame?: SerializedAssetInput } & Omit<GenerateVideoOptions, "startFrame" | "endFrame">)
+ | ({ type: "voice" } & GenerateVoiceOptions)
+ | ({ type: "audio" } & GenerateAudioOptions)
+ | { type: TransformType; input: SerializedAssetInput };
/**
* A declaration as data, for the edit protocol. An `AssetRef` itself cannot
* travel as a prop value (see `isPropValue`: an object literal would not read
* back as one), so an edit carries the spec in this tagged shape instead, and
- * the source writer spells it as the `generate.*` call that reproduces it.
+ * the source writer spells it as the call that reproduces it — inputs
+ * included, so a transform over a declaration travels as the nested call it
+ * is.
*/
-export type SerializedAssetRef = { $generate: FlatAssetSpec };
+export interface SerializedAssetRef {
+ $asset: SerializedAssetSpec;
+}
-/**
- * The wire form of a declaration, or undefined when the spec references
- * other declarations: nested refs only exist in authored code, where the
- * call spelling them already is — the editor declares over library assets.
- */
-export function serializeAssetRef(ref: AssetRef): SerializedAssetRef | undefined {
- const spec = ref.spec;
- const inputs = [
- ...(spec.type === "image" ? (spec.refs ?? []) : []),
- ...(spec.type === "video" ? [spec.startFrame, spec.endFrame] : []),
- ];
- if (inputs.some((input) => isAssetRef(input))) return undefined;
+export function serializeAssetRef(ref: AssetRef): SerializedAssetRef {
+ const serializeInput = (input: AssetInput): SerializedAssetInput =>
+ isAssetRef(input) ? serializeAssetRef(input) : input;
+ const spec = ref.spec;
+ const serialized: Record<string, unknown> = {};
// Without the undefined entries: they are not part of the declaration, and
// the writer spells every entry it is handed.
- const entries = Object.entries(spec).filter(([, value]) => value !== undefined);
- return { $generate: Object.fromEntries(entries) as FlatAssetSpec };
+ for (const [key, value] of Object.entries(spec)) {
+ if (value === undefined) continue;
+ serialized[key] = key === "refs"
+ ? (value as AssetInput[]).map(serializeInput)
+ : key === "startFrame" || key === "endFrame" || key === "input"
+ ? serializeInput(value as AssetInput)
+ : value;
+ }
+ return { $asset: serialized as SerializedAssetSpec };
}
-const SPEC_TYPES = ["image", "video", "voice", "audio"];
-
export function isSerializedAssetRef(value: unknown): value is SerializedAssetRef {
- if (typeof value !== "object" || value === null || !("$generate" in value)) return false;
- const spec = (value as SerializedAssetRef).$generate;
- return (
- typeof spec === "object" && spec !== null &&
- SPEC_TYPES.includes(spec.type) && typeof spec.prompt === "string"
- );
+ if (typeof value !== "object" || value === null || !("$asset" in value)) return false;
+ return isSerializedSpec((value as SerializedAssetRef).$asset);
}
-/** Host-side accessor for a declaration's spec. */
-export function getAssetSpec(ref: AssetRef): AssetSpecInput {
- return ref.spec;
+function isSerializedInput(value: unknown): value is SerializedAssetInput {
+ return typeof value === "string" || isSerializedAssetRef(value);
+}
+
+function isSerializedSpec(spec: unknown): spec is SerializedAssetSpec {
+ if (typeof spec !== "object" || spec === null) return false;
+ const record = spec as Record<string, unknown>;
+ const type = record.type;
+ if (typeof type !== "string") return false;
+
+ if (isTransformType(type)) return isSerializedInput(record.input);
+ if (!GENERATE_TYPES.includes(type) || typeof record.prompt !== "string") return false;
+ if (record.refs !== undefined && !(Array.isArray(record.refs) && record.refs.every(isSerializedInput))) return false;
+ if (record.startFrame !== undefined && !isSerializedInput(record.startFrame)) return false;
+ if (record.endFrame !== undefined && !isSerializedInput(record.endFrame)) return false;
+ return true;
}
-function requirePrompt(value: unknown, kind: string): string {
+// ---------------------------------------------------------------------------
+// The declarations
+
+function requirePrompt(value: unknown, call: string): string {
if (typeof value !== "string" || value.trim().length === 0) {
- throw new Error(`generate.${kind} requires a non-empty string prompt`);
+ throw new Error(`${call} requires a non-empty string prompt`);
}
return value;
}
-function checkAssetInput(value: unknown, kind: string, option: string): void {
+function checkAssetInput(value: unknown, call: string, option: string): void {
if (value === undefined) return;
if (typeof value === "string" && value.trim().length > 0) return;
if (isAssetRef(value)) return;
- throw new Error(`generate.${kind}: ${option} must be a path, URL, asset id, or an AssetRef`);
+ throw new Error(`${call}: ${option} must be a path, URL, asset id, or an AssetRef`);
+}
+
+function requireAssetInput(value: unknown, call: string): AssetInput {
+ if (value === undefined) throw new Error(`${call} requires an input: a path, URL, asset id, or an AssetRef`);
+ checkAssetInput(value, call, "input");
+ return value as AssetInput;
}
export const generate = {
image(opts: GenerateImageOptions): AssetRef {
- requirePrompt(opts.prompt, "image");
- for (const ref of opts.refs ?? []) checkAssetInput(ref, "image", "refs");
+ requirePrompt(opts.prompt, "generate.image");
+ for (const ref of opts.refs ?? []) checkAssetInput(ref, "generate.image", "refs");
return new AssetRef({ type: "image", ...opts });
},
video(opts: GenerateVideoOptions): AssetRef {
- requirePrompt(opts.prompt, "video");
- checkAssetInput(opts.startFrame, "video", "startFrame");
- checkAssetInput(opts.endFrame, "video", "endFrame");
+ requirePrompt(opts.prompt, "generate.video");
+ checkAssetInput(opts.startFrame, "generate.video", "startFrame");
+ checkAssetInput(opts.endFrame, "generate.video", "endFrame");
if (opts.duration !== undefined && (!Number.isInteger(opts.duration) || opts.duration <= 0)) {
throw new Error("generate.video: duration must be a positive whole number of seconds");
}
@@ -160,15 +272,38 @@ export const generate = {
},
voice(opts: GenerateVoiceOptions): AssetRef {
- requirePrompt(opts.prompt, "voice");
+ requirePrompt(opts.prompt, "generate.voice");
return new AssetRef({ type: "voice", ...opts });
},
audio(opts: GenerateAudioOptions): AssetRef {
- requirePrompt(opts.prompt, "audio");
+ requirePrompt(opts.prompt, "generate.audio");
if (opts.duration !== undefined && (!Number.isFinite(opts.duration) || opts.duration <= 0)) {
throw new Error("generate.audio: duration must be a positive number of seconds");
}
return new AssetRef({ type: "audio", ...opts });
},
};
+
+/**
+ * Assets made from assets. Each takes what it works on as its argument — a
+ * source string or another declaration — so a chain reads inside out, and
+ * runs in that order. The result is cached by step and input like any
+ * generation: the same call on the same asset is the same asset.
+ */
+export const transform = {
+ /** `input` at twice the pixels. A picture or footage; the box it is shown in is unchanged. */
+ upscale(input: AssetInput): AssetRef {
+ return new AssetRef({ type: "upscale", input: requireAssetInput(input, "transform.upscale") });
+ },
+
+ /** The subject of the picture `input` cut out, the rest transparent. */
+ removeBackground(input: AssetInput): AssetRef {
+ return new AssetRef({ type: "removeBackground", input: requireAssetInput(input, "transform.removeBackground") });
+ },
+
+ /** The footage `input` scored with a generated soundtrack. */
+ addAudio(input: AssetInput): AssetRef {
+ return new AssetRef({ type: "addAudio", input: requireAssetInput(input, "transform.addAudio") });
+ },
+};
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.ts
index 8e2a15d3..dc964fca 100644
--- a/packages/jsx/src/index.ts
+++ b/packages/jsx/src/index.ts
@@ -12,9 +12,14 @@
export {
generate,
+ transform,
AssetRef,
isAssetRef,
getAssetSpec,
+ getAssetInputs,
+ mapAssetInputs,
+ isTransformSpec,
+ isTransformType,
serializeAssetRef,
isSerializedAssetRef,
} from "./generate";
@@ -22,8 +27,12 @@ export type {
AspectRatio,
AssetInput,
AssetSpecInput,
- FlatAssetSpec,
+ GenerateSpec,
+ TransformSpec,
+ TransformType,
+ SerializedAssetInput,
SerializedAssetRef,
+ SerializedAssetSpec,
GenerateAudioOptions,
GenerateImageOptions,
GenerateVideoOptions,
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 9acc1ec7..3b8374af 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -286,7 +286,7 @@ type FillProps = {
type MediaProps = {
/**
- * Path, URL, asset id, or a `generate.*` declaration. A path naming a
+ * Path, URL, asset id, or a `generate.*` / `transform.*` declaration. A path naming a
* directory of numbered frames (`shot_001.png`, `shot_002.png`, ...) is an
* image sequence, and plays on `<Video>` or `<Image>` as footage does — see
* `frameRate` for how long it lasts. On `<Captions>` a transcript source
@@ -296,22 +296,6 @@ type MediaProps = {
src: string | AssetRef;
};
-/**
- * Model calls the source is put through before the element shows it. The
- * `src` goes on naming what it was made from, so taking a modifier off gives
- * the original back; what they made is cached by source and modifiers, so it
- * is made once however many elements ask for it, and adding a second
- * modifier does not re-run the first. Applied in the order below.
- */
-type UpscaleProps = {
- /**
- * Resolution multiplier: 2 asks for twice the pixels. Enlarges the source,
- * not the box — the element keeps the width and height it was given, and
- * renders sharper. Default 1, the source as it is.
- */
- upscale?: number;
-};
-
type FitProps = {
/** How the source maps into the box. Default "cover" on `<Video>`, "contain" on `<Image>`. */
objectFit?: Fit;
@@ -655,24 +639,12 @@ export type ColorStopProps = ColorProps & OpacityProps & TrackChildren & {
*/
export type MediaPaintProps = PaintProps & MediaProps & FitProps & FrameRateProps & TrackChildren;
-export type VideoProps = CommonProps & MediaProps & FitProps & FrameRateProps & AudioTrackProps & UpscaleProps & {
- /**
- * Scores the footage: a generated soundtrack for a clip that has none. See
- * `UpscaleProps` for what a modifier is; applied last, after `upscale`, so
- * a re-encode cannot drop the track. Independent of `volume` and `muted`,
- * which mix whatever track the clip ends up with.
- */
- addAudio?: boolean;
+export type VideoProps = CommonProps & MediaProps & FitProps & FrameRateProps & AudioTrackProps & {
/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */
children?: SolidJSX.Element;
};
-export type ImageProps = CommonProps & MediaProps & FitProps & FrameRateProps & UpscaleProps & {
- /**
- * Cuts the subject out, leaving the rest of the picture transparent. See
- * `UpscaleProps` for what a modifier is; applied before `upscale`.
- */
- removeBackground?: boolean;
+export type ImageProps = CommonProps & MediaProps & FitProps & FrameRateProps & {
/** Paint children, stacked over the media paint created by `src`; `<Stroke>`, `<Shadow>`, `<Effect>`, `<Animation>` and `<KeyframeTrack>` children. */
children?: SolidJSX.Element;
};
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 6375ae42..51c7a934 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -3,7 +3,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CAPTION_PRESET_FILLS, CAPTION_PRESET_STYLES, CaptionType, Chars, ClipHeight, ClipsContent, Computed, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Delay, Effect, EffectType, Expanded, FontStyle, FramePromises, FrameRate, Generating, GenerationRequest, getActiveEntity, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, Host, IsMask, isText, ItemIndex, KeepAspectRatio, Keyframe, KeyframeTrack, MixedCornerRadius, Mode, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, PendingSync, Playback, PlaybackRate, Position, removeChild, RenderSurface, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceFrameRate, setCameraMatrix, setTimelineView, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SyncRequest, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, Trim, UniformScale, Volume, Workarea } from '@diffusionstudio/runtime';
import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
import { createSignal } from 'solid-js';
import { SVGElements } from 'solid-js/web';
@@ -26,15 +26,6 @@ const UNAUTHORED_PROPS: ReadonlySet<string> = new Set([SOURCE_ATTR, LOOP_ATTR, '
*/
const HOLD_TIMEOUT_MS = 30_000;
-/** What an element with no `SourceModifiers` trait is asking for: nothing. */
-const NO_MODIFIERS = { removeBackground: false, upscale: 1, addAudio: false };
-
-/** `upscale` as a factor; anything that is not one above 1 is natural size. */
-function upscaleFactor(value: unknown): number {
- const factor = typeof value === 'number' ? value : Number(value);
- return Number.isFinite(factor) && factor > 1 ? factor : 1;
-}
-
export interface AuthoredElement {
/** The camelCase tag the project used. */
tag: string;
@@ -1325,47 +1316,6 @@ export class RuntimeDocument implements ProjectDocument<SceneNode> {
});
return;
}
- case 'removeBackground':
- case 'addAudio':
- case 'upscale': {
- const current = entity.get(SourceModifiers) ?? NO_MODIFIERS;
- const next = {
- ...current,
- [name]: name === 'upscale' ? upscaleFactor(value) : value === true,
- };
-
- if (
- next.removeBackground === current.removeBackground
- && next.upscale === current.upscale
- && next.addAudio === current.addAudio
- ) return;
-
- if (hasModifier(next)) {
- entity.add(SourceModifiers);
- entity.set(SourceModifiers, next);
- } else {
- entity.remove(SourceModifiers);
- }
-
- if (entity.has(LoadRequest) || entity.has(GenerationRequest)) return;
-
- const src = node.props.src;
- if (src === undefined || src === null || src === '') return;
-
- // A resolution running for the old modifiers must not bind late.
- entity.remove(PendingSource, Generating);
-
- if (typeof src === 'string') {
- entity.add(LoadRequest);
- entity.set(LoadRequest, { value: src });
- return;
- }
-
- entity.add(GenerationRequest);
- entity.set(GenerationRequest, { ref: src as AssetRef });
-
- return;
- }
case 'seed': {
if (!entity.has(Caption)) return;
// An authored src mounts a transcript directly; there is no
diff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.ts
index 14c73ef5..e0a88cc5 100644
--- a/packages/runtime/src/actions/assets.ts
+++ b/packages/runtime/src/actions/assets.ts
@@ -6,7 +6,7 @@
// goes through the AssetLibrary (@diffusionstudio/assets); these are the
// lookups the decoders and hosts share.
-import { Ai, AssetId, Library, Paint, SourceFrameRate, SourceModifiers } from '../traits';
+import { Ai, AssetId, Library, Paint, SourceFrameRate } from '../traits';
import { PaintType } from '../constants';
import type { Entity, World } from 'koota';
@@ -27,28 +27,6 @@ export function getAi(world: World): GenAi {
return ai;
}
-/** What a source is put through after it resolves (see `SourceModifiers`). */
-export interface SourceModifierValues {
- removeBackground: boolean;
- /** Factor, 1 = natural size. */
- upscale: number;
- addAudio: boolean;
-}
-
-/** Whether a set of modifiers asks for anything at all. */
-export const hasModifier = (modifiers: SourceModifierValues): boolean =>
- modifiers.removeBackground || modifiers.upscale > 1 || modifiers.addAudio;
-
-/**
- * The modifiers `entity` asks its source to be put through, or undefined
- * when it asks for none — which is what the trait's absence means, so this
- * is also the question "is this element showing a derived source".
- */
-export function getModifiers(entity: Entity): SourceModifierValues | undefined {
- const modifiers = entity.get(SourceModifiers);
- return modifiers && hasModifier(modifiers) ? modifiers : undefined;
-}
-
/**
* Binds an entity to an asset: stamps its AssetId, and follows the asset with
* the paint. A media paint is whichever of the two the asset can be drawn
diff --git a/packages/runtime/src/ai.ts b/packages/runtime/src/ai.ts
index f0a2ae2c..21c81db8 100644
--- a/packages/runtime/src/ai.ts
+++ b/packages/runtime/src/ai.ts
@@ -2,12 +2,12 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { generate } from '@diffusionstudio/jsx';
+import { generate, transform } from '@diffusionstudio/jsx';
import type { Entity, World } from 'koota';
import type { Asset } from '@diffusionstudio/assets';
-import type { SourceModifierValues } from './actions/assets';
import type {
+ AssetInput,
AssetRef,
GenerateAudioOptions,
GenerateImageOptions,
@@ -26,11 +26,13 @@ import type {
*/
export abstract class GenAi {
/**
- * The declaration, made real: an asset whose bytes the model produced.
- * Content-addressed — the fully-resolved spec's hash is the asset's
- * `generation.key`, so the same spec is the same asset in this session
- * and the next, and identical concurrent declarations share one run.
- * Rejects with the recorded reason when the key stands in error.
+ * The declaration, made real: an asset whose bytes the model produced —
+ * from a prompt (`generate.*`) or from another asset (`transform.*`), its
+ * inputs resolved first. Content-addressed — the fully-resolved spec's
+ * hash is the asset's `generation.key`, so the same spec is the same
+ * asset in this session and the next, and identical concurrent
+ * declarations share one run. Rejects with the recorded reason when the
+ * key stands in error.
*/
public abstract resolve(ref: AssetRef): Promise<Asset>;
@@ -42,14 +44,6 @@ export abstract class GenAi {
*/
public abstract transcribe(world: World, scene: Entity, seed: number): Promise<Asset>;
- /**
- * `asset` put through the modifiers an element asked of its source (see
- * `SourceModifiers`): the same asset and the same modifiers are the same
- * result, in this session and the next, and every step is cached on its
- * own, so adding one to an element does not re-run the others.
- */
- public abstract derive(asset: Asset, modifiers: SourceModifierValues): Promise<Asset>;
-
/**
* The imperative surface: `ai.generate.image({...})` declares and resolves
* in one call, returning the finished asset.
@@ -60,4 +54,11 @@ export abstract class GenAi {
voice: (options: GenerateVoiceOptions): Promise<Asset> => this.resolve(generate.voice(options)),
audio: (options: GenerateAudioOptions): Promise<Asset> => this.resolve(generate.audio(options)),
};
+
+ /** The same for `transform.*`: `ai.transform.upscale(input)` is the upscaled asset. */
+ public readonly transform = {
+ upscale: (input: AssetInput): Promise<Asset> => this.resolve(transform.upscale(input)),
+ removeBackground: (input: AssetInput): Promise<Asset> => this.resolve(transform.removeBackground(input)),
+ addAudio: (input: AssetInput): Promise<Asset> => this.resolve(transform.addAudio(input)),
+ };
}
diff --git a/packages/runtime/src/systems/assets.ts b/packages/runtime/src/systems/assets.ts
index cc81aa7f..a0e2e647 100644
--- a/packages/runtime/src/systems/assets.ts
+++ b/packages/runtime/src/systems/assets.ts
@@ -5,7 +5,7 @@
import { parseSource } from '@diffusionstudio/jsx';
import { Ai, Cache, Computed, Delay, FrameRate, FramePromises, Generating, GenerationRequest, Host, Library, LoadRequest, PendingSource, PendingSync, PlaybackRate, Source, SourceError, SyncRequest, TranscriptionRequest, Trim } from '../traits';
-import { bindAsset, getAssetFile, getModifiers } from '../actions/assets';
+import { bindAsset, getAssetFile } from '../actions/assets';
import { getEntityTree, getParentNode, getSceneAncestor } from '../queries/hierarchy';
import { findAssetDuration, findGeometryAsset } from '../utils/time';
import { computeAudioSyncOffsetCached } from '../media/audio-sync';
@@ -30,7 +30,7 @@ export function assetSystem(world: World): void {
for (const entity of world.query(LoadRequest)) {
const source = entity.get(LoadRequest)!.value;
entity.remove(LoadRequest);
- start(world, entity, library.resolve(source));
+ resolve(world, entity, library.resolve(source));
}
}
@@ -41,7 +41,7 @@ export function assetSystem(world: World): void {
entity.remove(GenerationRequest);
if (ref === null) continue;
if (isDomImage(entity)) pointDomImageAt(entity, null);
- start(world, entity, ai.resolve(ref), true);
+ resolve(world, entity, ai.resolve(ref), true);
}
for (const entity of world.query(TranscriptionRequest)) {
@@ -215,24 +215,6 @@ function currentSync(entity: Entity, token: object): boolean {
return entity.isAlive() && entity.get(PendingSync)?.value === token;
}
-/**
- * Starts a resolution for whatever the element's src named, putting it
- * through the modifiers the element asks of it (see `SourceModifiers`). The
- * two are one wait: the element binds the asset it is going to show, not
- * first the one it was made from.
- */
-function start(world: World, entity: Entity, base: Promise<Asset>, generating = false): void {
- const modifiers = getModifiers(entity);
- const ai = world.get(Ai);
-
- if (!modifiers || !ai) {
- resolve(world, entity, base, generating);
- return;
- }
-
- resolve(world, entity, base.then((asset) => ai.derive(asset, modifiers)), true);
-}
-
/**
* Whether anything in `scene`'s subtree (besides `except`, the requesting
* element itself) is still waiting on a source: a request this system has
@@ -254,8 +236,7 @@ function hasPendingSources(world: World, scene: Entity, except: Entity): boolean
* Tracks one started resolution: the entity remembers which one it is waiting
* on (PendingSource), so of overlapping resolutions only the latest binds, and
* one that outlives its element (or its src) is dropped. The wait itself is
- * the identity rather than what was asked for — the same src put through
- * different modifiers is a different answer, and the later question is always
+ * the identity rather than what was asked for — the later question is always
* the one being asked.
*/
function resolve(world: World, entity: Entity, promise: Promise<Asset>, generating = false): void {
diff --git a/packages/runtime/src/traits/node.ts b/packages/runtime/src/traits/node.ts
index 7c90436d..5960b674 100644
--- a/packages/runtime/src/traits/node.ts
+++ b/packages/runtime/src/traits/node.ts
@@ -85,16 +85,6 @@ export const GenerationRequest = trait(() => ({ ref: null as AssetRef | null }))
// through the library.
export const LoadRequest = trait({ value: '' });
-// Model calls the src is put through once it resolves: the element shows
-// what they made of what it named, and the src goes on naming the original,
-// so taking a modifier off gives it back. Every field's default is off —
-// `upscale` 1 is natural size — and the trait is only present while one is
-// not. Applied in the order declared here: the matte before the enlarging
-// (cheaper, and the model wants a normally-sized picture), the audio last so
-// a re-encode cannot drop it. Each step is cached on its own, so turning one
-// on does not re-pay for the ones already applied.
-export const SourceModifiers = trait({ removeBackground: false, upscale: 1, addAudio: false });
-
// On a `<captions>` element without a src: transcribe the scene it sits
// under, through the world's Ai. The seed keys the take — the transcript is
// cached by scene id + seed, so re-running with a seed replays that take and