Skip to content

The VS Code of video editing, built for agents.

TypeScript

Refactor image and video generation components to use new rendering structure

konstantin-paulus2d27b6deb92c
diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsx
index ac5c8acf..d0c50d0d 100644
--- a/apps/web/src/components/genai/use-generate-image.tsx
+++ b/apps/web/src/components/genai/use-generate-image.tsx
@@ -2,15 +2,9 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Image generation as the runtime means it: the prompt box declares a
-// `generate.image` source per variant and inserts an <Image> consuming it,
-// the way any drawn element is inserted. The asset system resolves the
-// declaration through the world's Ai (see `EditorGenAi`), and the element
-// lands in the project file with the `generate.image({...})` call that
-// reproduces it — same spec, same asset, this session and the next.
import { generate } from "@diffusionstudio/jsx";
-import { Image } from "@diffusionstudio/reconciler";
+import { ImagePaint, Rect } from "@diffusionstudio/reconciler";
import { Library } from "@diffusionstudio/runtime";
import { useWorld } from "@diffusionstudio/koota-solid";
import { useEditor } from "@/engine/hooks";
@@ -45,7 +39,9 @@ export function useGenerateImage() {
}));
insertGenerated(world, editor, dims, (box, index) => (
- <Image src={sources[index]} x={box.x} y={box.y} width={box.width} height={box.height} />
+ <Rect x={box.x} y={box.y} width={box.width} height={box.height}>
+ <ImagePaint src={sources[index]} />
+ </Rect>
), config.count);
};
diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsx
index 25e23bf5..85c510f2 100644
--- a/apps/web/src/components/genai/use-generate-video.tsx
+++ b/apps/web/src/components/genai/use-generate-video.tsx
@@ -2,13 +2,8 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Video generation, declared rather than performed: the prompt box writes a
-// `generate.video` source and inserts a <Video> consuming it (see ./insert).
-// The frames a model animates between are library assets, named by path, so
-// the call in the file says what it was made from as well as what to make.
-
import { generate } from "@diffusionstudio/jsx";
-import { Video } from "@diffusionstudio/reconciler";
+import { Rect, VideoPaint } from "@diffusionstudio/reconciler";
import { Library } from "@diffusionstudio/runtime";
import { useWorld } from "@diffusionstudio/koota-solid";
import { useEditor } from "@/engine/hooks";
@@ -41,7 +36,9 @@ export function useGenerateVideo() {
});
insertGenerated(world, editor, dims, (box) => (
- <Video src={src} x={box.x} y={box.y} width={box.width} height={box.height} />
+ <Rect x={box.x} y={box.y} width={box.width} height={box.height}>
+ <VideoPaint src={src} />
+ </Rect>
));
};
diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.ts
index 559d0207..ff439c4b 100644
--- a/apps/web/src/engine/hud/hud-system.ts
+++ b/apps/web/src/engine/hud/hud-system.ts
@@ -2,19 +2,6 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-/**
- * Everything drawn over the document rather than in it: snap guides, hover
- * outlines, scene headers, the selection mask with its handles, and the
- * marquee. Runs after the render system, on the same canvas and in device
- * pixels (the transform is reset per piece, so nothing here inherits the
- * camera).
- *
- * Drawing is also where the HUD's hit regions come from: a control is pushed
- * with the handler it answers to right after it is painted, so the regions
- * come out in paint order, on top of the entity regions the render system
- * left, and the input system can test them back to front next frame.
- */
-
import { Not, Or } from 'koota';
import {
Active, Computed, Culled, FrameRate, Generating, Geometry, Group, Hidden,
@@ -54,7 +41,7 @@ export function hudSystem(world: World): void {
drawSnapLines(world, ctx, resolution);
drawHoverOutlines(world, ctx, resolution);
- for (const entity of world.query(Or(Name, Generating, SourceError), ChildOf(world.get(Root)!), Not(Culled), Not(Hidden))) {
+ for (const entity of world.query(ChildOf(world.get(Root)!), Not(Culled), Not(Hidden), Name)) {
drawHeader(world, ctx, entity, resolution);
}
@@ -119,9 +106,10 @@ function drawHoverOutlines(world: World, ctx: Ctx2D, resolution: number): void {
* It sits in the node's own rotation, one line above its top edge.
*/
function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number): void {
- const failure = entity.get(SourceError)?.value;
- const generating = entity.has(Generating);
- const label = failure || (generating ? 'Generating...' : entity.get(Name)?.value);
+ const errored = world.query(ChildOf(entity), SourceError);
+ const generating = world.query(ChildOf(entity), Generating);
+ const failure = errored.at(0)?.get(SourceError)?.value;
+ const label = failure || (generating.length ? 'Generating...' : entity.get(Name)?.value);
if (!label) return;
const header = getHeaderLayout(world, entity, resolution);
diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsx
index 3ebeca91..1a4b442e 100644
--- a/apps/web/src/engine/insert-asset.tsx
+++ b/apps/web/src/engine/insert-asset.tsx
@@ -2,11 +2,8 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-// Putting an asset on the canvas: an element in the project's JSX whose
-// `src` is the asset's library path, inserted the way every drawn element
-// is (see DocumentEditor.insertElement), so it lands in the file too.
-import { Audio, Captions, Image, Video } from '@diffusionstudio/reconciler';
+import { Audio, Captions, ImagePaint, Rect, VideoPaint } from '@diffusionstudio/reconciler';
import { Computed, getActiveEntity, getNextName, Root, Source, store } from '@diffusionstudio/runtime';
import { assetName } from '@diffusionstudio/assets';
@@ -50,9 +47,17 @@ export function insertAsset(world: World, asset: Asset, options: InsertAssetOpti
switch (asset.type) {
case 'VIDEO':
case 'SEQUENCE':
- return <Video name={name} src={src} {...position} {...size} {...timing} />;
+ return (
+ <Rect name={name} {...position} {...size} {...timing}>
+ <VideoPaint src={src} />
+ </Rect>
+ );
case 'IMAGE':
- return <Image name={name} src={src} {...position} {...size} {...timing} />;
+ return (
+ <Rect name={name} {...position} {...size} {...timing}>
+ <ImagePaint src={src} />
+ </Rect>
+ );
case 'AUDIO':
return <Audio name={name} src={src} {...position} {...size} {...timing} />;
case 'TRANSCRIPT':
diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.ts
index 30c5e9aa..eefeb56c 100644
--- a/apps/web/src/engine/timeline/render/clip.ts
+++ b/apps/web/src/engine/timeline/render/clip.ts
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Generating, Hidden, Name, Selected, SourceError, TrimDragOrigin, fitsChildren, getGeneratingColor, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';
+import { Caption, CaptionType, Chars, ChildOf, ClipDragOrigin, Computed, Generating, Hidden, Name, Selected, SourceError, TrimDragOrigin, fitsChildren, getGeneratingColor, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';
import { getDocumentEditor } from '../../editor';
import {
@@ -56,8 +56,8 @@ export function renderClip(
pointer.scope(String(entity.id()));
const asset = getClipAsset(world, entity);
- const failed = entity.has(SourceError);
- const style = getClipStyle(entity, asset);
+ const error = world.query(ChildOf(entity), SourceError).at(0)?.get(SourceError)?.value;
+ const style = getClipStyle(entity, asset, !!error?.length);
handleBody(world, surface, entity, left, width, row, resolution);
@@ -67,7 +67,7 @@ export function renderClip(
left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);
}
- const generating = entity.has(Generating);
+ const generating = world.query(ChildOf(entity), Generating).length > 0;
ctx.save();
ctx.beginPath();
@@ -76,11 +76,42 @@ export function renderClip(
ctx.fill();
ctx.restore();
- if (!generating && !failed) {
+ if (!generating && !error) {
renderContent(world, scene, surface, entity, asset, style, row);
}
- renderLabel(world, scene, surface, entity, style.foreground, left, width, row);
+
+ // Render the label
+ {
+ let label = entity.get(Name)?.value ?? '';
+
+ if (error) label = error;
+ if (generating) label = 'Generating...';
+ if (isCaption(entity)) label = label || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;
+ if (isText(entity)) label = entity.get(Chars)?.value ?? label ?? '';
+
+ const [viewportLeft] = getViewport(world, scene, surface.layout.width);
+ const x = Math.max(left, viewportLeft);
+ const maxWidth = left + width - x - 2 * CLIP_LABEL_X;
+
+ ctx.save();
+ ctx.translate(x, 0);
+
+ ctx.fillStyle = style.foreground;
+ ctx.fontStretch = 'ultra-condensed';
+ ctx.font = CLIP_FONT;
+ ctx.textAlign = 'left';
+ ctx.textBaseline = 'middle';
+
+ const fitted = truncateText(ctx, label, maxWidth, CLIP_FONT);
+ if (fitted) {
+ // Tall enough for the label to sit at the top of the clip; below that
+ // there is only the middle.
+ ctx.fillText(fitted, CLIP_LABEL_X, row.height >= CLIP_BREAKPOINTS.xs ? CLIP_LABEL_Y : row.height / 2);
+ }
+
+ ctx.restore();
+ }
// Last, so it sits over whatever the clip drew inside itself.
const selected = entity.has(Selected);
@@ -236,57 +267,6 @@ function handleBody(
else editor.select(entity);
}
-/**
- * The clip's name, or what a text clip says. Pinned to the left edge of the
- * viewport while the clip runs off it, so a clip longer than the screen is
- * still labelled.
- */
-function renderLabel(
- world: World,
- scene: Entity,
- surface: TimelineSurfaceState,
- entity: Entity,
- color: string,
- left: number,
- width: number,
- row: RowCursor,
-): void {
- const ctx = surface.ctx!;
-
- // A caption's row is mostly its words; below a certain height the label
- // would be all there is room for, so the words win. One still generating
- // (or one that failed) has no words yet, so there is nothing for the label
- // to crowd out.
- if (
- isCaption(entity) && !entity.has(Generating) && !entity.has(SourceError)
- && row.height < CLIP_BREAKPOINTS.sm
- ) return;
-
- const label = clipLabel(entity);
- if (!label) return;
-
- const [viewportLeft] = getViewport(world, scene, surface.layout.width);
- const x = Math.max(left, viewportLeft);
- const maxWidth = left + width - x - 2 * CLIP_LABEL_X;
-
- ctx.save();
- ctx.translate(x, 0);
-
- ctx.fillStyle = color;
- ctx.fontStretch = 'ultra-condensed';
- ctx.font = CLIP_FONT;
- ctx.textAlign = 'left';
- ctx.textBaseline = 'middle';
-
- const fitted = truncateText(ctx, label, maxWidth, CLIP_FONT);
- if (fitted) {
- // Tall enough for the label to sit at the top of the clip; below that
- // there is only the middle.
- ctx.fillText(fitted, CLIP_LABEL_X, row.height >= CLIP_BREAKPOINTS.xs ? CLIP_LABEL_Y : row.height / 2);
- }
-
- ctx.restore();
-}
const CAPTION_PRESETS: Record<CaptionType, string> = {
[CaptionType.CLASSIC]: 'Classic',
@@ -298,28 +278,6 @@ const CAPTION_PRESETS: Record<CaptionType, string> = {
[CaptionType.STARK]: 'Stark',
};
-/**
- * What the clip calls itself. A plain text clip goes by what it says, and a
- * caption that has not been named goes by its preset — read here rather than
- * written to the entity, which is what the old timeline did: a renderer
- * naming the thing it is drawing is an edit nobody asked for.
- */
-function clipLabel(entity: Entity): string {
- const failure = entity.get(SourceError)?.value;
- if (failure) return failure;
-
- if (entity.has(Generating)) return 'Generating...';
-
- const name = entity.get(Name)?.value;
-
- if (isCaption(entity)) {
- return name || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;
- }
-
- if (isText(entity)) return entity.get(Chars)?.value ?? name ?? '';
-
- return name ?? '';
-}
/** Whether a clip is drawn faded: hidden nodes still take up their row. */
export function getClipAlpha(entity: Entity): number {
diff --git a/apps/web/src/engine/timeline/style.ts b/apps/web/src/engine/timeline/style.ts
index 69310281..65f41739 100644
--- a/apps/web/src/engine/timeline/style.ts
+++ b/apps/web/src/engine/timeline/style.ts
@@ -13,7 +13,6 @@ import {
isMask,
isScene,
isText,
- SourceError,
} from '@diffusionstudio/runtime';
import { COLORS } from './constants';
@@ -24,12 +23,11 @@ import type { Entity, World } from 'koota';
export type ClipStyle = {
background: string;
foreground: string;
- /** The colour of what is drawn inside: a waveform, a group's children. */
primary?: string;
};
-export function getClipStyle(entity: Entity, asset: Asset | null): ClipStyle {
- if (entity.has(SourceError) && !entity.has(AssetId)) return COLORS.clip.failed;
+export function getClipStyle(entity: Entity, asset: Asset | null, errored = false): ClipStyle {
+ if (errored && !entity.has(AssetId)) return COLORS.clip.failed;
if (isCaption(entity)) return COLORS.clip.caption;
if (isText(entity)) return COLORS.clip.text;
if (isScene(entity)) return COLORS.clip.scene;
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index 6230d1d8..7d199cfe 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -16,7 +16,7 @@ import {
TransitionType,
} from '../constants';
import {
- AssetId, ChildOf, Hidden, Culled, Generating, Interactive, IsMask,
+ ChildOf, Hidden, Culled, Generating, Interactive, IsMask,
ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,
SourceError,
BlendMode, Effect, Transition, MixedCornerRadius,
@@ -580,11 +580,13 @@ function renderStrokes(world: World, entity: Entity): void {
* The pulse a node waiting on a generation is filled with
*/
function renderGenerating(world: World, entity: Entity): void {
- const failed = entity.has(SourceError) && !entity.has(AssetId);
- if (!failed && !entity.has(Generating)) return;
+ const invariants = world.query(ChildOf(entity), SourceError, Generating);
+ const errored = invariants.find(invariant => invariant.get(SourceError));
+ const generating = invariants.find(invariant => invariant.get(Generating));
+ if (!errored && !generating) return;
const ctx = getCtx(world);
- ctx.fillStyle = failed ? FAILED_COLOR : getGeneratingColor(world);
+ ctx.fillStyle = errored ? FAILED_COLOR : getGeneratingColor(world);
ctx.fill();
}