The VS Code of video editing, built for agents.
TypeScript
Refactor image and video generation components to use new rendering structure
konstantin-paulus2d27b6deb92c
diff --git a/apps/web/src/components/genai/use-generate-image.tsx b/apps/web/src/components/genai/use-generate-image.tsxindex ac5c8acf..d0c50d0d 100644--- a/apps/web/src/components/genai/use-generate-image.tsx+++ b/apps/web/src/components/genai/use-generate-image.tsx@@ -2,15 +2,9 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Image generation as the runtime means it: the prompt box declares a-// `generate.image` source per variant and inserts an <Image> consuming it,-// the way any drawn element is inserted. The asset system resolves the-// declaration through the world's Ai (see `EditorGenAi`), and the element-// lands in the project file with the `generate.image({...})` call that-// reproduces it — same spec, same asset, this session and the next.import { generate } from "@diffusionstudio/jsx";-import { Image } from "@diffusionstudio/reconciler";+import { ImagePaint, Rect } from "@diffusionstudio/reconciler";import { Library } from "@diffusionstudio/runtime";import { useWorld } from "@diffusionstudio/koota-solid";import { useEditor } from "@/engine/hooks";@@ -45,7 +39,9 @@ export function useGenerateImage() {}));insertGenerated(world, editor, dims, (box, index) => (- <Image src={sources[index]} x={box.x} y={box.y} width={box.width} height={box.height} />+ <Rect x={box.x} y={box.y} width={box.width} height={box.height}>+ <ImagePaint src={sources[index]} />+ </Rect>), config.count);};diff --git a/apps/web/src/components/genai/use-generate-video.tsx b/apps/web/src/components/genai/use-generate-video.tsxindex 25e23bf5..85c510f2 100644--- a/apps/web/src/components/genai/use-generate-video.tsx+++ b/apps/web/src/components/genai/use-generate-video.tsx@@ -2,13 +2,8 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Video generation, declared rather than performed: the prompt box writes a-// `generate.video` source and inserts a <Video> consuming it (see ./insert).-// The frames a model animates between are library assets, named by path, so-// the call in the file says what it was made from as well as what to make.-import { generate } from "@diffusionstudio/jsx";-import { Video } from "@diffusionstudio/reconciler";+import { Rect, VideoPaint } from "@diffusionstudio/reconciler";import { Library } from "@diffusionstudio/runtime";import { useWorld } from "@diffusionstudio/koota-solid";import { useEditor } from "@/engine/hooks";@@ -41,7 +36,9 @@ export function useGenerateVideo() {});insertGenerated(world, editor, dims, (box) => (- <Video src={src} x={box.x} y={box.y} width={box.width} height={box.height} />+ <Rect x={box.x} y={box.y} width={box.width} height={box.height}>+ <VideoPaint src={src} />+ </Rect>));};diff --git a/apps/web/src/engine/hud/hud-system.ts b/apps/web/src/engine/hud/hud-system.tsindex 559d0207..ff439c4b 100644--- a/apps/web/src/engine/hud/hud-system.ts+++ b/apps/web/src/engine/hud/hud-system.ts@@ -2,19 +2,6 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-/**- * Everything drawn over the document rather than in it: snap guides, hover- * outlines, scene headers, the selection mask with its handles, and the- * marquee. Runs after the render system, on the same canvas and in device- * pixels (the transform is reset per piece, so nothing here inherits the- * camera).- *- * Drawing is also where the HUD's hit regions come from: a control is pushed- * with the handler it answers to right after it is painted, so the regions- * come out in paint order, on top of the entity regions the render system- * left, and the input system can test them back to front next frame.- */-import { Not, Or } from 'koota';import {Active, Computed, Culled, FrameRate, Generating, Geometry, Group, Hidden,@@ -54,7 +41,7 @@ export function hudSystem(world: World): void {drawSnapLines(world, ctx, resolution);drawHoverOutlines(world, ctx, resolution);- for (const entity of world.query(Or(Name, Generating, SourceError), ChildOf(world.get(Root)!), Not(Culled), Not(Hidden))) {+ for (const entity of world.query(ChildOf(world.get(Root)!), Not(Culled), Not(Hidden), Name)) {drawHeader(world, ctx, entity, resolution);}@@ -119,9 +106,10 @@ function drawHoverOutlines(world: World, ctx: Ctx2D, resolution: number): void {* It sits in the node's own rotation, one line above its top edge.*/function drawHeader(world: World, ctx: Ctx2D, entity: Entity, resolution: number): void {- const failure = entity.get(SourceError)?.value;- const generating = entity.has(Generating);- const label = failure || (generating ? 'Generating...' : entity.get(Name)?.value);+ const errored = world.query(ChildOf(entity), SourceError);+ const generating = world.query(ChildOf(entity), Generating);+ const failure = errored.at(0)?.get(SourceError)?.value;+ const label = failure || (generating.length ? 'Generating...' : entity.get(Name)?.value);if (!label) return;const header = getHeaderLayout(world, entity, resolution);diff --git a/apps/web/src/engine/insert-asset.tsx b/apps/web/src/engine/insert-asset.tsxindex 3ebeca91..1a4b442e 100644--- a/apps/web/src/engine/insert-asset.tsx+++ b/apps/web/src/engine/insert-asset.tsx@@ -2,11 +2,8 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-// Putting an asset on the canvas: an element in the project's JSX whose-// `src` is the asset's library path, inserted the way every drawn element-// is (see DocumentEditor.insertElement), so it lands in the file too.-import { Audio, Captions, Image, Video } from '@diffusionstudio/reconciler';+import { Audio, Captions, ImagePaint, Rect, VideoPaint } from '@diffusionstudio/reconciler';import { Computed, getActiveEntity, getNextName, Root, Source, store } from '@diffusionstudio/runtime';import { assetName } from '@diffusionstudio/assets';@@ -50,9 +47,17 @@ export function insertAsset(world: World, asset: Asset, options: InsertAssetOptiswitch (asset.type) {case 'VIDEO':case 'SEQUENCE':- return <Video name={name} src={src} {...position} {...size} {...timing} />;+ return (+ <Rect name={name} {...position} {...size} {...timing}>+ <VideoPaint src={src} />+ </Rect>+ );case 'IMAGE':- return <Image name={name} src={src} {...position} {...size} {...timing} />;+ return (+ <Rect name={name} {...position} {...size} {...timing}>+ <ImagePaint src={src} />+ </Rect>+ );case 'AUDIO':return <Audio name={name} src={src} {...position} {...size} {...timing} />;case 'TRANSCRIPT':diff --git a/apps/web/src/engine/timeline/render/clip.ts b/apps/web/src/engine/timeline/render/clip.tsindex 30c5e9aa..eefeb56c 100644--- a/apps/web/src/engine/timeline/render/clip.ts+++ b/apps/web/src/engine/timeline/render/clip.ts@@ -2,7 +2,7 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Caption, CaptionType, Chars, ClipDragOrigin, Computed, Generating, Hidden, Name, Selected, SourceError, TrimDragOrigin, fitsChildren, getGeneratingColor, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';+import { Caption, CaptionType, Chars, ChildOf, ClipDragOrigin, Computed, Generating, Hidden, Name, Selected, SourceError, TrimDragOrigin, fitsChildren, getGeneratingColor, isCaption, isGroup, isText, store } from '@diffusionstudio/runtime';import { getDocumentEditor } from '../../editor';import {@@ -56,8 +56,8 @@ export function renderClip(pointer.scope(String(entity.id()));const asset = getClipAsset(world, entity);- const failed = entity.has(SourceError);- const style = getClipStyle(entity, asset);+ const error = world.query(ChildOf(entity), SourceError).at(0)?.get(SourceError)?.value;+ const style = getClipStyle(entity, asset, !!error?.length);handleBody(world, surface, entity, left, width, row, resolution);@@ -67,7 +67,7 @@ export function renderClip(left = framesToPixels(computed.start[entity.id()] ?? 0, resolution);}- const generating = entity.has(Generating);+ const generating = world.query(ChildOf(entity), Generating).length > 0;ctx.save();ctx.beginPath();@@ -76,11 +76,42 @@ export function renderClip(ctx.fill();ctx.restore();- if (!generating && !failed) {+ if (!generating && !error) {renderContent(world, scene, surface, entity, asset, style, row);}- renderLabel(world, scene, surface, entity, style.foreground, left, width, row);++ // Render the label+ {+ let label = entity.get(Name)?.value ?? '';++ if (error) label = error;+ if (generating) label = 'Generating...';+ if (isCaption(entity)) label = label || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;+ if (isText(entity)) label = entity.get(Chars)?.value ?? label ?? '';++ const [viewportLeft] = getViewport(world, scene, surface.layout.width);+ const x = Math.max(left, viewportLeft);+ const maxWidth = left + width - x - 2 * CLIP_LABEL_X;++ ctx.save();+ ctx.translate(x, 0);++ ctx.fillStyle = style.foreground;+ ctx.fontStretch = 'ultra-condensed';+ ctx.font = CLIP_FONT;+ ctx.textAlign = 'left';+ ctx.textBaseline = 'middle';++ const fitted = truncateText(ctx, label, maxWidth, CLIP_FONT);+ if (fitted) {+ // Tall enough for the label to sit at the top of the clip; below that+ // there is only the middle.+ ctx.fillText(fitted, CLIP_LABEL_X, row.height >= CLIP_BREAKPOINTS.xs ? CLIP_LABEL_Y : row.height / 2);+ }++ ctx.restore();+ }// Last, so it sits over whatever the clip drew inside itself.const selected = entity.has(Selected);@@ -236,57 +267,6 @@ function handleBody(else editor.select(entity);}-/**- * The clip's name, or what a text clip says. Pinned to the left edge of the- * viewport while the clip runs off it, so a clip longer than the screen is- * still labelled.- */-function renderLabel(- world: World,- scene: Entity,- surface: TimelineSurfaceState,- entity: Entity,- color: string,- left: number,- width: number,- row: RowCursor,-): void {- const ctx = surface.ctx!;-- // A caption's row is mostly its words; below a certain height the label- // would be all there is room for, so the words win. One still generating- // (or one that failed) has no words yet, so there is nothing for the label- // to crowd out.- if (- isCaption(entity) && !entity.has(Generating) && !entity.has(SourceError)- && row.height < CLIP_BREAKPOINTS.sm- ) return;-- const label = clipLabel(entity);- if (!label) return;-- const [viewportLeft] = getViewport(world, scene, surface.layout.width);- const x = Math.max(left, viewportLeft);- const maxWidth = left + width - x - 2 * CLIP_LABEL_X;-- ctx.save();- ctx.translate(x, 0);-- ctx.fillStyle = color;- ctx.fontStretch = 'ultra-condensed';- ctx.font = CLIP_FONT;- ctx.textAlign = 'left';- ctx.textBaseline = 'middle';-- const fitted = truncateText(ctx, label, maxWidth, CLIP_FONT);- if (fitted) {- // Tall enough for the label to sit at the top of the clip; below that- // there is only the middle.- ctx.fillText(fitted, CLIP_LABEL_X, row.height >= CLIP_BREAKPOINTS.xs ? CLIP_LABEL_Y : row.height / 2);- }-- ctx.restore();-}const CAPTION_PRESETS: Record<CaptionType, string> = {[CaptionType.CLASSIC]: 'Classic',@@ -298,28 +278,6 @@ const CAPTION_PRESETS: Record<CaptionType, string> = {[CaptionType.STARK]: 'Stark',};-/**- * What the clip calls itself. A plain text clip goes by what it says, and a- * caption that has not been named goes by its preset — read here rather than- * written to the entity, which is what the old timeline did: a renderer- * naming the thing it is drawing is an edit nobody asked for.- */-function clipLabel(entity: Entity): string {- const failure = entity.get(SourceError)?.value;- if (failure) return failure;-- if (entity.has(Generating)) return 'Generating...';-- const name = entity.get(Name)?.value;-- if (isCaption(entity)) {- return name || `${CAPTION_PRESETS[entity.get(Caption)?.type ?? CaptionType.CLASSIC]} Captions`;- }-- if (isText(entity)) return entity.get(Chars)?.value ?? name ?? '';-- return name ?? '';-}/** Whether a clip is drawn faded: hidden nodes still take up their row. */export function getClipAlpha(entity: Entity): number {diff --git a/apps/web/src/engine/timeline/style.ts b/apps/web/src/engine/timeline/style.tsindex 69310281..65f41739 100644--- a/apps/web/src/engine/timeline/style.ts+++ b/apps/web/src/engine/timeline/style.ts@@ -13,7 +13,6 @@ import {isMask,isScene,isText,- SourceError,} from '@diffusionstudio/runtime';import { COLORS } from './constants';@@ -24,12 +23,11 @@ import type { Entity, World } from 'koota';export type ClipStyle = {background: string;foreground: string;- /** The colour of what is drawn inside: a waveform, a group's children. */primary?: string;};-export function getClipStyle(entity: Entity, asset: Asset | null): ClipStyle {- if (entity.has(SourceError) && !entity.has(AssetId)) return COLORS.clip.failed;+export function getClipStyle(entity: Entity, asset: Asset | null, errored = false): ClipStyle {+ if (errored && !entity.has(AssetId)) return COLORS.clip.failed;if (isCaption(entity)) return COLORS.clip.caption;if (isText(entity)) return COLORS.clip.text;if (isScene(entity)) return COLORS.clip.scene;diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 6230d1d8..7d199cfe 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -16,7 +16,7 @@ import {TransitionType,} from '../constants';import {- AssetId, ChildOf, Hidden, Culled, Generating, Interactive, IsMask,+ ChildOf, Hidden, Culled, Generating, Interactive, IsMask,ClipsContent, Geometry, Group, Paint, Color, Caption, ScaleMode, Shader,SourceError,BlendMode, Effect, Transition, MixedCornerRadius,@@ -580,11 +580,13 @@ function renderStrokes(world: World, entity: Entity): void {* The pulse a node waiting on a generation is filled with*/function renderGenerating(world: World, entity: Entity): void {- const failed = entity.has(SourceError) && !entity.has(AssetId);- if (!failed && !entity.has(Generating)) return;+ const invariants = world.query(ChildOf(entity), SourceError, Generating);+ const errored = invariants.find(invariant => invariant.get(SourceError));+ const generating = invariants.find(invariant => invariant.get(Generating));+ if (!errored && !generating) return;const ctx = getCtx(world);- ctx.fillStyle = failed ? FAILED_COLOR : getGeneratingColor(world);+ ctx.fillStyle = errored ? FAILED_COLOR : getGeneratingColor(world);ctx.fill();}