The VS Code of video editing, built for agents.
TypeScript
Enhance entity management and rendering capabilities in the runtime
konstantin-paulus47e364d3d0f8
diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.tsindex 4936af43..97a5835c 100644--- a/packages/encoder/src/encoder.ts+++ b/packages/encoder/src/encoder.ts@@ -8,7 +8,7 @@ import {AudioSample,AudioSampleSource,} from 'mediabunny';-import { Or } from 'koota';+import { Not, Or } from 'koota';import {createRuntimeWorld, setActive, serializeEntity, cloneFromRecords,getEntityTree, propagateTimeRangeDown, framesToSeconds,@@ -153,7 +153,7 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {bitrate: audioBitrate,});- const sceneFills = [...world.query(Paint, ChildOf(scene))];+ const sceneFills = [...world.query(Paint, Not(Geometry), ChildOf(scene))];const videoSource = new CanvasSource(offscreenCanvas, {codec: videoCodec,diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 88b3bd7c..f741723b 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -6,6 +6,9 @@import {Active,appendChild,+ AssetId,+ Assets,+ Audio,Background,Chars,ClipsContent,@@ -23,13 +26,18 @@ import {getParentNode,isText,ItemIndex,+ Muted,Name,Offset,+ Paint,+ PaintType,parseColor,Playback,Position,removeChild,resizeEntity,+ ScaleMode,+ ScaleModeType,secondsToFrames,getEntityChildren,Group,@@ -47,6 +55,7 @@ import {TextAlign,TextBaseline,TextStyle,+ Volume,} from '@diffusionstudio/runtime';import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';@@ -180,6 +189,12 @@ const TIME_TRAITS = {sourceOut: SourceOut,} as const;+const SCALE_MODES: Record<string, ScaleModeType> = {+ cover: ScaleModeType.COVER,+ contain: ScaleModeType.FIT,+ fill: ScaleModeType.FILL,+};+const FONT_STYLES: Record<string, FontStyle> = {normal: FontStyle.NORMAL,italic: FontStyle.ITALIC,@@ -287,9 +302,32 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.add(TextStyle);break;}+ case 'video': {+ entity = createEntity(this.world);+ entity.add(Geometry);+ entity.set(Geometry, { value: GeometryType.RECT });+ entity.add(Position);+ entity.set(Position, { x: 0, y: 0 });+ entity.add(Paint);+ entity.set(Paint, { value: PaintType.VIDEO });+ resizeEntity(this.world, entity, { width: 1920, height: 1080 });+ break;+ }+ case 'audio': {+ entity = createEntity(this.world);+ entity.add(Geometry);+ entity.set(Geometry, { value: GeometryType.RECT });+ entity.add(Position);+ entity.set(Position, { x: 0, y: 0 });+ entity.add(Audio);+ entity.add(Paint);+ entity.set(Paint, { value: PaintType.WAVEFORM });+ resizeEntity(this.world, entity, { width: 500, height: 150 });+ break;+ }default:throw new Error(- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect> and <text>).`,+ `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video> and <audio>).`,);}@@ -421,6 +459,47 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(trait, { value: secondsToFrames(seconds, fps) });return;}+ case 'src': {+ if (typeof value !== 'string' || !this.world.get(Assets)?.has(value)) {+ entity.remove(AssetId);+ return;+ }++ entity.add(AssetId);+ entity.set(AssetId, { value });+ return;+ }+ case 'objectFit': {+ const mode = typeof value === 'string' ? SCALE_MODES[value] : undefined;+ if (mode === undefined) {+ entity.remove(ScaleMode);+ return;+ }++ entity.add(ScaleMode);+ entity.set(ScaleMode, { value: mode });+ return;+ }+ case 'volume': {+ // Decibels; -Infinity is silence, not an invalid number.+ const decibels = value === -Infinity ? value : toNumber(value);+ if (decibels === undefined) {+ entity.remove(Volume);+ return;+ }++ entity.add(Volume);+ entity.set(Volume, { value: decibels });+ return;+ }+ case 'muted': {+ if (value === true) {+ entity.add(Muted);+ } else {+ entity.remove(Muted);+ }+ return;+ }case 'fill':case 'color': {const color = parseColor(value);diff --git a/packages/runtime/src/actions/cache.ts b/packages/runtime/src/actions/cache.tsindex 910bac29..1abcb1e3 100644--- a/packages/runtime/src/actions/cache.ts+++ b/packages/runtime/src/actions/cache.ts@@ -65,8 +65,10 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nulresetAnimatedValues(world, parent);}- if (entity.has(Paint)) {- cache.fills[pid] = collect(Paint, ChildOf(parent))+ // A geometry carrying Paint is not a fill of its parent: that is its own+ // intrinsic paint (see getIntrinsicPaint).+ if (entity.has(Paint) && !entity.has(Geometry)) {+ cache.fills[pid] = collect(Paint, Not(Geometry), ChildOf(parent)).sort(sortByItemIndex);}diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.tsindex 95357688..81fa6490 100644--- a/packages/runtime/src/actions/timing.ts+++ b/packages/runtime/src/actions/timing.ts@@ -9,12 +9,12 @@import { store } from '../world/store';import { DEFAULT_DURATION_FRAMES } from '../constants';import {- Geometry, Group, Paint, Audio, Caption, Cache, Computed,+ Geometry, Group, Paint, Caption, Cache, Computed, AssetId,Start, End, SourceIn, SourceOut, PlaybackRate, CaptionDecoderHandle,} from '../traits';import { isGroupLike } from '../queries/predicates';import { getParentNode } from '../queries/hierarchy';-import { findAssetDuration, getSourceFrameAt, getTimelineOrigin } from '../utils/time';+import { findAssetDuration, getSourceFrameAt, getTimelineOrigin, isPaintEntity } from '../utils/time';import type { Entity, World } from 'koota';@@ -24,13 +24,19 @@ import type { Entity, World } from 'koota';*/type TimeTrait = typeof Start;+/**+ * A trait a recompute may read as absent: one of the authored time traits, or+ * the node's own AssetId (its intrinsic media, another source of a length).+ */+type Ignorable = TimeTrait | typeof AssetId;+/*** The authored value, or undefined when the node doesn't carry it. `ignore`* reads one trait as absent: koota fires onRemove before it clears the trait,* so a removal handler has to recompute against the value that is going away* (the same reason rebuildCaches takes an exclude).*/-function authored(entity: Entity, trait: TimeTrait, ignore?: TimeTrait): number | undefined {+function authored(entity: Entity, trait: TimeTrait, ignore?: Ignorable): number | undefined {return trait === ignore ? undefined : entity.get(trait)?.value;}@@ -51,10 +57,10 @@ function resolveDuration(entity: Entity,start: number,sourceIn: number,- ignore?: TimeTrait,+ ignore?: Ignorable,): number {const playbackRate = authored(entity, PlaybackRate, ignore) || 1;- const assetDuration = findAssetDuration(world, entity);+ const assetDuration = findAssetDuration(world, entity, ignore === AssetId);let duration = Infinity;@@ -84,7 +90,7 @@ function resolveDuration(* comparing two nodes works in one space. The rate places source frame 0, which* is the origin every descendant is measured against.*/-export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?: TimeTrait): void {+export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?: Ignorable): void {const computed = store(world, Computed);const eid = entity.id();@@ -141,7 +147,7 @@ export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?:* the entity's origin or changes its rate has changed; every child below is* placed against that origin and has to be re-derived.*/-export function propagateTimeRangeDown(world: World, entity: Entity, ignore?: TimeTrait): void {+export function propagateTimeRangeDown(world: World, entity: Entity, ignore?: Ignorable): void {recomputeEntityTimeRange(world, entity, ignore);for (const child of entity.get(Cache)?.children ?? []) {propagateTimeRangeDown(world, child);@@ -167,7 +173,7 @@ export function bubbleTimeRangeUp(world: World, child: Entity): void {}/** Whether the node takes its bounds from its children rather than authoring them. */-export function fitsChildren(entity: Entity, ignore?: TimeTrait): boolean {+export function fitsChildren(entity: Entity, ignore?: Ignorable): boolean {return isGroupLike(entity) && authored(entity, End, ignore) === undefined;}@@ -225,11 +231,13 @@ export function trimEntityOut(_world: World, entity: Entity, frame: number): voi/*** Re-derive the time range when the asset id of an entity changes.*/-export function reactToAssetChange(world: World, entity: Entity) {- if (entity.has(Paint)) {+/**+ * `removing`: the id is on its way off the entity (koota fires onRemove before+ * clearing it), so recompute as if it were already gone.+ */+export function reactToAssetChange(world: World, entity: Entity, removing = false) {+ if (isPaintEntity(entity)) {reactToPaintChange(world, entity);- } else if (entity.has(Audio)) {- reactToGeometryDurationChange(world, entity);} else if (entity.has(Caption)) {// Re-pointed transcript: drop the decoder so it re-resolves with the// new asset.@@ -237,6 +245,11 @@ export function reactToAssetChange(world: World, entity: Entity) {entity.get(CaptionDecoderHandle)?.dispose();entity.set(CaptionDecoderHandle, null);}+ } else if (entity.has(Geometry)) {+ // A geometry's own asset backs its intrinsic paint (a video's footage)+ // or, on an audio clip, its recording: a new one is a new source length.+ recomputeEntityTimeRange(world, entity, removing ? AssetId : undefined);+ bubbleTimeRangeUp(world, entity);}}@@ -247,7 +260,8 @@ export function reactToAssetChange(world: World, entity: Entity) {* and a longer one gives back whatever the authored bounds still allow.*/export function reactToPaintChange(world: World, paint: Entity) {- const entity = getParentNode(paint);+ // A geometry's own Paint is its intrinsic paint: the geometry is the clip.+ const entity = paint.has(Geometry) ? paint : getParentNode(paint);if (entity === null || !entity.has(Geometry)) return;recomputeEntityTimeRange(world, entity);@@ -256,8 +270,8 @@ export function reactToPaintChange(world: World, paint: Entity) {/*** Recompute a geometry's bounds and bubble. Used when something other than its- * own authored time shifts its duration (e.g. an attached asset id on an Audio- * entity).+ * own authored time shifts its duration (e.g. the intrinsic media its own+ * asset id names).*/export function reactToGeometryDurationChange(world: World, entity: Entity) {if (!entity.has(Geometry)) return;diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.tsindex 968cc7fd..f6ccb33b 100644--- a/packages/runtime/src/systems/playback.ts+++ b/packages/runtime/src/systems/playback.ts@@ -21,7 +21,7 @@ import {Root,} from '../traits';import { getParentNode } from '../queries/hierarchy';-import { getSourceWindow } from '../utils/time';+import { getIntrinsicPaint, getSourceWindow } from '../utils/time';import { clamp } from '../math/common';import { getTransitionWindow } from '../utils/transition';import {@@ -250,9 +250,23 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {const hidden = entity.has(Hidden) || entity.has(Culled);const visualsEnabled = !hidden && world.get(Mode)?.value !== 'offline-audio';+ let intrinsicVideo = false;let paintAudioSource: Entity | undefined;+if (!hidden) {- for (const fill of world.query(ChildOf(entity), Paint, Not(Hidden))) {+ const intrinsic = getIntrinsicPaint(entity);+ if (intrinsic === PaintType.VIDEO) {+ if (visualsEnabled) {+ forwardVideoDecoder(world, scene, entity, entity);+ }+ intrinsicVideo = true;+ } else if (intrinsic === PaintType.SEQUENCE && visualsEnabled) {+ forwardSequenceDecoder(world, scene, entity, entity);+ } else if (intrinsic === PaintType.IMAGE && visualsEnabled) {+ forwardImageDecoder(world, scene, entity, entity);+ }++ for (const fill of world.query(ChildOf(entity), Paint, Not(Geometry), Not(Hidden))) {const paint = paintStore.value[fill.id()];if (paint === PaintType.VIDEO) {@@ -285,7 +299,7 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {forwardCaptionDecoder(world, scene, entity);}- if (entity.has(Audio) || paintAudioSource) {+ if (intrinsicVideo || entity.has(Audio) || paintAudioSource) {forwardAudioDecoder(world, scene, entity, paintAudioSource);}diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 8d9f6c2a..fbdcb7f4 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -31,6 +31,7 @@ import { getViewMatrix } from '../queries/camera';import { colorToHex } from '../utils/color';import { renderText } from '../utils/text';import { getTransitionWindow } from '../utils/transition';+import { getIntrinsicPaint } from '../utils/time';import { createLinearGradient, createRadialGradient } from './gradients';import {resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,@@ -191,15 +192,81 @@ function buildEffects(world: World, entity: Entity): string | null {}/**- * The geometry's intrinsic solid fill (its own Color trait), if any. Drawn- * into the current path before the Paint sub-entities so it always sits at- * the bottom of the fill stack. Reads Computed.color, so it animates.+ * The geometry's intrinsic fill: its own Color trait (a solid, read from+ * Computed.color so it animates) and its own Paint trait (see+ * `getIntrinsicPaint`), if any, in that order. Drawn into the current path+ * before the Paint sub-entities so it always sits at the bottom of the fill+ * stack. A shader paint first in the stack takes an intrinsic image/video as+ * its input instead (see `renderShaderFill`), in which case the media is not+ * drawn here. Only media paints are intrinsic so far; a waveform (an audio+ * clip's) has no picture on the canvas.*/export function renderIntrinsicFill(world: World, entity: Entity): void {- if (!entity.has(Color)) return;+ if (entity.has(Color)) {+ const ctx = getCtx(world);+ ctx.fillStyle = colorToHex(store(world, Computed).color[entity.id()] ?? 0);+ ctx.fill();+ }++ const kind = mediaKind(getIntrinsicPaint(entity));+ if (kind === null) return;+ if (shaderInput(world, entity, store(world, Cache).fills[entity.id()] ?? [], 0) === entity) return;+ renderMedia(world, entity, entity, kind);+}++type MediaKind = 'IMAGE' | 'VIDEO' | 'SEQUENCE';++function mediaKind(paint: PaintType | undefined): MediaKind | null {+ if (paint === PaintType.IMAGE) return 'IMAGE';+ if (paint === PaintType.VIDEO) return 'VIDEO';+ if (paint === PaintType.SEQUENCE) return 'SEQUENCE';+ return null;+}++/** What the image/video/sequence decoders hand out to draw. */+type MediaFrame = ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;++/**+ * Draws the current frame of `source` (an image/video/sequence paint, or the+ * geometry itself for intrinsic media) into `entity`'s box, fitted by the+ * source's ScaleMode; a failed decoder paints the missing-asset color.+ */+function renderMedia(world: World, entity: Entity, source: Entity, kind: MediaKind): void {const ctx = getCtx(world);- ctx.fillStyle = colorToHex(store(world, Computed).color[entity.id()] ?? 0);- ctx.fill();+ const computed = store(world, Computed);+ const eid = entity.id();+ const w = computed.width[eid]!;+ const h = computed.height[eid]!;++ let frame: MediaFrame | null | undefined;+ let failed = false;+ if (kind === 'IMAGE') {+ const decoder = resolveImageDecoder(world, source)?.decoder;+ frame = decoder?.getBitmap(w, h);+ failed = decoder?.failed ?? false;+ } else if (kind === 'VIDEO') {+ const decoder = resolveVideoDecoder(world, source);+ frame = decoder?.toBitmap();+ failed = decoder?.errored ?? false;+ } else {+ const decoder = resolveSequenceDecoder(world, source);+ frame = decoder?.toBitmap();+ failed = decoder?.errored ?? false;+ }++ if (frame) {+ ctx.save();+ ctx.clip();++ const mode = store(world, ScaleMode).value[source.id()] ?? 0;+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);+ ctx.drawImage(frame, dx, dy, sw, sh);++ ctx.restore();+ } else if (failed) {+ ctx.fillStyle = MISSING_ASSET_COLOR;+ ctx.fill();+ }}export function renderFills(world: World, entity: Entity): void {@@ -207,7 +274,6 @@ export function renderFills(world: World, entity: Entity): void {const computed = store(world, Computed);const paintStore = store(world, Paint);const appearance = store(world, Appearance);- const scaleMode = store(world, ScaleMode);const eid = entity.id();const fills = store(world, Cache).fills[eid] ?? [];@@ -227,64 +293,11 @@ export function renderFills(world: World, entity: Entity): void {const paint = paintStore.value[fid];if (paint === PaintType.IMAGE) {- const decoder = resolveImageDecoder(world, fill)?.decoder;- const w = computed.width[eid]!;- const h = computed.height[eid]!;- const bitmap = decoder?.getBitmap(w, h);-- if (bitmap) {- ctx.save();- ctx.clip();-- const mode = scaleMode.value[fid] ?? 0;- const [dx, dy, sw, sh] = getScaledImageProps(mode, bitmap.width, bitmap.height, w, h);- ctx.drawImage(bitmap, dx, dy, sw, sh);-- ctx.restore();- } else if (decoder?.failed) {- ctx.fillStyle = MISSING_ASSET_COLOR;- ctx.fill();- }+ renderMedia(world, entity, fill, 'IMAGE');} else if (paint === PaintType.VIDEO) {- const decoder = resolveVideoDecoder(world, fill);- const frame = decoder?.toBitmap();-- if (frame) {- const w = computed.width[eid]!;- const h = computed.height[eid]!;-- ctx.save();- ctx.clip();-- const mode = scaleMode.value[fid] ?? 0;- const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);- ctx.drawImage(frame, dx, dy, sw, sh);-- ctx.restore();- } else if (decoder?.errored) {- ctx.fillStyle = MISSING_ASSET_COLOR;- ctx.fill();- }+ renderMedia(world, entity, fill, 'VIDEO');} else if (paint === PaintType.SEQUENCE) {- const decoder = resolveSequenceDecoder(world, fill);- const frame = decoder?.toBitmap();-- if (frame) {- const w = computed.width[eid]!;- const h = computed.height[eid]!;-- ctx.save();- ctx.clip();-- const mode = scaleMode.value[fid] ?? 0;- const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);- ctx.drawImage(frame, dx, dy, sw, sh);-- ctx.restore();- } else if (decoder?.errored) {- ctx.fillStyle = MISSING_ASSET_COLOR;- ctx.fill();- }+ renderMedia(world, entity, fill, 'SEQUENCE');} else if (paint === PaintType.HTML) {const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;@@ -327,61 +340,91 @@ export function renderFills(world: World, entity: Entity): void {}}-/** The current frame of a video/image paint, as a GPU-uploadable source. */+/**+ * Whether `source` is a picture a shader can sample: an image/video paint,+ * be it a paint sub-entity or the geometry's own intrinsic paint.+ */+function shaderMediaKind(world: World, source: Entity): 'IMAGE' | 'VIDEO' | null {+ if (!source.has(Paint)) return null;+ const paint = store(world, Paint).value[source.id()];+ if (paint === PaintType.IMAGE) return 'IMAGE';+ if (paint === PaintType.VIDEO) return 'VIDEO';+ return null;+}++/** The current frame of a video/image source, as a GPU-uploadable source. */function shaderSourceBitmap(world: World,- fill: Entity,+ source: Entity,w: number,h: number,): { source: GPUCopyExternalImageSource; width: number; height: number } | null {- const paint = store(world, Paint).value[fill.id()];+ const kind = shaderMediaKind(world, source);- if (paint === PaintType.IMAGE) {- const bitmap = resolveImageDecoder(world, fill)?.decoder?.getBitmap(w, h);+ if (kind === 'IMAGE') {+ const bitmap = resolveImageDecoder(world, source)?.decoder?.getBitmap(w, h);return bitmap ? { source: bitmap, width: bitmap.width, height: bitmap.height } : null;}- if (paint === PaintType.VIDEO) {- const frame = resolveVideoDecoder(world, fill)?.toBitmap();+ if (kind === 'VIDEO') {+ const frame = resolveVideoDecoder(world, source)?.toBitmap();return frame ? { source: frame, width: frame.width, height: frame.height } : null;}return null;}/**- * Whether the fill at `index` is the input of a ready shader paint directly- * above it. Only the immediate neighbor counts (a hidden paint in between- * decouples the pair). Must mirror `renderShaderFill`'s input selection- * exactly — a consumed media paint that the shader then fails to draw would- * blank the element, so both sides check pipeline readiness and frame- * availability.+ * The media directly below the fill at `index`, if it is one a shader could+ * take as input: the visible image/video paint right before it, or, for the+ * first fill, the geometry's own intrinsic image/video. Only the immediate+ * neighbor counts (a hidden paint in between decouples the pair).*/-function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {- const paintStore = store(world, Paint);- const computed = store(world, Computed);- const fill = fills[index]!;- const paint = paintStore.value[fill.id()];- if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return false;+function mediaBelow(world: World, entity: Entity, fills: Entity[], index: number): Entity | null {+ const source = index === 0 ? entity : fills[index - 1]!;+ if (source.has(Hidden)) return null;+ return shaderMediaKind(world, source) === null ? null : source;+}++/**+ * The media the shader paint at `index` will sample this frame, or null when+ * it has none to (there is no shader there, it is not ready, nothing samplable+ * sits below it, or that has no frame yet). Whatever it returns is drawn by+ * the shader and not on its own — a consumed media that the shader then fails+ * to draw would blank the element, so this checks pipeline readiness and frame+ * availability, and `renderShaderFill` draws exactly what it says.+ */+function shaderInput(world: World, entity: Entity, fills: Entity[], index: number): Entity | null {+ const shader = fills[index];+ if (shader === undefined || store(world, Paint).value[shader.id()] !== PaintType.SHADER) return null;+ if (shader.has(Hidden)) return null;+ if (!resolveShaderHost(world, shader)?.ready) return null;- const next = fills[index + 1];- if (next === undefined || paintStore.value[next.id()] !== PaintType.SHADER) return false;- if (next.has(Hidden)) return false;- if (!resolveShaderHost(world, next)?.ready) return false;+ const media = mediaBelow(world, entity, fills, index);+ if (media === null) return null;+ const computed = store(world, Computed);const w = computed.width[entity.id()]!;const h = computed.height[entity.id()]!;- return shaderSourceBitmap(world, fill, w, h) !== null;+ return shaderSourceBitmap(world, media, w, h) === null ? null : media;}/**- * Draws a shader paint: the media paint directly below it in the fill stack- * is sampled as the shader's `source` texture and its output lands in the- * parent's box in the media paint's place. Without a media paint below the+ * Whether the fill at `index` is the input of a ready shader paint directly+ * above it (see `shaderInput`).+ */+function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {+ return shaderInput(world, entity, fills, index + 1) === fills[index];+}++/**+ * Draws a shader paint: the media directly below it — the image/video paint+ * before it in the fill stack, or the geometry's intrinsic media under the+ * first fill — is sampled as the shader's `source` texture and its output+ * lands in the parent's box in the media's place. Without media below the* shader runs procedurally over a transparent source; before the pipeline is* ready it draws nothing and the media, if any, draws normally.*/function renderShaderFill(world: World, entity: Entity, fills: Entity[], index: number): void {const ctx = getCtx(world);- const paintStore = store(world, Paint);const computed = store(world, Computed);const host = resolveShaderHost(world, fills[index]!);@@ -391,15 +434,12 @@ function renderShaderFill(world: World, entity: Entity, fills: Entity[], index:const w = computed.width[eid]!;const h = computed.height[eid]!;- // A visible media paint directly below is the shader's input. Anything- // else (no fill below, a hidden one, a solid/gradient) runs the shader- // procedurally over a transparent source, stacking like a normal paint.- const media = fills[index - 1];- const isMedia = media !== undefined- && !media.has(Hidden)- && (paintStore.value[media.id()] === PaintType.IMAGE || paintStore.value[media.id()] === PaintType.VIDEO);+ // Anything but samplable media below (no fill below, a hidden one, a+ // solid/gradient) runs the shader procedurally over a transparent source,+ // stacking like a normal paint.+ const media = mediaBelow(world, entity, fills, index);let input: ReturnType<typeof shaderSourceBitmap> = null;- if (isMedia) {+ if (media !== null) {input = shaderSourceBitmap(world, media, w, h);if (!input) return;}diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.tsindex 7f5b2360..75a9f019 100644--- a/packages/runtime/src/utils/time.ts+++ b/packages/runtime/src/utils/time.ts@@ -4,7 +4,7 @@import { CONONICAL_TIME_BASE, PaintType } from '../constants';import {- Audio, AssetId, Cache, Computed, Paint, SourceIn, Assets, FrameRate,+ Audio, AssetId, Cache, Computed, Geometry, Paint, SourceIn, Assets, FrameRate,} from '../traits';import { getParentNode } from '../queries/hierarchy';@@ -91,6 +91,23 @@ export function getSourceWindow(entity: Entity): { in: number; out: number } {return { in: sourceIn, out: sourceIn + Math.round(duration * playbackRate) };}+/**+ * The paint a geometry is intrinsically made of: its own Paint trait, when it+ * carries one. Like its own Color, an intrinsic paint is drawn (and played) by+ * the geometry itself, from its own AssetId, beneath any paint children — a+ * paint child is a sub-entity carrying Paint and an asset of its own. Only a+ * geometry can have an intrinsic paint; a paint sub-entity has no Geometry.+ */+export function getIntrinsicPaint(entity: Entity): PaintType | undefined {+ if (!entity.has(Geometry)) return undefined;+ return entity.get(Paint)?.value;+}++/** Whether `entity` is a paint sub-entity (as opposed to a geometry with an intrinsic paint). */+export function isPaintEntity(entity: Entity): boolean {+ return entity.has(Paint) && !entity.has(Geometry);+}+/** The asset backing a geometry: its own AssetId, or the first fill's. */export function findGeometryAsset(world: World, entity: Entity): Asset | null {const assets = world.get(Assets);@@ -115,18 +132,20 @@ export function findGeometryAsset(world: World, entity: Entity): Asset | null {/*** Intrinsic duration (in project frames) of the media asset attached to an- * entity: the audio asset on Audio clips, else the first video/sequence fill.- * Null when nothing time-based is attached.+ * entity: its own asset when the entity is an audio clip or its intrinsic+ * paint is a video/sequence, else the first video/sequence fill. Null when+ * nothing time-based is attached. `ignoreOwn` reads the entity's own asset as+ * absent (for a handler of its removal).*/-export function findAssetDuration(world: World, entity: Entity): number | null {+export function findAssetDuration(world: World, entity: Entity, ignoreOwn = false): number | null {const assets = world.get(Assets);if (!assets) return null;const frameRate = world.get(FrameRate)?.value ?? 30;- if (entity.has(Audio)) {+ const paint = getIntrinsicPaint(entity);+ if (!ignoreOwn && (entity.has(Audio) || paint === PaintType.VIDEO || paint === PaintType.SEQUENCE)) {const asset = assets.get(entity.get(AssetId)?.value ?? '');-- if (asset?.type === 'AUDIO' || asset?.type === 'VIDEO') {+ if (asset?.type === 'AUDIO' || asset?.type === 'VIDEO' || asset?.type === 'SEQUENCE') {return secondsToFrames(asset.duration, frameRate);}}diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex dc24132a..b237a96f 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -139,6 +139,10 @@ export function observeWorld(world: World): () => void {reactToAssetChange(world, entity);}));+ subs.push(world.onRemove(AssetId, (entity) => {+ reactToAssetChange(world, entity, true);+ }));+const propagateAndBubble = (entity: Entity) => {propagateTimeRangeDown(world, entity);bubbleTimeRangeUp(world, entity);