Skip to content

The VS Code of video editing, built for agents.

TypeScript

Enhance entity management and rendering capabilities in the runtime

konstantin-paulus47e364d3d0f8
diff --git a/packages/encoder/src/encoder.ts b/packages/encoder/src/encoder.ts
index 4936af43..97a5835c 100644
--- a/packages/encoder/src/encoder.ts
+++ b/packages/encoder/src/encoder.ts
@@ -8,7 +8,7 @@ import {
AudioSample,
AudioSampleSource,
} from 'mediabunny';
-import { Or } from 'koota';
+import { Not, Or } from 'koota';
import {
createRuntimeWorld, setActive, serializeEntity, cloneFromRecords,
getEntityTree, propagateTimeRangeDown, framesToSeconds,
@@ -153,7 +153,7 @@ export async function createEncoder(sourceWorld: World, config: EncoderConfig) {
bitrate: audioBitrate,
});
- const sceneFills = [...world.query(Paint, ChildOf(scene))];
+ const sceneFills = [...world.query(Paint, Not(Geometry), ChildOf(scene))];
const videoSource = new CanvasSource(offscreenCanvas, {
codec: videoCodec,
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 88b3bd7c..f741723b 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -6,6 +6,9 @@
import {
Active,
appendChild,
+ AssetId,
+ Assets,
+ Audio,
Background,
Chars,
ClipsContent,
@@ -23,13 +26,18 @@ import {
getParentNode,
isText,
ItemIndex,
+ Muted,
Name,
Offset,
+ Paint,
+ PaintType,
parseColor,
Playback,
Position,
removeChild,
resizeEntity,
+ ScaleMode,
+ ScaleModeType,
secondsToFrames,
getEntityChildren,
Group,
@@ -47,6 +55,7 @@ import {
TextAlign,
TextBaseline,
TextStyle,
+ Volume,
} from '@diffusionstudio/runtime';
import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
@@ -180,6 +189,12 @@ const TIME_TRAITS = {
sourceOut: SourceOut,
} as const;
+const SCALE_MODES: Record<string, ScaleModeType> = {
+ cover: ScaleModeType.COVER,
+ contain: ScaleModeType.FIT,
+ fill: ScaleModeType.FILL,
+};
+
const FONT_STYLES: Record<string, FontStyle> = {
normal: FontStyle.NORMAL,
italic: FontStyle.ITALIC,
@@ -287,9 +302,32 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.add(TextStyle);
break;
}
+ case 'video': {
+ entity = createEntity(this.world);
+ entity.add(Geometry);
+ entity.set(Geometry, { value: GeometryType.RECT });
+ entity.add(Position);
+ entity.set(Position, { x: 0, y: 0 });
+ entity.add(Paint);
+ entity.set(Paint, { value: PaintType.VIDEO });
+ resizeEntity(this.world, entity, { width: 1920, height: 1080 });
+ break;
+ }
+ case 'audio': {
+ entity = createEntity(this.world);
+ entity.add(Geometry);
+ entity.set(Geometry, { value: GeometryType.RECT });
+ entity.add(Position);
+ entity.set(Position, { x: 0, y: 0 });
+ entity.add(Audio);
+ entity.add(Paint);
+ entity.set(Paint, { value: PaintType.WAVEFORM });
+ resizeEntity(this.world, entity, { width: 500, height: 150 });
+ break;
+ }
default:
throw new Error(
- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect> and <text>).`,
+ `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video> and <audio>).`,
);
}
@@ -421,6 +459,47 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.set(trait, { value: secondsToFrames(seconds, fps) });
return;
}
+ case 'src': {
+ if (typeof value !== 'string' || !this.world.get(Assets)?.has(value)) {
+ entity.remove(AssetId);
+ return;
+ }
+
+ entity.add(AssetId);
+ entity.set(AssetId, { value });
+ return;
+ }
+ case 'objectFit': {
+ const mode = typeof value === 'string' ? SCALE_MODES[value] : undefined;
+ if (mode === undefined) {
+ entity.remove(ScaleMode);
+ return;
+ }
+
+ entity.add(ScaleMode);
+ entity.set(ScaleMode, { value: mode });
+ return;
+ }
+ case 'volume': {
+ // Decibels; -Infinity is silence, not an invalid number.
+ const decibels = value === -Infinity ? value : toNumber(value);
+ if (decibels === undefined) {
+ entity.remove(Volume);
+ return;
+ }
+
+ entity.add(Volume);
+ entity.set(Volume, { value: decibels });
+ return;
+ }
+ case 'muted': {
+ if (value === true) {
+ entity.add(Muted);
+ } else {
+ entity.remove(Muted);
+ }
+ return;
+ }
case 'fill':
case 'color': {
const color = parseColor(value);
diff --git a/packages/runtime/src/actions/cache.ts b/packages/runtime/src/actions/cache.ts
index 910bac29..1abcb1e3 100644
--- a/packages/runtime/src/actions/cache.ts
+++ b/packages/runtime/src/actions/cache.ts
@@ -65,8 +65,10 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul
resetAnimatedValues(world, parent);
}
- if (entity.has(Paint)) {
- cache.fills[pid] = collect(Paint, ChildOf(parent))
+ // A geometry carrying Paint is not a fill of its parent: that is its own
+ // intrinsic paint (see getIntrinsicPaint).
+ if (entity.has(Paint) && !entity.has(Geometry)) {
+ cache.fills[pid] = collect(Paint, Not(Geometry), ChildOf(parent))
.sort(sortByItemIndex);
}
diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.ts
index 95357688..81fa6490 100644
--- a/packages/runtime/src/actions/timing.ts
+++ b/packages/runtime/src/actions/timing.ts
@@ -9,12 +9,12 @@
import { store } from '../world/store';
import { DEFAULT_DURATION_FRAMES } from '../constants';
import {
- Geometry, Group, Paint, Audio, Caption, Cache, Computed,
+ Geometry, Group, Paint, Caption, Cache, Computed, AssetId,
Start, End, SourceIn, SourceOut, PlaybackRate, CaptionDecoderHandle,
} from '../traits';
import { isGroupLike } from '../queries/predicates';
import { getParentNode } from '../queries/hierarchy';
-import { findAssetDuration, getSourceFrameAt, getTimelineOrigin } from '../utils/time';
+import { findAssetDuration, getSourceFrameAt, getTimelineOrigin, isPaintEntity } from '../utils/time';
import type { Entity, World } from 'koota';
@@ -24,13 +24,19 @@ import type { Entity, World } from 'koota';
*/
type TimeTrait = typeof Start;
+/**
+ * A trait a recompute may read as absent: one of the authored time traits, or
+ * the node's own AssetId (its intrinsic media, another source of a length).
+ */
+type Ignorable = TimeTrait | typeof AssetId;
+
/**
* The authored value, or undefined when the node doesn't carry it. `ignore`
* reads one trait as absent: koota fires onRemove before it clears the trait,
* so a removal handler has to recompute against the value that is going away
* (the same reason rebuildCaches takes an exclude).
*/
-function authored(entity: Entity, trait: TimeTrait, ignore?: TimeTrait): number | undefined {
+function authored(entity: Entity, trait: TimeTrait, ignore?: Ignorable): number | undefined {
return trait === ignore ? undefined : entity.get(trait)?.value;
}
@@ -51,10 +57,10 @@ function resolveDuration(
entity: Entity,
start: number,
sourceIn: number,
- ignore?: TimeTrait,
+ ignore?: Ignorable,
): number {
const playbackRate = authored(entity, PlaybackRate, ignore) || 1;
- const assetDuration = findAssetDuration(world, entity);
+ const assetDuration = findAssetDuration(world, entity, ignore === AssetId);
let duration = Infinity;
@@ -84,7 +90,7 @@ function resolveDuration(
* comparing two nodes works in one space. The rate places source frame 0, which
* is the origin every descendant is measured against.
*/
-export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?: TimeTrait): void {
+export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?: Ignorable): void {
const computed = store(world, Computed);
const eid = entity.id();
@@ -141,7 +147,7 @@ export function recomputeEntityTimeRange(world: World, entity: Entity, ignore?:
* the entity's origin or changes its rate has changed; every child below is
* placed against that origin and has to be re-derived.
*/
-export function propagateTimeRangeDown(world: World, entity: Entity, ignore?: TimeTrait): void {
+export function propagateTimeRangeDown(world: World, entity: Entity, ignore?: Ignorable): void {
recomputeEntityTimeRange(world, entity, ignore);
for (const child of entity.get(Cache)?.children ?? []) {
propagateTimeRangeDown(world, child);
@@ -167,7 +173,7 @@ export function bubbleTimeRangeUp(world: World, child: Entity): void {
}
/** Whether the node takes its bounds from its children rather than authoring them. */
-export function fitsChildren(entity: Entity, ignore?: TimeTrait): boolean {
+export function fitsChildren(entity: Entity, ignore?: Ignorable): boolean {
return isGroupLike(entity) && authored(entity, End, ignore) === undefined;
}
@@ -225,11 +231,13 @@ export function trimEntityOut(_world: World, entity: Entity, frame: number): voi
/**
* Re-derive the time range when the asset id of an entity changes.
*/
-export function reactToAssetChange(world: World, entity: Entity) {
- if (entity.has(Paint)) {
+/**
+ * `removing`: the id is on its way off the entity (koota fires onRemove before
+ * clearing it), so recompute as if it were already gone.
+ */
+export function reactToAssetChange(world: World, entity: Entity, removing = false) {
+ if (isPaintEntity(entity)) {
reactToPaintChange(world, entity);
- } else if (entity.has(Audio)) {
- reactToGeometryDurationChange(world, entity);
} else if (entity.has(Caption)) {
// Re-pointed transcript: drop the decoder so it re-resolves with the
// new asset.
@@ -237,6 +245,11 @@ export function reactToAssetChange(world: World, entity: Entity) {
entity.get(CaptionDecoderHandle)?.dispose();
entity.set(CaptionDecoderHandle, null);
}
+ } else if (entity.has(Geometry)) {
+ // A geometry's own asset backs its intrinsic paint (a video's footage)
+ // or, on an audio clip, its recording: a new one is a new source length.
+ recomputeEntityTimeRange(world, entity, removing ? AssetId : undefined);
+ bubbleTimeRangeUp(world, entity);
}
}
@@ -247,7 +260,8 @@ export function reactToAssetChange(world: World, entity: Entity) {
* and a longer one gives back whatever the authored bounds still allow.
*/
export function reactToPaintChange(world: World, paint: Entity) {
- const entity = getParentNode(paint);
+ // A geometry's own Paint is its intrinsic paint: the geometry is the clip.
+ const entity = paint.has(Geometry) ? paint : getParentNode(paint);
if (entity === null || !entity.has(Geometry)) return;
recomputeEntityTimeRange(world, entity);
@@ -256,8 +270,8 @@ export function reactToPaintChange(world: World, paint: Entity) {
/**
* Recompute a geometry's bounds and bubble. Used when something other than its
- * own authored time shifts its duration (e.g. an attached asset id on an Audio
- * entity).
+ * own authored time shifts its duration (e.g. the intrinsic media its own
+ * asset id names).
*/
export function reactToGeometryDurationChange(world: World, entity: Entity) {
if (!entity.has(Geometry)) return;
diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.ts
index 968cc7fd..f6ccb33b 100644
--- a/packages/runtime/src/systems/playback.ts
+++ b/packages/runtime/src/systems/playback.ts
@@ -21,7 +21,7 @@ import {
Root,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
-import { getSourceWindow } from '../utils/time';
+import { getIntrinsicPaint, getSourceWindow } from '../utils/time';
import { clamp } from '../math/common';
import { getTransitionWindow } from '../utils/transition';
import {
@@ -250,9 +250,23 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {
const hidden = entity.has(Hidden) || entity.has(Culled);
const visualsEnabled = !hidden && world.get(Mode)?.value !== 'offline-audio';
+ let intrinsicVideo = false;
let paintAudioSource: Entity | undefined;
+
if (!hidden) {
- for (const fill of world.query(ChildOf(entity), Paint, Not(Hidden))) {
+ const intrinsic = getIntrinsicPaint(entity);
+ if (intrinsic === PaintType.VIDEO) {
+ if (visualsEnabled) {
+ forwardVideoDecoder(world, scene, entity, entity);
+ }
+ intrinsicVideo = true;
+ } else if (intrinsic === PaintType.SEQUENCE && visualsEnabled) {
+ forwardSequenceDecoder(world, scene, entity, entity);
+ } else if (intrinsic === PaintType.IMAGE && visualsEnabled) {
+ forwardImageDecoder(world, scene, entity, entity);
+ }
+
+ for (const fill of world.query(ChildOf(entity), Paint, Not(Geometry), Not(Hidden))) {
const paint = paintStore.value[fill.id()];
if (paint === PaintType.VIDEO) {
@@ -285,7 +299,7 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {
forwardCaptionDecoder(world, scene, entity);
}
- if (entity.has(Audio) || paintAudioSource) {
+ if (intrinsicVideo || entity.has(Audio) || paintAudioSource) {
forwardAudioDecoder(world, scene, entity, paintAudioSource);
}
diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.ts
index 8d9f6c2a..fbdcb7f4 100644
--- a/packages/runtime/src/systems/render.ts
+++ b/packages/runtime/src/systems/render.ts
@@ -31,6 +31,7 @@ import { getViewMatrix } from '../queries/camera';
import { colorToHex } from '../utils/color';
import { renderText } from '../utils/text';
import { getTransitionWindow } from '../utils/transition';
+import { getIntrinsicPaint } from '../utils/time';
import { createLinearGradient, createRadialGradient } from './gradients';
import {
resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,
@@ -191,15 +192,81 @@ function buildEffects(world: World, entity: Entity): string | null {
}
/**
- * The geometry's intrinsic solid fill (its own Color trait), if any. Drawn
- * into the current path before the Paint sub-entities so it always sits at
- * the bottom of the fill stack. Reads Computed.color, so it animates.
+ * The geometry's intrinsic fill: its own Color trait (a solid, read from
+ * Computed.color so it animates) and its own Paint trait (see
+ * `getIntrinsicPaint`), if any, in that order. Drawn into the current path
+ * before the Paint sub-entities so it always sits at the bottom of the fill
+ * stack. A shader paint first in the stack takes an intrinsic image/video as
+ * its input instead (see `renderShaderFill`), in which case the media is not
+ * drawn here. Only media paints are intrinsic so far; a waveform (an audio
+ * clip's) has no picture on the canvas.
*/
export function renderIntrinsicFill(world: World, entity: Entity): void {
- if (!entity.has(Color)) return;
+ if (entity.has(Color)) {
+ const ctx = getCtx(world);
+ ctx.fillStyle = colorToHex(store(world, Computed).color[entity.id()] ?? 0);
+ ctx.fill();
+ }
+
+ const kind = mediaKind(getIntrinsicPaint(entity));
+ if (kind === null) return;
+ if (shaderInput(world, entity, store(world, Cache).fills[entity.id()] ?? [], 0) === entity) return;
+ renderMedia(world, entity, entity, kind);
+}
+
+type MediaKind = 'IMAGE' | 'VIDEO' | 'SEQUENCE';
+
+function mediaKind(paint: PaintType | undefined): MediaKind | null {
+ if (paint === PaintType.IMAGE) return 'IMAGE';
+ if (paint === PaintType.VIDEO) return 'VIDEO';
+ if (paint === PaintType.SEQUENCE) return 'SEQUENCE';
+ return null;
+}
+
+/** What the image/video/sequence decoders hand out to draw. */
+type MediaFrame = ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;
+
+/**
+ * Draws the current frame of `source` (an image/video/sequence paint, or the
+ * geometry itself for intrinsic media) into `entity`'s box, fitted by the
+ * source's ScaleMode; a failed decoder paints the missing-asset color.
+ */
+function renderMedia(world: World, entity: Entity, source: Entity, kind: MediaKind): void {
const ctx = getCtx(world);
- ctx.fillStyle = colorToHex(store(world, Computed).color[entity.id()] ?? 0);
- ctx.fill();
+ const computed = store(world, Computed);
+ const eid = entity.id();
+ const w = computed.width[eid]!;
+ const h = computed.height[eid]!;
+
+ let frame: MediaFrame | null | undefined;
+ let failed = false;
+ if (kind === 'IMAGE') {
+ const decoder = resolveImageDecoder(world, source)?.decoder;
+ frame = decoder?.getBitmap(w, h);
+ failed = decoder?.failed ?? false;
+ } else if (kind === 'VIDEO') {
+ const decoder = resolveVideoDecoder(world, source);
+ frame = decoder?.toBitmap();
+ failed = decoder?.errored ?? false;
+ } else {
+ const decoder = resolveSequenceDecoder(world, source);
+ frame = decoder?.toBitmap();
+ failed = decoder?.errored ?? false;
+ }
+
+ if (frame) {
+ ctx.save();
+ ctx.clip();
+
+ const mode = store(world, ScaleMode).value[source.id()] ?? 0;
+ const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);
+ ctx.drawImage(frame, dx, dy, sw, sh);
+
+ ctx.restore();
+ } else if (failed) {
+ ctx.fillStyle = MISSING_ASSET_COLOR;
+ ctx.fill();
+ }
}
export function renderFills(world: World, entity: Entity): void {
@@ -207,7 +274,6 @@ export function renderFills(world: World, entity: Entity): void {
const computed = store(world, Computed);
const paintStore = store(world, Paint);
const appearance = store(world, Appearance);
- const scaleMode = store(world, ScaleMode);
const eid = entity.id();
const fills = store(world, Cache).fills[eid] ?? [];
@@ -227,64 +293,11 @@ export function renderFills(world: World, entity: Entity): void {
const paint = paintStore.value[fid];
if (paint === PaintType.IMAGE) {
- const decoder = resolveImageDecoder(world, fill)?.decoder;
- const w = computed.width[eid]!;
- const h = computed.height[eid]!;
- const bitmap = decoder?.getBitmap(w, h);
-
- if (bitmap) {
- ctx.save();
- ctx.clip();
-
- const mode = scaleMode.value[fid] ?? 0;
- const [dx, dy, sw, sh] = getScaledImageProps(mode, bitmap.width, bitmap.height, w, h);
- ctx.drawImage(bitmap, dx, dy, sw, sh);
-
- ctx.restore();
- } else if (decoder?.failed) {
- ctx.fillStyle = MISSING_ASSET_COLOR;
- ctx.fill();
- }
+ renderMedia(world, entity, fill, 'IMAGE');
} else if (paint === PaintType.VIDEO) {
- const decoder = resolveVideoDecoder(world, fill);
- const frame = decoder?.toBitmap();
-
- if (frame) {
- const w = computed.width[eid]!;
- const h = computed.height[eid]!;
-
- ctx.save();
- ctx.clip();
-
- const mode = scaleMode.value[fid] ?? 0;
- const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);
- ctx.drawImage(frame, dx, dy, sw, sh);
-
- ctx.restore();
- } else if (decoder?.errored) {
- ctx.fillStyle = MISSING_ASSET_COLOR;
- ctx.fill();
- }
+ renderMedia(world, entity, fill, 'VIDEO');
} else if (paint === PaintType.SEQUENCE) {
- const decoder = resolveSequenceDecoder(world, fill);
- const frame = decoder?.toBitmap();
-
- if (frame) {
- const w = computed.width[eid]!;
- const h = computed.height[eid]!;
-
- ctx.save();
- ctx.clip();
-
- const mode = scaleMode.value[fid] ?? 0;
- const [dx, dy, sw, sh] = getScaledImageProps(mode, frame.width, frame.height, w, h);
- ctx.drawImage(frame, dx, dy, sw, sh);
-
- ctx.restore();
- } else if (decoder?.errored) {
- ctx.fillStyle = MISSING_ASSET_COLOR;
- ctx.fill();
- }
+ renderMedia(world, entity, fill, 'SEQUENCE');
} else if (paint === PaintType.HTML) {
const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;
@@ -327,61 +340,91 @@ export function renderFills(world: World, entity: Entity): void {
}
}
-/** The current frame of a video/image paint, as a GPU-uploadable source. */
+/**
+ * Whether `source` is a picture a shader can sample: an image/video paint,
+ * be it a paint sub-entity or the geometry's own intrinsic paint.
+ */
+function shaderMediaKind(world: World, source: Entity): 'IMAGE' | 'VIDEO' | null {
+ if (!source.has(Paint)) return null;
+ const paint = store(world, Paint).value[source.id()];
+ if (paint === PaintType.IMAGE) return 'IMAGE';
+ if (paint === PaintType.VIDEO) return 'VIDEO';
+ return null;
+}
+
+/** The current frame of a video/image source, as a GPU-uploadable source. */
function shaderSourceBitmap(
world: World,
- fill: Entity,
+ source: Entity,
w: number,
h: number,
): { source: GPUCopyExternalImageSource; width: number; height: number } | null {
- const paint = store(world, Paint).value[fill.id()];
+ const kind = shaderMediaKind(world, source);
- if (paint === PaintType.IMAGE) {
- const bitmap = resolveImageDecoder(world, fill)?.decoder?.getBitmap(w, h);
+ if (kind === 'IMAGE') {
+ const bitmap = resolveImageDecoder(world, source)?.decoder?.getBitmap(w, h);
return bitmap ? { source: bitmap, width: bitmap.width, height: bitmap.height } : null;
}
- if (paint === PaintType.VIDEO) {
- const frame = resolveVideoDecoder(world, fill)?.toBitmap();
+ if (kind === 'VIDEO') {
+ const frame = resolveVideoDecoder(world, source)?.toBitmap();
return frame ? { source: frame, width: frame.width, height: frame.height } : null;
}
return null;
}
/**
- * Whether the fill at `index` is the input of a ready shader paint directly
- * above it. Only the immediate neighbor counts (a hidden paint in between
- * decouples the pair). Must mirror `renderShaderFill`'s input selection
- * exactly — a consumed media paint that the shader then fails to draw would
- * blank the element, so both sides check pipeline readiness and frame
- * availability.
+ * The media directly below the fill at `index`, if it is one a shader could
+ * take as input: the visible image/video paint right before it, or, for the
+ * first fill, the geometry's own intrinsic image/video. Only the immediate
+ * neighbor counts (a hidden paint in between decouples the pair).
*/
-function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {
- const paintStore = store(world, Paint);
- const computed = store(world, Computed);
- const fill = fills[index]!;
- const paint = paintStore.value[fill.id()];
- if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return false;
+function mediaBelow(world: World, entity: Entity, fills: Entity[], index: number): Entity | null {
+ const source = index === 0 ? entity : fills[index - 1]!;
+ if (source.has(Hidden)) return null;
+ return shaderMediaKind(world, source) === null ? null : source;
+}
+
+/**
+ * The media the shader paint at `index` will sample this frame, or null when
+ * it has none to (there is no shader there, it is not ready, nothing samplable
+ * sits below it, or that has no frame yet). Whatever it returns is drawn by
+ * the shader and not on its own — a consumed media that the shader then fails
+ * to draw would blank the element, so this checks pipeline readiness and frame
+ * availability, and `renderShaderFill` draws exactly what it says.
+ */
+function shaderInput(world: World, entity: Entity, fills: Entity[], index: number): Entity | null {
+ const shader = fills[index];
+ if (shader === undefined || store(world, Paint).value[shader.id()] !== PaintType.SHADER) return null;
+ if (shader.has(Hidden)) return null;
+ if (!resolveShaderHost(world, shader)?.ready) return null;
- const next = fills[index + 1];
- if (next === undefined || paintStore.value[next.id()] !== PaintType.SHADER) return false;
- if (next.has(Hidden)) return false;
- if (!resolveShaderHost(world, next)?.ready) return false;
+ const media = mediaBelow(world, entity, fills, index);
+ if (media === null) return null;
+ const computed = store(world, Computed);
const w = computed.width[entity.id()]!;
const h = computed.height[entity.id()]!;
- return shaderSourceBitmap(world, fill, w, h) !== null;
+ return shaderSourceBitmap(world, media, w, h) === null ? null : media;
}
/**
- * Draws a shader paint: the media paint directly below it in the fill stack
- * is sampled as the shader's `source` texture and its output lands in the
- * parent's box in the media paint's place. Without a media paint below the
+ * Whether the fill at `index` is the input of a ready shader paint directly
+ * above it (see `shaderInput`).
+ */
+function shaderConsumesFill(world: World, entity: Entity, fills: Entity[], index: number): boolean {
+ return shaderInput(world, entity, fills, index + 1) === fills[index];
+}
+
+/**
+ * Draws a shader paint: the media directly below it — the image/video paint
+ * before it in the fill stack, or the geometry's intrinsic media under the
+ * first fill — is sampled as the shader's `source` texture and its output
+ * lands in the parent's box in the media's place. Without media below the
* shader runs procedurally over a transparent source; before the pipeline is
* ready it draws nothing and the media, if any, draws normally.
*/
function renderShaderFill(world: World, entity: Entity, fills: Entity[], index: number): void {
const ctx = getCtx(world);
- const paintStore = store(world, Paint);
const computed = store(world, Computed);
const host = resolveShaderHost(world, fills[index]!);
@@ -391,15 +434,12 @@ function renderShaderFill(world: World, entity: Entity, fills: Entity[], index:
const w = computed.width[eid]!;
const h = computed.height[eid]!;
- // A visible media paint directly below is the shader's input. Anything
- // else (no fill below, a hidden one, a solid/gradient) runs the shader
- // procedurally over a transparent source, stacking like a normal paint.
- const media = fills[index - 1];
- const isMedia = media !== undefined
- && !media.has(Hidden)
- && (paintStore.value[media.id()] === PaintType.IMAGE || paintStore.value[media.id()] === PaintType.VIDEO);
+ // Anything but samplable media below (no fill below, a hidden one, a
+ // solid/gradient) runs the shader procedurally over a transparent source,
+ // stacking like a normal paint.
+ const media = mediaBelow(world, entity, fills, index);
let input: ReturnType<typeof shaderSourceBitmap> = null;
- if (isMedia) {
+ if (media !== null) {
input = shaderSourceBitmap(world, media, w, h);
if (!input) return;
}
diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.ts
index 7f5b2360..75a9f019 100644
--- a/packages/runtime/src/utils/time.ts
+++ b/packages/runtime/src/utils/time.ts
@@ -4,7 +4,7 @@
import { CONONICAL_TIME_BASE, PaintType } from '../constants';
import {
- Audio, AssetId, Cache, Computed, Paint, SourceIn, Assets, FrameRate,
+ Audio, AssetId, Cache, Computed, Geometry, Paint, SourceIn, Assets, FrameRate,
} from '../traits';
import { getParentNode } from '../queries/hierarchy';
@@ -91,6 +91,23 @@ export function getSourceWindow(entity: Entity): { in: number; out: number } {
return { in: sourceIn, out: sourceIn + Math.round(duration * playbackRate) };
}
+/**
+ * The paint a geometry is intrinsically made of: its own Paint trait, when it
+ * carries one. Like its own Color, an intrinsic paint is drawn (and played) by
+ * the geometry itself, from its own AssetId, beneath any paint children — a
+ * paint child is a sub-entity carrying Paint and an asset of its own. Only a
+ * geometry can have an intrinsic paint; a paint sub-entity has no Geometry.
+ */
+export function getIntrinsicPaint(entity: Entity): PaintType | undefined {
+ if (!entity.has(Geometry)) return undefined;
+ return entity.get(Paint)?.value;
+}
+
+/** Whether `entity` is a paint sub-entity (as opposed to a geometry with an intrinsic paint). */
+export function isPaintEntity(entity: Entity): boolean {
+ return entity.has(Paint) && !entity.has(Geometry);
+}
+
/** The asset backing a geometry: its own AssetId, or the first fill's. */
export function findGeometryAsset(world: World, entity: Entity): Asset | null {
const assets = world.get(Assets);
@@ -115,18 +132,20 @@ export function findGeometryAsset(world: World, entity: Entity): Asset | null {
/**
* Intrinsic duration (in project frames) of the media asset attached to an
- * entity: the audio asset on Audio clips, else the first video/sequence fill.
- * Null when nothing time-based is attached.
+ * entity: its own asset when the entity is an audio clip or its intrinsic
+ * paint is a video/sequence, else the first video/sequence fill. Null when
+ * nothing time-based is attached. `ignoreOwn` reads the entity's own asset as
+ * absent (for a handler of its removal).
*/
-export function findAssetDuration(world: World, entity: Entity): number | null {
+export function findAssetDuration(world: World, entity: Entity, ignoreOwn = false): number | null {
const assets = world.get(Assets);
if (!assets) return null;
const frameRate = world.get(FrameRate)?.value ?? 30;
- if (entity.has(Audio)) {
+ const paint = getIntrinsicPaint(entity);
+ if (!ignoreOwn && (entity.has(Audio) || paint === PaintType.VIDEO || paint === PaintType.SEQUENCE)) {
const asset = assets.get(entity.get(AssetId)?.value ?? '');
-
- if (asset?.type === 'AUDIO' || asset?.type === 'VIDEO') {
+ if (asset?.type === 'AUDIO' || asset?.type === 'VIDEO' || asset?.type === 'SEQUENCE') {
return secondsToFrames(asset.duration, frameRate);
}
}
diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.ts
index dc24132a..b237a96f 100644
--- a/packages/runtime/src/world/observers.ts
+++ b/packages/runtime/src/world/observers.ts
@@ -139,6 +139,10 @@ export function observeWorld(world: World): () => void {
reactToAssetChange(world, entity);
}));
+ subs.push(world.onRemove(AssetId, (entity) => {
+ reactToAssetChange(world, entity, true);
+ }));
+
const propagateAndBubble = (entity: Entity) => {
propagateTimeRangeDown(world, entity);
bubbleTimeRangeUp(world, entity);