The VS Code of video editing, built for agents.
TypeScript
Refactor media handling and remove sequence support
konstantin-pauluse6dbd415aa53
diff --git a/apps/web/src/components/sidebar-right/inspector/fill-picker.tsx b/apps/web/src/components/sidebar-right/inspector/fill-picker.tsxindex be622557..a4512a8d 100644--- a/apps/web/src/components/sidebar-right/inspector/fill-picker.tsx+++ b/apps/web/src/components/sidebar-right/inspector/fill-picker.tsx@@ -169,11 +169,10 @@ export function FillPicker(props: FillPickerProps) {};const handleSelectAsset = (asset: Asset) => {+ // A frames directory plays, and plays through the video paint.const plays = asset.type === "VIDEO" || asset.type === "SEQUENCE";const current = paintType();- const sameKind = plays- ? current === PaintType.VIDEO || current === PaintType.SEQUENCE- : current === PaintType.IMAGE;+ const sameKind = plays ? current === PaintType.VIDEO : current === PaintType.IMAGE;// Same element, another source: a `src` write, so the fit and any tracks// under it survive the swap of picture.diff --git a/apps/web/src/components/sidebar-right/inspector/fill-row.tsx b/apps/web/src/components/sidebar-right/inspector/fill-row.tsxindex be5d61ea..76ebe927 100644--- a/apps/web/src/components/sidebar-right/inspector/fill-row.tsx+++ b/apps/web/src/components/sidebar-right/inspector/fill-row.tsx@@ -30,7 +30,6 @@ const FILL_TYPE_LABELS: Record<PaintType, string> = {[PaintType.SOLID]: "Solid",[PaintType.IMAGE]: "Image",[PaintType.VIDEO]: "Video",- [PaintType.SEQUENCE]: "Sequence",[PaintType.LINEAR_GRADIENT]: "Gradient",[PaintType.RADIAL_GRADIENT]: "Gradient",[PaintType.WAVEFORM]: "Waveform",diff --git a/apps/web/src/components/sidebar-right/inspector/time.tsx b/apps/web/src/components/sidebar-right/inspector/time.tsxindex 4ad9eab6..95119036 100644--- a/apps/web/src/components/sidebar-right/inspector/time.tsx+++ b/apps/web/src/components/sidebar-right/inspector/time.tsx@@ -37,15 +37,16 @@ import {FrameRate,Geometry,PlaybackRate,+ SourceFrameRate,findGeometryAsset,+ findGeometryAssetSource,+ getSequenceFrameRate,isGroupLike,isScene,secondsToFrames,} from "@diffusionstudio/runtime";import { useDerived, useEditor } from "@/engine/hooks";import { editTime, trimIn, trimOut } from "@/engine/timing";-import { useLibrary } from "@/engine/library";-import { setSequenceFrameRate } from "@/engine/asset-actions";import type { Entity } from "koota";@@ -104,7 +105,6 @@ type TimeSettingsProps = {export function TimeSettings(props: TimeSettingsProps) {const world = useWorld();const editor = useEditor();- const library = useLibrary();const entity = () => props.selection[0]!;const [addons, setAddons] = createStoredSignal(@@ -131,13 +131,23 @@ export function TimeSettings(props: TimeSettingsProps) {const supportsPlaybackRate = createMemo(() => entity().has(Geometry) && !isScene(entity()));- // The sequence the node shows, if any; its frame rate is the asset's, so- // it is sampled too (`library.update` changes it in place).+ // The frames directory the node shows, if any, and the element carrying it:+ // the clip itself for a <video>, the paint for a <rect> with a fill.const sequenceAsset = useDerived(() => {const asset = findGeometryAsset(world, entity());return asset?.type === 'SEQUENCE' ? asset : null;});- const sequenceFrameRate = useDerived(() => sequenceAsset()?.frameRate ?? null);+ const sequenceSource = useDerived(() => findGeometryAssetSource(world, entity()));+ // The element's own rate when it sets one, else the asset's. Both are+ // sampled: the trait is what the prop writes, the asset what it falls back to.+ const authoredFrameRate = useTrait(sequenceSource, SourceFrameRate);+ const sequenceFrameRate = createMemo(() => {+ const asset = sequenceAsset();+ const source = sequenceSource();+ if (!asset || !source) return null;++ return authoredFrameRate()?.value || getSequenceFrameRate(source, asset);+ });const handleAddAddon = (addon: TimeAddon) => setAddons({ ...addons(), [addon]: true });const handleRemoveAddon = (addon: TimeAddon) => setAddons({ ...addons(), [addon]: false });@@ -174,9 +184,11 @@ export function TimeSettings(props: TimeSettingsProps) {const handleFrameRateChange = (newRate: number) => {const asset = sequenceAsset();- const lib = library();- if (!asset || !lib) return;- setSequenceFrameRate(world, lib, asset, newRate);+ const source = sequenceSource();+ if (!asset || !source) return;++ const clamped = Math.max(1, Math.min(240, Math.round(newRate)));+ editor.editProperty(source, 'frameRate', clamped === asset.frameRate ? false : clamped);};return (diff --git a/apps/web/src/components/timeline/layers/sub-item.tsx b/apps/web/src/components/timeline/layers/sub-item.tsxindex 977a9358..5d5d0603 100644--- a/apps/web/src/components/timeline/layers/sub-item.tsx+++ b/apps/web/src/components/timeline/layers/sub-item.tsx@@ -32,7 +32,6 @@ const PAINT_NAMES: Partial<Record<PaintType, string>> = {[PaintType.SOLID]: 'Solid',[PaintType.IMAGE]: 'Image',[PaintType.VIDEO]: 'Video',- [PaintType.SEQUENCE]: 'Sequence',[PaintType.WAVEFORM]: 'Waveform',[PaintType.HTML]: 'HTML',[PaintType.SURFACE]: 'Surface',diff --git a/apps/web/src/components/ui/fill-item.tsx b/apps/web/src/components/ui/fill-item.tsxindex 5925c20e..a8c9121b 100644--- a/apps/web/src/components/ui/fill-item.tsx+++ b/apps/web/src/components/ui/fill-item.tsx@@ -384,7 +384,7 @@ function PaintItemIcon(props: PaintItemIconProps) {return (<>- <Show when={asset() && (type() === PaintType.VIDEO || type() === PaintType.IMAGE || type() === PaintType.SEQUENCE)}>+ <Show when={asset() && (type() === PaintType.VIDEO || type() === PaintType.IMAGE)}><AssetThumbnailasset={asset()!}class="size-full"diff --git a/apps/web/src/engine/asset-actions.ts b/apps/web/src/engine/asset-actions.tsindex df1da213..69b274c1 100644--- a/apps/web/src/engine/asset-actions.ts+++ b/apps/web/src/engine/asset-actions.ts@@ -7,11 +7,9 @@import { toast } from "somoto";import { importFiles as importFilesInto, pickFiles, saveAssetAs as saveAs } from "@diffusionstudio/assets";-import { AssetId, reactToAssetChange } from "@diffusionstudio/runtime";import { insertAsset } from "./insert-asset";import { forgetPoster } from "./timeline/media";-import type { SequenceAsset } from "@diffusionstudio/assets";import type { World } from "koota";import type { Asset, AssetLibrary } from "@diffusionstudio/assets";@@ -73,19 +71,3 @@ export function insertAssetAtPlayhead(world: World, asset: Asset): void {}}-/**- * Sets the rate an image sequence plays its frames at. The frame count is- * what the sequence is, so its duration follows; every clip showing it is- * re-derived (the asset length is one of the caps on a clip's span).- */-export function setSequenceFrameRate(world: World, library: AssetLibrary, asset: SequenceAsset, frameRate: number): void {- const clamped = Math.max(1, Math.min(240, Math.round(frameRate)));- if (clamped === asset.frameRate) return;-- const frames = Math.max(1, Math.round(asset.duration * asset.frameRate));- library.update(asset, { frameRate: clamped, duration: frames / clamped });-- for (const entity of world.query(AssetId)) {- if (entity.get(AssetId)?.value === asset.id) reactToAssetChange(world, entity);- }-}diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 12a874ba..1ec307d2 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -278,9 +278,12 @@ type FillProps = {type MediaProps = {/**- * Path, URL, asset id, or a `generate.*` declaration. On `<Captions>` a- * transcript source (.srt, .vtt, or transcript .json) mounted instead of- * transcribing the scene; `generate.*` is not accepted there.+ * Path, URL, asset id, or a `generate.*` declaration. A path naming a+ * directory of numbered frames (`shot_001.png`, `shot_002.png`, ...) is an+ * image sequence, and plays on `<Video>` or `<Image>` as footage does — see+ * `frameRate` for how long it lasts. On `<Captions>` a transcript source+ * (.srt, .vtt, or transcript .json) mounted instead of transcribing the+ * scene; `generate.*` is not accepted there.*/src: string | AssetRef;/**@@ -316,6 +319,21 @@ type FitProps = {objectFit?: Fit;};+type FrameRateProps = {+ /**+ * Frames per second for a `src` naming a directory of numbered frames — a+ * folder of pictures has a count, not a duration, so this is what says how+ * long the clip runs (600 frames at 24 is 25 seconds, at 60 is 10). Default+ * 30. Nothing for encoded video or a still to read: a file carries its own+ * rate, and neither has a frame count to divide.+ *+ * Not `playbackRate`, which retimes a source against the timeline whatever+ * its natural speed is; this is what that natural speed is. Unrelated to the+ * composition's own frame rate, which the export sets.+ */+ frameRate?: number;+};+type AudioTrackProps = {/** Decibels: 0 = unity gain, negative attenuates (-6 ≈ half as loud), -Infinity = silence. Use `muted` to silence. */volume?: number;@@ -582,13 +600,13 @@ export type ColorStopProps = ColorProps & OpacityProps & TrackChildren & {* geometry's box, a paint child like a solid or a gradient (several stack in* document order). The node tags `<image>` / `<video>` are the same media as* an element of its own; these fill something else with it, so a rect or a- * text can be filled with a picture. Which tag it is says what the source is- * played as, exactly as on the node tags: a frames directory plays as a- * sequence under either.+ * text can be filled with a picture. Which tag it is only says what the source+ * is expected to be: the paint follows what the src turns out to name, so a+ * frames directory plays under either.*/-export type MediaPaintProps = PaintProps & MediaProps & FitProps & TrackChildren;+export type MediaPaintProps = PaintProps & MediaProps & FitProps & FrameRateProps & TrackChildren;-export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps & UpscaleProps & {+export type VideoProps = CommonProps & MediaProps & FitProps & FrameRateProps & AudioTrackProps & UpscaleProps & {/*** Scores the footage: a generated soundtrack for a clip that has none. See* `UpscaleProps` for what a modifier is; applied last, after `upscale`, so@@ -600,7 +618,7 @@ export type VideoProps = CommonProps & MediaProps & FitProps & AudioTrackProps &children?: SolidJSX.Element;};-export type ImageProps = CommonProps & MediaProps & FitProps & UpscaleProps & {+export type ImageProps = CommonProps & MediaProps & FitProps & FrameRateProps & UpscaleProps & {/*** Cuts the subject out, leaving the rest of the picture transparent. See* `UpscaleProps` for what a modifier is; applied before `upscale`.diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 3ffcf46f..4c7f5cf5 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -3,7 +3,7 @@* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';+import { Active, AdjustmentLayer, Animation, AnimationPhase, AnimationType, appendChild, AssetId, Audio, Background, bindAsset, BlendMode, BlendModeType, Blur, Caption, CaptionAlign, CaptionType, Chars, ClipHeight, ClipsContent, CornerRadius, createEntity, DEFAULT_BACKGROUND, Color, ColorStop, Effect, EffectType, End, Expanded, FontStyle, FrameRate, Generating, GenerationRequest, Loop, LoadRequest, Geometry, GeometryType, getEntityTree, getParentEntity, getParentNode, Hidden, IsMask, isText, ItemIndex, Keyframe, KeyframeTrack, MixedCornerRadius, Muted, Name, Offset, Opacity, Paint, PaintType, parseColor, PendingSource, Playback, PlaybackRate, Position, removeChild, resizeEntity, Scale, ScaleMode, ScaleModeType, secondsToFrames, getAsset, getEntityChildren, Group, Sequential, Shader, Size, Stage, Root, Rotation, Scene, Selected, Shadow, Source, SourceError, SourceIn, SourceOut, SourceFrameRate, SourceModifiers, hasModifier, setCameraMatrix, Start, Stroke, StrokeCap, StrokeJoin, StrokeStyle, SurfaceHost, SurfaceHostHandle, TextAlign, TextBaseline, TextCase, TextRange, TextStyle, TranscriptionRequest, Transition, TransitionType, UniformScale, Volume } from '@diffusionstudio/runtime';import { LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';@@ -990,6 +990,21 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(trait, { value: this.toFrames(seconds) });return;}+ case 'frameRate': {+ // Frames per second for a frames-directory source, which is the+ // only thing that says how long it lasts. The world's own frame+ // rate is the composition's and is not authored per element, so+ // this name is free to mean the source's.+ const fps = toNumber(value);+ if (fps === undefined || fps <= 0) {+ entity.remove(SourceFrameRate);+ return;+ }++ entity.add(SourceFrameRate);+ entity.set(SourceFrameRate, { value: fps });+ return;+ }case 'src': {// Generating goes with the wait it belongs to: the resolution// running for the old src will not clear it, having beendiff --git a/packages/runtime/src/actions/assets.ts b/packages/runtime/src/actions/assets.tsindex 785b4fa9..14c73ef5 100644--- a/packages/runtime/src/actions/assets.ts+++ b/packages/runtime/src/actions/assets.ts@@ -6,11 +6,11 @@// goes through the AssetLibrary (@diffusionstudio/assets); these are the// lookups the decoders and hosts share.-import { Ai, AssetId, Library, Paint, SourceModifiers } from '../traits';+import { Ai, AssetId, Library, Paint, SourceFrameRate, SourceModifiers } from '../traits';import { PaintType } from '../constants';import type { Entity, World } from 'koota';-import type { Asset, AssetLibrary } from '@diffusionstudio/assets';+import type { Asset, AssetLibrary, SequenceAsset } from '@diffusionstudio/assets';import type { GenAi } from '../ai';/** The world's asset library; throws when the host attached none. */@@ -50,20 +50,53 @@ export function getModifiers(entity: Entity): SourceModifierValues | undefined {}/**- * Binds an entity to an asset: stamps its AssetId, and follows the asset- * with the paint — a frames directory on a `<video>` or `<image>` plays as- * a sequence, and a sequence paint goes back when the asset is not one.+ * Binds an entity to an asset: stamps its AssetId, and follows the asset with+ * the paint. A media paint is whichever of the two the asset can be drawn+ * through — a frames directory decodes and draws exactly as a video does, so+ * it paints as one — and it is re-decided on every bind, so an element handed+ * a different kind of source shows it rather than nothing at all. Paints that+ * are not media (a gradient, a shader) are left alone: their fill is not the+ * asset.*/export function bindAsset(entity: Entity, asset: Asset): void {entity.add(AssetId);entity.set(AssetId, { value: asset.id });const paint = entity.get(Paint)?.value;- if (asset.type === 'SEQUENCE' && (paint === PaintType.VIDEO || paint === PaintType.IMAGE)) {- entity.set(Paint, { value: PaintType.SEQUENCE });- } else if (asset.type !== 'SEQUENCE' && paint === PaintType.SEQUENCE) {- entity.set(Paint, { value: asset.type === 'IMAGE' ? PaintType.IMAGE : PaintType.VIDEO });+ if (paint !== PaintType.IMAGE && paint !== PaintType.VIDEO) return;++ const wanted = asset.type === 'IMAGE' ? PaintType.IMAGE+ : asset.type === 'VIDEO' || asset.type === 'SEQUENCE' ? PaintType.VIDEO+ // Audio and transcripts have nothing to draw; leave the paint as authored.+ : paint;++ if (wanted !== paint) entity.set(Paint, { value: wanted });+}++/**+ * The rate a frames directory is played at: the element's own `frameRate`+ * when it sets one, else the rate the library gave the asset. `ignoreAuthored`+ * reads the element's as absent, for a handler of its removal.+ */+export function getSequenceFrameRate(entity: Entity, asset: SequenceAsset, ignoreAuthored = false): number {+ const authored = ignoreAuthored ? 0 : entity.get(SourceFrameRate)?.value ?? 0;+ return authored > 0 ? authored : asset.frameRate;+}++/**+ * How long the asset lasts as `entity` plays it, in seconds, or null when it+ * is not something with a duration. Only a sequence's answer depends on the+ * entity: its frames are a count, and the rate turning that into a duration+ * is the element's to set.+ */+export function getSourceDuration(entity: Entity, asset: Asset, ignoreAuthoredRate = false): number | null {+ if (asset.type === 'SEQUENCE') {+ const frames = asset.duration * asset.frameRate;+ return frames / getSequenceFrameRate(entity, asset, ignoreAuthoredRate);}++ if (asset.type === 'AUDIO' || asset.type === 'VIDEO') return asset.duration;+ return null;}/** The asset with `id` (or at that library path), or undefined without a library. */diff --git a/packages/runtime/src/actions/timing.ts b/packages/runtime/src/actions/timing.tsindex 725c15f5..70381864 100644--- a/packages/runtime/src/actions/timing.ts+++ b/packages/runtime/src/actions/timing.ts@@ -9,7 +9,7 @@import { store } from '../world/store';import { DEFAULT_DURATION_FRAMES } from '../constants';import {- Geometry, Group, AdjustmentLayer, Paint, Caption, Cache, Computed, AssetId, IsMask,+ Geometry, Group, AdjustmentLayer, Paint, Caption, Cache, Computed,Start, End, SourceIn, SourceOut, PlaybackRate, CaptionDecoderHandle,} from '../traits';import { isGroupLike } from '../queries/predicates';@@ -17,18 +17,7 @@ import { getParentNode } from '../queries/hierarchy';import { findAssetDuration, getSourceFrameAt, getTimelineOrigin, isPaintEntity } from '../utils/time';import type { Entity, World } from 'koota';--/**- * One of the authored time traits: Start, End, SourceIn, SourceOut,- * PlaybackRate. They all carry a single frame count (or multiplier).- */-type TimeTrait = typeof Start;--/**- * A trait a recompute may read as absent: one of the authored time traits, or- * the node's own AssetId (its intrinsic media, another source of a length).- */-type Ignorable = TimeTrait | typeof AssetId | typeof IsMask;+import type { Ignorable, TimeTrait } from '../utils/time';/*** The authored value, or undefined when the node doesn't carry it. `ignore`@@ -60,7 +49,7 @@ function resolveDuration(ignore?: Ignorable,): number {const playbackRate = authored(entity, PlaybackRate, ignore) || 1;- const assetDuration = findAssetDuration(world, entity, ignore === AssetId);+ const assetDuration = findAssetDuration(world, entity, ignore);let duration = Infinity;@@ -228,13 +217,13 @@ export function trimEntityOut(_world: World, entity: Entity, frame: number): voi}/**- * Re-derive the time range when the asset id of an entity changes.- */-/**- * `removing`: the id is on its way off the entity (koota fires onRemove before- * clearing it), so recompute as if it were already gone.+ * Re-derive the time range when what an entity plays changes: a new asset id,+ * or a new rate to play a frames directory at.+ *+ * `ignore`: the trait is on its way off the entity (koota fires onRemove+ * before clearing it), so recompute as if it were already gone.*/-export function reactToAssetChange(world: World, entity: Entity, removing = false) {+export function reactToAssetChange(world: World, entity: Entity, ignore?: Ignorable) {if (isPaintEntity(entity)) {reactToPaintChange(world, entity);} else if (entity.has(Caption)) {@@ -247,7 +236,7 @@ export function reactToAssetChange(world: World, entity: Entity, removing = fals} else if (entity.has(Geometry)) {// A geometry's own asset backs its intrinsic paint (a video's footage)// or, on an audio clip, its recording: a new one is a new source length.- recomputeEntityTimeRange(world, entity, removing ? AssetId : undefined);+ recomputeEntityTimeRange(world, entity, ignore);bubbleTimeRangeUp(world, entity);}}diff --git a/packages/runtime/src/constants.ts b/packages/runtime/src/constants.tsindex 2df9f483..917d41b1 100644--- a/packages/runtime/src/constants.ts+++ b/packages/runtime/src/constants.ts@@ -73,7 +73,6 @@ export enum PaintType {VIDEO,LINEAR_GRADIENT,RADIAL_GRADIENT,- SEQUENCE,WAVEFORM,HTML,SURFACE,diff --git a/packages/runtime/src/media/dispose.ts b/packages/runtime/src/media/dispose.tsindex 3b0aad51..28025e7c 100644--- a/packages/runtime/src/media/dispose.ts+++ b/packages/runtime/src/media/dispose.ts@@ -9,7 +9,7 @@import {ChildOf,- ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,+ ImageDecoderHandle, VideoDecoderHandle,AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle, AudioBusHandle,} from '../traits';@@ -24,10 +24,6 @@ export function disposeDecoders(world: World, entity: Entity): void {entity.get(VideoDecoderHandle)?.dispose();entity.set(VideoDecoderHandle, null);}- if (entity.has(SequenceDecoderHandle)) {- entity.get(SequenceDecoderHandle)?.dispose();- entity.set(SequenceDecoderHandle, null);- }if (entity.has(AudioDecoderHandle)) {entity.get(AudioDecoderHandle)?.reset();entity.set(AudioDecoderHandle, null);diff --git a/packages/runtime/src/media/handoff.ts b/packages/runtime/src/media/handoff.tsindex 5ce1f01f..3471dd0d 100644--- a/packages/runtime/src/media/handoff.ts+++ b/packages/runtime/src/media/handoff.ts@@ -10,13 +10,13 @@// seek lands; moving the handle means the copy draws its first frame from the// cache that was already warm for it.-import { AssetId, SequenceDecoderHandle, VideoDecoderHandle } from '../traits';+import { AssetId, VideoDecoderHandle } from '../traits';import { getEntityChildren } from '../queries/hierarchy';import type { Entity, World } from 'koota';/**- * Moves the live video and sequence decoders of `from`'s subtree to `to`'s.+ * Moves the live video decoders of `from`'s subtree to `to`'s.** For a split: the original ends at the playhead and the copy starts there,* so the copy is what renders next and the original is the one that can@@ -58,12 +58,4 @@ function handOff(from: Entity, to: Entity): void {to.add(VideoDecoderHandle);to.set(VideoDecoderHandle, video);}-- const sequence = from.get(SequenceDecoderHandle);- if (sequence) {- from.set(SequenceDecoderHandle, null);- to.get(SequenceDecoderHandle)?.dispose();- to.add(SequenceDecoderHandle);- to.set(SequenceDecoderHandle, sequence);- }}diff --git a/packages/runtime/src/media/sequence.ts b/packages/runtime/src/media/sequence.tsindex bfd16ef5..9610ac46 100644--- a/packages/runtime/src/media/sequence.ts+++ b/packages/runtime/src/media/sequence.ts@@ -2,12 +2,9 @@* License, v. 2.0. If a copy of the MPL was not distributed with this* file, You can obtain one at http://mozilla.org/MPL/2.0/. */-import { AssetId, SequenceDecoderHandle } from '../traits';-import { getAsset } from '../actions/assets';import { assert } from '../utils/assert';import { FrameCache } from './frame-cache';-import type { Entity, World } from 'koota';import type { SequenceAsset } from '@diffusionstudio/assets';const FRAME_EXTENSIONS = /\.(png|jpe?g|webp|avif|bmp|gif)$/i;@@ -24,6 +21,14 @@ export class SequenceDecoder {public mode: SequenceBufferMode = 'alive';public initialized: Promise<void>;+ /**+ * Frames per second the folder is played at (see the SourceFrameRate+ * trait). Read on every seek rather than baked in, so an element retiming+ * its sequence does not cost a rebuild — and the frames the decoder holds+ * are the right ones either way.+ */+ public frameRate: number;+private frames: File[] = [];private lastRenderedFrame: number = -1;private seekGeneration = 0;@@ -37,6 +42,7 @@ export class SequenceDecoder {public constructor(asset: SequenceAsset, hasCache: boolean) {this.hasCache = hasCache;this.asset = asset;+ this.frameRate = asset.frameRate;this.initialized = this.initialize();}@@ -143,7 +149,7 @@ export class SequenceDecoder {}public async seekTo(frame: number, frameRate: number) {- const targetFrame = Math.round((frame / frameRate) * this.asset.frameRate);+ const targetFrame = Math.round((frame / frameRate) * this.frameRate);// Path without cacheif (!this.hasCache) {@@ -242,26 +248,3 @@ export class SequenceDecoder {this.lastRenderedFrame = -1;}}--export function resolveSequenceDecoder(world: World, entity: Entity, hasCache = true): SequenceDecoder | null {- const assetId = entity.get(AssetId)?.value;- if (!assetId) return null;-- const asset = getAsset(world, assetId);- if (!asset || asset.type !== 'SEQUENCE') return null;-- // Identity check (not just id) so decoders rebuild when asset metadata —- // e.g. frameRate — changes via `assets.save({ ...asset, frameRate: ... })`.- const existing = entity.get(SequenceDecoderHandle);- if (existing && existing.asset === asset) {- existing.hasCache = hasCache;- return existing;- }-- existing?.dispose();-- const decoder = new SequenceDecoder(asset, hasCache);- entity.add(SequenceDecoderHandle);- entity.set(SequenceDecoderHandle, decoder);- return decoder;-}diff --git a/packages/runtime/src/media/video.ts b/packages/runtime/src/media/video.tsindex 2abeb518..7540149c 100644--- a/packages/runtime/src/media/video.ts+++ b/packages/runtime/src/media/video.ts@@ -6,9 +6,10 @@ import { BlobSource, ALL_FORMATS, Input, InputVideoTrack, EncodedPacketSink, Encimport { AssetId, VideoDecoderHandle, Mode } from '../traits';import { assert } from '../utils/assert';-import { getAsset, getAssetFile } from '../actions/assets';+import { getAsset, getAssetFile, getSequenceFrameRate } from '../actions/assets';import { FrameCache } from './frame-cache';import { getKeyframeIndex } from './keyframe-index';+import { SequenceDecoder } from './sequence';import type { Entity, World } from 'koota';import type { KeyframeIndex } from './keyframe-index';@@ -675,7 +676,14 @@ export class VideoExporter {}}-export type VideoDecoderInstance = VideoBuffer | VideoExporter;+/**+ * What a video paint decodes through. Three implementations of one interface+ * — `seekTo`, `toBitmap`, `idle`, `dispose` — picked by what the source turns+ * out to be and what the world is doing with it: a demuxed buffer for playing+ * a file, an exact-seeking reader for encoding one, and a frames directory+ * read off disk. Nothing downstream of `resolveVideoDecoder` asks which.+ */+export type VideoDecoderInstance = VideoBuffer | VideoExporter | SequenceDecoder;const videoTrackCache = new Map<string, Promise<InputVideoTrack | null>>();@@ -707,12 +715,30 @@ export function getVideoTrack(source: VideoAsset) {return promise;}+/**+ * The decoder `entity`'s video paint draws from, built on first use and kept+ * until the asset it was built for is no longer the one asked for.+ *+ * A sequence's rate is the element's to set, so it is pushed on every call+ * rather than fixed at construction — re-reading a folder to play it slower+ * would be a rebuild for nothing.+ */export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderInstance | null {const assetId = entity.get(AssetId)?.value;if (!assetId) return null;+ // Only a live preview keeps frames around it; an export reads each frame+ // once, in order, and a cache would be a window it never looks back into.+ const hasCache = world.get(Mode)?.value === 'realtime';++ // The id is the only thing that can go stale: a library edit assigns onto+ // the asset in place, so the object a live decoder holds is the library's.const existing = entity.get(VideoDecoderHandle);if (existing && existing.asset.id === assetId) {+ if (existing instanceof SequenceDecoder) {+ existing.hasCache = hasCache;+ existing.frameRate = getSequenceFrameRate(entity, existing.asset);+ }return existing;}@@ -720,11 +746,18 @@ export function resolveVideoDecoder(world: World, entity: Entity): VideoDecoderIexisting?.dispose();const asset = getAsset(world, assetId);- if (!asset || asset.type !== 'VIDEO') return null;+ if (!asset) return null;++ let decoder: VideoDecoderInstance;+ if (asset.type === 'SEQUENCE') {+ decoder = new SequenceDecoder(asset, hasCache);+ decoder.frameRate = getSequenceFrameRate(entity, asset);+ } else if (asset.type === 'VIDEO') {+ decoder = hasCache ? new VideoBuffer(asset) : new VideoExporter(asset);+ } else {+ return null;+ }- const decoder = world.get(Mode)?.value === 'realtime'- ? new VideoBuffer(asset)- : new VideoExporter(asset);entity.add(VideoDecoderHandle);entity.set(VideoDecoderHandle, decoder);return decoder;diff --git a/packages/runtime/src/systems/playback.ts b/packages/runtime/src/systems/playback.tsindex f6ccb33b..3b9f4c15 100644--- a/packages/runtime/src/systems/playback.ts+++ b/packages/runtime/src/systems/playback.ts@@ -26,7 +26,7 @@ import { clamp } from '../math/common';import { getTransitionWindow } from '../utils/transition';import {resolveAudioDecoder, resolveCaptionDecoder, resolveImageDecoder,- resolveSequenceDecoder, resolveShaderHost, resolveVideoDecoder,+ resolveShaderHost, resolveVideoDecoder,} from '../media';import { AudioBus } from '../media/audio-bus';@@ -138,30 +138,6 @@ function forwardVideoDecoder(world: World, scene: Entity, entity: Entity, fill:}}-function forwardSequenceDecoder(world: World, scene: Entity, entity: Entity, fill: Entity): void {- const computed = store(world, Computed);- const eid = entity.id();- const fps = world.get(FrameRate)?.value ?? 30;-- const globalFrame = computed.localTime[scene.id()]!;- const localFrame = computed.localTime[eid]!;- const start = computed.start[eid]!;- const end = computed.end[eid]!;- const hasCache = world.get(Mode)?.value === 'realtime';- const warmupDecoder = globalFrame >= start - WARMUP_FRAMES && globalFrame < end + WARMUP_FRAMES && hasCache;-- const source = getSourceWindow(entity);-- const decoder = resolveSequenceDecoder(world, fill, hasCache);- if (!decoder) return;-- if (computed.visibility[eid] === 1 || warmupDecoder) {- const seekFrame = clamp(localFrame, source.in, source.out);- const seekPromise = decoder.seekTo(seekFrame, fps);- framePromises(world)?.push(seekPromise);- }-}-function forwardCaptionDecoder(world: World, _scene: Entity, entity: Entity): void {const localFrame = store(world, Computed).localTime[entity.id()]!;const fps = world.get(FrameRate)?.value ?? 30;@@ -260,8 +236,6 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {forwardVideoDecoder(world, scene, entity, entity);}intrinsicVideo = true;- } else if (intrinsic === PaintType.SEQUENCE && visualsEnabled) {- forwardSequenceDecoder(world, scene, entity, entity);} else if (intrinsic === PaintType.IMAGE && visualsEnabled) {forwardImageDecoder(world, scene, entity, entity);}@@ -276,10 +250,6 @@ function forwardDecoders(world: World, scene: Entity, entity: Entity): void {paintAudioSource = fill;}- if (paint === PaintType.SEQUENCE && visualsEnabled) {- forwardSequenceDecoder(world, scene, entity, fill);- }-if (paint === PaintType.IMAGE && visualsEnabled) {forwardImageDecoder(world, scene, entity, fill);}diff --git a/packages/runtime/src/systems/render.ts b/packages/runtime/src/systems/render.tsindex 1180549a..ff68a6d1 100644--- a/packages/runtime/src/systems/render.ts+++ b/packages/runtime/src/systems/render.ts@@ -36,7 +36,7 @@ import { getTransitionWindow } from '../utils/transition';import { getIntrinsicPaint } from '../utils/time';import { createLinearGradient, createRadialGradient } from './gradients';import {- resolveImageDecoder, resolveVideoDecoder, resolveSequenceDecoder,+ resolveImageDecoder, resolveVideoDecoder,resolveCaptionDecoder, resolveShaderHost, resolveWaveformPeaks,} from '../media';@@ -234,20 +234,19 @@ export function renderIntrinsicFill(world: World, entity: Entity): void {renderMedia(world, entity, entity, kind);}-type MediaKind = 'IMAGE' | 'VIDEO' | 'SEQUENCE';+type MediaKind = 'IMAGE' | 'VIDEO';function mediaKind(paint: PaintType | undefined): MediaKind | null {if (paint === PaintType.IMAGE) return 'IMAGE';if (paint === PaintType.VIDEO) return 'VIDEO';- if (paint === PaintType.SEQUENCE) return 'SEQUENCE';return null;}-/** What the image/video/sequence decoders hand out to draw. */+/** What the image and video decoders hand out to draw. */type MediaFrame = ImageBitmap | HTMLImageElement | HTMLCanvasElement | OffscreenCanvas;/**- * Draws the current frame of `source` (an image/video/sequence paint, or the+ * Draws the current frame of `source` (an image or video paint, or the* geometry itself for intrinsic media) into `entity`'s box, fitted by the* source's ScaleMode; a failed decoder paints the missing-asset color.*/@@ -264,12 +263,8 @@ function renderMedia(world: World, entity: Entity, source: Entity, kind: MediaKiconst decoder = resolveImageDecoder(world, source)?.decoder;frame = decoder?.getBitmap(w, h);failed = decoder?.failed ?? false;- } else if (kind === 'VIDEO') {- const decoder = resolveVideoDecoder(world, source);- frame = decoder?.toBitmap();- failed = decoder?.errored ?? false;} else {- const decoder = resolveSequenceDecoder(world, source);+ const decoder = resolveVideoDecoder(world, source);frame = decoder?.toBitmap();failed = decoder?.errored ?? false;}@@ -316,8 +311,6 @@ export function renderFills(world: World, entity: Entity): void {renderMedia(world, entity, fill, 'IMAGE');} else if (paint === PaintType.VIDEO) {renderMedia(world, entity, fill, 'VIDEO');- } else if (paint === PaintType.SEQUENCE) {- renderMedia(world, entity, fill, 'SEQUENCE');} else if (paint === PaintType.HTML) {const host = fill.has(HtmlHostHandle) ? fill.get(HtmlHostHandle) : null;@@ -568,7 +561,7 @@ function renderGenerating(world: World, entity: Entity): void {//// Renders an audio asset's pre-computed peaks as a bar chart inside the// parent geometry's bounds. Sourced from the paint's own AssetId — the paint-// carries its asset reference, exactly like IMAGE/VIDEO/SEQUENCE paints.+// carries its asset reference, exactly like IMAGE and VIDEO paints.const WAVEFORM_BAR_WIDTH = 6;const WAVEFORM_BAR_GAP = 6;diff --git a/packages/runtime/src/traits/media.ts b/packages/runtime/src/traits/media.tsindex 01b5d9a4..80bd135d 100644--- a/packages/runtime/src/traits/media.ts+++ b/packages/runtime/src/traits/media.ts@@ -11,7 +11,6 @@ import type { SurfaceHost } from '../media/surface';import type { ShaderHost } from '../media/shader';import type { ImageDecoder } from '../media/image';import type { VideoDecoderInstance } from '../media/video';-import type { SequenceDecoder } from '../media/sequence';import type { AudioDecoder } from '../media/audio';import type { CaptionDecoder } from '../media/caption/types';import type { Waveform } from '../media/audio-peaks';@@ -38,8 +37,6 @@ export const ImageDecoderHandle = trait(() => null as ImageDecoder | null);export const VideoDecoderHandle = trait(() => null as VideoDecoderInstance | null);-export const SequenceDecoderHandle = trait(() => null as SequenceDecoder | null);-export const AudioDecoderHandle = trait(() => null as AudioDecoder | null);export const WaveformHandle = trait(() => null as Waveform | null);diff --git a/packages/runtime/src/traits/timing.ts b/packages/runtime/src/traits/timing.tsindex 8b8ef17b..127c89a1 100644--- a/packages/runtime/src/traits/timing.ts+++ b/packages/runtime/src/traits/timing.ts@@ -22,6 +22,14 @@ export const End = trait({ value: 0 });export const SourceIn = trait({ value: 0 });export const SourceOut = trait({ value: 0 });+// Frames per second a frames-directory source is played at, which is the only+// thing that says how long it lasts: a folder of pictures has a count, not a+// duration. Absent means the rate the library gave the asset. Nothing for+// encoded media to read — a video file carries its own rate — and unrelated to+// PlaybackRate, which retimes whatever the source's natural speed turns out to+// be; this is what that speed is.+export const SourceFrameRate = trait({ value: 0 });+// Speed multiplier for the node's local time (1 = normal). Scales the source// window against the timeline window: at 2, twice the source frames fit into// the same stretch of timeline.diff --git a/packages/runtime/src/utils/time.ts b/packages/runtime/src/utils/time.tsindex 60e9f3a7..0d33116c 100644--- a/packages/runtime/src/utils/time.ts+++ b/packages/runtime/src/utils/time.ts@@ -5,8 +5,10 @@import { CONONICAL_TIME_BASE, PaintType } from '../constants';import {Audio, AssetId, Cache, Computed, Geometry, Paint, SourceIn, Library, FrameRate,+ Start, IsMask, SourceFrameRate,} from '../traits';import { getParentNode } from '../queries/hierarchy';+import { getSourceDuration } from '../actions/assets';import type { Entity, World } from 'koota';import type { Asset } from '@diffusionstudio/assets';@@ -108,56 +110,78 @@ export function isPaintEntity(entity: Entity): boolean {return entity.has(Paint) && !entity.has(Geometry);}-/** The asset backing a geometry: its own AssetId, or the first fill's. */-export function findGeometryAsset(world: World, entity: Entity): Asset | null {+/**+ * The element a geometry shows its media through: itself when its own paint is+ * the media, else the first fill that is. This is the element the source props+ * belong to — a `<video>` carries its own `src`, a `<rect><videoPaint>` carries+ * it on the paint — so an editor writing one has to write it here.+ */+export function findGeometryAssetSource(world: World, entity: Entity): Entity | null {const assets = world.get(Library);if (!assets) return null;const ownId = entity.get(AssetId)?.value;- if (ownId) {- const asset = assets.get(ownId);- if (asset) return asset;- }+ if (ownId && assets.get(ownId)) return entity;for (const fill of entity.get(Cache)?.fills ?? []) {const fillId = fill.get(AssetId)?.value;- if (fillId) {- const asset = assets.get(fillId);- if (asset) return asset;- }+ if (fillId && assets.get(fillId)) return fill;}return null;}+/** The asset backing a geometry: its own AssetId, or the first fill's. */+export function findGeometryAsset(world: World, entity: Entity): Asset | null {+ const source = findGeometryAssetSource(world, entity);+ if (source === null) return null;++ return world.get(Library)?.get(source.get(AssetId)?.value ?? '') ?? null;+}++/**+ * One of the authored time traits: Start, End, SourceIn, SourceOut,+ * PlaybackRate, SourceFrameRate. They all carry a single frame count (or rate).+ */+export type TimeTrait = typeof Start;++/**+ * A trait a recompute may read as absent: one of the authored time traits, or+ * the node's own AssetId (its intrinsic media, another source of a length).+ * koota fires onRemove before it clears the trait, so a removal handler has to+ * ask for the length the node will have once it is gone.+ */+export type Ignorable = TimeTrait | typeof AssetId | typeof IsMask;+/*** Intrinsic duration (in project frames) of the media asset attached to an* entity: its own asset when the entity is an audio clip or its intrinsic- * paint is a video/sequence, else the first video/sequence fill. Null when- * nothing time-based is attached. `ignoreOwn` reads the entity's own asset as- * absent (for a handler of its removal).+ * paint is a video, else the first video fill. Null when nothing time-based is+ * attached.+ *+ * The duration is asked of the holder rather than read off the asset, because+ * a frames directory only lasts as long as the rate it is played at says (see+ * `getSourceDuration`).*/-export function findAssetDuration(world: World, entity: Entity, ignoreOwn = false): number | null {+export function findAssetDuration(world: World, entity: Entity, ignore?: Ignorable): number | null {const assets = world.get(Library);if (!assets) return null;const frameRate = world.get(FrameRate)?.value ?? 30;+ const ignoreAuthoredRate = ignore === SourceFrameRate;const paint = getIntrinsicPaint(entity);- if (!ignoreOwn && (entity.has(Audio) || paint === PaintType.VIDEO || paint === PaintType.SEQUENCE)) {+ if (ignore !== AssetId && (entity.has(Audio) || paint === PaintType.VIDEO)) {const asset = assets.get(entity.get(AssetId)?.value ?? '');- if (asset?.type === 'AUDIO' || asset?.type === 'VIDEO' || asset?.type === 'SEQUENCE') {- return secondsToFrames(asset.duration, frameRate);- }+ const duration = asset ? getSourceDuration(entity, asset, ignoreAuthoredRate) : null;+ if (duration !== null) return secondsToFrames(duration, frameRate);}for (const fill of entity.get(Cache)?.fills ?? []) {- const paint = fill.get(Paint)?.value;- if (paint === PaintType.VIDEO || paint === PaintType.SEQUENCE) {- const asset = assets.get(fill.get(AssetId)?.value ?? '');- if (asset && (asset.type === 'VIDEO' || asset.type === 'SEQUENCE')) {- return secondsToFrames(asset.duration, frameRate);- }- }+ if (fill.get(Paint)?.value !== PaintType.VIDEO) continue;++ const asset = assets.get(fill.get(AssetId)?.value ?? '');+ const duration = asset ? getSourceDuration(fill, asset, ignoreAuthoredRate) : null;+ if (duration !== null) return secondsToFrames(duration, frameRate);}return null;diff --git a/packages/runtime/src/world/observers.ts b/packages/runtime/src/world/observers.tsindex a5a278cd..e48ac428 100644--- a/packages/runtime/src/world/observers.ts+++ b/packages/runtime/src/world/observers.ts@@ -20,11 +20,11 @@import { store } from './store';import {ChildOf, Culled, Sequential, Group, Scene, Audio, Paint, AssetId,- Start, End, SourceIn, SourceOut, PlaybackRate, Keyframe, ItemIndex,+ Start, End, SourceIn, SourceOut, PlaybackRate, SourceFrameRate, Keyframe, ItemIndex,Position, Offset, Rotation, Scale, UniformScale, Skew, Anchor, Flip,Opacity, Color, Blur, Volume, Effect, CornerRadius, MixedCornerRadius,ColorStop, StrokeStyle, Size, Computed, Active, Stage, IsMask,- ImageDecoderHandle, VideoDecoderHandle, SequenceDecoderHandle,+ ImageDecoderHandle, VideoDecoderHandle,AudioDecoderHandle, CaptionDecoderHandle, WaveformHandle,HtmlHostHandle, SurfaceHostHandle, ShaderHostHandle, AudioBusHandle,} from '../traits';@@ -80,7 +80,6 @@ export function observeWorld(world: World): () => void {// readable.subs.push(world.onRemove(ImageDecoderHandle, (e) => e.get(ImageDecoderHandle)?.dispose()));subs.push(world.onRemove(VideoDecoderHandle, (e) => e.get(VideoDecoderHandle)?.dispose()));- subs.push(world.onRemove(SequenceDecoderHandle, (e) => e.get(SequenceDecoderHandle)?.dispose()));subs.push(world.onRemove(AudioDecoderHandle, (e) => e.get(AudioDecoderHandle)?.reset()));subs.push(world.onRemove(CaptionDecoderHandle, (e) => e.get(CaptionDecoderHandle)?.dispose()));subs.push(world.onRemove(WaveformHandle, (e) => e.get(WaveformHandle)?.dispose()));@@ -155,7 +154,22 @@ export function observeWorld(world: World): () => void {}));subs.push(world.onRemove(AssetId, (entity) => {- reactToAssetChange(world, entity, true);+ reactToAssetChange(world, entity, AssetId);+ }));++ // The rate a frames directory plays at is the whole of how long it is, so a+ // change to it is a change of source length — the same recompute a new+ // asset gets, not the propagate the other authored time traits get.+ subs.push(world.onAdd(SourceFrameRate, (entity) => {+ reactToAssetChange(world, entity);+ }));++ subs.push(world.onChange(SourceFrameRate, (entity) => {+ reactToAssetChange(world, entity);+ }));++ subs.push(world.onRemove(SourceFrameRate, (entity) => {+ reactToAssetChange(world, entity, SourceFrameRate);}));const propagateAndBubble = (entity: Entity) => {diff --git a/packages/runtime/src/world/serialize.ts b/packages/runtime/src/world/serialize.tsindex 1c03c693..90ec17e5 100644--- a/packages/runtime/src/world/serialize.ts+++ b/packages/runtime/src/world/serialize.ts@@ -12,7 +12,7 @@ import {Opacity, BlendMode, Color, CornerRadius, MixedCornerRadius, Blur, ScaleMode, Effect,ColorStop, StrokeStyle, Shader,Chars, TextStyle,- Start, End, SourceIn, SourceOut, PlaybackRate,+ Start, End, SourceIn, SourceOut, PlaybackRate, SourceFrameRate,Playback, Sequential, Transition, ClipHeight, Expanded,Volume, Muted,KeyframeTrack, Keyframe, Animation, Stage,@@ -132,6 +132,7 @@ export interface EntityRecord {SourceIn?: number;SourceOut?: number;PlaybackRate?: number;+ SourceFrameRate?: number;Constraint?: {horizontal?: number;vertical?: number;@@ -345,6 +346,9 @@ export function serializeEntity(entity: Entity): EntityRecord {if (entity.has(SourceOut)) {record.SourceOut = entity.get(SourceOut)!.value;}+ if (entity.has(SourceFrameRate)) {+ record.SourceFrameRate = entity.get(SourceFrameRate)!.value;+ }if (entity.has(PlaybackRate)) {record.PlaybackRate = entity.get(PlaybackRate)!.value;}@@ -592,6 +596,9 @@ export function deserializeEntity(entity: Entity, e: Partial<EntityRecord>): voiif (e.SourceOut !== undefined) {entity.add(SourceOut({ value: e.SourceOut }));}+ if (e.SourceFrameRate !== undefined) {+ entity.add(SourceFrameRate({ value: e.SourceFrameRate }));+ }if (e.PlaybackRate !== undefined) {entity.add(PlaybackRate({ value: e.PlaybackRate }));}