The VS Code of video editing, built for agents.
TypeScript
Add keyframe and keyframe track support in JSX and runtime
konstantin-paulus5b690f4ba3ad
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.tsindex c5f0ba1a..ef2fe729 100644--- a/packages/jsx/src/index.ts+++ b/packages/jsx/src/index.ts@@ -38,6 +38,7 @@ export { useTicker, useFile } from "./hooks";export type { Ticker } from "./hooks";export type {Animatable,+ AnimatableProperty,AnimationSpec,AnimationType,AudioProps,@@ -55,6 +56,8 @@ export type {HtmlProps,ImageProps,Keyframe,+ KeyframeProps,+ KeyframeTrackProps,PatchProps,RectProps,SceneProps,diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.tsindex 3d7e5e43..3067b004 100644--- a/packages/jsx/src/jsx-runtime.ts+++ b/packages/jsx/src/jsx-runtime.ts@@ -30,6 +30,8 @@ import type {HtmlPaintProps,HtmlProps,ImageProps,+ KeyframeProps,+ KeyframeTrackProps,RectProps,SceneProps,SequenceProps,@@ -88,6 +90,8 @@ export declare namespace JSX {stroke: StrokeProps & SourceProps;shadow: ShadowProps & SourceProps;effect: EffectProps & SourceProps;+ keyframeTrack: KeyframeTrackProps & SourceProps;+ keyframe: KeyframeProps & SourceProps;htmlPaint: HtmlPaintProps & SourceProps;html: HtmlProps & SourceProps;shaderPaint: ShaderPaintProps & SourceProps;diff --git a/packages/jsx/src/source.ts b/packages/jsx/src/source.tsindex b044367e..4f117bb7 100644--- a/packages/jsx/src/source.ts+++ b/packages/jsx/src/source.ts@@ -106,6 +106,8 @@ export const COMPOSITION_TAGS = ["stroke","shadow","effect",+ "keyframeTrack",+ "keyframe","htmlPaint","html","shaderPaint",diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.tsindex 0ff741b9..9485baab 100644--- a/packages/jsx/src/types.ts+++ b/packages/jsx/src/types.ts@@ -68,6 +68,28 @@ export type Keyframe<T> = {*/export type Animatable<T> = T | Keyframe<T>[];+/**+ * The props a `<keyframeTrack>` can drive, by name: every `Animatable` prop.+ * Which entity's prop is the track's parent's (`width` under a `<stroke>` is+ * the line width, `value` under an `<effect>` its amount, `color`/`opacity`+ * under a paint the paint's).+ */+export type AnimatableProperty =+ | "x"+ | "y"+ | "offsetX"+ | "offsetY"+ | "width"+ | "height"+ | "rotation"+ | "opacity"+ | "cornerRadius"+ | "volume"+ | "color"+ | "offset"+ | "blur"+ | "value";+/** Transition styles — the editor's transition inspector options. */export type TransitionType =| "dissolve"@@ -188,8 +210,18 @@ export type PatchProps = {miterLimit?: number;/** Which filter an `<effect>` applies. */type?: EffectType;- /** An `<effect>`'s amount: px for "blur", degrees for "hueRotate", 0–1 otherwise. Animatable. */- value?: Animatable<number>;+ /**+ * An `<effect>`'s amount: px for "blur", degrees for "hueRotate", 0–1+ * otherwise (animatable). On a `<keyframe>`, the value at its `time`: a+ * number, or any CSS color on a `color` track.+ */+ value?: Animatable<number> | string;+ /** Which prop a `<keyframeTrack>` animates, on the element holding the track. */+ property?: AnimatableProperty;+ /** A `<keyframe>`'s time, node-local: 0 is where the clip begins (its `start`). Any `Time` format. */+ time?: Time;+ /** Shapes the segment from a `<keyframe>` to the next one; ignored on the last. Default "linear". */+ easing?: Easing;/** Parent-timeline time at which the node begins. Default 0. */start?: Time;/** Parent-timeline time at which the node ends. Alternative to `sourceOut`. */@@ -355,23 +387,50 @@ export type RectProps = CommonProps & Pick<PatchProps, "fill"> & {* its line style. Several stack in document order, later ones on top.*/export type StrokeProps = Required<Pick<PatchProps, "color">> &- Pick<PatchProps, "opacity" | "width" | "join" | "cap" | "miterLimit">;+ Pick<PatchProps, "opacity" | "width" | "join" | "cap" | "miterLimit"> & {+ /** `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+ };/*** `<shadow>` — a drop shadow beneath the parent's box (or glyphs): a blurred,* offset copy of its silhouette in `color`. Several stack in document order.*/export type ShadowProps = Required<Pick<PatchProps, "color">> &- Pick<PatchProps, "opacity" | "blur" | "offsetX" | "offsetY">;+ Pick<PatchProps, "opacity" | "blur" | "offsetX" | "offsetY"> & {+ /** `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+ };/*** `<effect>` — a filter over the parent's rendered pixels (its fills, strokes* and children together), a sub-entity like a paint. Several stack in* document order.*/-export type EffectProps = Required<Pick<PatchProps, "type" | "value">>;+export type EffectProps = Required<Pick<PatchProps, "type">> & {+ value: Animatable<number>;+ /** `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+};-export type SolidPaintProps = Required<Pick<PatchProps, "color">> & Pick<PatchProps, "opacity">;+/**+ * `<keyframeTrack>` — the keyframes of one prop of the element holding it,+ * as elements: the addressable form of an `Animatable` prop's keyframe list,+ * so an editor moving a keyframe has an element to write it to. One track+ * per prop; a static value of that prop is what holds outside the keyframes.+ */+export type KeyframeTrackProps = Required<Pick<PatchProps, "property">> & {+ /** `<Keyframe>` children, in any order; they sort by `time`. */+ children?: SolidJSX.Element;+};++/** `<keyframe>` — one keyframe of the `<keyframeTrack>` holding it. */+export type KeyframeProps = Required<Pick<PatchProps, "time" | "value">> & Pick<PatchProps, "easing">;++export type SolidPaintProps = Required<Pick<PatchProps, "color">> & Pick<PatchProps, "opacity"> & {+ /** `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+};export type GradientPaintProps = Pick<PatchProps, "opacity"> & {/** Gradient rotation in degrees. Defaults to 0 (left to right). */@@ -381,7 +440,10 @@ export type GradientPaintProps = Pick<PatchProps, "opacity"> & {};export type ColorStopProps = Required<Pick<PatchProps, "offset" | "color">> &- Pick<PatchProps, "opacity">;+ Pick<PatchProps, "opacity"> & {+ /** `<KeyframeTrack>` children. */+ children?: SolidJSX.Element;+ };export type VideoProps = CommonProps &Required<Pick<PatchProps, "src">> &@@ -434,7 +496,10 @@ export type SurfaceProps = CommonProps & Pick<SurfacePaintProps, "ref">;export type AudioProps = TimingProps &Required<Pick<PatchProps, "src">> &- Pick<PatchProps, "name" | "volume" | "muted" | "syncTo" | "animations">;+ Pick<PatchProps, "name" | "volume" | "muted" | "syncTo" | "animations"> & {+ /** `<KeyframeTrack>` children (a `volume` track). */+ children?: SolidJSX.Element;+ };export type TextProps = CommonProps &Pick<diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.tsindex 4815764e..19af9d05 100644--- a/packages/reconciler/src/document.ts+++ b/packages/reconciler/src/document.ts@@ -30,6 +30,8 @@ import {getParentNode,isText,ItemIndex,+ Keyframe,+ KeyframeTrack,Muted,Name,Offset,@@ -69,7 +71,7 @@ import {} from '@diffusionstudio/runtime';import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';-import type { CameraMatrix } from '@diffusionstudio/runtime';+import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';import type { PropValue } from '@diffusionstudio/jsx';import { trait, type Entity, type World } from 'koota';import type { ProjectDocument } from './host';@@ -205,6 +207,43 @@ const PAINT_TYPES: Record<string, PaintType> = {radialGradientPaint: PaintType.RADIAL_GRADIENT,};+/**+ * A `<keyframeTrack>`'s `property` (a prop name) as the runtime's property+ * path. `width` depends on the holder: a stroke's is its line width.+ */+const TRACK_PROPERTIES: Record<string, PropertyPath> = {+ x: 'position.x',+ y: 'position.y',+ offsetX: 'offset.x',+ offsetY: 'offset.y',+ width: 'width',+ height: 'height',+ rotation: 'rotation',+ opacity: 'opacity',+ cornerRadius: 'vertexRadius',+ volume: 'volume',+ color: 'color',+ offset: 'stop.offset',+ blur: 'blur',+ value: 'effect.value',+};++/**+ * Named easings as the descriptors the runtime (and the editor's+ * interpolation inspector) speak; the descriptor forms pass through with+ * their whitespace dropped, linear is the empty string.+ */+const EASINGS: Record<string, string> = {+ linear: '',+ easeIn: 'cubicBezier(0.42,0,1,1)',+ easeOut: 'cubicBezier(0,0,0.58,1)',+ easeInOut: 'cubicBezier(0.42,0,0.58,1)',+ gentle: 'spring(0.5,628)',+ snappy: 'spring(0.15,300)',+ bouncy: 'spring(0.4,500)',+ strong: 'spring(0.65,400)',+};+const EFFECT_TYPES: Record<string, EffectType> = {blur: EffectType.LAYER_BLUR,brightness: EffectType.BRIGHTNESS,@@ -410,9 +449,22 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {entity.set(Effect, { type: EffectType.LAYER_BLUR, value: 0 });break;}+ case 'keyframeTrack': {+ // `property` names the prop; the path it resolves to depends on the+ // holder, so it is (re)resolved when the track is inserted.+ entity = createEntity(this.world);+ entity.add(KeyframeTrack);+ break;+ }+ case 'keyframe': {+ entity = createEntity(this.world);+ entity.add(Keyframe);+ entity.set(Keyframe, { easing: '' });+ break;+ }default:throw new Error(- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <colorStop>, <stroke>, <shadow> and <effect>).`,+ `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <colorStop>, <stroke>, <shadow>, <effect>, <keyframeTrack> and <keyframe>).`,);}@@ -556,10 +608,33 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {return;}case 'value': {+ if (entity.has(Keyframe)) {+ // A number, or a color on a color track; either is a number to the trait.+ entity.set(Keyframe, { value: toNumber(value) ?? parseColor(value) ?? 0 });+ return;+ }if (!entity.has(Effect)) return;entity.set(Effect, { value: toNumber(value) ?? 0 });return;}+ case 'property': {+ if (!entity.has(KeyframeTrack)) return;+ this.resolveTrackProperty(entity, value);+ return;+ }+ case 'time': {+ if (!entity.has(Keyframe)) return;+ const seconds = toSeconds(value);+ const fps = this.world.get(FrameRate)?.value ?? 30;+ entity.set(Keyframe, { time: seconds === undefined ? 0 : secondsToFrames(seconds, fps) });+ return;+ }+ case 'easing': {+ if (!entity.has(Keyframe)) return;+ const easing = typeof value === 'string' ? value.replace(/\s+/g, '') : '';+ entity.set(Keyframe, { easing: EASINGS[easing] ?? easing });+ return;+ }case 'blur': {const blur = toNumber(value);if (blur === undefined) {@@ -715,6 +790,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}}+ /**+ * Writes a track's runtime property path from its authored `property`,+ * against the entity holding it (a `width` track under a stroke drives the+ * line width). Called when the prop is set and again on insertion, since+ * Solid sets props before the track has a parent.+ */+ private resolveTrackProperty(track: Entity, property: unknown): void {+ let path = typeof property === 'string' ? TRACK_PROPERTIES[property] : undefined;+ if (path === 'width' && getParentEntity(track)?.has(Stroke)) path = 'stroke.width';+ track.set(KeyframeTrack, { property: path ?? '' });+ }+/*** Entity handles are minted per lookup, so every comparison here is between* entities: two handles for the same node are never the same object. Text@@ -760,6 +847,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {}appendChild(this.world, node.entity, parent.entity);++ if (node.entity.has(KeyframeTrack)) {+ this.resolveTrackProperty(node.entity, node.entity.get(Authored)?.props.property);+ }}const siblings = getEntityChildren(this.world, parent.entity).filter((sibling) => sibling !== node.entity);diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.tsindex 0c7c8151..1643c8b9 100644--- a/packages/reconciler/src/elements.ts+++ b/packages/reconciler/src/elements.ts@@ -28,6 +28,8 @@ import type {HtmlPaintProps,HtmlProps,ImageProps,+ KeyframeProps,+ KeyframeTrackProps,RectProps,SceneProps,SequenceProps,@@ -94,6 +96,8 @@ export const ColorStop = hostElement<ColorStopProps>("ColorStop");export const Stroke = hostElement<StrokeProps>("Stroke");export const Shadow = hostElement<ShadowProps>("Shadow");export const Effect = hostElement<EffectProps>("Effect");+export const KeyframeTrack = hostElement<KeyframeTrackProps>("KeyframeTrack");+export const Keyframe = hostElement<KeyframeProps>("Keyframe");export const HtmlPaint = hostElement<HtmlPaintProps>("HtmlPaint");export const Html = hostElement<HtmlProps>("Html");export const ShaderPaint = hostElement<ShaderPaintProps>("ShaderPaint");diff --git a/packages/reconciler/src/patch.ts b/packages/reconciler/src/patch.tsindex 245607b7..7befec7c 100644--- a/packages/reconciler/src/patch.ts+++ b/packages/reconciler/src/patch.ts@@ -29,6 +29,9 @@ export const PATCH_PROP_KEYS = Object.keys({miterLimit: true,type: true,value: true,+ property: true,+ time: true,+ easing: true,start: true,end: true,sourceIn: true,diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.mdindex 8f0741d9..d05e71af 100644--- a/packages/runtime/MIGRATION.md+++ b/packages/runtime/MIGRATION.md@@ -149,6 +149,8 @@ apps/web now depends on `@diffusionstudio/runtime` and `@diffusionstudio/koota-s**`<effect>` is hosted** (`EffectProps` in `@diffusionstudio/jsx`, `effect` in `IntrinsicElements` and `COMPOSITION_TAGS`, `Effect` component in the reconciler): one element for every filter effect, `<effect type="blur" value={8} />`, a sub-entity child of any node (`<group>` included: the filter applies to the group's rendered pixels as a whole), stacking in document order like the CSS filter list `buildEffects` turns them into. `type` (`blur`/`brightness`/`contrast`/`grayscale`/`hueRotate`/`invert`/`saturate`/`sepia`, new in `PatchProps` as `EffectType`) maps onto the runtime's `EffectType` enum minus `DROP_SHADOW`, which is `<shadow>`'s; `value` (new in `PatchProps`, animatable: the `effect.value` keyframe target) is px for a blur, degrees for a hue rotation, 0-1 otherwise, and both are required, since the trait's zero value is meaningless for most types (a brightness of 0 is black). The recipe is the bare `Effect` trait, created as a blur of 0 (a no-op) until `type`/`value` arrive; an unknown `type` falls back to that too. Per-type tags (`<blur>`, `<sepia>`, ...) were considered and rejected: `blur` is already the name of a prop, and one tag with a `type` keeps the effect list a list.+ **`<keyframeTrack>` and `<keyframe>` are hosted** (`KeyframeTrackProps`/`KeyframeProps` and `AnimatableProperty` in `@diffusionstudio/jsx`, both tags in `IntrinsicElements` and `COMPOSITION_TAGS`, `KeyframeTrack`/`Keyframe` components in the reconciler): the addressable form of an `Animatable` prop's keyframe list, so an editor moving a keyframe has an element (and id) to write it to. `<keyframeTrack property="x">` sits under the element whose prop it drives (any node, paint, color stop, stroke, shadow or effect: the runtime's `KeyframeTrack` is `ChildOf` its target) and holds `<keyframe time value easing />` children in any order (`Cache.keyframes` sorts by frame). `property` is the prop's JSX name and resolves through `TRACK_PROPERTIES` to the runtime's `PropertyPath` (`x` -> `position.x`, `cornerRadius` -> `vertexRadius`, `offset` -> `stop.offset`, `value` -> `effect.value`, ...); `width` under a `Stroke` is `stroke.width`, so the path is (re)resolved on insertion, since Solid sets props before the track has a parent (`resolveTrackProperty`). `time` is node-local and goes through `parseTime` + `secondsToFrames` like the timing props; `value` is a number, or a CSS color on a color track (`parseColor`); `easing` maps the named presets to the descriptors the runtime's `resolveEasing` and the interpolation inspector speak (`easeIn` -> `cubicBezier(0.42,0,1,1)`, `gentle` -> `spring(0.5,628)`, ..., `linear` -> `""`; descriptors pass through with whitespace dropped). `PatchProps` gained `property`, `time`, `easing`, and `value` widened to `Animatable<number> | string` (`EffectProps` narrows it back). One runtime fix came with it (`actions/cache.ts`): `rebuildCaches` re-aggregates a node's `Cache.keyframeTracks` not only when a track is attached but also when a non-node sub-entity carrying tracks is (`carriesKeyframeTrack`), because JSX builds bottom-up: a paint's tracks are attached to the paint before the paint meets its rect, and the old (bitecs) code only ever saw tracks attached to already-placed targets. Not decided yet: whether the inline `Animatable<T>` keyframe arrays stay (the host does not expand them; a static value is what `setProperty` handles) or the tags become the only spelling.+**Left stale on purpose** (deferred to a docs/examples pass): all eight `examples/*.tsx` still open with `<rect scene="...">` and no `<stage>`, so `tsc -p examples --noEmit` fails on them; `reference/jsx/{scene,roots}.md` still describes the pre-`<stage>` model where a mount grafts roots into an existing document; and `reference/jsx/{audio-sync,video,audio,elements}.md` still document `key` as the `syncTo` target. Deliberately not half-migrated: `reference/jsx` is stale as a whole, and patching the `key` rows alone would hide that.## Remaining work (suggested order)diff --git a/packages/runtime/src/actions/cache.ts b/packages/runtime/src/actions/cache.tsindex c70aaa33..a0ce2ab5 100644--- a/packages/runtime/src/actions/cache.ts+++ b/packages/runtime/src/actions/cache.ts@@ -48,7 +48,10 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul.sort(sortByItemIndex);}- if (entity.has(KeyframeTrack)) {+ // A track attached directly, or carried by a sub-entity (a paint, stroke,+ // effect...) that is attached with tracks already under it: JSX builds+ // bottom-up, so a paint's tracks exist before the paint meets its node.+ if (entity.has(KeyframeTrack) || (!isNodeEntity(entity) && carriesKeyframeTrack(world, entity))) {const node = findClosestParentGeometry(parent);aggregateKeyframeTracks(world, node, exclude);resetAnimatedValues(world, node);@@ -94,11 +97,25 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul}}+function isNodeEntity(entity: Entity): boolean {+ return entity.has(Geometry) || entity.has(Group) || entity.has(AdjustmentLayer);+}++/** Whether a KeyframeTrack sits under `entity`, short of any nested node. */+function carriesKeyframeTrack(world: World, entity: Entity): boolean {+ for (const child of world.query(ChildOf(entity))) {+ if (child.has(KeyframeTrack)) return true;+ if (isNodeEntity(child)) continue;+ if (carriesKeyframeTrack(world, child)) return true;+ }+ return false;+}+/** Nearest self-or-ancestor node entity (geometry, group, adjustment layer). */export function findClosestParentGeometry(entity: Entity): Entity | null {let current: Entity | null = entity;while (current) {- if (current.has(Geometry) || current.has(Group) || current.has(AdjustmentLayer)) {+ if (isNodeEntity(current)) {return current;}current = getParentNode(current);@@ -128,7 +145,7 @@ export function aggregateKeyframeTracks(world: World, node: Entity | null, exclucontinue;}// Stop at nested nodes; each owns its own cache entry.- if (child.has(Geometry) || child.has(Group) || child.has(AdjustmentLayer)) continue;+ if (isNodeEntity(child)) continue;walk(child);}};