Skip to content

The VS Code of video editing, built for agents.

TypeScript

Add keyframe and keyframe track support in JSX and runtime

konstantin-paulus5b690f4ba3ad
diff --git a/packages/jsx/src/index.ts b/packages/jsx/src/index.ts
index c5f0ba1a..ef2fe729 100644
--- a/packages/jsx/src/index.ts
+++ b/packages/jsx/src/index.ts
@@ -38,6 +38,7 @@ export { useTicker, useFile } from "./hooks";
export type { Ticker } from "./hooks";
export type {
Animatable,
+ AnimatableProperty,
AnimationSpec,
AnimationType,
AudioProps,
@@ -55,6 +56,8 @@ export type {
HtmlProps,
ImageProps,
Keyframe,
+ KeyframeProps,
+ KeyframeTrackProps,
PatchProps,
RectProps,
SceneProps,
diff --git a/packages/jsx/src/jsx-runtime.ts b/packages/jsx/src/jsx-runtime.ts
index 3d7e5e43..3067b004 100644
--- a/packages/jsx/src/jsx-runtime.ts
+++ b/packages/jsx/src/jsx-runtime.ts
@@ -30,6 +30,8 @@ import type {
HtmlPaintProps,
HtmlProps,
ImageProps,
+ KeyframeProps,
+ KeyframeTrackProps,
RectProps,
SceneProps,
SequenceProps,
@@ -88,6 +90,8 @@ export declare namespace JSX {
stroke: StrokeProps & SourceProps;
shadow: ShadowProps & SourceProps;
effect: EffectProps & SourceProps;
+ keyframeTrack: KeyframeTrackProps & SourceProps;
+ keyframe: KeyframeProps & SourceProps;
htmlPaint: HtmlPaintProps & SourceProps;
html: HtmlProps & SourceProps;
shaderPaint: ShaderPaintProps & SourceProps;
diff --git a/packages/jsx/src/source.ts b/packages/jsx/src/source.ts
index b044367e..4f117bb7 100644
--- a/packages/jsx/src/source.ts
+++ b/packages/jsx/src/source.ts
@@ -106,6 +106,8 @@ export const COMPOSITION_TAGS = [
"stroke",
"shadow",
"effect",
+ "keyframeTrack",
+ "keyframe",
"htmlPaint",
"html",
"shaderPaint",
diff --git a/packages/jsx/src/types.ts b/packages/jsx/src/types.ts
index 0ff741b9..9485baab 100644
--- a/packages/jsx/src/types.ts
+++ b/packages/jsx/src/types.ts
@@ -68,6 +68,28 @@ export type Keyframe<T> = {
*/
export type Animatable<T> = T | Keyframe<T>[];
+/**
+ * The props a `<keyframeTrack>` can drive, by name: every `Animatable` prop.
+ * Which entity's prop is the track's parent's (`width` under a `<stroke>` is
+ * the line width, `value` under an `<effect>` its amount, `color`/`opacity`
+ * under a paint the paint's).
+ */
+export type AnimatableProperty =
+ | "x"
+ | "y"
+ | "offsetX"
+ | "offsetY"
+ | "width"
+ | "height"
+ | "rotation"
+ | "opacity"
+ | "cornerRadius"
+ | "volume"
+ | "color"
+ | "offset"
+ | "blur"
+ | "value";
+
/** Transition styles — the editor's transition inspector options. */
export type TransitionType =
| "dissolve"
@@ -188,8 +210,18 @@ export type PatchProps = {
miterLimit?: number;
/** Which filter an `<effect>` applies. */
type?: EffectType;
- /** An `<effect>`'s amount: px for "blur", degrees for "hueRotate", 0–1 otherwise. Animatable. */
- value?: Animatable<number>;
+ /**
+ * An `<effect>`'s amount: px for "blur", degrees for "hueRotate", 0–1
+ * otherwise (animatable). On a `<keyframe>`, the value at its `time`: a
+ * number, or any CSS color on a `color` track.
+ */
+ value?: Animatable<number> | string;
+ /** Which prop a `<keyframeTrack>` animates, on the element holding the track. */
+ property?: AnimatableProperty;
+ /** A `<keyframe>`'s time, node-local: 0 is where the clip begins (its `start`). Any `Time` format. */
+ time?: Time;
+ /** Shapes the segment from a `<keyframe>` to the next one; ignored on the last. Default "linear". */
+ easing?: Easing;
/** Parent-timeline time at which the node begins. Default 0. */
start?: Time;
/** Parent-timeline time at which the node ends. Alternative to `sourceOut`. */
@@ -355,23 +387,50 @@ export type RectProps = CommonProps & Pick<PatchProps, "fill"> & {
* its line style. Several stack in document order, later ones on top.
*/
export type StrokeProps = Required<Pick<PatchProps, "color">> &
- Pick<PatchProps, "opacity" | "width" | "join" | "cap" | "miterLimit">;
+ Pick<PatchProps, "opacity" | "width" | "join" | "cap" | "miterLimit"> & {
+ /** `<KeyframeTrack>` children. */
+ children?: SolidJSX.Element;
+ };
/**
* `<shadow>` — a drop shadow beneath the parent's box (or glyphs): a blurred,
* offset copy of its silhouette in `color`. Several stack in document order.
*/
export type ShadowProps = Required<Pick<PatchProps, "color">> &
- Pick<PatchProps, "opacity" | "blur" | "offsetX" | "offsetY">;
+ Pick<PatchProps, "opacity" | "blur" | "offsetX" | "offsetY"> & {
+ /** `<KeyframeTrack>` children. */
+ children?: SolidJSX.Element;
+ };
/**
* `<effect>` — a filter over the parent's rendered pixels (its fills, strokes
* and children together), a sub-entity like a paint. Several stack in
* document order.
*/
-export type EffectProps = Required<Pick<PatchProps, "type" | "value">>;
+export type EffectProps = Required<Pick<PatchProps, "type">> & {
+ value: Animatable<number>;
+ /** `<KeyframeTrack>` children. */
+ children?: SolidJSX.Element;
+};
-export type SolidPaintProps = Required<Pick<PatchProps, "color">> & Pick<PatchProps, "opacity">;
+/**
+ * `<keyframeTrack>` — the keyframes of one prop of the element holding it,
+ * as elements: the addressable form of an `Animatable` prop's keyframe list,
+ * so an editor moving a keyframe has an element to write it to. One track
+ * per prop; a static value of that prop is what holds outside the keyframes.
+ */
+export type KeyframeTrackProps = Required<Pick<PatchProps, "property">> & {
+ /** `<Keyframe>` children, in any order; they sort by `time`. */
+ children?: SolidJSX.Element;
+};
+
+/** `<keyframe>` — one keyframe of the `<keyframeTrack>` holding it. */
+export type KeyframeProps = Required<Pick<PatchProps, "time" | "value">> & Pick<PatchProps, "easing">;
+
+export type SolidPaintProps = Required<Pick<PatchProps, "color">> & Pick<PatchProps, "opacity"> & {
+ /** `<KeyframeTrack>` children. */
+ children?: SolidJSX.Element;
+};
export type GradientPaintProps = Pick<PatchProps, "opacity"> & {
/** Gradient rotation in degrees. Defaults to 0 (left to right). */
@@ -381,7 +440,10 @@ export type GradientPaintProps = Pick<PatchProps, "opacity"> & {
};
export type ColorStopProps = Required<Pick<PatchProps, "offset" | "color">> &
- Pick<PatchProps, "opacity">;
+ Pick<PatchProps, "opacity"> & {
+ /** `<KeyframeTrack>` children. */
+ children?: SolidJSX.Element;
+ };
export type VideoProps = CommonProps &
Required<Pick<PatchProps, "src">> &
@@ -434,7 +496,10 @@ export type SurfaceProps = CommonProps & Pick<SurfacePaintProps, "ref">;
export type AudioProps = TimingProps &
Required<Pick<PatchProps, "src">> &
- Pick<PatchProps, "name" | "volume" | "muted" | "syncTo" | "animations">;
+ Pick<PatchProps, "name" | "volume" | "muted" | "syncTo" | "animations"> & {
+ /** `<KeyframeTrack>` children (a `volume` track). */
+ children?: SolidJSX.Element;
+ };
export type TextProps = CommonProps &
Pick<
diff --git a/packages/reconciler/src/document.ts b/packages/reconciler/src/document.ts
index 4815764e..19af9d05 100644
--- a/packages/reconciler/src/document.ts
+++ b/packages/reconciler/src/document.ts
@@ -30,6 +30,8 @@ import {
getParentNode,
isText,
ItemIndex,
+ Keyframe,
+ KeyframeTrack,
Muted,
Name,
Offset,
@@ -69,7 +71,7 @@ import {
} from '@diffusionstudio/runtime';
import { isPropValue, LOOP_ATTR, parseTime, SOURCE_ATTR } from '@diffusionstudio/jsx';
-import type { CameraMatrix } from '@diffusionstudio/runtime';
+import type { CameraMatrix, PropertyPath } from '@diffusionstudio/runtime';
import type { PropValue } from '@diffusionstudio/jsx';
import { trait, type Entity, type World } from 'koota';
import type { ProjectDocument } from './host';
@@ -205,6 +207,43 @@ const PAINT_TYPES: Record<string, PaintType> = {
radialGradientPaint: PaintType.RADIAL_GRADIENT,
};
+/**
+ * A `<keyframeTrack>`'s `property` (a prop name) as the runtime's property
+ * path. `width` depends on the holder: a stroke's is its line width.
+ */
+const TRACK_PROPERTIES: Record<string, PropertyPath> = {
+ x: 'position.x',
+ y: 'position.y',
+ offsetX: 'offset.x',
+ offsetY: 'offset.y',
+ width: 'width',
+ height: 'height',
+ rotation: 'rotation',
+ opacity: 'opacity',
+ cornerRadius: 'vertexRadius',
+ volume: 'volume',
+ color: 'color',
+ offset: 'stop.offset',
+ blur: 'blur',
+ value: 'effect.value',
+};
+
+/**
+ * Named easings as the descriptors the runtime (and the editor's
+ * interpolation inspector) speak; the descriptor forms pass through with
+ * their whitespace dropped, linear is the empty string.
+ */
+const EASINGS: Record<string, string> = {
+ linear: '',
+ easeIn: 'cubicBezier(0.42,0,1,1)',
+ easeOut: 'cubicBezier(0,0,0.58,1)',
+ easeInOut: 'cubicBezier(0.42,0,0.58,1)',
+ gentle: 'spring(0.5,628)',
+ snappy: 'spring(0.15,300)',
+ bouncy: 'spring(0.4,500)',
+ strong: 'spring(0.65,400)',
+};
+
const EFFECT_TYPES: Record<string, EffectType> = {
blur: EffectType.LAYER_BLUR,
brightness: EffectType.BRIGHTNESS,
@@ -410,9 +449,22 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
entity.set(Effect, { type: EffectType.LAYER_BLUR, value: 0 });
break;
}
+ case 'keyframeTrack': {
+ // `property` names the prop; the path it resolves to depends on the
+ // holder, so it is (re)resolved when the track is inserted.
+ entity = createEntity(this.world);
+ entity.add(KeyframeTrack);
+ break;
+ }
+ case 'keyframe': {
+ entity = createEntity(this.world);
+ entity.add(Keyframe);
+ entity.set(Keyframe, { easing: '' });
+ break;
+ }
default:
throw new Error(
- `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <colorStop>, <stroke>, <shadow> and <effect>).`,
+ `<${tag}> is not supported yet (only <stage>, <scene>, <group>, <sequence>, <rect>, <text>, <video>, <image>, <audio>, <solidPaint>, <linearGradientPaint>, <radialGradientPaint>, <colorStop>, <stroke>, <shadow>, <effect>, <keyframeTrack> and <keyframe>).`,
);
}
@@ -556,10 +608,33 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
return;
}
case 'value': {
+ if (entity.has(Keyframe)) {
+ // A number, or a color on a color track; either is a number to the trait.
+ entity.set(Keyframe, { value: toNumber(value) ?? parseColor(value) ?? 0 });
+ return;
+ }
if (!entity.has(Effect)) return;
entity.set(Effect, { value: toNumber(value) ?? 0 });
return;
}
+ case 'property': {
+ if (!entity.has(KeyframeTrack)) return;
+ this.resolveTrackProperty(entity, value);
+ return;
+ }
+ case 'time': {
+ if (!entity.has(Keyframe)) return;
+ const seconds = toSeconds(value);
+ const fps = this.world.get(FrameRate)?.value ?? 30;
+ entity.set(Keyframe, { time: seconds === undefined ? 0 : secondsToFrames(seconds, fps) });
+ return;
+ }
+ case 'easing': {
+ if (!entity.has(Keyframe)) return;
+ const easing = typeof value === 'string' ? value.replace(/\s+/g, '') : '';
+ entity.set(Keyframe, { easing: EASINGS[easing] ?? easing });
+ return;
+ }
case 'blur': {
const blur = toNumber(value);
if (blur === undefined) {
@@ -715,6 +790,18 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
}
}
+ /**
+ * Writes a track's runtime property path from its authored `property`,
+ * against the entity holding it (a `width` track under a stroke drives the
+ * line width). Called when the prop is set and again on insertion, since
+ * Solid sets props before the track has a parent.
+ */
+ private resolveTrackProperty(track: Entity, property: unknown): void {
+ let path = typeof property === 'string' ? TRACK_PROPERTIES[property] : undefined;
+ if (path === 'width' && getParentEntity(track)?.has(Stroke)) path = 'stroke.width';
+ track.set(KeyframeTrack, { property: path ?? '' });
+ }
+
/**
* Entity handles are minted per lookup, so every comparison here is between
* entities: two handles for the same node are never the same object. Text
@@ -760,6 +847,10 @@ export class RuntimeDocument implements ProjectDocument<HostNode> {
}
appendChild(this.world, node.entity, parent.entity);
+
+ if (node.entity.has(KeyframeTrack)) {
+ this.resolveTrackProperty(node.entity, node.entity.get(Authored)?.props.property);
+ }
}
const siblings = getEntityChildren(this.world, parent.entity).filter((sibling) => sibling !== node.entity);
diff --git a/packages/reconciler/src/elements.ts b/packages/reconciler/src/elements.ts
index 0c7c8151..1643c8b9 100644
--- a/packages/reconciler/src/elements.ts
+++ b/packages/reconciler/src/elements.ts
@@ -28,6 +28,8 @@ import type {
HtmlPaintProps,
HtmlProps,
ImageProps,
+ KeyframeProps,
+ KeyframeTrackProps,
RectProps,
SceneProps,
SequenceProps,
@@ -94,6 +96,8 @@ export const ColorStop = hostElement<ColorStopProps>("ColorStop");
export const Stroke = hostElement<StrokeProps>("Stroke");
export const Shadow = hostElement<ShadowProps>("Shadow");
export const Effect = hostElement<EffectProps>("Effect");
+export const KeyframeTrack = hostElement<KeyframeTrackProps>("KeyframeTrack");
+export const Keyframe = hostElement<KeyframeProps>("Keyframe");
export const HtmlPaint = hostElement<HtmlPaintProps>("HtmlPaint");
export const Html = hostElement<HtmlProps>("Html");
export const ShaderPaint = hostElement<ShaderPaintProps>("ShaderPaint");
diff --git a/packages/reconciler/src/patch.ts b/packages/reconciler/src/patch.ts
index 245607b7..7befec7c 100644
--- a/packages/reconciler/src/patch.ts
+++ b/packages/reconciler/src/patch.ts
@@ -29,6 +29,9 @@ export const PATCH_PROP_KEYS = Object.keys({
miterLimit: true,
type: true,
value: true,
+ property: true,
+ time: true,
+ easing: true,
start: true,
end: true,
sourceIn: true,
diff --git a/packages/runtime/MIGRATION.md b/packages/runtime/MIGRATION.md
index 8f0741d9..d05e71af 100644
--- a/packages/runtime/MIGRATION.md
+++ b/packages/runtime/MIGRATION.md
@@ -149,6 +149,8 @@ apps/web now depends on `@diffusionstudio/runtime` and `@diffusionstudio/koota-s
**`<effect>` is hosted** (`EffectProps` in `@diffusionstudio/jsx`, `effect` in `IntrinsicElements` and `COMPOSITION_TAGS`, `Effect` component in the reconciler): one element for every filter effect, `<effect type="blur" value={8} />`, a sub-entity child of any node (`<group>` included: the filter applies to the group's rendered pixels as a whole), stacking in document order like the CSS filter list `buildEffects` turns them into. `type` (`blur`/`brightness`/`contrast`/`grayscale`/`hueRotate`/`invert`/`saturate`/`sepia`, new in `PatchProps` as `EffectType`) maps onto the runtime's `EffectType` enum minus `DROP_SHADOW`, which is `<shadow>`'s; `value` (new in `PatchProps`, animatable: the `effect.value` keyframe target) is px for a blur, degrees for a hue rotation, 0-1 otherwise, and both are required, since the trait's zero value is meaningless for most types (a brightness of 0 is black). The recipe is the bare `Effect` trait, created as a blur of 0 (a no-op) until `type`/`value` arrive; an unknown `type` falls back to that too. Per-type tags (`<blur>`, `<sepia>`, ...) were considered and rejected: `blur` is already the name of a prop, and one tag with a `type` keeps the effect list a list.
+ **`<keyframeTrack>` and `<keyframe>` are hosted** (`KeyframeTrackProps`/`KeyframeProps` and `AnimatableProperty` in `@diffusionstudio/jsx`, both tags in `IntrinsicElements` and `COMPOSITION_TAGS`, `KeyframeTrack`/`Keyframe` components in the reconciler): the addressable form of an `Animatable` prop's keyframe list, so an editor moving a keyframe has an element (and id) to write it to. `<keyframeTrack property="x">` sits under the element whose prop it drives (any node, paint, color stop, stroke, shadow or effect: the runtime's `KeyframeTrack` is `ChildOf` its target) and holds `<keyframe time value easing />` children in any order (`Cache.keyframes` sorts by frame). `property` is the prop's JSX name and resolves through `TRACK_PROPERTIES` to the runtime's `PropertyPath` (`x` -> `position.x`, `cornerRadius` -> `vertexRadius`, `offset` -> `stop.offset`, `value` -> `effect.value`, ...); `width` under a `Stroke` is `stroke.width`, so the path is (re)resolved on insertion, since Solid sets props before the track has a parent (`resolveTrackProperty`). `time` is node-local and goes through `parseTime` + `secondsToFrames` like the timing props; `value` is a number, or a CSS color on a color track (`parseColor`); `easing` maps the named presets to the descriptors the runtime's `resolveEasing` and the interpolation inspector speak (`easeIn` -> `cubicBezier(0.42,0,1,1)`, `gentle` -> `spring(0.5,628)`, ..., `linear` -> `""`; descriptors pass through with whitespace dropped). `PatchProps` gained `property`, `time`, `easing`, and `value` widened to `Animatable<number> | string` (`EffectProps` narrows it back). One runtime fix came with it (`actions/cache.ts`): `rebuildCaches` re-aggregates a node's `Cache.keyframeTracks` not only when a track is attached but also when a non-node sub-entity carrying tracks is (`carriesKeyframeTrack`), because JSX builds bottom-up: a paint's tracks are attached to the paint before the paint meets its rect, and the old (bitecs) code only ever saw tracks attached to already-placed targets. Not decided yet: whether the inline `Animatable<T>` keyframe arrays stay (the host does not expand them; a static value is what `setProperty` handles) or the tags become the only spelling.
+
**Left stale on purpose** (deferred to a docs/examples pass): all eight `examples/*.tsx` still open with `<rect scene="...">` and no `<stage>`, so `tsc -p examples --noEmit` fails on them; `reference/jsx/{scene,roots}.md` still describes the pre-`<stage>` model where a mount grafts roots into an existing document; and `reference/jsx/{audio-sync,video,audio,elements}.md` still document `key` as the `syncTo` target. Deliberately not half-migrated: `reference/jsx` is stale as a whole, and patching the `key` rows alone would hide that.
## Remaining work (suggested order)
diff --git a/packages/runtime/src/actions/cache.ts b/packages/runtime/src/actions/cache.ts
index c70aaa33..a0ce2ab5 100644
--- a/packages/runtime/src/actions/cache.ts
+++ b/packages/runtime/src/actions/cache.ts
@@ -48,7 +48,10 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul
.sort(sortByItemIndex);
}
- if (entity.has(KeyframeTrack)) {
+ // A track attached directly, or carried by a sub-entity (a paint, stroke,
+ // effect...) that is attached with tracks already under it: JSX builds
+ // bottom-up, so a paint's tracks exist before the paint meets its node.
+ if (entity.has(KeyframeTrack) || (!isNodeEntity(entity) && carriesKeyframeTrack(world, entity))) {
const node = findClosestParentGeometry(parent);
aggregateKeyframeTracks(world, node, exclude);
resetAnimatedValues(world, node);
@@ -94,11 +97,25 @@ export function rebuildCaches(world: World, entity: Entity, parent: Entity | nul
}
}
+function isNodeEntity(entity: Entity): boolean {
+ return entity.has(Geometry) || entity.has(Group) || entity.has(AdjustmentLayer);
+}
+
+/** Whether a KeyframeTrack sits under `entity`, short of any nested node. */
+function carriesKeyframeTrack(world: World, entity: Entity): boolean {
+ for (const child of world.query(ChildOf(entity))) {
+ if (child.has(KeyframeTrack)) return true;
+ if (isNodeEntity(child)) continue;
+ if (carriesKeyframeTrack(world, child)) return true;
+ }
+ return false;
+}
+
/** Nearest self-or-ancestor node entity (geometry, group, adjustment layer). */
export function findClosestParentGeometry(entity: Entity): Entity | null {
let current: Entity | null = entity;
while (current) {
- if (current.has(Geometry) || current.has(Group) || current.has(AdjustmentLayer)) {
+ if (isNodeEntity(current)) {
return current;
}
current = getParentNode(current);
@@ -128,7 +145,7 @@ export function aggregateKeyframeTracks(world: World, node: Entity | null, exclu
continue;
}
// Stop at nested nodes; each owns its own cache entry.
- if (child.has(Geometry) || child.has(Group) || child.has(AdjustmentLayer)) continue;
+ if (isNodeEntity(child)) continue;
walk(child);
}
};