Skip to content

Code-first video & image, rendered natively on WebGPU.

feat(types): phase 3 schema and pipeline unification with generic node operations

okanasl9674ab657f10
diff --git a/packages/compositions/src/program/schema.ts b/packages/compositions/src/program/schema.ts
index 5b9d1d53..9c191ada 100644
--- a/packages/compositions/src/program/schema.ts
+++ b/packages/compositions/src/program/schema.ts
@@ -14,10 +14,11 @@
* - no backwards compatibility: v1 shapes fail fast with E-codes
*/
-import type {
- MeshAudioDeformConfig,
- TextSpan,
- TextSpanMark,
+import {
+ Effect,
+ type MeshAudioDeformConfig,
+ type TextSpan,
+ type TextSpanMark,
} from "@framefields/core";
import { ColorSchema } from "@framefields/node-sdk";
export type { TextSpan, TextSpanMark };
@@ -59,6 +60,33 @@ export const ShapeTypeSchema = z.enum([
]);
export type ShapeType = z.infer<typeof ShapeTypeSchema>;
+export interface EffectSpec<TOp extends string = string> {
+ op: TOp;
+ [key: string]: unknown;
+}
+
+export type ProgramEffect =
+ | Effect<object>
+ | EffectSpec
+ | (Record<string, unknown> & { op: string });
+
+export const EffectSpecSchema = z.union([
+ z.instanceof(Effect),
+ z
+ .object({
+ op: z.string().min(1),
+ })
+ .passthrough(),
+ z.custom<Effect<object>>((val) =>
+ Boolean(
+ val &&
+ typeof val === "object" &&
+ "op" in val &&
+ typeof (val as { op: unknown }).op === "string",
+ ),
+ ),
+]);
+
export const AnimatablePropSchema = z.enum([
"x",
"y",
@@ -526,7 +554,7 @@ const BaseNodeFields = {
opacity: z.number().min(0).max(1).default(1),
blendMode: z.string().optional(),
/** Per-node WebGPU effect chain (`layer.withEffect(s)`); resolved to operations in to-virtual-media. */
- effects: z.array(z.unknown()).optional(),
+ effects: z.array(EffectSpecSchema).optional(),
rotation: z.number().default(0),
scale: z.number().min(0).default(1),
/** Transform pivot, 0–1 fraction of the node box (0.5 = center). */
@@ -671,6 +699,10 @@ export interface CompositionNodeBase {
durationFrames?: number;
motionBlurShutter?: number;
animation?: LayerAnimationSpec;
+ /** Attached effect chain. */
+ effects?: ProgramEffect[];
+ /** Crop options attached directly to the node. */
+ crop?: unknown;
vision?: unknown;
relighting?: unknown;
}
@@ -1602,7 +1634,7 @@ export const CompositorProgramSchema = z
fonts: z.array(z.string()).optional(),
signals: z.record(z.string(), z.unknown()).optional(),
/** Composition-wide effect chain (`comp.apply(effect)`); resolved to operations downstream. */
- effects: z.array(z.unknown()).optional(),
+ effects: z.array(EffectSpecSchema).optional(),
/** Composition-wide vision config or bundle. */
vision: z.unknown().optional(),
/** Per-frame notification hook. */
diff --git a/packages/compositions/src/program/to-virtual-media.ts b/packages/compositions/src/program/to-virtual-media.ts
index bc70e23b..6d60e170 100644
--- a/packages/compositions/src/program/to-virtual-media.ts
+++ b/packages/compositions/src/program/to-virtual-media.ts
@@ -15,12 +15,17 @@
import {
createVirtualMedia,
type DataType,
+ type FrameContext,
getActiveMediaMetadata,
type VirtualMediaData,
} from "@framefields/core";
import { collectInlineSignals } from "./inline-signals.js";
import { hoistMattes } from "./mattes.js";
-import type { CompositorProgramConfig, LayoutNode } from "./schema.js";
+import type {
+ CompositorProgramConfig,
+ LayoutNode,
+ ProgramEffect,
+} from "./schema.js";
import { computeStaggerDelay } from "./stagger.js";
@@ -30,9 +35,15 @@ export interface CompositorToProgramOptions {
/** Comp duration in ms (metadata only; derived upstream). */
durationMs?: number;
/** Bind a media node's `inputHandleId` to its graph source item. */
- resolveMedia?: (inputHandleId: string) => VirtualMediaData | undefined;
+ resolveMedia?: (inputHandleId?: string) => VirtualMediaData | undefined;
/** External signals mapped by inputHandleId. */
- signals?: Record<string, any>;
+ signals?: Record<string, unknown>;
+ /** Per-frame notification hook. */
+ onRequestFrame?: ((ctx: FrameContext) => void) | unknown;
+ /** Composition-wide effect chain (`comp.apply(effect)`). */
+ effects?: ProgramEffect[];
+ /** Composition-wide vision config or bundle. */
+ vision?: unknown;
}
function nodeToVirtualMedia(
@@ -44,7 +55,10 @@ function nodeToVirtualMedia(
inheritedDurationFrames?: number,
): VirtualMediaData {
let children: VirtualMediaData[] = [];
- const sourceRef = (node as any).src ?? (node as any).inputHandleId;
+ const sourceRef =
+ node.kind === "media"
+ ? (node.src ?? node.inputHandleId)
+ : node.inputHandleId;
const isDirectMediaSource = (s: unknown) =>
typeof s === "string" &&
(s.startsWith("http://") ||
@@ -60,13 +74,13 @@ function nodeToVirtualMedia(
let media: VirtualMediaData | undefined;
if (node.kind === "media") {
- if (resolveMedia) {
+ if (resolveMedia && node.inputHandleId) {
media = resolveMedia(node.inputHandleId);
}
if (
!media &&
sourceRef &&
- (isDirectMediaSource(sourceRef) || (node as any).src)
+ (isDirectMediaSource(sourceRef) || Boolean(node.src))
) {
media = createVirtualMedia(
sourceRef,
@@ -83,8 +97,10 @@ function nodeToVirtualMedia(
`[Compositor] Media layer '${node.id ?? "?"}' source '${sourceRef}' could not be resolved.`,
);
}
- if (media && (node as any).crop) {
- const cropRaw = (node as any).crop;
+ if (media && node.crop) {
+ const cropRaw = node.crop as
+ | { toOperation?: () => Record<string, unknown> }
+ | Record<string, unknown>;
const cropOp =
typeof cropRaw.toOperation === "function"
? cropRaw.toOperation()
@@ -97,28 +113,33 @@ function nodeToVirtualMedia(
dataType: media.operation.dataType,
},
children: [media],
- } as unknown as VirtualMediaData;
+ };
}
- if (media && Array.isArray((node as any).effects)) {
- for (const eff of (node as any).effects) {
+ if (media && Array.isArray(node.effects)) {
+ for (const eff of node.effects) {
const effOp =
- typeof eff.toOperation === "function" ? eff.toOperation() : eff;
+ eff &&
+ typeof (eff as { toOperation?: unknown }).toOperation === "function"
+ ? (
+ eff as { toOperation: () => Record<string, unknown> }
+ ).toOperation()
+ : (eff as Record<string, unknown>);
media = {
metadata: { ...media.metadata },
operation: {
- id: (node as any).id
- ? `${(node as any).id}-${effOp.op ?? "effect"}`
- : undefined,
+ id: node.id ? `${node.id}-${effOp.op ?? "effect"}` : undefined,
effect: eff,
...effOp,
dataType: media.operation.dataType,
},
children: [media],
- } as unknown as VirtualMediaData;
+ };
}
}
- if (media && (node as any).vision) {
- const visionRaw = (node as any).vision;
+ if (media && node.vision) {
+ const visionRaw = node.vision as
+ | { toOperation?: () => Record<string, unknown> }
+ | Record<string, unknown>;
const visionOp =
typeof visionRaw.toOperation === "function"
? visionRaw.toOperation()
@@ -127,13 +148,13 @@ function nodeToVirtualMedia(
metadata: { ...media.metadata },
operation: {
op: "Vision",
- id: (node as any).id,
+ id: node.id,
effect: visionRaw,
...visionOp,
dataType: media.operation.dataType,
},
children: [media],
- } as unknown as VirtualMediaData;
+ };
}
}
@@ -141,13 +162,12 @@ function nodeToVirtualMedia(
children = media ? [media] : [];
} else if ("children" in node && node.children) {
const staggerFrames =
- "staggerFrames" in node && typeof (node as any).staggerFrames === "number"
- ? (node as any).staggerFrames
+ node.kind === "flex" && typeof node.staggerFrames === "number"
+ ? node.staggerFrames
: 0;
const staggerDirection =
- "staggerDirection" in node &&
- typeof (node as any).staggerDirection === "string"
- ? (node as any).staggerDirection
+ node.kind === "flex" && typeof node.staggerDirection === "string"
+ ? node.staggerDirection
: "forward";
const nodeStart =
@@ -159,10 +179,11 @@ function nodeToVirtualMedia(
Number.isFinite(node.durationFrames)
? Math.round(node.durationFrames)
: inheritedDurationFrames;
- children = node.children.map((c, idx) => {
+ const nodeChildren = node.children;
+ children = nodeChildren.map((c, idx) => {
const delay = computeStaggerDelay(
idx,
- node.children!.length,
+ nodeChildren.length,
staggerFrames,
staggerDirection,
);
@@ -219,8 +240,8 @@ function nodeToVirtualMedia(
if (dur === undefined) {
const childEnds = children.map(
(c) =>
- ((c.operation as any)?.startFrame ?? 0) +
- ((c.operation as any)?.durationFrames ?? defaultCompDuration),
+ (c.operation.startFrame ?? 0) +
+ (c.operation.durationFrames ?? defaultCompDuration),
);
dur = Math.max(1, Math.max(...childEnds) - startFrame);
}
@@ -317,27 +338,30 @@ export function compositorToProgram(
// their tracks; fold them in under their generated handle ids.
signals: {
...collectInlineSignals(config.layout),
- ...(signals ?? (config as any).signals ?? {}),
+ ...(signals ?? config.signals ?? {}),
},
- ...(((config as any).onRequestFrame ||
- (options as any).onRequestFrame) && {
- onRequestFrame:
- (config as any).onRequestFrame ?? (options as any).onRequestFrame,
- }),
+ ...(config.onRequestFrame || options.onRequestFrame
+ ? {
+ onRequestFrame: config.onRequestFrame ?? options.onRequestFrame,
+ }
+ : {}),
},
children: hoistMattes(config.layout).map((n) =>
nodeToVirtualMedia(n, resolveMedia, fps, 0, durationMs),
),
} as unknown as VirtualMediaData;
- const effectsList: any[] = Array.isArray((options as any).effects)
- ? (options as any).effects
- : Array.isArray((config as any).effects)
- ? (config as any).effects
+ const effectsList: ProgramEffect[] = Array.isArray(options.effects)
+ ? options.effects
+ : Array.isArray(config.effects)
+ ? config.effects
: [];
for (const eff of effectsList) {
const effOp =
- typeof eff.toOperation === "function" ? eff.toOperation() : eff;
+ eff &&
+ typeof (eff as { toOperation?: unknown }).toOperation === "function"
+ ? (eff as { toOperation: () => Record<string, unknown> }).toOperation()
+ : (eff as Record<string, unknown>);
rootMedia = {
metadata: { ...rootMedia.metadata },
operation: {
@@ -348,12 +372,14 @@ export function compositorToProgram(
} as unknown as VirtualMediaData;
}
- const visionRaw = (config as any).vision ?? (options as any).vision;
+ const visionRaw = config.vision ?? options.vision;
if (visionRaw) {
const visionOp =
- typeof visionRaw.toOperation === "function"
- ? visionRaw.toOperation()
- : visionRaw;
+ typeof (visionRaw as { toOperation?: unknown }).toOperation === "function"
+ ? (
+ visionRaw as { toOperation: () => Record<string, unknown> }
+ ).toOperation()
+ : (visionRaw as Record<string, unknown>);
rootMedia = {
metadata: { ...rootMedia.metadata },
operation: {
diff --git a/packages/core/src/effects/index.ts b/packages/core/src/effects/index.ts
index 69e2c153..01b46a4a 100644
--- a/packages/core/src/effects/index.ts
+++ b/packages/core/src/effects/index.ts
@@ -6,6 +6,7 @@ import {
signal,
type WritableSignal,
} from "../signals/index.js";
+import type { NodeOperation } from "../types/index.js";
export type EffectConfigValue<T> =
| T
@@ -126,7 +127,7 @@ export abstract class Effect<
/**
* Converts the effect into an operation node for the VirtualMediaData AST.
*/
- public toOperation(): Record<string, unknown> {
+ public toOperation(): NodeOperation<TOp, TConfig> {
const opPayload: Record<string, unknown> = {
op: this.op,
effect: this,
@@ -148,6 +149,6 @@ export abstract class Effect<
}
}
- return opPayload;
+ return opPayload as NodeOperation<TOp, TConfig>;
}
}
diff --git a/packages/core/src/types/video/virtual-video.ts b/packages/core/src/types/video/virtual-video.ts
index e75ef8a8..168de410 100644
--- a/packages/core/src/types/video/virtual-video.ts
+++ b/packages/core/src/types/video/virtual-video.ts
@@ -1,4 +1,5 @@
import { z } from "zod";
+import type { FrameContext } from "../../signals/index.js";
import { type DataType, DataTypeVal, FileAssetSchema } from "../base.js";
export const MediaSourceSchema = z.object({
@@ -46,6 +47,7 @@ export const BaseMediaOperationPropsSchema = z.object({
volume: z.number().optional(),
opacity: z.number().optional(),
startFrame: z.number().optional(),
+ durationFrames: z.number().optional(),
timeline: TimelineTransformSchema.optional(),
});
@@ -53,6 +55,28 @@ export type BaseMediaOperationProps = z.infer<
typeof BaseMediaOperationPropsSchema
>;
+export type NodeInputConnection = {
+ connectionValid: boolean;
+ outputItem: {
+ type: string;
+ data: unknown;
+ } | null;
+};
+
+export type NodeOperation<
+ TOp extends string = string,
+ TConfig extends object = Record<string, unknown>,
+> = BaseMediaOperationProps & {
+ op: TOp;
+ id?: string;
+ metadata?: MediaMetadata;
+ dataType?: DataType;
+ effect?: unknown;
+ signals?: Record<string, unknown>;
+ inputs?: Record<string, NodeInputConnection>;
+ onRequestFrame?: (ctx: FrameContext) => void;
+} & TConfig;
+
/** Original source file (leaf node) */
export const SourceOperationSchema = BaseMediaOperationPropsSchema.extend({
op: z.literal("source"),
@@ -84,18 +108,16 @@ export type MediaOperation = BaseMediaOperationProps &
(
| ({ op: "source" } & z.infer<typeof SourceOperationSchema>)
| ({ op: "text" } & z.infer<typeof TextOperationSchema>)
- | ({
- op: string;
- metadata?: MediaMetadata;
- dataType: DataType;
- } & Record<string, any>)
+ | NodeOperation<string, Record<string, unknown>>
);
// --- VirtualMediaData: THE recursive data type for all media outputs ---
-export type VirtualMediaData = {
+export type VirtualMediaData<
+ TOperation extends MediaOperation = MediaOperation,
+> = {
metadata: MediaMetadata;
- operation: MediaOperation;
+ operation: TOperation;
children: VirtualMediaData[];
};
diff --git a/packages/framefields/src/type-safety.test.ts b/packages/framefields/src/type-safety.test.ts
index 3ff2ff78..02929606 100644
--- a/packages/framefields/src/type-safety.test.ts
+++ b/packages/framefields/src/type-safety.test.ts
@@ -1,3 +1,7 @@
+import {
+ CompositorProgramSchema,
+ EffectSpecSchema,
+} from "@framefields/compositions/program";
import { describe, expect, expectTypeOf, it } from "vitest";
import {
type AnimatableNode,
@@ -26,6 +30,7 @@ import {
ms,
type NodeCapability,
type NodeId,
+ type NodeOperation,
type Normalized,
nodeId,
normalized,
@@ -33,6 +38,7 @@ import {
type SectionOptions,
seconds,
Vignette,
+ type VirtualMediaData,
} from "./index.js";
describe("Type Safety & Branded Types (Phase 1)", () => {
@@ -177,3 +183,71 @@ describe("Discriminated Effects & Port Capabilities (Phase 2)", () => {
expect(img2.effects?.length).toBe(2);
});
});
+
+describe("Schema & Pipeline Unification (Phase 3)", () => {
+ it("strongly types NodeOperation from effect toOperation", () => {
+ const blur = new Blur({ strength: 10 });
+ const op = blur.toOperation();
+
+ expect(op.op).toBe("Blur");
+ expectTypeOf(op.op).toEqualTypeOf<"Blur">();
+ expectTypeOf(op).toMatchTypeOf<NodeOperation<"Blur">>();
+ });
+
+ it("validates effects via EffectSpecSchema and CompositorProgramSchema", () => {
+ // Valid Effect class instance
+ const effectInstance = new FilmGrain({ strength: 12 });
+ expect(EffectSpecSchema.safeParse(effectInstance).success).toBe(true);
+
+ // Valid plain effect spec object
+ const plainSpec = { op: "Blur", strength: 8 };
+ expect(EffectSpecSchema.safeParse(plainSpec).success).toBe(true);
+
+ // Invalid effect without op
+ const invalidSpec = { strength: 8 };
+ expect(EffectSpecSchema.safeParse(invalidSpec).success).toBe(false);
+
+ // Valid compositor program with effects
+ const programWithEffects = {
+ width: 1920,
+ height: 1080,
+ fps: 30,
+ layout: [
+ {
+ id: "hero",
+ kind: "box",
+ effects: [plainSpec, effectInstance],
+ },
+ ],
+ effects: [plainSpec],
+ };
+ expect(CompositorProgramSchema.safeParse(programWithEffects).success).toBe(
+ true,
+ );
+
+ // Invalid compositor program with invalid effect
+ const programWithBadEffect = {
+ width: 1920,
+ height: 1080,
+ fps: 30,
+ layout: [
+ {
+ id: "hero",
+ kind: "box",
+ effects: [invalidSpec],
+ },
+ ],
+ };
+ expect(
+ CompositorProgramSchema.safeParse(programWithBadEffect).success,
+ ).toBe(false);
+ });
+
+ it("propagates typed operations in VirtualMediaData without any", () => {
+ type BlurMedia = VirtualMediaData<
+ NodeOperation<"Blur", { strength: number }>
+ >;
+ expectTypeOf<BlurMedia["operation"]["op"]>().toEqualTypeOf<"Blur">();
+ expectTypeOf<BlurMedia["operation"]["strength"]>().toEqualTypeOf<number>();
+ });
+});
diff --git a/packages/node-sdk/renderer/index.ts b/packages/node-sdk/renderer/index.ts
index 7901bf85..17df8a31 100644
--- a/packages/node-sdk/renderer/index.ts
+++ b/packages/node-sdk/renderer/index.ts
@@ -1,5 +1,10 @@
/// <reference types="@webgpu/types" />
-import type { VirtualMediaData } from "@framefields/core";
+import type {
+ MediaOperation,
+ NodeInputConnection,
+ NodeOperation,
+ VirtualMediaData,
+} from "@framefields/core";
import type { RenderContextValue } from "@framefields/webgpu-renderers";
export type GPUCommandEncoder = globalThis.GPUCommandEncoder;
@@ -7,9 +12,13 @@ export type GPURenderPassEncoder = globalThis.GPURenderPassEncoder;
export type GPUTextureView = globalThis.GPUTextureView;
export type GPUTexture = globalThis.GPUTexture;
-export interface NodeRenderProps {
+export type { MediaOperation, NodeInputConnection, NodeOperation };
+
+export interface NodeRenderProps<
+ TOperation extends MediaOperation = MediaOperation,
+> {
renderId: string;
- virtualMedia: VirtualMediaData;
+ virtualMedia: VirtualMediaData<TOperation>;
volume?: number;
playbackRateOverride?: number;
trimStartOverride?: number;
@@ -39,7 +48,9 @@ export interface NodeRenderProps {
) => unknown;
}
-export type WebGPUNodeRenderer = (args: {
+export type WebGPUNodeRenderer<
+ TOperation extends MediaOperation = MediaOperation,
+> = (args: {
ctx: RenderContextValue;
encoder: GPUCommandEncoder;
pass: GPURenderPassEncoder;
@@ -47,7 +58,7 @@ export type WebGPUNodeRenderer = (args: {
targetTexture: GPUTexture;
targetWidth: number;
targetHeight: number;
- props: NodeRenderProps;
+ props: NodeRenderProps<TOperation>;
drawChild: (
child: VirtualMediaData,
overrides?: Partial<NodeRenderProps>,
@@ -68,12 +79,13 @@ export interface AudioProcessorContext {
renderId?: string;
}
-export type AudioProcessor = (
- channels: Float32Array[],
- sampleRate: number,
- virtualMedia: VirtualMediaData,
- ctx?: AudioProcessorContext,
-) => void | Promise<void>;
+export type AudioProcessor<TOperation extends MediaOperation = MediaOperation> =
+ (
+ channels: Float32Array[],
+ sampleRate: number,
+ virtualMedia: VirtualMediaData<TOperation>,
+ ctx?: AudioProcessorContext,
+ ) => void | Promise<void>;
export {
isSignal,