Code-first video & image, rendered natively on WebGPU.
feat(types): phase 3 schema and pipeline unification with generic node operations
okanasl9674ab657f10
diff --git a/packages/compositions/src/program/schema.ts b/packages/compositions/src/program/schema.tsindex 5b9d1d53..9c191ada 100644--- a/packages/compositions/src/program/schema.ts+++ b/packages/compositions/src/program/schema.ts@@ -14,10 +14,11 @@* - no backwards compatibility: v1 shapes fail fast with E-codes*/-import type {- MeshAudioDeformConfig,- TextSpan,- TextSpanMark,+import {+ Effect,+ type MeshAudioDeformConfig,+ type TextSpan,+ type TextSpanMark,} from "@framefields/core";import { ColorSchema } from "@framefields/node-sdk";export type { TextSpan, TextSpanMark };@@ -59,6 +60,33 @@ export const ShapeTypeSchema = z.enum([]);export type ShapeType = z.infer<typeof ShapeTypeSchema>;+export interface EffectSpec<TOp extends string = string> {+ op: TOp;+ [key: string]: unknown;+}++export type ProgramEffect =+ | Effect<object>+ | EffectSpec+ | (Record<string, unknown> & { op: string });++export const EffectSpecSchema = z.union([+ z.instanceof(Effect),+ z+ .object({+ op: z.string().min(1),+ })+ .passthrough(),+ z.custom<Effect<object>>((val) =>+ Boolean(+ val &&+ typeof val === "object" &&+ "op" in val &&+ typeof (val as { op: unknown }).op === "string",+ ),+ ),+]);+export const AnimatablePropSchema = z.enum(["x","y",@@ -526,7 +554,7 @@ const BaseNodeFields = {opacity: z.number().min(0).max(1).default(1),blendMode: z.string().optional(),/** Per-node WebGPU effect chain (`layer.withEffect(s)`); resolved to operations in to-virtual-media. */- effects: z.array(z.unknown()).optional(),+ effects: z.array(EffectSpecSchema).optional(),rotation: z.number().default(0),scale: z.number().min(0).default(1),/** Transform pivot, 0–1 fraction of the node box (0.5 = center). */@@ -671,6 +699,10 @@ export interface CompositionNodeBase {durationFrames?: number;motionBlurShutter?: number;animation?: LayerAnimationSpec;+ /** Attached effect chain. */+ effects?: ProgramEffect[];+ /** Crop options attached directly to the node. */+ crop?: unknown;vision?: unknown;relighting?: unknown;}@@ -1602,7 +1634,7 @@ export const CompositorProgramSchema = zfonts: z.array(z.string()).optional(),signals: z.record(z.string(), z.unknown()).optional(),/** Composition-wide effect chain (`comp.apply(effect)`); resolved to operations downstream. */- effects: z.array(z.unknown()).optional(),+ effects: z.array(EffectSpecSchema).optional(),/** Composition-wide vision config or bundle. */vision: z.unknown().optional(),/** Per-frame notification hook. */diff --git a/packages/compositions/src/program/to-virtual-media.ts b/packages/compositions/src/program/to-virtual-media.tsindex bc70e23b..6d60e170 100644--- a/packages/compositions/src/program/to-virtual-media.ts+++ b/packages/compositions/src/program/to-virtual-media.ts@@ -15,12 +15,17 @@import {createVirtualMedia,type DataType,+ type FrameContext,getActiveMediaMetadata,type VirtualMediaData,} from "@framefields/core";import { collectInlineSignals } from "./inline-signals.js";import { hoistMattes } from "./mattes.js";-import type { CompositorProgramConfig, LayoutNode } from "./schema.js";+import type {+ CompositorProgramConfig,+ LayoutNode,+ ProgramEffect,+} from "./schema.js";import { computeStaggerDelay } from "./stagger.js";@@ -30,9 +35,15 @@ export interface CompositorToProgramOptions {/** Comp duration in ms (metadata only; derived upstream). */durationMs?: number;/** Bind a media node's `inputHandleId` to its graph source item. */- resolveMedia?: (inputHandleId: string) => VirtualMediaData | undefined;+ resolveMedia?: (inputHandleId?: string) => VirtualMediaData | undefined;/** External signals mapped by inputHandleId. */- signals?: Record<string, any>;+ signals?: Record<string, unknown>;+ /** Per-frame notification hook. */+ onRequestFrame?: ((ctx: FrameContext) => void) | unknown;+ /** Composition-wide effect chain (`comp.apply(effect)`). */+ effects?: ProgramEffect[];+ /** Composition-wide vision config or bundle. */+ vision?: unknown;}function nodeToVirtualMedia(@@ -44,7 +55,10 @@ function nodeToVirtualMedia(inheritedDurationFrames?: number,): VirtualMediaData {let children: VirtualMediaData[] = [];- const sourceRef = (node as any).src ?? (node as any).inputHandleId;+ const sourceRef =+ node.kind === "media"+ ? (node.src ?? node.inputHandleId)+ : node.inputHandleId;const isDirectMediaSource = (s: unknown) =>typeof s === "string" &&(s.startsWith("http://") ||@@ -60,13 +74,13 @@ function nodeToVirtualMedia(let media: VirtualMediaData | undefined;if (node.kind === "media") {- if (resolveMedia) {+ if (resolveMedia && node.inputHandleId) {media = resolveMedia(node.inputHandleId);}if (!media &&sourceRef &&- (isDirectMediaSource(sourceRef) || (node as any).src)+ (isDirectMediaSource(sourceRef) || Boolean(node.src))) {media = createVirtualMedia(sourceRef,@@ -83,8 +97,10 @@ function nodeToVirtualMedia(`[Compositor] Media layer '${node.id ?? "?"}' source '${sourceRef}' could not be resolved.`,);}- if (media && (node as any).crop) {- const cropRaw = (node as any).crop;+ if (media && node.crop) {+ const cropRaw = node.crop as+ | { toOperation?: () => Record<string, unknown> }+ | Record<string, unknown>;const cropOp =typeof cropRaw.toOperation === "function"? cropRaw.toOperation()@@ -97,28 +113,33 @@ function nodeToVirtualMedia(dataType: media.operation.dataType,},children: [media],- } as unknown as VirtualMediaData;+ };}- if (media && Array.isArray((node as any).effects)) {- for (const eff of (node as any).effects) {+ if (media && Array.isArray(node.effects)) {+ for (const eff of node.effects) {const effOp =- typeof eff.toOperation === "function" ? eff.toOperation() : eff;+ eff &&+ typeof (eff as { toOperation?: unknown }).toOperation === "function"+ ? (+ eff as { toOperation: () => Record<string, unknown> }+ ).toOperation()+ : (eff as Record<string, unknown>);media = {metadata: { ...media.metadata },operation: {- id: (node as any).id- ? `${(node as any).id}-${effOp.op ?? "effect"}`- : undefined,+ id: node.id ? `${node.id}-${effOp.op ?? "effect"}` : undefined,effect: eff,...effOp,dataType: media.operation.dataType,},children: [media],- } as unknown as VirtualMediaData;+ };}}- if (media && (node as any).vision) {- const visionRaw = (node as any).vision;+ if (media && node.vision) {+ const visionRaw = node.vision as+ | { toOperation?: () => Record<string, unknown> }+ | Record<string, unknown>;const visionOp =typeof visionRaw.toOperation === "function"? visionRaw.toOperation()@@ -127,13 +148,13 @@ function nodeToVirtualMedia(metadata: { ...media.metadata },operation: {op: "Vision",- id: (node as any).id,+ id: node.id,effect: visionRaw,...visionOp,dataType: media.operation.dataType,},children: [media],- } as unknown as VirtualMediaData;+ };}}@@ -141,13 +162,12 @@ function nodeToVirtualMedia(children = media ? [media] : [];} else if ("children" in node && node.children) {const staggerFrames =- "staggerFrames" in node && typeof (node as any).staggerFrames === "number"- ? (node as any).staggerFrames+ node.kind === "flex" && typeof node.staggerFrames === "number"+ ? node.staggerFrames: 0;const staggerDirection =- "staggerDirection" in node &&- typeof (node as any).staggerDirection === "string"- ? (node as any).staggerDirection+ node.kind === "flex" && typeof node.staggerDirection === "string"+ ? node.staggerDirection: "forward";const nodeStart =@@ -159,10 +179,11 @@ function nodeToVirtualMedia(Number.isFinite(node.durationFrames)? Math.round(node.durationFrames): inheritedDurationFrames;- children = node.children.map((c, idx) => {+ const nodeChildren = node.children;+ children = nodeChildren.map((c, idx) => {const delay = computeStaggerDelay(idx,- node.children!.length,+ nodeChildren.length,staggerFrames,staggerDirection,);@@ -219,8 +240,8 @@ function nodeToVirtualMedia(if (dur === undefined) {const childEnds = children.map((c) =>- ((c.operation as any)?.startFrame ?? 0) +- ((c.operation as any)?.durationFrames ?? defaultCompDuration),+ (c.operation.startFrame ?? 0) ++ (c.operation.durationFrames ?? defaultCompDuration),);dur = Math.max(1, Math.max(...childEnds) - startFrame);}@@ -317,27 +338,30 @@ export function compositorToProgram(// their tracks; fold them in under their generated handle ids.signals: {...collectInlineSignals(config.layout),- ...(signals ?? (config as any).signals ?? {}),+ ...(signals ?? config.signals ?? {}),},- ...(((config as any).onRequestFrame ||- (options as any).onRequestFrame) && {- onRequestFrame:- (config as any).onRequestFrame ?? (options as any).onRequestFrame,- }),+ ...(config.onRequestFrame || options.onRequestFrame+ ? {+ onRequestFrame: config.onRequestFrame ?? options.onRequestFrame,+ }+ : {}),},children: hoistMattes(config.layout).map((n) =>nodeToVirtualMedia(n, resolveMedia, fps, 0, durationMs),),} as unknown as VirtualMediaData;- const effectsList: any[] = Array.isArray((options as any).effects)- ? (options as any).effects- : Array.isArray((config as any).effects)- ? (config as any).effects+ const effectsList: ProgramEffect[] = Array.isArray(options.effects)+ ? options.effects+ : Array.isArray(config.effects)+ ? config.effects: [];for (const eff of effectsList) {const effOp =- typeof eff.toOperation === "function" ? eff.toOperation() : eff;+ eff &&+ typeof (eff as { toOperation?: unknown }).toOperation === "function"+ ? (eff as { toOperation: () => Record<string, unknown> }).toOperation()+ : (eff as Record<string, unknown>);rootMedia = {metadata: { ...rootMedia.metadata },operation: {@@ -348,12 +372,14 @@ export function compositorToProgram(} as unknown as VirtualMediaData;}- const visionRaw = (config as any).vision ?? (options as any).vision;+ const visionRaw = config.vision ?? options.vision;if (visionRaw) {const visionOp =- typeof visionRaw.toOperation === "function"- ? visionRaw.toOperation()- : visionRaw;+ typeof (visionRaw as { toOperation?: unknown }).toOperation === "function"+ ? (+ visionRaw as { toOperation: () => Record<string, unknown> }+ ).toOperation()+ : (visionRaw as Record<string, unknown>);rootMedia = {metadata: { ...rootMedia.metadata },operation: {diff --git a/packages/core/src/effects/index.ts b/packages/core/src/effects/index.tsindex 69e2c153..01b46a4a 100644--- a/packages/core/src/effects/index.ts+++ b/packages/core/src/effects/index.ts@@ -6,6 +6,7 @@ import {signal,type WritableSignal,} from "../signals/index.js";+import type { NodeOperation } from "../types/index.js";export type EffectConfigValue<T> =| T@@ -126,7 +127,7 @@ export abstract class Effect</*** Converts the effect into an operation node for the VirtualMediaData AST.*/- public toOperation(): Record<string, unknown> {+ public toOperation(): NodeOperation<TOp, TConfig> {const opPayload: Record<string, unknown> = {op: this.op,effect: this,@@ -148,6 +149,6 @@ export abstract class Effect<}}- return opPayload;+ return opPayload as NodeOperation<TOp, TConfig>;}}diff --git a/packages/core/src/types/video/virtual-video.ts b/packages/core/src/types/video/virtual-video.tsindex e75ef8a8..168de410 100644--- a/packages/core/src/types/video/virtual-video.ts+++ b/packages/core/src/types/video/virtual-video.ts@@ -1,4 +1,5 @@import { z } from "zod";+import type { FrameContext } from "../../signals/index.js";import { type DataType, DataTypeVal, FileAssetSchema } from "../base.js";export const MediaSourceSchema = z.object({@@ -46,6 +47,7 @@ export const BaseMediaOperationPropsSchema = z.object({volume: z.number().optional(),opacity: z.number().optional(),startFrame: z.number().optional(),+ durationFrames: z.number().optional(),timeline: TimelineTransformSchema.optional(),});@@ -53,6 +55,28 @@ export type BaseMediaOperationProps = z.infer<typeof BaseMediaOperationPropsSchema>;+export type NodeInputConnection = {+ connectionValid: boolean;+ outputItem: {+ type: string;+ data: unknown;+ } | null;+};++export type NodeOperation<+ TOp extends string = string,+ TConfig extends object = Record<string, unknown>,+> = BaseMediaOperationProps & {+ op: TOp;+ id?: string;+ metadata?: MediaMetadata;+ dataType?: DataType;+ effect?: unknown;+ signals?: Record<string, unknown>;+ inputs?: Record<string, NodeInputConnection>;+ onRequestFrame?: (ctx: FrameContext) => void;+} & TConfig;+/** Original source file (leaf node) */export const SourceOperationSchema = BaseMediaOperationPropsSchema.extend({op: z.literal("source"),@@ -84,18 +108,16 @@ export type MediaOperation = BaseMediaOperationProps &(| ({ op: "source" } & z.infer<typeof SourceOperationSchema>)| ({ op: "text" } & z.infer<typeof TextOperationSchema>)- | ({- op: string;- metadata?: MediaMetadata;- dataType: DataType;- } & Record<string, any>)+ | NodeOperation<string, Record<string, unknown>>);// --- VirtualMediaData: THE recursive data type for all media outputs ----export type VirtualMediaData = {+export type VirtualMediaData<+ TOperation extends MediaOperation = MediaOperation,+> = {metadata: MediaMetadata;- operation: MediaOperation;+ operation: TOperation;children: VirtualMediaData[];};diff --git a/packages/framefields/src/type-safety.test.ts b/packages/framefields/src/type-safety.test.tsindex 3ff2ff78..02929606 100644--- a/packages/framefields/src/type-safety.test.ts+++ b/packages/framefields/src/type-safety.test.ts@@ -1,3 +1,7 @@+import {+ CompositorProgramSchema,+ EffectSpecSchema,+} from "@framefields/compositions/program";import { describe, expect, expectTypeOf, it } from "vitest";import {type AnimatableNode,@@ -26,6 +30,7 @@ import {ms,type NodeCapability,type NodeId,+ type NodeOperation,type Normalized,nodeId,normalized,@@ -33,6 +38,7 @@ import {type SectionOptions,seconds,Vignette,+ type VirtualMediaData,} from "./index.js";describe("Type Safety & Branded Types (Phase 1)", () => {@@ -177,3 +183,71 @@ describe("Discriminated Effects & Port Capabilities (Phase 2)", () => {expect(img2.effects?.length).toBe(2);});});++describe("Schema & Pipeline Unification (Phase 3)", () => {+ it("strongly types NodeOperation from effect toOperation", () => {+ const blur = new Blur({ strength: 10 });+ const op = blur.toOperation();++ expect(op.op).toBe("Blur");+ expectTypeOf(op.op).toEqualTypeOf<"Blur">();+ expectTypeOf(op).toMatchTypeOf<NodeOperation<"Blur">>();+ });++ it("validates effects via EffectSpecSchema and CompositorProgramSchema", () => {+ // Valid Effect class instance+ const effectInstance = new FilmGrain({ strength: 12 });+ expect(EffectSpecSchema.safeParse(effectInstance).success).toBe(true);++ // Valid plain effect spec object+ const plainSpec = { op: "Blur", strength: 8 };+ expect(EffectSpecSchema.safeParse(plainSpec).success).toBe(true);++ // Invalid effect without op+ const invalidSpec = { strength: 8 };+ expect(EffectSpecSchema.safeParse(invalidSpec).success).toBe(false);++ // Valid compositor program with effects+ const programWithEffects = {+ width: 1920,+ height: 1080,+ fps: 30,+ layout: [+ {+ id: "hero",+ kind: "box",+ effects: [plainSpec, effectInstance],+ },+ ],+ effects: [plainSpec],+ };+ expect(CompositorProgramSchema.safeParse(programWithEffects).success).toBe(+ true,+ );++ // Invalid compositor program with invalid effect+ const programWithBadEffect = {+ width: 1920,+ height: 1080,+ fps: 30,+ layout: [+ {+ id: "hero",+ kind: "box",+ effects: [invalidSpec],+ },+ ],+ };+ expect(+ CompositorProgramSchema.safeParse(programWithBadEffect).success,+ ).toBe(false);+ });++ it("propagates typed operations in VirtualMediaData without any", () => {+ type BlurMedia = VirtualMediaData<+ NodeOperation<"Blur", { strength: number }>+ >;+ expectTypeOf<BlurMedia["operation"]["op"]>().toEqualTypeOf<"Blur">();+ expectTypeOf<BlurMedia["operation"]["strength"]>().toEqualTypeOf<number>();+ });+});diff --git a/packages/node-sdk/renderer/index.ts b/packages/node-sdk/renderer/index.tsindex 7901bf85..17df8a31 100644--- a/packages/node-sdk/renderer/index.ts+++ b/packages/node-sdk/renderer/index.ts@@ -1,5 +1,10 @@/// <reference types="@webgpu/types" />-import type { VirtualMediaData } from "@framefields/core";+import type {+ MediaOperation,+ NodeInputConnection,+ NodeOperation,+ VirtualMediaData,+} from "@framefields/core";import type { RenderContextValue } from "@framefields/webgpu-renderers";export type GPUCommandEncoder = globalThis.GPUCommandEncoder;@@ -7,9 +12,13 @@ export type GPURenderPassEncoder = globalThis.GPURenderPassEncoder;export type GPUTextureView = globalThis.GPUTextureView;export type GPUTexture = globalThis.GPUTexture;-export interface NodeRenderProps {+export type { MediaOperation, NodeInputConnection, NodeOperation };++export interface NodeRenderProps<+ TOperation extends MediaOperation = MediaOperation,+> {renderId: string;- virtualMedia: VirtualMediaData;+ virtualMedia: VirtualMediaData<TOperation>;volume?: number;playbackRateOverride?: number;trimStartOverride?: number;@@ -39,7 +48,9 @@ export interface NodeRenderProps {) => unknown;}-export type WebGPUNodeRenderer = (args: {+export type WebGPUNodeRenderer<+ TOperation extends MediaOperation = MediaOperation,+> = (args: {ctx: RenderContextValue;encoder: GPUCommandEncoder;pass: GPURenderPassEncoder;@@ -47,7 +58,7 @@ export type WebGPUNodeRenderer = (args: {targetTexture: GPUTexture;targetWidth: number;targetHeight: number;- props: NodeRenderProps;+ props: NodeRenderProps<TOperation>;drawChild: (child: VirtualMediaData,overrides?: Partial<NodeRenderProps>,@@ -68,12 +79,13 @@ export interface AudioProcessorContext {renderId?: string;}-export type AudioProcessor = (- channels: Float32Array[],- sampleRate: number,- virtualMedia: VirtualMediaData,- ctx?: AudioProcessorContext,-) => void | Promise<void>;+export type AudioProcessor<TOperation extends MediaOperation = MediaOperation> =+ (+ channels: Float32Array[],+ sampleRate: number,+ virtualMedia: VirtualMediaData<TOperation>,+ ctx?: AudioProcessorContext,+ ) => void | Promise<void>;export {isSignal,