Code-first video & image, rendered natively on WebGPU.
Add media time remapping (#57)
Okan Aslankan18aece79fd84
diff --git a/.changeset/media-time-remapping.md b/.changeset/media-time-remapping.mdnew file mode 100644index 00000000..32a65db9--- /dev/null+++ b/.changeset/media-time-remapping.md@@ -0,0 +1,12 @@+---+"@framefields/core": minor+"@framefields/compositions": minor+"@framefields/node-compositor": minor+"@framefields/node-sdk": minor+"@framefields/node-vision": patch+"@framefields/renderer": minor+"@framefields/webgpu-renderers": minor+"framefields": minor+---++Add forward and reverse media time remapping for video and audio layers, including smooth ramps, video holds, silent audio holds, source range checks and clip vision synchronization. Reverse video decoding uses a bounded frame cache.diff --git a/nodes/node-compositor/SKILL.md b/nodes/node-compositor/SKILL.mdindex ce6c08c8..5f9e188d 100644--- a/nodes/node-compositor/SKILL.md+++ b/nodes/node-compositor/SKILL.md@@ -134,6 +134,7 @@ Per-kind fields:- **Charts**: there is no chart node kind. `Layer.chart` in the `framefields` SDK expands a chart into `box`, `shape` (`"path"`/`"circle"`) and `text` nodes, so a program document describes a chart with those kinds.- **`media`**: `inputHandleId` (string, required — must match a connected input handle).- **Standard Media (`Video`, `Image`, `GIF`, `SVG`, `Lottie`)**: `fit` (`"cover"` | `"contain"` | `"fill"` | `"none"`, default `"contain"`), `volume` (0–1), `muted`, `borderRadius`, `borderColor`, `borderWidth`.+ - **Video and audio timing**: `timeRemap` maps layer-local frames to source seconds, with an explicit `durationFrames`. Use `{ kind: "rate", sourceStartSec, rate }` with a nonzero rate (negative plays in reverse), or `{ kind: "keyframes", keyframes: [{ frame, sourceSec, interpolation? }, ...] }`. Keyframes span frame 0 through `durationFrames` and increase by frame; source time may increase, decrease, or hold. `interpolation` on the starting key applies to the following segment: `linear`, `ease-in`, `ease-out`, or `smooth`; all are monotonic. Equal source times hold video and silence audio. Rate changes alter audio pitch. Do not combine with `trimStartSec`; effects retain the composition clock.- **Captions & Subtitles (`Caption` DataType)**: When bound to a `Caption` input (SRT source), `kind: "media"` renders time-synchronized subtitle cues:- `fontSize` (number, default `48`): Font size in pixels.- `fontFamily` (string, default `"Inter"`): Font family.@@ -368,4 +369,4 @@ If a layer or the final composite renders blank, check the following common fail}]}-```\ No newline at end of file+```diff --git a/nodes/node-compositor/src/renderers/webgpu-renderer.ts b/nodes/node-compositor/src/renderers/webgpu-renderer.tsindex 8e04ef74..36fd97b4 100644--- a/nodes/node-compositor/src/renderers/webgpu-renderer.ts+++ b/nodes/node-compositor/src/renderers/webgpu-renderer.ts@@ -13,6 +13,7 @@ import type { MeshAudioDeformConfig, TextSpan } from "@framefields/core";import {DEFAULT_DURATION_MS,getActiveMediaMetadata,+ mediaSourceTime,type VirtualMediaData,} from "@framefields/core";import {@@ -3871,14 +3872,26 @@ export const CompositorWebGPURenderer: WebGPUNodeRenderer = async (args) => {// or the raw frame lands there wherever the clip does not cover it.excludeTextures: [activeTex, targetTexture, ...excludes],isVideoMode: isVideoMode && !contentIsImageCompositor,+ ...(isVideoMode &&+ lop.timeRemap &&+ kind === "media" &&+ lop.dataType === "Video" && {+ sourceTimeSec: mediaSourceTime(+ lop.timeRemap,+ Math.max(0, shiftedFrame - (lop.startFrame ?? 0)),+ fps ?? 24,+ ),+ }),...(isVideoMode && {frame:- Math.max(0, shiftedFrame - (lop.startFrame ?? 0)) +- (lop.mediaStartFrame ??- lop.trimStartFrames ??- (lop.trimStartSec != null- ? Math.round(lop.trimStartSec * (fps ?? 24))- : 0)),+ lop.timeRemap && kind === "media" && lop.dataType === "Video"+ ? Math.max(0, shiftedFrame - (lop.startFrame ?? 0))+ : Math.max(0, shiftedFrame - (lop.startFrame ?? 0)) ++ (lop.mediaStartFrame ??+ lop.trimStartFrames ??+ (lop.trimStartSec != null+ ? Math.round(lop.trimStartSec * (fps ?? 24))+ : 0)),}),compositionFrame,...(isCaption && {diff --git a/nodes/node-vision/src/renderers/webgpu-renderer.ts b/nodes/node-vision/src/renderers/webgpu-renderer.tsindex fb2ddda2..d865ad7c 100644--- a/nodes/node-vision/src/renderers/webgpu-renderer.ts+++ b/nodes/node-vision/src/renderers/webgpu-renderer.ts@@ -634,7 +634,15 @@ export const VisionWebGPURenderer: WebGPUNodeRenderer = async (args) => {!wantsPose &&!wantsSelfie &&op.keyBackground !== true- ? clipSubjectFor(effectKey, frameIdx, fps, targetWidth, targetHeight)+ ? clipSubjectFor(+ effectKey,+ props.sourceTimeSec === undefined+ ? frameIdx+ : props.sourceTimeSec * fps,+ fps,+ targetWidth,+ targetHeight,+ ): undefined;const fromClip = clipSubject !== undefined;diff --git a/packages/compositions/src/compositions/canvas-composition.ts b/packages/compositions/src/compositions/canvas-composition.tsindex 4748f2f5..df59d35a 100644--- a/packages/compositions/src/compositions/canvas-composition.ts+++ b/packages/compositions/src/compositions/canvas-composition.ts@@ -589,6 +589,7 @@ export async function drawCompositionTree(accumulatedSeekOffsetSec,);const rawTimestampSec =+ props.sourceTimeSec ??(segmentLocalFrame / fps) * finalPlaybackRate + effectiveTrimSec;const timestampSec = Math.max(0,@@ -597,7 +598,7 @@ export async function drawCompositionTree(await drawVideoNode(ctx, pass, {nodeId: props.renderId,- frameKey: `${props.renderId ?? "video"}-${params.sourceUrl}-${Math.round(timestampSec * fps)}`,+ frameKey: `${props.renderId ?? "video"}-${params.sourceUrl}-${timestampSec}`,sourceUrl: params.sourceUrl,timestampSec,dstRect: {diff --git a/packages/compositions/src/index.ts b/packages/compositions/src/index.tsindex 72bdb4d8..f421d63d 100644--- a/packages/compositions/src/index.ts+++ b/packages/compositions/src/index.ts@@ -3,5 +3,6 @@ export * from "./compositions/composition-state.js";export * from "./utils/apply-operations.js";export * from "./utils/audio-extractor.js";export * from "./utils/layer-logic.js";+export * from "./utils/media-time-preflight.js";export * from "./utils/normalization.js";export * from "./utils/timing.js";diff --git a/packages/compositions/src/program/schema.test.ts b/packages/compositions/src/program/schema.test.tsindex 32d7d27a..ddcd94be 100644--- a/packages/compositions/src/program/schema.test.ts+++ b/packages/compositions/src/program/schema.test.ts@@ -4,6 +4,7 @@ import {EaseRefSchema,type FlexNode,type LayoutNode,+ LayoutNodeSchema,} from "./schema.js";import {collectMediaBindings,@@ -1085,3 +1086,42 @@ describe("validateLayoutProgram (E-code pre-parse scan)", () => {}});});++describe("media time remapping schema", () => {+ it("accepts forward and reverse maps and rejects malformed layer timing", () => {+ const node = {+ id: "clip",+ kind: "media",+ inputHandleId: "a.mp4",+ dataType: "Video",+ durationFrames: 30,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 },+ { frame: 15, sourceSec: 1 },+ { frame: 30, sourceSec: 2 },+ ],+ },+ };+ expect(LayoutNodeSchema.safeParse(node).success).toBe(true);+ expect(+ LayoutNodeSchema.safeParse({ ...node, trimStartSec: 1 }).success,+ ).toBe(false);+ expect(+ LayoutNodeSchema.safeParse({ ...node, durationFrames: 29 }).success,+ ).toBe(false);+ expect(+ LayoutNodeSchema.safeParse({+ ...node,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 },+ { frame: 30, sourceSec: 0 },+ ],+ },+ }).success,+ ).toBe(true);+ });+});diff --git a/packages/compositions/src/program/schema.ts b/packages/compositions/src/program/schema.tsindex 30f3350b..84bd6489 100644--- a/packages/compositions/src/program/schema.ts+++ b/packages/compositions/src/program/schema.ts@@ -17,9 +17,12 @@import {Effect,type EffectDomain,+ type MediaTimeRemap,+ MediaTimeRemapSchema,type MeshAudioDeformConfig,type TextSpan,type TextSpanMark,+ validateMediaTimeRemap,} from "@framefields/core";import { ColorSchema } from "@framefields/node-sdk";export type { TextSpan, TextSpanMark };@@ -1123,6 +1126,8 @@ export interface MediaNode extends CompositionNodeBase {muted?: boolean;/** Source in-point: seconds into the clip that play at the layer's startFrame. */trimStartSec?: number;+ /** Layer-local frame to absolute source time mapping. */+ timeRemap?: MediaTimeRemap;// Caption stylingfontFamily?: string;fontSize?: number;@@ -1640,6 +1645,7 @@ export const MediaNodeSchema: z.ZodType<MediaNode> = zvolume: z.number().min(0).max(1).default(1),muted: z.boolean().default(false),trimStartSec: z.number().min(0).optional(),+ timeRemap: MediaTimeRemapSchema.optional(),// Caption stylingfontFamily: z.string().optional(),fontSize: z.number().positive().optional(),@@ -1659,7 +1665,28 @@ export const MediaNodeSchema: z.ZodType<MediaNode> = zshadows: z.array(TextShadowSchema).optional(),maxWidth: z.number().optional(),})- .strict() as unknown as z.ZodType<MediaNode>;+ .strict()+ .superRefine((node, ctx) => {+ if (!node.timeRemap) return;+ if (node.trimStartSec !== undefined)+ ctx.addIssue({+ code: "custom",+ path: ["timeRemap"],+ message: "timeRemap cannot be combined with trimStartSec",+ });+ if (node.dataType !== "Video" && node.dataType !== "Audio")+ ctx.addIssue({+ code: "custom",+ path: ["timeRemap"],+ message: "timeRemap requires video or audio media",+ });+ const error = validateMediaTimeRemap(+ node.timeRemap,+ node.durationFrames ?? 0,+ );+ if (error)+ ctx.addIssue({ code: "custom", path: ["timeRemap"], message: error });+ }) as unknown as z.ZodType<MediaNode>;export const ShapeFields = {shapeType: ShapeTypeSchema.default("rect"),diff --git a/packages/compositions/src/program/to-virtual-media.test.ts b/packages/compositions/src/program/to-virtual-media.test.tsindex 1a3ebd26..e2305065 100644--- a/packages/compositions/src/program/to-virtual-media.test.ts+++ b/packages/compositions/src/program/to-virtual-media.test.ts@@ -40,6 +40,36 @@ const mediaVM = (id: string): VirtualMediaData =>}) as unknown as VirtualMediaData;describe("compositorToProgram", () => {+ it("preserves the remapped layer duration when the resolved source is shorter", () => {+ const video = {+ metadata: { durationMs: 1000 },+ operation: { op: "source", dataType: "Video" },+ children: [],+ } as unknown as VirtualMediaData;+ const config = {+ width: 64,+ height: 64,+ fps: 30,+ mode: "Video",+ layout: [+ {+ id: "slow",+ kind: "media",+ inputHandleId: "source",+ dataType: "Video",+ durationFrames: 60,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ },+ ],+ } as unknown as CompositorProgramConfig;+ const vm = compositorToProgram(config, {+ isVideoMode: true,+ fps: 30,+ resolveMedia: () => video,+ });+ expect(vm.children[0].operation.durationFrames).toBe(60);+ expect(vm.children[0].metadata.durationMs).toBe(2000);+ });it("builds the render tree 1:1 from the document", () => {const vm = compositorToProgram(doc, { isVideoMode: true, fps: 30 });const op = vm.operation as any;diff --git a/packages/compositions/src/program/to-virtual-media.ts b/packages/compositions/src/program/to-virtual-media.tsindex e4bd1f28..83999b12 100644--- a/packages/compositions/src/program/to-virtual-media.ts+++ b/packages/compositions/src/program/to-virtual-media.ts@@ -17,7 +17,9 @@ import {type DataType,type FrameContext,getActiveMediaMetadata,+ MediaTimeRemapSchema,type VirtualMediaData,+ validateMediaTimeRemap,} from "@framefields/core";import { collectInlineSignals } from "./inline-signals.js";import { hoistMattes } from "./mattes.js";@@ -54,6 +56,26 @@ function nodeToVirtualMedia(compDurationMs?: number,inheritedDurationFrames?: number,): VirtualMediaData {+ if (node.kind === "media" && node.timeRemap !== undefined) {+ const parsed = MediaTimeRemapSchema.safeParse(node.timeRemap);+ if (!parsed.success)+ throw new Error(+ `Invalid timeRemap on '${node.id}': ${parsed.error.message}`,+ );+ const problem = validateMediaTimeRemap(+ parsed.data,+ node.durationFrames ?? 0,+ );+ if (problem)+ throw new Error(`Invalid timeRemap on '${node.id}': ${problem}`);+ if (+ node.trimStartSec !== undefined ||+ (node.dataType !== "Video" && node.dataType !== "Audio")+ )+ throw new Error(+ `Invalid timeRemap on '${node.id}': use audio or video without trimStartSec`,+ );+ }let children: VirtualMediaData[] = [];const sourceRef =node.kind === "media"@@ -256,6 +278,7 @@ function nodeToVirtualMedia(} else if (dur !== undefined) {if ((dataType === "Video" || dataType === "Audio") &&+ !(node.kind === "media" && node.timeRemap) &&limit !== undefined &&limit > 0) {diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.tsindex 2efc4065..627a3e65 100644--- a/packages/compositions/src/utils/audio-extractor.ts+++ b/packages/compositions/src/utils/audio-extractor.ts@@ -1,6 +1,10 @@import {getActiveMediaMetadata,getMediaType,+ type MediaTimeRemap,+ mediaSourceIsHold,+ mediaSourceRange,+ mediaSourceTime,type VirtualMediaData,} from "@framefields/core";import { type AudioProcessor, audioRegistry } from "@framefields/node-sdk";@@ -439,6 +443,31 @@ async function decodeAudioRange(}}+/** Samples the mapped source at the audio clock; holds are silent. */+async function resampleMappedPCM(+ input: Float32Array,+ sourceRate: number,+ outputRate: number,+ length: number,+ inputStart: number,+ map: MediaTimeRemap,+ startFrame: number,+ fps: number,+): Promise<Float32Array> {+ const output = new Float32Array(length);+ for (let i = 0; i < length; i++) {+ if (i > 0 && i % 100_000 === 0) await yieldToMain();+ const frame = startFrame + (i / outputRate) * fps;+ if (mediaSourceIsHold(map, frame)) continue;+ const position = mediaSourceTime(map, frame, fps) * sourceRate - inputStart;+ const index = Math.floor(position);+ if (index < 0 || index >= input.length) continue;+ const a = input[index];+ output[i] = a + ((input[index + 1] ?? a) - a) * (position - index);+ }+ return output;+}+/*** Resamples a PCM buffer using linear interpolation to target sample rate & speed.* `trimOffsetSamples` is a whole-file index; `inputStartIndex` is the@@ -612,18 +641,22 @@ export async function mixAudioTracks(url: string,startSec: number,endSec: number,+ cache = true,): Promise<DecodedAudioRange | null> {if (undecodable.has(url)) return null;let ranges = decodedCache.get(url);- const hit = ranges?.find(- (r) => r.validFromSec <= startSec && r.validToSec >= endSec,- );+ const hit = cache+ ? ranges?.find(+ (r) => r.validFromSec <= startSec && r.validToSec >= endSec,+ )+ : undefined;if (hit) return hit;const decoded = await decodeAudioRange(url, { startSec, endSec });if (!decoded) {undecodable.add(url);return null;}+ if (!cache) return decoded;if (!ranges) {ranges = [];decodedCache.set(url, ranges);@@ -632,6 +665,71 @@ export async function mixAudioTracks(return decoded;}+ // Limit each remapped decode by both output and source duration; never retain+ // the temporary ranges after the output chunk has been sampled.+ async function mappedChannels(+ url: string,+ length: number,+ map: MediaTimeRemap,+ startFrame: number,+ ): Promise<Float32Array[] | null> {+ const left = new Float32Array(length),+ right = new Float32Array(length);+ for (let offset = 0; offset < length; ) {+ let count = Math.min(length - offset, targetSampleRate * 2);+ const first = startFrame + (offset / targetSampleRate) * fps;+ let range = mediaSourceRange(+ map,+ first,+ first + ((count - 1) / targetSampleRate) * fps,+ fps,+ );+ while (range.max - range.min > 3 && count > 1) {+ count = Math.max(1, Math.floor(count / 2));+ range = mediaSourceRange(+ map,+ first,+ first + ((count - 1) / targetSampleRate) * fps,+ fps,+ );+ }+ const decoded = await getDecodedRange(+ url,+ range.min,+ range.max + RANGE_MARGIN_SEC,+ false,+ );+ if (!decoded) return null;+ const a = await resampleMappedPCM(+ decoded.channels[0],+ decoded.sampleRate,+ targetSampleRate,+ count,+ decoded.startIndex,+ map,+ first,+ fps,+ );+ const b =+ decoded.channels.length > 1+ ? await resampleMappedPCM(+ decoded.channels[1],+ decoded.sampleRate,+ targetSampleRate,+ count,+ decoded.startIndex,+ map,+ first,+ fps,+ )+ : a;+ left.set(a, offset);+ right.set(b, offset);+ offset += count;+ }+ return [left, right];+ }+// A subtree without a source or a sound-producing processor mixes to// silence (text, images, shapes), so it is skipped instead of being// allocated, processed and summed at full length.@@ -665,6 +763,7 @@ export async function mixAudioTracks(inheritedVolume: number,inheritedSpeed: number,inheritedDurationFrames: number | null,+ inheritedRemap?: { map: MediaTimeRemap; offsetFrames: number },): Promise<Float32Array[] | null> {if (!node || !isAudible(node)) return null;const op = node.operation;@@ -672,11 +771,13 @@ export async function mixAudioTracks(const durationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;const durationFrames = Math.round((durationMs / 1000) * fps);const actualDurationFrames =- inheritedDurationFrames !== null- ? durationFrames > 0- ? Math.min(inheritedDurationFrames, durationFrames)- : inheritedDurationFrames- : durationFrames;+ inheritedRemap && inheritedDurationFrames !== null+ ? inheritedDurationFrames+ : inheritedDurationFrames !== null+ ? durationFrames > 0+ ? Math.min(inheritedDurationFrames, durationFrames)+ : inheritedDurationFrames+ : durationFrames;const localSamplesCount = Math.ceil((actualDurationFrames / fps) * targetSampleRate,@@ -693,6 +794,7 @@ export async function mixAudioTracks(inheritedVolume,inheritedSpeed,actualDurationFrames,+ inheritedRemap,);if (childChannels) {const childOp = child.operation;@@ -715,7 +817,20 @@ export async function mixAudioTracks(const speedMultiplier = inheritedSpeed;const nodeStartFrame = op.startFrame ?? op.timeline?.startFrame ?? 0;- const localClockFrames = inheritedClockFrames + nodeStartFrame;+ const ownRemap = (op as { timeRemap?: MediaTimeRemap }).timeRemap;+ const remap = ownRemap+ ? { map: ownRemap, offsetFrames: 0 }+ : inheritedRemap+ ? {+ map: inheritedRemap.map,+ offsetFrames: inheritedRemap.offsetFrames + nodeStartFrame,+ }+ : undefined;+ // Compositor layer starts are flattened to composition frames by the compiler.+ const localClockFrames =+ op.op === "CompositorLayer"+ ? nodeStartFrame+ : inheritedClockFrames + nodeStartFrame;const segments = op.timeline?.segments || [];@@ -738,11 +853,13 @@ export async function mixAudioTracks(const nodeDurationMs = getActiveMediaMetadata(node)?.durationMs ?? 0;const nodeDurationFrames = Math.round((nodeDurationMs / 1000) * fps);const nodeActualDurationFrames =- inheritedDurationFrames !== null- ? nodeDurationFrames > 0- ? Math.min(inheritedDurationFrames, nodeDurationFrames)- : inheritedDurationFrames- : nodeDurationFrames;+ inheritedRemap && !ownRemap && inheritedDurationFrames !== null+ ? inheritedDurationFrames+ : inheritedDurationFrames !== null+ ? nodeDurationFrames > 0+ ? Math.min(inheritedDurationFrames, nodeDurationFrames)+ : inheritedDurationFrames+ : nodeDurationFrames;const localSamplesCount = Math.ceil((nodeActualDurationFrames / fps) * targetSampleRate,);@@ -769,9 +886,11 @@ export async function mixAudioTracks(}const actualDurationFrames =- inheritedDurationFrames !== null- ? Math.min(inheritedDurationFrames, segDurationFrames)- : segDurationFrames;+ inheritedRemap && !ownRemap && !seg && inheritedDurationFrames !== null+ ? inheritedDurationFrames+ : inheritedDurationFrames !== null+ ? Math.min(inheritedDurationFrames, segDurationFrames)+ : segDurationFrames;const segmentStartSample = Math.round((windowStartFrameOffset / fps) * targetSampleRate,@@ -801,9 +920,19 @@ export async function mixAudioTracks(Math.ceil(clipDurationSec * targetSampleRate),);- // Decode (or reuse) only the stretch of the source this clip plays+ const mapped =+ remap && clipSamplesCount > 0+ ? await mappedChannels(+ params.sourceUrl,+ clipSamplesCount,+ remap.map,+ remap.offsetFrames + windowStartFrameOffset,+ )+ : null;const decoded =- clipSamplesCount > 0 && segmentStartSample < localSamplesCount+ !remap &&+ clipSamplesCount > 0 &&+ segmentStartSample < localSamplesCount? await getDecodedRange(params.sourceUrl,effectiveTrimSec,@@ -812,37 +941,39 @@ export async function mixAudioTracks(RANGE_MARGIN_SEC,): null;-- if (decoded) {- const trimOffsetSamples = Math.round(- effectiveTrimSec * decoded.sampleRate,- );-- // Resample left & right channels- const stretchedLeft = await resampleAndStretchPCM(- decoded.channels[0],- decoded.sampleRate,- targetSampleRate,- finalSpeed,- clipSamplesCount,- trimOffsetSamples,- decoded.startIndex,- );-- // A mono source plays the same samples on both sides- const stretchedRight =- decoded.channels.length > 1- ? await resampleAndStretchPCM(- decoded.channels[1],- decoded.sampleRate,- targetSampleRate,- finalSpeed,- clipSamplesCount,- trimOffsetSamples,- decoded.startIndex,- )- : stretchedLeft;-+ if (mapped || decoded) {+ let stretchedLeft: Float32Array;+ let stretchedRight: Float32Array;+ if (mapped) {+ [stretchedLeft, stretchedRight] = mapped;+ } else if (decoded) {+ const trimOffsetSamples = Math.round(+ effectiveTrimSec * decoded.sampleRate,+ );+ stretchedLeft = await resampleAndStretchPCM(+ decoded.channels[0],+ decoded.sampleRate,+ targetSampleRate,+ finalSpeed,+ clipSamplesCount,+ trimOffsetSamples,+ decoded.startIndex,+ );+ stretchedRight =+ decoded.channels.length > 1+ ? await resampleAndStretchPCM(+ decoded.channels[1],+ decoded.sampleRate,+ targetSampleRate,+ finalSpeed,+ clipSamplesCount,+ trimOffsetSamples,+ decoded.startIndex,+ )+ : stretchedLeft;+ } else {+ throw new Error("No decoded audio");+ }const stretched = [stretchedLeft, stretchedRight];if (bus.canAdopt(stretched, segmentStartSample)) {// The clip fills the silent bus: scale it in place and keep it@@ -894,13 +1025,18 @@ export async function mixAudioTracks(volume,speedMultiplier,actualDurationFrames,+ remap,);if (childChannels) {const childOp = child.operation;const childStartFrame =childOp?.startFrame ?? childOp?.timeline?.startFrame ?? 0;+ const relativeStartFrame =+ op.op === "CompositorLayer" && childOp?.op === "CompositorLayer"+ ? childStartFrame - nodeStartFrame+ : childStartFrame;const childStartSample = Math.round(- (childStartFrame / fps) * targetSampleRate,+ (relativeStartFrame / fps) * targetSampleRate,);bus.add(childChannels, segmentStartSample + childStartSample);}diff --git a/packages/compositions/src/utils/media-time-preflight.ts b/packages/compositions/src/utils/media-time-preflight.tsnew file mode 100644index 00000000..4a336390--- /dev/null+++ b/packages/compositions/src/utils/media-time-preflight.ts@@ -0,0 +1,76 @@+import {+ type MediaTimeRemap,+ mediaSourceRange,+ type VirtualMediaData,+} from "@framefields/core";+import { inputStore } from "@framefields/webgpu-renderers";+import { computeRenderParams } from "./apply-operations.js";++/** Resolve the source bound to a compositor layer, including effect wrappers. */+function sourceOf(node: VirtualMediaData): string | undefined {+ if (node.operation?.op === "source")+ return computeRenderParams(node).sourceUrl;+ for (const child of node.children ?? []) {+ const source = sourceOf(child);+ if (source) return source;+ }+}++/** Verify mapped source intervals before decoding or mixing. */+export async function preflightMediaTime(+ root: VirtualMediaData,+ fps: number,+): Promise<void> {+ const walk = async (node: VirtualMediaData): Promise<void> => {+ const op = node.operation as unknown as {+ op?: string;+ id?: string;+ dataType?: string;+ muted?: boolean;+ volume?: number;+ durationFrames?: number;+ timeRemap?: MediaTimeRemap;+ };+ if (op?.op === "CompositorLayer" && op.timeRemap && op.durationFrames) {+ const source = sourceOf(node);+ if (!source)+ throw new Error(`Time remap on '${op.id}' has no media source`);+ const range = mediaSourceRange(op.timeRemap, 0, op.durationFrames, fps);+ const input = await inputStore.acquire(source);+ try {+ const video =+ op.dataType === "Video" ? await input.getPrimaryVideoTrack() : null;+ const audio =+ op.dataType === "Video" && (op.muted || op.volume === 0)+ ? null+ : await input.getPrimaryAudioTrack();+ if (op.dataType === "Video" && !video)+ throw new Error(+ `Time remap on '${op.id}': source has no video track`,+ );+ if (op.dataType === "Audio" && !audio)+ throw new Error(+ `Time remap on '${op.id}': source has no audio track`,+ );+ for (const track of [video, audio]) {+ if (!track) continue;+ const first = await input.getFirstTimestamp([track]);+ const last = await track.computeDuration();+ if (+ range.min < first - 1e-8 ||+ range.max > last + 1e-8 ||+ (range.maxIncluded && range.max >= last - 1e-8)+ ) {+ throw new Error(+ `Time remap on '${op.id}' requests source ${range.min.toFixed(6)}–${range.max.toFixed(6)}s outside ${first.toFixed(6)}–${last.toFixed(6)}s`,+ );+ }+ }+ } finally {+ inputStore.release(source);+ }+ }+ for (const child of node.children ?? []) await walk(child);+ };+ await walk(root);+}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex e3287830..cecbd497 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1,6 +1,7 @@export * from "./effects/index.js";export * from "./errors.js";export * from "./id-gen.js";+export * from "./media-time.js";export * from "./node-metadata.js";export * from "./render-diagnostics.js";export * from "./signals/index.js";diff --git a/packages/core/src/media-time.test.ts b/packages/core/src/media-time.test.tsnew file mode 100644index 00000000..4a0ffd6a--- /dev/null+++ b/packages/core/src/media-time.test.ts@@ -0,0 +1,133 @@+import { describe, expect, it } from "vitest";+import {+ MediaTimeRemapSchema,+ mediaSourceIsHold,+ mediaSourceRange,+ mediaSourceRate,+ mediaSourceTime,+ validateMediaTimeRemap,+} from "./media-time.js";++describe("media time mapping", () => {+ it("maps fractional frames at a constant rate", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "rate",+ sourceStartSec: 2,+ rate: 0.25,+ });+ expect(mediaSourceTime(map, 15.5, 30)).toBeCloseTo(2 + 15.5 / 120);+ expect(validateMediaTimeRemap(map, 90)).toBeUndefined();+ });+ it("maps non-integer frame rates without accumulating drift", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "rate",+ sourceStartSec: 0.125,+ rate: 0.5,+ });+ const fps = 30000 / 1001;+ expect(mediaSourceTime(map, fps * 3600, fps)).toBeCloseTo(1800.125, 9);+ });+ it("maps piecewise source times and holds", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 },+ { frame: 30, sourceSec: 2 },+ { frame: 60, sourceSec: 2 },+ { frame: 90, sourceSec: 4 },+ ],+ });+ expect(validateMediaTimeRemap(map, 90)).toBeUndefined();+ expect(mediaSourceTime(map, 15, 30)).toBe(1.5);+ expect(mediaSourceTime(map, 45, 30)).toBe(2);+ expect(mediaSourceTime(map, 75, 30)).toBe(3);+ });+ it("rejects zero rate and accepts reverse playback", () => {+ expect(+ MediaTimeRemapSchema.safeParse({+ kind: "rate",+ sourceStartSec: 0,+ rate: 0,+ }).success,+ ).toBe(false);+ const map = MediaTimeRemapSchema.parse({+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 },+ { frame: 30, sourceSec: 0 },+ ],+ });+ expect(validateMediaTimeRemap(map, 30)).toBeUndefined();+ expect(mediaSourceTime(map, 15, 30)).toBe(0.5);+ expect(validateMediaTimeRemap(map, 31)).toMatch(/durationFrames/);+ });+ it("requires strictly ordered keyframes and a full layer window", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0 },+ { frame: 10, sourceSec: 1 },+ { frame: 10, sourceSec: 1 },+ { frame: 30, sourceSec: 2 },+ ],+ });+ expect(validateMediaTimeRemap(map, 30)).toMatch(/strictly increasing/);+ expect(validateMediaTimeRemap(map, 29)).toMatch(/durationFrames/);+ });+ it("is independent of evaluation order", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 },+ { frame: 20, sourceSec: 3 },+ { frame: 40, sourceSec: 3 },+ { frame: 60, sourceSec: 4 },+ ],+ });+ const frames = [0, 12.5, 20, 39.5, 40, 59, 60];+ const expected = frames.map((frame) => mediaSourceTime(map, frame, 30));+ expect(+ frames+ .toReversed()+ .map((frame) => mediaSourceTime(map, frame, 30))+ .reverse(),+ ).toEqual(expected);+ });+ it("evaluates smooth ramps without overshoot in either direction", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1, interpolation: "ease-in" },+ { frame: 30, sourceSec: 3, interpolation: "smooth" },+ { frame: 60, sourceSec: 1, interpolation: "ease-out" },+ { frame: 90, sourceSec: 2 },+ ],+ });+ expect(validateMediaTimeRemap(map, 90)).toBeUndefined();+ expect(mediaSourceTime(map, 15, 30)).toBe(1.5);+ expect(mediaSourceTime(map, 45, 30)).toBe(2);+ expect(mediaSourceTime(map, 75, 30)).toBe(1.75);+ expect(mediaSourceRate(map, 0, 30)).toBe(0);+ expect(mediaSourceIsHold(map, 0)).toBe(false);+ expect(mediaSourceRange(map, 0, 90, 30)).toEqual({+ min: 1,+ max: 3,+ maxIncluded: true,+ });+ for (let frame = 0; frame <= 90; frame += 0.25)+ expect(mediaSourceTime(map, frame, 30)).toBeGreaterThanOrEqual(1);+ });+ it("maps constant reverse speed and bounds its source range", () => {+ const map = MediaTimeRemapSchema.parse({+ kind: "rate",+ sourceStartSec: 4,+ rate: -2,+ });+ expect(mediaSourceTime(map, 30, 30)).toBe(2);+ expect(mediaSourceRange(map, 0, 30, 30)).toEqual({+ min: 2,+ max: 4,+ maxIncluded: true,+ });+ });+});diff --git a/packages/core/src/media-time.ts b/packages/core/src/media-time.tsnew file mode 100644index 00000000..5b88a548--- /dev/null+++ b/packages/core/src/media-time.ts@@ -0,0 +1,171 @@+import { z } from "zod";++/** Maps layer-local frames to absolute source seconds. */+export const MediaTimeRemapSchema = z.discriminatedUnion("kind", [+ z+ .object({+ kind: z.literal("rate"),+ sourceStartSec: z.number().finite().nonnegative(),+ rate: z+ .number()+ .finite()+ .refine((v) => v !== 0, "rate must not be zero"),+ })+ .strict(),+ z+ .object({+ kind: z.literal("keyframes"),+ keyframes: z+ .array(+ z+ .object({+ frame: z.number().int().nonnegative(),+ sourceSec: z.number().finite().nonnegative(),+ interpolation: z+ .enum(["linear", "ease-in", "ease-out", "smooth"])+ .optional(),+ })+ .strict(),+ )+ .min(2),+ })+ .strict(),+]);++export type MediaTimeRemap = z.infer<typeof MediaTimeRemapSchema>;++/** Validate the map against its explicit, exclusive layer duration. */+export function validateMediaTimeRemap(+ map: MediaTimeRemap,+ durationFrames: number,+): string | undefined {+ if (!Number.isInteger(durationFrames) || durationFrames <= 0)+ return "durationFrames must be a positive integer";+ if (map.kind === "rate") return;+ const keys = map.keyframes;+ if (+ keys.length < 2 ||+ keys[0].frame !== 0 ||+ keys[keys.length - 1].frame !== durationFrames+ ) {+ return "time remap keyframes must start at frame 0 and end at durationFrames";+ }+ for (let i = 1; i < keys.length; i++) {+ if (keys[i].frame <= keys[i - 1].frame)+ return "time remap keyframes must have strictly increasing frames";+ }+}++/** Find the segment containing a layer-local frame. */+function segmentIndex(+ map: Extract<MediaTimeRemap, { kind: "keyframes" }>,+ frame: number,+): number {+ let lo = 0;+ let hi = map.keyframes.length - 1;+ while (hi - lo > 1) {+ const mid = Math.floor((lo + hi) / 2);+ if (map.keyframes[mid].frame <= frame) lo = mid;+ else hi = mid;+ }+ return lo;+}++/** Evaluate the monotonic easing assigned to the starting keyframe. */+function shapeTime(+ t: number,+ interpolation: "linear" | "ease-in" | "ease-out" | "smooth",+): number {+ switch (interpolation) {+ case "ease-in":+ return t * t;+ case "ease-out":+ return 1 - (1 - t) * (1 - t);+ case "smooth":+ return t * t * (3 - 2 * t);+ default:+ return t;+ }+}++/** Absolute source seconds at a fractional layer-local frame. */+export function mediaSourceTime(+ map: MediaTimeRemap,+ localFrame: number,+ fps: number,+): number {+ if (!Number.isFinite(localFrame) || !Number.isFinite(fps) || fps <= 0)+ throw new Error("Invalid media time or fps");+ if (map.kind === "rate")+ return map.sourceStartSec + (localFrame / fps) * map.rate;+ const keys = map.keyframes;+ if (keys.length < 2)+ throw new Error("Time remap needs at least two keyframes");+ if (localFrame <= keys[0].frame) return keys[0].sourceSec;+ if (localFrame >= keys[keys.length - 1].frame)+ return keys[keys.length - 1].sourceSec;+ const i = segmentIndex(map, localFrame);+ const a = keys[i];+ const b = keys[i + 1];+ const t = (localFrame - a.frame) / (b.frame - a.frame);+ return (+ a.sourceSec ++ (b.sourceSec - a.sourceSec) * shapeTime(t, a.interpolation ?? "linear")+ );+}++/** True only for a constant source-time interval, not a ramp with zero instantaneous velocity. */+export function mediaSourceIsHold(map: MediaTimeRemap, frame: number): boolean {+ if (map.kind === "rate") return false;+ const i = segmentIndex(map, frame);+ return map.keyframes[i].sourceSec === map.keyframes[i + 1].sourceSec;+}++/** Source seconds per output second at a layer-local frame. */+export function mediaSourceRate(+ map: MediaTimeRemap,+ frame: number,+ fps: number,+): number {+ if (map.kind === "rate") return map.rate;+ const i = segmentIndex(map, frame);+ const a = map.keyframes[i];+ const b = map.keyframes[i + 1];+ const t = Math.max(0, Math.min(1, (frame - a.frame) / (b.frame - a.frame)));+ const derivative =+ a.interpolation === "ease-in"+ ? 2 * t+ : a.interpolation === "ease-out"+ ? 2 * (1 - t)+ : a.interpolation === "smooth"+ ? 6 * t * (1 - t)+ : 1;+ return ((b.sourceSec - a.sourceSec) / (b.frame - a.frame)) * fps * derivative;+}++/** Bounds over a local-frame interval, including intermediate turns. */+export function mediaSourceRange(+ map: MediaTimeRemap,+ fromFrame: number,+ toFrame: number,+ fps: number,+): { min: number; max: number; maxIncluded: boolean } {+ const start = Math.min(fromFrame, toFrame),+ end = Math.max(fromFrame, toFrame);+ const first = mediaSourceTime(map, start, fps);+ const last = mediaSourceTime(map, end, fps);+ let min = Math.min(first, last);+ let max = Math.max(first, last);+ let maxIncluded = start < end && first === max;+ if (map.kind === "keyframes") {+ for (const key of map.keyframes) {+ if (key.frame <= start || key.frame >= end) continue;+ min = Math.min(min, key.sourceSec);+ if (key.sourceSec > max) {+ max = key.sourceSec;+ maxIncluded = true;+ } else if (key.sourceSec === max) maxIncluded = true;+ }+ }+ return { min, max, maxIncluded };+}diff --git a/packages/framefields/src/audio-mix-ranges.test.ts b/packages/framefields/src/audio-mix-ranges.test.tsindex 985d17e3..cce11c05 100644--- a/packages/framefields/src/audio-mix-ranges.test.ts+++ b/packages/framefields/src/audio-mix-ranges.test.ts@@ -49,6 +49,11 @@ beforeAll(async () => {sampleRate: SR,}),);+ sources.short = path.join(dir, "short.wav");+ await fs.writeFile(+ sources.short,+ encodeStereoWav([noise(1, 1), noise(1, 2)], { sampleRate: SR }),+ );if (hasFfmpeg) {for (const [ext, codec] of [["mp3", "libmp3lame"],@@ -149,3 +154,225 @@ describe("audio mix source ranges", () => {);}, 60_000);});++describe("media time remapping", () => {+ it("plays a constant negative rate backward", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 30,+ });+ comp.addAudio(+ Layer.audio(sources.short, {+ durationFrames: 30,+ timeRemap: { kind: "rate", sourceStartSec: 0.9, rate: -0.5 },+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ for (const i of [0, 12000, 24000, 36000]) {+ const sourceIndex = Math.round(0.9 * SR - i / 2);+ expect(channels[0][i]).toBeCloseTo(+ Math.tanh(whole.channels[0][sourceIndex]),+ 5,+ );+ }+ });+ it("keeps reverse samples continuous across bounded decode chunks", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 90,+ });+ comp.addAudio(+ Layer.audio(sources.short, {+ durationFrames: 90,+ timeRemap: { kind: "rate", sourceStartSec: 0.9, rate: -0.25 },+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ for (const i of [2 * SR - 4, 2 * SR - 1, 2 * SR, 2 * SR + 1, 2 * SR + 4]) {+ const position = 0.9 * SR - i * 0.25;+ const index = Math.floor(position);+ const expected =+ whole.channels[0][index] ++ (whole.channels[0][index + 1] - whole.channels[0][index]) *+ (position - index);+ expect(channels[0][i]).toBeCloseTo(Math.tanh(expected), 5);+ }+ });+ it("plays reverse audio through direction changes and smooth ramps", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 90,+ });+ comp.addAudio(+ Layer.audio(sources.short, {+ durationFrames: 90,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0.1, interpolation: "smooth" },+ { frame: 30, sourceSec: 0.8 },+ { frame: 60, sourceSec: 0.2 },+ { frame: 90, sourceSec: 0.2 },+ ],+ },+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ for (const [output, source] of [+ [0, 0.1],+ [0.5, 0.45],+ [1, 0.8],+ [1.5, 0.5],+ [1.9, 0.26],+ ]) {+ const sourceIndex = source * SR;+ const index = Math.floor(sourceIndex);+ const expected =+ whole.channels[0][index] ++ (whole.channels[0][index + 1] - whole.channels[0][index]) *+ (sourceIndex - index);+ expect(channels[0][Math.round(output * SR)]).toBeCloseTo(+ Math.tanh(expected),+ 4,+ );+ }+ for (const sample of [2 * SR, 2.5 * SR, 3 * SR - 1])+ expect(channels[0][sample]).toBe(0);+ });+ it("resolves nested layer start times without offsetting source time", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 90,+ });+ comp.add(+ Layer.box({+ startFrame: 15,+ durationFrames: 75,+ children: [+ Layer.audio(sources.short, {+ startFrame: 15,+ durationFrames: 30,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 1 },+ }),+ ],+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ expect(channels[0][SR - 1]).toBe(0);+ for (const i of [0, 1000, 24000])+ expect(channels[0][SR + i]).toBeCloseTo(+ Math.tanh(whole.channels[0][i]),+ 5,+ );+ });+ it("keeps slow audio audible beyond the original source duration", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 60,+ });+ comp.addAudio(+ Layer.audio(sources.short, {+ durationFrames: 60,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ for (const i of [48000, 72000, 90000]) {+ expect(channels[0][i]).toBeCloseTo(+ Math.tanh(whole.channels[0][i / 2]),+ 5,+ );+ }+ });+ it("does not truncate retimed audio when source metadata has a shorter duration", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 60,+ });+ comp.addAudio(+ Layer.audio(sources.short, {+ durationFrames: 60,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }),+ );+ const vm = comp.toVirtualMedia();+ vm.children[0].children[0].metadata.durationMs = 1000;+ const { channels } = await new HeadlessMediaRenderer().renderAudio(vm, {+ fps: FPS,+ sampleRate: SR,+ });+ const whole = await decodeAudioSource(sources.short);+ if (!whole) throw new Error("source audio did not decode");+ expect(channels[0][72000]).toBeCloseTo(+ Math.tanh(whole.channels[0][36000]),+ 5,+ );+ });+ it("maps source samples at sub-frame rates and outputs silence for holds", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationFrames: 90,+ });+ comp.addAudio(+ Layer.audio(sources.wav, {+ startFrame: 15,+ durationFrames: 60,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 2 },+ { frame: 15, sourceSec: 2.25 },+ { frame: 30, sourceSec: 2.25 },+ { frame: 60, sourceSec: 3.25 },+ ],+ },+ }),+ );+ const { channels } = await mix(comp);+ const whole = await decodeAudioSource(sources.wav);+ if (!whole) throw new Error("source audio did not decode");+ const base = SR / 2;+ for (const i of [0, 2, 200, 6000, 12000, 23998]) {+ const source = 2 * SR + i * 0.5;+ const index = Math.floor(source);+ const expected =+ whole.channels[0][index] ++ (whole.channels[0][index + 1] - whole.channels[0][index]) *+ (source - index);+ expect(channels[0][base + i]).toBeCloseTo(Math.tanh(expected), 5);+ }+ for (const i of [24000, 30000, 47999])+ expect(channels[0][base + i]).toBe(0);+ for (const i of [48000, 50000, 70000]) {+ const index = Math.round(2.25 * SR + (i - 48000));+ expect(channels[0][base + i]).toBeCloseTo(+ Math.tanh(whole.channels[0][index]),+ 5,+ );+ }+ });+});diff --git a/packages/framefields/src/clip-vision-matte.test.ts b/packages/framefields/src/clip-vision-matte.test.tsindex 13a234f1..1f7eb9a5 100644--- a/packages/framefields/src/clip-vision-matte.test.ts+++ b/packages/framefields/src/clip-vision-matte.test.ts@@ -68,6 +68,92 @@ const cutout = (clip: string, vision: Record<string, unknown> = {}) =>});describe("matte reuses comp.vision", () => {+ it("follows a clip's subject while its media plays in reverse", async () => {+ const file = await greenClip();+ const c = stage("#0000ff");+ const layer = Layer.video(file, {+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ durationFrames: 2,+ muted: true,+ timeRemap: { kind: "rate", sourceStartSec: 2 / FPS, rate: -1 },+ }).withVision({ mode: "matte", baseUrl: "http://127.0.0.1:9/" });+ c.add(layer);+ const clip = c.vision(layer, { cacheDir: false, smoothing: 0 });+ const right = leftHalf().map((v) => 255 - v);+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 40 }, (_, i) => ({+ kp: null,+ mask: null,+ subject: i === 2 ? right : leftHalf(),+ })),+ );+ const first = await decodePngPixels(await c.renderFrame({ frame: 0 }));+ const next = await decodePngPixels(await c.renderFrame({ frame: 1 }));+ expect(getPixel(first, 150, 100).g).toBeGreaterThan(240);+ expect(getPixel(next, 50, 100).g).toBeGreaterThan(240);+ });+ it("uses mapped source time to select the rendered matte", async () => {+ const file = await greenClip();+ const c = stage("#0000ff");+ const layer = Layer.video(file, {+ position: "absolute",+ x: 0,+ y: 0,+ width: SIZE,+ height: SIZE,+ durationFrames: 4,+ muted: true,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 1 / FPS },+ { frame: 4, sourceSec: 1 / FPS },+ ],+ },+ }).withVision({ mode: "matte", baseUrl: "http://127.0.0.1:9/" });+ c.add(layer);+ const clip = c.vision(layer, { cacheDir: false, smoothing: 0 });+ const right = leftHalf().map((v) => 255 - v);+ seedClipVision(+ clip,+ { width: SIZE, height: SIZE },+ Array.from({ length: 40 }, (_, i) => ({+ kp: null,+ mask: null,+ subject: i === 0 ? leftHalf() : right,+ })),+ );+ const img = await decodePngPixels(await c.renderFrame({ frame: 0 }));+ expect(getPixel(img, 50, 100).b).toBeGreaterThan(240);+ expect(getPixel(img, 150, 100).g).toBeGreaterThan(240);+ });+ it("links each remapped layer to its own vision clock", () => {+ const c = stage();+ const first = Layer.video("clip.mp4", {+ durationFrames: 10,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }).withVision({ mode: "matte" });+ const second = Layer.video("clip.mp4", {+ durationFrames: 10,+ timeRemap: { kind: "rate", sourceStartSec: 1, rate: 1 },+ }).withVision({ mode: "matte" });+ c.add([first, second]);+ const a = c.vision(first, { cacheDir: false });+ const b = c.vision(second, { cacheDir: false });+ (c as unknown as { linkClipMattes(): void }).linkClipMattes();+ expect((first.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(+ a,+ );+ expect((second.vision as { [CLIP_SUBJECT]?: unknown })[CLIP_SUBJECT]).toBe(+ b,+ );+ });it("cuts the layer out with the clip's cached subject, from the first frame", async () => {const file = await greenClip();const c = stage("#0000ff");diff --git a/packages/framefields/src/clip-vision.test.ts b/packages/framefields/src/clip-vision.test.tsindex 2ef6a822..5265f1d8 100644--- a/packages/framefields/src/clip-vision.test.ts+++ b/packages/framefields/src/clip-vision.test.ts@@ -1,5 +1,6 @@import { describe, expect, it } from "vitest";import { clipVisionFromResults } from "./clip-vision.js";+import { Composition, Layer } from "./index.js";/** A 160×80 clip: frame f has the left wrist at (20 + 20f, 10) and a 40×40 square silhouette moving right. */function clip(frames = 4, options = {}) {@@ -92,3 +93,71 @@ describe("ClipVision", () => {);});});++describe("remapped clip vision", () => {+ it("samples reverse motion and covers intermediate turns in a readiness window", () => {+ const c = clip(4, {+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0 },+ { frame: 3, sourceSec: 3 / 24, interpolation: "smooth" },+ { frame: 6, sourceSec: 1 / 24 },+ ],+ },+ });+ expect(c.person.point("leftWrist", 3).x).toBe(80);+ expect(c.person.point("leftWrist", 6).x).toBe(40);+ const range = (+ c as unknown as { window(from: number, to: number): [number, number] }+ ).window(0, 6);+ expect(range[1]).toBeGreaterThanOrEqual(3);+ });+ it("requires a layer to be in the composition before binding vision", () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: 24,+ durationFrames: 24,+ });+ expect(() =>+ comp.vision(Layer.video("clip.mp4", { durationFrames: 24 })),+ ).toThrow(/Add the media layer/);+ });+ it("binds to the absolute start of a nested media layer", () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: 24,+ durationFrames: 100,+ });+ const layer = Layer.video("clip.mp4", {+ startFrame: 7,+ durationFrames: 24,+ timeRemap: { kind: "rate", sourceStartSec: 1, rate: 0.5 },+ });+ comp.add(+ Layer.box({ startFrame: 10, durationFrames: 60, children: [layer] }),+ );+ const vision = comp.vision(layer);+ expect(vision.startFrame).toBe(17);+ expect(vision.clipFrame(19)).toBe(25);+ });+ it("samples the same source clock as the media layer", () => {+ const c = clip(4, {+ startFrame: 10,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0 },+ { frame: 2, sourceSec: 1 / 24 },+ { frame: 4, sourceSec: 1 / 24 },+ { frame: 6, sourceSec: 3 / 24 },+ ],+ },+ });+ expect(c.person.point("leftWrist", 12).x).toBe(40);+ expect(c.person.point("leftWrist", 13).x).toBe(40);+ expect(c.person.point("leftWrist", 16).x).toBe(80);+ });+});diff --git a/packages/framefields/src/clip-vision.ts b/packages/framefields/src/clip-vision.tsindex 2541978e..1b52fb97 100644--- a/packages/framefields/src/clip-vision.ts+++ b/packages/framefields/src/clip-vision.ts@@ -1,4 +1,10 @@-import { type FrameSignal, Signal } from "@framefields/core";+import {+ type FrameSignal,+ type MediaTimeRemap,+ mediaSourceRange,+ mediaSourceTime,+ Signal,+} from "@framefields/core";import {type ClipSubjectSource,COCO17_KEYPOINT_NAMES,@@ -16,6 +22,10 @@ export interface ClipVisionOptions {startFrame?: number;/** Seconds trimmed off the clip's start, as on its media layer (default 0). */trimStartSec?: number;+ /** The media layer's source-time mapping. */+ timeRemap?: MediaTimeRemap;+ /** Media layer duration in composition frames, when known. */+ durationFrames?: number;/** Analysis frame rate (default: the composition's). */fps?: number;/** Track 17 COCO keypoints (default true). */@@ -91,6 +101,7 @@ const CACHE_VERSION = 1;export class ClipVision implements ClipSubjectSource {readonly source: string;readonly fps: number;+ readonly compositionFps: number;readonly startFrame: number;readonly trimStartSec: number;readonly smoothing: number;@@ -119,6 +130,7 @@ export class ClipVision implements ClipSubjectSource {constructor(source: string, fps: number, options: ClipVisionOptions = {}) {this.source = source;this.options = options;+ this.compositionFps = fps;this.fps = options.fps ?? fps;this.startFrame = Math.round(options.startFrame ?? 0);this.trimStartSec = options.trimStartSec ?? 0;@@ -166,8 +178,12 @@ export class ClipVision implements ClipSubjectSource {*/subject(frame: number, fps: number): InstanceMask | null | undefined {const c = (frame * this.fps) / fps;- if (Math.abs(c - Math.round(c)) > 1e-6) return undefined;- const s = this.subjects.get(Math.round(c));+ if (!this.options.timeRemap && Math.abs(c - Math.round(c)) > 1e-6)+ return undefined;+ const sourceFrame = this.options.timeRemap+ ? Math.floor(c + 1e-6)+ : Math.round(c);+ const s = this.subjects.get(sourceFrame);if (s === undefined) return s;if (s === null || !this.sourceWidth) return null;const width = this.sourceWidth;@@ -185,12 +201,42 @@ export class ClipVision implements ClipSubjectSource {/** Composition frame → clip frame (fractional). */clipFrame(frame: number): number {- return frame - this.startFrame + this.trimStartSec * this.fps;+ if (this.options.timeRemap)+ return (+ mediaSourceTime(+ this.options.timeRemap,+ this.localFrame(frame),+ this.compositionFps,+ ) * this.fps+ );+ return (+ ((frame - this.startFrame) / this.compositionFps + this.trimStartSec) *+ this.fps+ );+ }++ private localFrame(frame: number): number {+ const local = Math.max(0, frame - this.startFrame);+ return this.options.durationFrames === undefined+ ? local+ : Math.min(local, Math.max(0, this.options.durationFrames - 1));}private window(frame: number, toFrame: number): [number, number] {- const a = Math.floor(this.clipFrame(Math.min(frame, toFrame)));- const b = Math.ceil(this.clipFrame(Math.max(frame, toFrame)));+ const range = this.options.timeRemap+ ? mediaSourceRange(+ this.options.timeRemap,+ this.localFrame(Math.min(frame, toFrame)),+ this.localFrame(Math.max(frame, toFrame)),+ this.compositionFps,+ )+ : undefined;+ const a = Math.floor(+ range ? range.min * this.fps : this.clipFrame(Math.min(frame, toFrame)),+ );+ const b = Math.ceil(+ range ? range.max * this.fps : this.clipFrame(Math.max(frame, toFrame)),+ );return [Math.max(0, a - HOLD - this.smoothing),Math.max(0, b + this.smoothing),diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 56d7ec53..b6b5bf0e 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -136,6 +136,7 @@ export {import fs from "node:fs/promises";import path from "node:path";+import { preflightMediaTime } from "@framefields/compositions";import {type AnimatableProp,type AnimatablePropsFor,@@ -144,6 +145,7 @@ import {type CameraNode,type CompositorProgramConfig,type CompositorToProgramOptions,+ collectNodeOps,compositorToProgram,type EnvironmentPreset,type FlexNode,@@ -172,12 +174,16 @@ import {type MediaMetadata,type MediaOperation,type MediaSourceInput,+ type MediaTimeRemap,type MeshAudioDeformConfig,ProgrammaticSignal,type RenderDiagnostic,type Signal,type VirtualMediaData,} from "@framefields/core";++export type { MediaTimeRemap } from "@framefields/core";+import {analyzeSequence as analyzeVisionFrames,CLIP_SUBJECT,@@ -1238,7 +1244,32 @@ export class Composition {* analysed just before it renders, so nothing runs up front. Results* persist on disk in Node. The same clip and options return one instance.*/- public vision(source: string, options: ClipVisionOptions = {}): ClipVision {+ public vision(+ sourceOrLayer: string | MediaNode,+ options: ClipVisionOptions = {},+ ): ClipVision {+ const source =+ typeof sourceOrLayer === "string"+ ? sourceOrLayer+ : (sourceOrLayer.src ?? sourceOrLayer.inputHandleId);+ if (typeof sourceOrLayer !== "string") {+ const layer = collectNodeOps(this.toVirtualMedia().children).find(+ (node) =>+ (node.operation as unknown as { id?: string }).id ===+ sourceOrLayer.id,+ );+ if (!layer)+ throw new Error(+ "Add the media layer to the composition before calling comp.vision(layer)",+ );+ options = {+ startFrame: layer.operation.startFrame ?? sourceOrLayer.startFrame,+ trimStartSec: sourceOrLayer.trimStartSec,+ timeRemap: sourceOrLayer.timeRemap,+ durationFrames: sourceOrLayer.durationFrames,+ ...options,+ };+ }const key = `${source}\u0000${JSON.stringify(options)}`;let clip = this._clipVision.get(key);if (!clip) {@@ -1268,12 +1299,18 @@ export class Composition {*/private linkClipMattes(): void {if (this._clipVision.size === 0) return;+ let starts: Map<string | undefined, number | undefined> | undefined;const walk = (node: unknown): void => {if (!node || typeof node !== "object") return;const n = node as {+ id?: string;kind?: string;dataType?: string;inputHandleId?: unknown;+ src?: string;+ startFrame?: number;+ trimStartSec?: number;+ timeRemap?: MediaTimeRemap;vision?: unknown;children?: unknown;};@@ -1284,7 +1321,16 @@ export class Composition {n.vision instanceof Vision &&!(CLIP_SUBJECT in n.vision)) {- const clip = this.clipForMatte(n.inputHandleId, n.vision);+ starts ??= new Map(+ collectNodeOps(this.toVirtualMedia().children).map((node) => [+ (node.operation as unknown as { id?: string }).id,+ node.operation.startFrame,+ ]),+ );+ const clip = this.clipForMatte(n.src ?? n.inputHandleId, n.vision, {+ ...n,+ startFrame: starts.get(n.id) ?? n.startFrame,+ });if (clip) {clip.keepSubjects();Object.defineProperty(n.vision, CLIP_SUBJECT, {@@ -1299,7 +1345,15 @@ export class Composition {for (const child of this.children) walk(child);}- private clipForMatte(source: string, v: Vision): ClipVision | undefined {+ private clipForMatte(+ source: string,+ v: Vision,+ layer: {+ startFrame?: number;+ trimStartSec?: number;+ timeRemap?: MediaTimeRemap;+ },+ ): ClipVision | undefined {const mode = v.mode ?? "passthrough";if (mode !== "matte" && mode !== "mask" && mode !== "crop") return;// Anything that changes the subject, or needs results the clip lacks.@@ -1319,6 +1373,9 @@ export class Composition {const o = clip.options;if (clip.source === source &&+ clip.startFrame === (layer.startFrame ?? 0) &&+ clip.trimStartSec === (layer.trimStartSec ?? 0) &&+ JSON.stringify(o.timeRemap) === JSON.stringify(layer.timeRemap) &&(o.variant ?? "s") === (v.variant ?? "s") &&(o.confidence ?? 0.3) === (v.confidence ?? 0.3)) {@@ -2013,7 +2070,7 @@ export class Composition {const durationMs =options.durationMs ?? this.durationMs ?? (await this.computeDuration());this.durationMs = durationMs;- return compositorToProgram(this.toSpec(), {+ const vm = compositorToProgram(this.toSpec(), {isVideoMode: true,fps: this.fps,durationMs,@@ -2022,6 +2079,8 @@ export class Composition {...(this._visionConfig && { vision: this._visionConfig }),...options,});+ await preflightMediaTime(vm, this.fps);+ return vm;}/**diff --git a/packages/framefields/src/media-time-render.test.ts b/packages/framefields/src/media-time-render.test.tsnew file mode 100644index 00000000..8d8b491a--- /dev/null+++ b/packages/framefields/src/media-time-render.test.ts@@ -0,0 +1,531 @@+import { execFileSync } from "node:child_process";+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { decodeAudioSource } from "@framefields/compositions";+import { beforeAll, describe, expect, it } from "vitest";+import {+ Blur,+ Composition,+ FilmGrain,+ HeadlessMediaRenderer,+ Layer,+ Signal,+} from "./index.js";+import { decodePngPixels, getPixel } from "./test-utils/pixels.js";++let source = "";+let sourceAv = "";+let sourceVfr = "";+let sourceShortAudio = "";+const hasFfmpeg = (() => {+ try {+ execFileSync("ffmpeg", ["-version"], { stdio: "ignore" });+ return true;+ } catch {+ return false;+ }+})();+beforeAll(async () => {+ if (!hasFfmpeg) return;+ source = path.join(+ await fs.mkdtemp(path.join(os.tmpdir(), "media-time-")),+ "source.mp4",+ );+ const colors = [+ [255, 0, 0],+ [0, 255, 0],+ [0, 0, 255],+ [255, 255, 255],+ ];+ const raw = Buffer.alloc(16 * 16 * 3 * colors.length);+ for (let f = 0; f < colors.length; f++)+ for (let i = 0; i < 16 * 16; i++)+ for (let c = 0; c < 3; c++) raw[(f * 16 * 16 + i) * 3 + c] = colors[f][c];+ execFileSync(+ "ffmpeg",+ [+ "-v",+ "error",+ "-y",+ "-f",+ "rawvideo",+ "-pix_fmt",+ "rgb24",+ "-s",+ "16x16",+ "-r",+ "4",+ "-i",+ "pipe:0",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-g",+ "1",+ source,+ ],+ { input: raw },+ );+ sourceAv = path.join(path.dirname(source), "source-av.mp4");+ execFileSync("ffmpeg", [+ "-v",+ "error",+ "-y",+ "-i",+ source,+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=48000:duration=1",+ "-c:v",+ "copy",+ "-c:a",+ "aac",+ "-shortest",+ sourceAv,+ ]);+ sourceVfr = path.join(path.dirname(source), "source-vfr.mp4");+ execFileSync(+ "ffmpeg",+ [+ "-v",+ "error",+ "-y",+ "-f",+ "rawvideo",+ "-pix_fmt",+ "rgb24",+ "-s",+ "16x16",+ "-r",+ "4",+ "-i",+ "pipe:0",+ "-vf",+ "setpts=N*N/(4*TB)",+ "-fps_mode",+ "vfr",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-g",+ "1",+ sourceVfr,+ ],+ { input: raw },+ );+ sourceShortAudio = path.join(path.dirname(source), "source-short-audio.mp4");+ execFileSync("ffmpeg", [+ "-v",+ "error",+ "-y",+ "-i",+ source,+ "-f",+ "lavfi",+ "-i",+ "sine=frequency=440:sample_rate=48000:duration=0.5",+ "-c:v",+ "copy",+ "-c:a",+ "aac",+ sourceShortAudio,+ ]);+});++describe.skipIf(!hasFfmpeg)("remapped video", () => {+ it("selects reverse variable-frame-rate samples by their presentation timestamps", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 4,+ });+ comp.add(+ Layer.video(sourceVfr, {+ width: 16,+ height: 16,+ durationFrames: 4,+ timeRemap: { kind: "rate", sourceStartSec: 1.25, rate: -1 },+ }),+ );+ const renderer = new HeadlessMediaRenderer();+ const frame = async (i: number) =>+ getPixel(+ await decodePngPixels(await comp.renderFrame({ frame: i, renderer })),+ 8,+ 8,+ );+ expect((await frame(0)).b).toBeGreaterThan(200);+ expect((await frame(2)).g).toBeGreaterThan(200);+ expect((await frame(1)).b).toBeGreaterThan(200);+ expect((await frame(3)).g).toBeGreaterThan(200);+ });+ it("rejects an out-of-range turning point even when the endpoints are valid", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(source, {+ width: 16,+ height: 16,+ durationFrames: 8,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0.2 },+ { frame: 4, sourceSec: 1.2 },+ { frame: 8, sourceSec: 0.2 },+ ],+ },+ }),+ );+ await expect(comp.renderFrame({ frame: 0 })).rejects.toThrow(/outside/);+ });+ it("renders reverse and a direction change with random-access seeks", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(source, {+ width: 16,+ height: 16,+ durationFrames: 8,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0 },+ { frame: 4, sourceSec: 0.75, interpolation: "smooth" },+ { frame: 8, sourceSec: 0 },+ ],+ },+ }),+ );+ const renderer = new HeadlessMediaRenderer();+ const pixel = async (frame: number) =>+ getPixel(+ await decodePngPixels(await comp.renderFrame({ frame, renderer })),+ 8,+ 8,+ );+ const blue = await pixel(5);+ const red = await pixel(0);+ const white = await pixel(4);+ const green = await pixel(6);+ expect(red.r).toBeGreaterThan(200);+ expect(red.g).toBeLessThan(40);+ expect(white.r).toBeGreaterThan(200);+ expect(white.g).toBeGreaterThan(200);+ expect(white.b).toBeGreaterThan(200);+ expect(blue.b).toBeGreaterThan(200);+ expect(blue.r).toBeLessThan(40);+ expect(green.g).toBeGreaterThan(200);+ expect(green.r).toBeLessThan(40);+ });+ it("does not require muted audio to span the picture mapping", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 4,+ });+ comp.add(+ Layer.video(sourceShortAudio, {+ width: 16,+ height: 16,+ durationFrames: 4,+ muted: true,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 1 },+ }),+ );+ await expect(comp.renderFrame({ frame: 3 })).resolves.toBeInstanceOf(+ Buffer,+ );+ });+ it("continues animated effects during a video hold", async () => {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: 4,+ durationFrames: 4,+ });+ comp.add(+ Layer.video(source, {+ width: 64,+ height: 64,+ durationFrames: 4,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0.25 },+ { frame: 4, sourceSec: 0.25 },+ ],+ },+ }).apply(new FilmGrain({ strength: 60, animated: true, speed: 50 })),+ );+ const renderer = new HeadlessMediaRenderer();+ const first = await comp.renderFrame({ frame: 0, renderer });+ const next = await comp.renderFrame({ frame: 1, renderer });+ expect(first.equals(next)).toBe(false);+ });+ it("selects variable-frame-rate samples by their timestamps", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 4,+ });+ comp.add(+ Layer.video(sourceVfr, {+ width: 16,+ height: 16,+ durationFrames: 4,+ timeRemap: { kind: "rate", sourceStartSec: 0.5, rate: 1 },+ }),+ );+ const green = getPixel(+ await decodePngPixels(await comp.renderFrame({ frame: 0 })),+ 8,+ 8,+ );+ const blue = getPixel(+ await decodePngPixels(await comp.renderFrame({ frame: 2 })),+ 8,+ 8,+ );+ expect(green.g).toBeGreaterThan(200);+ expect(blue.b).toBeGreaterThan(200);+ });+ it("retimes a video's embedded audio with its picture", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(sourceAv, {+ width: 16,+ height: 16,+ durationFrames: 8,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }),+ );+ const result = await new HeadlessMediaRenderer().renderAudio(+ comp.toVirtualMedia(),+ { fps: 4, sampleRate: 48000 },+ );+ const original = await decodeAudioSource(sourceAv);+ if (!original) throw new Error("embedded audio did not decode");+ for (const i of [48000, 72000, 90000])+ expect(result.channels[0][i]).toBeCloseTo(+ Math.tanh(original.channels[0][i / 2]),+ 4,+ );+ });+ it("maps nested layer timing from the layer's own start", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.box({+ width: 16,+ height: 16,+ startFrame: 1,+ durationFrames: 7,+ children: [+ Layer.video(source, {+ width: 16,+ height: 16,+ startFrame: 1,+ durationFrames: 4,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }),+ ],+ }),+ );+ const pixel = getPixel(+ await decodePngPixels(await comp.renderFrame({ frame: 4 })),+ 8,+ 8,+ );+ expect(pixel.g).toBeGreaterThan(200);+ expect(pixel.r).toBeLessThan(40);+ });+ it("renders the mapped source frame at a layer-local clock, including a hold", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 10,+ });+ comp.add(+ Layer.video(sourceAv, {+ width: 16,+ height: 16,+ startFrame: 2,+ durationFrames: 8,+ timeRemap: {+ kind: "keyframes",+ keyframes: [+ { frame: 0, sourceSec: 0 },+ { frame: 2, sourceSec: 0.25 },+ { frame: 4, sourceSec: 0.25 },+ { frame: 8, sourceSec: 0.75 },+ ],+ },+ }),+ );+ const renderer = new HeadlessMediaRenderer();+ const pixel = async (frame: number) =>+ getPixel(+ await decodePngPixels(await comp.renderFrame({ frame, renderer })),+ 8,+ 8,+ );+ const red = await pixel(2);+ const green = await pixel(4);+ const hold = await pixel(5);+ const blue = await pixel(8);+ expect(red.r).toBeGreaterThan(200);+ expect(red.g).toBeLessThan(40);+ expect(green.g).toBeGreaterThan(200);+ expect(green.r).toBeLessThan(40);+ expect(hold.g).toBeGreaterThan(200);+ expect(hold.r).toBeLessThan(40);+ expect(blue.b).toBeGreaterThan(200);+ expect(blue.g).toBeLessThan(40);+ });+ it("rejects a map beyond the source duration", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(source, {+ width: 16,+ height: 16,+ durationFrames: 8,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 1 },+ }),+ );+ await expect(comp.renderFrame({ frame: 0 })).rejects.toThrow(/outside/);+ await expect(comp.renderVideo()).rejects.toThrow(/outside/);+ });+ it("keeps effect signals on the composition clock", async () => {+ const frames: number[] = [];+ const strength = Signal.builder({+ type: "custom",+ fn: (ctx) => {+ frames.push(ctx.frame);+ return 2;+ },+ });+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(source, {+ width: 16,+ height: 16,+ startFrame: 2,+ durationFrames: 4,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }).apply(new Blur({ strength })),+ );+ await comp.renderFrame({ frame: 4 });+ expect(frames).toContain(4);+ expect(frames).not.toContain(1);+ });+ it("encodes the same retimed sequence", async () => {+ const comp = new Composition({+ width: 16,+ height: 16,+ fps: 4,+ durationFrames: 8,+ });+ comp.add(+ Layer.video(sourceAv, {+ width: 16,+ height: 16,+ durationFrames: 8,+ timeRemap: { kind: "rate", sourceStartSec: 0, rate: 0.5 },+ }),+ );+ const result = await comp.renderVideo();+ try {+ const probe = JSON.parse(+ execFileSync(+ "ffprobe",+ [+ "-v",+ "error",+ "-select_streams",+ "v:0",+ "-show_entries",+ "stream=nb_frames",+ "-of",+ "json",+ result.filePath,+ ],+ { encoding: "utf8" },+ ),+ ) as { streams: { nb_frames: string }[] };+ expect(Number(probe.streams[0].nb_frames)).toBe(8);+ const bytes = execFileSync("ffmpeg", [+ "-v",+ "error",+ "-i",+ result.filePath,+ "-f",+ "rawvideo",+ "-pix_fmt",+ "rgb24",+ "pipe:1",+ ]);+ const pixel = (frame: number, channel: number) =>+ bytes[(frame * 16 * 16 + 8 * 16 + 8) * 3 + channel];+ expect(pixel(0, 0)).toBeGreaterThan(200);+ expect(pixel(2, 1)).toBeGreaterThan(200);+ expect(pixel(4, 2)).toBeGreaterThan(200);+ const audio = execFileSync("ffmpeg", [+ "-v",+ "error",+ "-i",+ result.filePath,+ "-map",+ "0:a:0",+ "-f",+ "f32le",+ "-ac",+ "1",+ "-ar",+ "48000",+ "pipe:1",+ ]);+ let energy = 0;+ for (let i = 48000; i < 72000; i++)+ energy += audio.readFloatLE(i * 4) ** 2;+ expect(Math.sqrt(energy / 24000)).toBeGreaterThan(0.02);+ } finally {+ await result.cleanup();+ }+ });+});diff --git a/packages/node-sdk/renderer/index.ts b/packages/node-sdk/renderer/index.tsindex 13375218..2cfdaf40 100644--- a/packages/node-sdk/renderer/index.ts+++ b/packages/node-sdk/renderer/index.ts@@ -28,6 +28,8 @@ export interface NodeRenderProps<opacity?: number;frame?: number;compositionFrame?: number;+ /** Absolute source time for a remapped media leaf; effect clocks stay unchanged. */+ sourceTimeSec?: number;fps?: number;elapsedMs?: number;durationMs?: number;diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex b6bcfb9d..f08b7c0f 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -7,6 +7,7 @@ import {compositionStateStore,drawCompositionTree,mixAudioTracks,+ preflightMediaTime,} from "@framefields/compositions";import {getMediaType,@@ -599,6 +600,8 @@ export class HeadlessWebGPURenderer {): Promise<Buffer> {await HeadlessWebGPURenderer.initialize();+ await preflightMediaTime(virtualMedia, fps);+const width = toEvenDimension(virtualMedia.metadata.width ?? 0);const height = toEvenDimension(virtualMedia.metadata.height ?? 0);if (!width || !height)@@ -802,6 +805,7 @@ export class HeadlessWebGPURenderer {): Promise<{ channels: Float32Array[]; sampleRate: number }> {await HeadlessWebGPURenderer.initialize();const fps = options.fps ?? (virtualMedia.metadata?.fps || 30);+ await preflightMediaTime(virtualMedia, fps);await renderSemaphore.acquire();try {const device = await ensureDevice();@@ -1091,6 +1095,7 @@ export class HeadlessWebGPURenderer {`FPS is missing from virtualMedia metadata (dataType: "${dataType}", operation: "${op}")`,);}+ await preflightMediaTime(virtualMedia, fps);const width = toEvenDimension(virtualMedia.metadata.width ?? (isAudioOnly ? 1280 : 0),diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts b/packages/webgpu-renderers/src/nodes/media-decoder.test.tsindex 93c97368..ef863152 100644--- a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts+++ b/packages/webgpu-renderers/src/nodes/media-decoder.test.ts@@ -14,20 +14,15 @@ vi.mock("mediabunny", () => {});const mockSamples = vi.fn().mockImplementation(async function* (ts: number) {- yield {- timestamp: ts,- displayWidth: 100,- displayHeight: 100,- copyTo: vi.fn().mockResolvedValue(undefined),- close: vi.fn(),- };- yield {- timestamp: ts + 0.1,- displayWidth: 100,- displayHeight: 100,- copyTo: vi.fn().mockResolvedValue(undefined),- close: vi.fn(),- };+ for (let i = Math.floor(ts * 10); i < 30; i++)+ yield {+ timestamp: i / 10,+ duration: 0.2,+ displayWidth: 100,+ displayHeight: 100,+ copyTo: vi.fn().mockResolvedValue(undefined),+ close: vi.fn(),+ };});const mockBuffers = vi.fn().mockImplementation(async function* (ts: number) {@@ -106,6 +101,29 @@ describe("Media Decoder", () => {decoder.destroy();});+ it("reuses and releases the bounded cache for reverse frames", async () => {+ const decoder = new VideoDecoder("test-url", true);+ await decoder.getFrame(1);+ await decoder.getFrame(0.9);+ const internals = decoder as unknown as {+ backwardFrames: { close: ReturnType<typeof vi.fn> }[];+ videoSampleSink: { samples: ReturnType<typeof vi.fn> };+ backwardBytes: number;+ };+ const frames = [...internals.backwardFrames];+ expect(frames.length).toBeGreaterThan(1);+ const calls = internals.videoSampleSink.samples.mock.calls.length;+ await decoder.getFrame(0.8);+ expect(internals.videoSampleSink.samples.mock.calls.length).toBe(calls);+ expect(internals.backwardBytes).toBeLessThanOrEqual(48 * 1024 * 1024);+ await decoder.getFrame(1);+ expect(+ internals.videoSampleSink.samples.mock.calls.length,+ ).toBeGreaterThan(calls);+ decoder.destroy();+ for (const frame of frames) expect(frame.close).toHaveBeenCalledOnce();+ });+it("should queue concurrent decode requests", async () => {const decoder = new VideoDecoder("test-url", false);const p1 = decoder.getFrame(0.1);diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.tsindex 65a37e1a..6c6e09ba 100644--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts@@ -86,6 +86,11 @@ export class VideoDecoder extends BaseMediaDecoder<private currentSample: VideoSample | null = null;private nextSample: VideoSample | null = null;private headlessQueue: VideoSample[] = [];+ /** Recently decoded source frames for backwards access, bounded by RGBA size. */+ private backwardFrames: VideoSample[] = [];+ private backwardBytes = 0;+ private backwardEnd: number | undefined;+ private lastHeadlessTimestamp: number | undefined;private isDecoding = false;/*** Reused RGBA staging array for headless decode. Only one decode is in@@ -157,10 +162,105 @@ export class VideoDecoder extends BaseMediaDecoder<} catch (_) {}}this.headlessQueue = [];+ this.clearBackwardFrames();+ this.lastHeadlessTimestamp = undefined;this.decodeBuffer = null;this.iteratorStartTimestamp = -1;}+ private clearBackwardFrames(): void {+ for (const sample of this.backwardFrames) sample.close();+ this.backwardFrames = [];+ this.backwardBytes = 0;+ this.backwardEnd = undefined;+ }++ private async copyHeadlessSample(+ sample: VideoSample,+ fetchId: number,+ ): Promise<VideoFrameData | null> {+ const width = sample.displayWidth;+ const height = sample.displayHeight;+ const byteLength = width * height * 4;+ let buffer = this.decodeBuffer;+ if (!buffer || buffer.length !== byteLength) {+ buffer = new Uint8Array(byteLength);+ this.decodeBuffer = buffer;+ }+ await sample.copyTo(buffer, { format: "RGBA" });+ if (this.currentFetchId !== fetchId || this.destroyed || !this.input)+ return null;+ premultiplyPixels(buffer);+ return { width, height, buffer };+ }++ private cachedBackwardFrame(timestamp: number): VideoSample | undefined {+ let selected: VideoSample | undefined;+ let index = -1;+ for (const [i, sample] of this.backwardFrames.entries()) {+ if (sample.timestamp > timestamp + 0.0001) break;+ selected = sample;+ index = i;+ }+ const next = this.backwardFrames[index + 1]?.timestamp ?? this.backwardEnd;+ if (+ !selected ||+ !(selected.duration > 0) ||+ timestamp + 0.0001 >=+ Math.min(selected.timestamp + selected.duration, next ?? Infinity)+ )+ return undefined;+ return selected;+ }++ /** Decode a small forward window once and retain its last frames for reverse playback. */+ private async fillBackwardFrames(+ timestamp: number,+ fetchId: number,+ ): Promise<void> {+ if (!this.videoSampleSink) return;+ await this.closeIterator(this.iterator);+ this.iterator = null;+ for (const sample of this.headlessQueue) sample.close();+ this.headlessQueue = [];+ this.clearBackwardFrames();+ const iterator = this.videoSampleSink.samples(Math.max(0, timestamp - 0.5));+ try {+ for await (const sample of iterator) {+ if (sample.timestamp > timestamp + 0.0001)+ this.backwardEnd = sample.timestamp;+ if (+ this.destroyed ||+ this.currentFetchId !== fetchId ||+ sample.timestamp > timestamp + 0.0001+ ) {+ sample.close();+ break;+ }+ const sampleBytes = sample.displayWidth * sample.displayHeight * 4;+ if (sampleBytes > 48 * 1024 * 1024) {+ sample.close();+ break;+ }+ this.backwardFrames.push(sample);+ this.backwardBytes += sampleBytes;+ while (+ this.backwardBytes > 48 * 1024 * 1024 &&+ this.backwardFrames.length > 1+ ) {+ const oldest = this.backwardFrames.shift();+ if (oldest) {+ this.backwardBytes -=+ oldest.displayWidth * oldest.displayHeight * 4;+ oldest.close();+ }+ }+ }+ } finally {+ await this.closeIterator(iterator);+ }+ }+protected onDestroy(): void {this.resetSinks();}@@ -275,6 +375,16 @@ export class VideoDecoder extends BaseMediaDecoder<const eps = 0.0001;const forwardSeekThreshold = 1.0;const backwardSeekThreshold = 0.01;+ const backwards =+ this.lastHeadlessTimestamp !== undefined &&+ safeTimestamp < this.lastHeadlessTimestamp - 0.0001;+ this.lastHeadlessTimestamp = safeTimestamp;+ let cached = this.cachedBackwardFrame(safeTimestamp);+ if (!cached && backwards) {+ await this.fillBackwardFrames(safeTimestamp, fetchId);+ cached = this.cachedBackwardFrame(safeTimestamp);+ }+ if (cached) return this.copyHeadlessSample(cached, fetchId);if (!this.iterator ||@@ -309,20 +419,7 @@ export class VideoDecoder extends BaseMediaDecoder<const currentSample = this.headlessQueue[0];if (!currentSample) return null;- const width = currentSample.displayWidth;- const height = currentSample.displayHeight;- const byteLength = width * height * 4;- let buffer = this.decodeBuffer;- if (!buffer || buffer.length !== byteLength) {- buffer = new Uint8Array(byteLength);- this.decodeBuffer = buffer;- }- await currentSample.copyTo(buffer, { format: "RGBA" });- if (this.currentFetchId !== fetchId || this.destroyed || !this.input)- return null;- // WebCodecs RGBA is straight alpha; the compositor expects premultiplied.- premultiplyPixels(buffer);- return { width, height, buffer };+ return this.copyHeadlessSample(currentSample, fetchId);}private async performDecodeInternal(diff --git a/plugins/framefields/skills/framefields-compose/SKILL.md b/plugins/framefields/skills/framefields-compose/SKILL.mdindex f82c5c64..625fba71 100644--- a/plugins/framefields/skills/framefields-compose/SKILL.md+++ b/plugins/framefields/skills/framefields-compose/SKILL.md@@ -106,6 +106,26 @@ All `Layer.*` builders return `AnimatableNode<T>` instances with chainable `.ani---+## Media time remapping++Video and audio layers accept `timeRemap` with an explicit `durationFrames`. Map frames are layer-local; `sourceSec` is absolute source time. Increasing source times play forward; decreasing source times play in reverse; equal source times hold the picture and silence the audio. Rate changes also change audio pitch. Do not combine this with `trimStartSec`.++```ts+const clip = Layer.video("assets/clip.mp4", {+ durationFrames: 90,+ timeRemap: { kind: "keyframes", keyframes: [+ { frame: 0, sourceSec: 1, interpolation: "smooth" },+ { frame: 30, sourceSec: 2 }, { frame: 45, sourceSec: 2 },+ { frame: 90, sourceSec: 1 },+ ] },+});+comp.add(clip);+const vision = comp.vision(clip);+// Or: timeRemap: { kind: "rate", sourceStartSec: 3, rate: -0.5 }+```++Keyframes must start at frame 0, end at `durationFrames`, and increase in frame. `interpolation` belongs to the starting key of a segment: `"linear"` (default), `"ease-in"`, `"ease-out"`, or `"smooth"`. These curves are monotonic and do not overshoot; `smooth` starts and ends at zero speed. Adjacent segments may have different speeds at their join. `comp.vision(layer)` requires the layer to have already been added; clip analysis runs in source order and is sampled on the mapped clock. Effects stay on the composition clock. Source ranges are checked before rendering. Audio uses resampling rather than pitch-preserving time stretch. Reverse video can render more slowly due to source seeking; headless decoding retains at most 48 MiB of cached frames per decoder.+## Animation System (`LayerAnimation`)Create animations using `LayerAnimation.create()` or fluent factory methods. Animations are composed of keyframed tracks or procedural drivers.