Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
fix(engine): MP4, MOV and TS exports keep sound and picture in sync when streams start apart (#5455)
Miguel Ángelb211fc7cb845
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.tsindex 69d284b9cf..8a69146f7a 100644--- a/packages/engine/src/index.ts+++ b/packages/engine/src/index.ts@@ -213,6 +213,7 @@ export {extractionFrameCountForDuration,resolveProjectRelativeSrc,getFrameAtTime,+ isVideoHiddenBeforeStreamStart,createFrameLookupTable,FrameLookupTable,analyzeClipMediaFit,diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.tsindex 1f97d2bf3f..ad1dc0edc2 100644--- a/packages/engine/src/services/audioFxRender.ts+++ b/packages/engine/src/services/audioFxRender.ts@@ -176,14 +176,7 @@ export function writeWav(writeFileSync(path, buf);}-/**- * Split an interleaved buffer into one array per channel.- *- * The graph used to fold everything to mono, which collapsed a stereo bed's- * width for the render only — and cost ~3 dB through the very mono-to-stereo- * rematrix that `prepareAudioTrack`'s pan filter exists to avoid. Preview kept- * the track stereo, so the two diverged the moment any effect was enabled.- */+/** Split an interleaved buffer into one array per channel, keeping stereo width as preview does. */function deinterleave(samples: Float32Array, channels: number): Float32Array[] {if (channels <= 1) return [samples];const frames = Math.floor(samples.length / channels);diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.tsindex a529255658..cdbdaf63e4 100644--- a/packages/engine/src/services/audioMixer.ts+++ b/packages/engine/src/services/audioMixer.ts@@ -1,15 +1,9 @@// fallow-ignore-file complexity code-duplication-/**- * Audio Mixer Service- *- * Processes and mixes audio tracks using FFmpeg.- */-import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";import { join, dirname, isAbsolute, relative } from "path";import { parseHTML } from "linkedom";-import { extractAudioMetadata } from "../utils/ffprobe.js";+import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";import { isNotMediaPayload } from "../utils/notMediaPayload.js";import { clampAudioGain } from "@hyperframes/core/audio-gain";import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";@@ -61,19 +55,8 @@ import type { AudioVolumeKeyframe } from "./audioMixer.types.js";export type { AudioElement, MixResult } from "./audioMixer.types.js";-/**- * Filename every caller must use for the mixed-audio artifact.- *- * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,- * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record- * the encoder's priming delay, so those leading samples decode as real silence- * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a- * frame-accurate video track. An MP4-family container carries the delay as an- * edit list, which every decoder then strips, so the mix lands on its authored- * start. Keep the choice here rather than at each call site: the same file is- * muxed into the video, shipped in a distributed plan, and handed to users as- * the PNG-sequence sidecar, and all three have to agree.- */+/** An MP4-family container records AAC priming as an edit list; a raw .aac plays ~21 ms late.+ * The video mux, the distributed plan and the PNG-sequence sidecar all read this one name. */export const MIXED_AUDIO_FILENAME = "audio.m4a";function clampVolume(volume: number): number {@@ -129,12 +112,13 @@ const MAX_RAMP_SLICES = 240;function buildRampFilterComplex(lane: RateSpec & object,duration: number,+ headFilter: string | null,tailFilter: string | null,): string {const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS)));const step = duration / slices;const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join("");- const parts = [`[0:a]asplit=${slices}${split}`];+ const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];for (let i = 0; i < slices; i += 1) {const from = sourceTimeAt(lane, i * step);const to = sourceTimeAt(lane, (i + 1) * step);@@ -156,27 +140,20 @@ function buildRampFilterComplex(}function preparedAudioOutputArgs(- srcPath: string,+ metadata: AudioMetadata | null,playbackRate: RateSpec,- duration = 0,-): Promise<string[]> {- return stereoOutputArgs(srcPath).then((channelArgs) => {- const filters: string[] = [];- const outputArgs: string[] = [];- if (channelArgs[0] === "-af" && channelArgs[1]) {- filters.push(channelArgs[1]);- } else {- outputArgs.push(...channelArgs);- }- if (typeof playbackRate === "object") {- const graph = buildRampFilterComplex(playbackRate, duration, filters.join(",") || null);- return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];- }- const atempo = buildAtempoFilter(playbackRate);- if (atempo) filters.push(atempo);- if (filters.length > 0) outputArgs.push("-af", filters.join(","));- return outputArgs;- });+ duration: number,+ headFilter: string | null,+): string[] {+ const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null;+ const outputArgs = channelFilter ? [] : ["-ac", "2"];+ if (typeof playbackRate === "object") {+ const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter);+ return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];+ }+ const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean);+ if (filters.length > 0) outputArgs.push("-af", filters.join(","));+ return outputArgs;}function escapeExpressionCommas(expression: string): string {@@ -217,16 +194,6 @@ const VOLUME_SIMPLIFY_EPSILON = 0.005;// native stereo sources have FL/FR and pass through unchanged.const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC";-async function stereoOutputArgs(srcPath: string): Promise<string[]> {- try {- const { channels } = await extractAudioMetadata(srcPath);- if (channels === 1) return ["-af", STEREO_CHANNEL_FILTER];- } catch {- // Preserve the previous FFmpeg conversion path when metadata probing fails.- }- return ["-ac", "2"];-}-/*** Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear* envelope with a bounded segment count.@@ -654,65 +621,12 @@ export function parseAudioElements(html: string): AudioElement[] {return elements;}-async function extractAudioFromVideo(- videoPath: string,- outputPath: string,- options?: { startTime?: number; duration?: number; playbackRate?: RateSpec },- signal?: AbortSignal,- config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,-): Promise<ExtractResult> {- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;- const outputDir = dirname(outputPath);- if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });-- const playbackRate = normalizeRateSpec(options?.playbackRate);- const args: string[] = [];- if (options?.startTime !== undefined) args.push("-ss", String(options.startTime));- if (options?.duration !== undefined) {- args.push("-t", String(sourceTimeAt(playbackRate, options.duration)));- }- args.push("-i", videoPath);- const outputArgs = await preparedAudioOutputArgs(videoPath, playbackRate, options?.duration);- args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);- if (playbackRate !== 1 && options?.duration !== undefined) {- args.push("-t", String(options.duration));- }- args.push("-y", outputPath);-- const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });-- if (signal?.aborted) {- const failure: AudioProcessingFailure = {- stage: "cancelled",- reason: "cancelled",- owner: "user",- retryable: false,- detail: "Audio extract cancelled",- };- return {- success: false,- outputPath,- durationMs: result.durationMs,- error: failure.detail,- failure,- };- }- if (!result.success) {- const failure = ffmpegFailure("extract", result);- return {- success: false,- outputPath,- durationMs: result.durationMs,- error: failure.detail,- failure,- };- }- return { success: true, outputPath, durationMs: result.durationMs };-}--async function prepareAudioTrack(+/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has+ * started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */+async function extractAudioSegment(srcPath: string,outputPath: string,+ stage: "extract" | "prepare",mediaStart: number,duration: number,playbackRate: RateSpec = 1,@@ -723,21 +637,25 @@ async function prepareAudioTrack(const outputDir = dirname(outputPath);if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });const normalizedPlaybackRate = normalizeRateSpec(playbackRate);- const outputArgs = await preparedAudioOutputArgs(srcPath, normalizedPlaybackRate, duration);+ const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);+ // A failed probe keeps FFmpeg's default channel conversion and the input seek.+ const metadata = await extractAudioMetadata(srcPath).catch(() => null);+ const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);+ const start = formatFilterNumber(mediaStart);+ const timestampTrim = inputSeek+ ? null+ : `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;+ const outputArgs = preparedAudioOutputArgs(+ metadata,+ normalizedPlaybackRate,+ duration,+ timestampTrim,+ );- const args = [- "-ss",- String(mediaStart),- "-t",- String(sourceTimeAt(normalizedPlaybackRate, duration)),- "-i",- srcPath,- "-acodec",- "pcm_s16le",- "-ar",- "48000",- ...outputArgs,- ];+ const args = inputSeek+ ? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]+ : ["-i", srcPath];+ args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));args.push("-y", outputPath);@@ -749,7 +667,7 @@ async function prepareAudioTrack(reason: "cancelled",owner: "user",retryable: false,- detail: "Audio prepare cancelled",+ detail: `Audio ${stage} cancelled`,};return {success: false,@@ -759,7 +677,7 @@ async function prepareAudioTrack(failure,};}- const failure = !result.success ? ffmpegFailure("prepare", result) : undefined;+ const failure = !result.success ? ffmpegFailure(stage, result) : undefined;return {success: result.success,outputPath,@@ -1243,14 +1161,13 @@ export async function processCompositionAudio(let audioSrcPath = srcPath;if (element.type === "video") {const extractedPath = join(workDir, `${element.id}-extracted.wav`);- const extractResult = await extractAudioFromVideo(+ const extractResult = await extractAudioSegment(srcPath,extractedPath,- {- startTime: element.mediaStart,- duration: element.end - element.start,- playbackRate: element.playbackRate,- },+ "extract",+ element.mediaStart,+ element.end - element.start,+ element.playbackRate,effectiveSignal,config,);@@ -1272,9 +1189,10 @@ export async function processCompositionAudio(audioSrcPath = extractedPath;} else {const trimmedPath = join(workDir, `${element.id}-trimmed.wav`);- const prepResult = await prepareAudioTrack(+ const prepResult = await extractAudioSegment(srcPath,trimmedPath,+ "prepare",element.mediaStart,element.end - element.start,element.playbackRate,diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.tsindex f7459f2e25..c1d9f77845 100644--- a/packages/engine/src/services/audioVolumeEnvelope.ts+++ b/packages/engine/src/services/audioVolumeEnvelope.ts@@ -11,7 +11,7 @@* output (and the golden baselines) only change where a fade is actually applied.** The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see- * `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so+ * `extractAudioSegment`). Anything else is rejected so* the caller can fall back to the expression path rather than corrupting audio.*/diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.tsnew file mode 100644index 0000000000..2d87f1f7c7--- /dev/null+++ b/packages/engine/src/services/streamStartOffset.test.ts@@ -0,0 +1,246 @@+import { spawnSync } from "node:child_process";+import { mkdtempSync, rmSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";+import { processCompositionAudio } from "./audioMixer.js";+import { extractMediaMetadata } from "../utils/ffprobe.js";+import {+ extractVideoFramesRange,+ getFrameAtTime,+ resolveFinalFrameExtractionWindow,+ resolveVideoExtractionWindow,+} from "./videoFrameExtractor.js";++const FFMPEG = getFfmpegBinary();+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;+const SAMPLE_RATE = 48_000;+const LEAD_SECONDS = 7;+const CLIP_SECONDS = 12;+const PICTURE_SECONDS = 20;++function ffmpeg(args: string[]): void {+ const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {+ encoding: "utf-8",+ });+ if (result.status !== 0) throw new Error(result.stderr);+}++/** Per output second, the source second heard (the tone is 200 + 100 * source second Hz), or null for silence. */+function sourceSecondsHeard(path: string, seconds: number): Array<number | null> {+ const decoded = spawnSync(+ FFMPEG,+ ["-v", "error", "-i", path, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"],+ { maxBuffer: 1 << 28 },+ );+ if (decoded.status !== 0) throw new Error(decoded.stderr.toString());+ const pcm = decoded.stdout;+ const heard: Array<number | null> = [];+ const edge = 2_000;+ for (let second = 0; second < seconds; second += 1) {+ let crossings = 0;+ let energy = 0;+ let previous = 0;+ for (let i = second * SAMPLE_RATE + edge; i < (second + 1) * SAMPLE_RATE - edge; i += 1) {+ const sample = pcm.readInt16LE(i * 2);+ energy += sample * sample;+ if (i > second * SAMPLE_RATE + edge && previous < 0 !== sample < 0) crossings += 1;+ previous = sample;+ }+ const windowSeconds = (SAMPLE_RATE - 2 * edge) / SAMPLE_RATE;+ const rms = Math.sqrt(energy / (SAMPLE_RATE - 2 * edge));+ heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100) + 0);+ }+ return heard;+}++describe.skipIf(!HAS_FFMPEG)(+ "sources whose audio and video streams start at different times",+ () => {+ const dir = mkdtempSync(join(tmpdir(), "hf-stream-start-"));++ beforeAll(() => {+ ffmpeg([+ "-f",+ "lavfi",+ "-i",+ `testsrc=size=64x36:rate=30:duration=${PICTURE_SECONDS}`,+ "-c:v",+ "libx264",+ "-g",+ "300",+ "-pix_fmt",+ "yuv420p",+ join(dir, "picture.mp4"),+ ]);+ ffmpeg([+ "-f",+ "lavfi",+ "-i",+ `aevalsrc='0.1*sin(2*PI*(200+100*floor(t))*t)':s=${SAMPLE_RATE}:d=20`,+ "-c:a",+ "aac",+ join(dir, "steps.m4a"),+ ]);+ const lead = String(LEAD_SECONDS);+ const picture = join(dir, "picture.mp4");+ const steps = join(dir, "steps.m4a");+ const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];+ ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);+ ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);+ // MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.+ ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);+ }, 60_000);++ afterAll(() => rmSync(dir, { recursive: true, force: true }));++ const flatLane = {+ target: "rate",+ points: [+ { t: 0, v: 1 },+ { t: CLIP_SECONDS, v: 1 },+ ],+ };+ it.each([+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 0 },+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },+ { source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },+ { source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 },+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane },+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },+ {+ source: "audio-late.mp4",+ audioLead: LEAD_SECONDS,+ type: "video",+ mediaStart: 0,+ seconds: 5,+ },+ ] as const)(+ "%#: $source as <$type> from $mediaStart s plays each second where the browser plays it",+ async ({ source, audioLead, type, mediaStart, ...rest }) => {+ const seconds = "seconds" in rest ? rest.seconds : CLIP_SECONDS;+ const workDir = mkdtempSync(join(dir, "work-"));+ const outputPath = `${workDir}.m4a`;+ const result = await processCompositionAudio(+ [+ {+ id: "clip",+ src: source,+ start: 0,+ end: seconds,+ mediaStart,+ ...("rate" in rest ? { playbackRate: rest.rate } : {}),+ layer: 0,+ volume: 1,+ type,+ },+ ],+ dir,+ workDir,+ outputPath,+ seconds,+ );+ expect(result.success, JSON.stringify(result)).toBe(true);+ const expected = Array.from({ length: seconds }, (_, second) => {+ const mediaTime = mediaStart + second;+ return mediaTime < audioLead ? null : mediaTime - audioLead;+ });+ expect(sourceSecondsHeard(outputPath, seconds)).toEqual(expected);+ },+ 30_000,+ );++ // Chrome, seeking a late video stream, paints nothing until its first frame is under 1 s away.+ it("shows no video frame while a late video stream's first frame is a second or more away", async () => {+ const extracted = await extractVideoFramesRange(+ join(dir, "video-late.mp4"),+ "video-late",+ 0,+ CLIP_SECONDS,+ { fps: 30, outputDir: join(dir, "frames") },+ );+ expect(getFrameAtTime(extracted, 1, 0)).toBeNull();+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 1.1, 0)).toBeNull();+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.9, 0)).not.toBeNull();+ expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();+ }, 30_000);++ it("keeps every frame of a video stream that starts first", async () => {+ const extracted = await extractVideoFramesRange(+ join(dir, "audio-late.mp4"),+ "audio-late",+ 0,+ CLIP_SECONDS,+ { fps: 30, outputDir: join(dir, "frames") },+ );+ expect(getFrameAtTime(extracted, 0, 0)).not.toBeNull();+ expect(getFrameAtTime(extracted, 1, 0)).not.toBeNull();+ }, 30_000);++ it("hides a late video stream's lead in a transport whose clock starts at 1.4 s", async () => {+ const extracted = await extractVideoFramesRange(+ join(dir, "video-late.ts"),+ "video-late-ts",+ 0,+ CLIP_SECONDS,+ { fps: 30, outputDir: join(dir, "frames") },+ );+ expect(getFrameAtTime(extracted, 1, 0)).toBeNull();+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull();+ }, 30_000);++ it("plays a late video stream to its end on the media timeline and loops on its full length", async () => {+ const source = join(dir, "video-late.mp4");+ const mediaSeconds = LEAD_SECONDS + PICTURE_SECONDS;+ const window = resolveVideoExtractionWindow(+ { start: 0, end: mediaSeconds, mediaStart: 0, loop: false },+ await extractMediaMetadata(source),+ mediaSeconds,+ );+ expect(window.durationSeconds).toBeCloseTo(mediaSeconds, 1);++ const extracted = await extractVideoFramesRange(source, "video-late-full", 0, mediaSeconds, {+ fps: 10,+ outputDir: join(dir, "frames"),+ });+ expect(getFrameAtTime(extracted, PICTURE_SECONDS + 1.5, 0, true)).not.toBeNull();+ expect(getFrameAtTime(extracted, mediaSeconds + 1.5, 0, true)).toBeNull();+ }, 30_000);++ it.each([6.5, 10])(+ "shows a late video stream from a trim at %s s on the media timeline",+ async (mediaStart) => {+ const extracted = await extractVideoFramesRange(+ join(dir, "video-late.mp4"),+ `video-late-from-${mediaStart}`,+ mediaStart,+ 5,+ { fps: 10, outputDir: join(dir, "frames") },+ );+ expect(getFrameAtTime(extracted, 0.05, 0, false, mediaStart)).not.toBeNull();+ },+ 30_000,+ );++ it("holds a late video stream's last frame past its end", async () => {+ const source = join(dir, "video-late.mp4");+ const metadata = await extractMediaMetadata(source);+ const video = {+ start: 0,+ end: 5,+ mediaStart: LEAD_SECONDS + PICTURE_SECONDS + 0.5,+ loop: false,+ };+ const window = await resolveFinalFrameExtractionWindow(+ source,+ video,+ metadata,+ resolveVideoExtractionWindow(video, metadata, 5),+ );+ expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1);+ }, 30_000);+ },+);diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.tsindex e5c9aac861..0a38fa3852 100644--- a/packages/engine/src/services/videoFrameExtractor.ts+++ b/packages/engine/src/services/videoFrameExtractor.ts@@ -1229,9 +1229,10 @@ export async function extractVideoFramesRange(if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {- const correctedDuration = startTime + framePaths.size / fps;- if (correctedDuration < metadata.videoStreamDurationSeconds) {- metadata.videoStreamDurationSeconds = correctedDuration;+ const correctedStreamDuration =+ startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);+ if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {+ metadata.videoStreamDurationSeconds = correctedStreamDuration;}}}@@ -1302,7 +1303,7 @@ function resolveSegmentDuration(export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {return Number.isFinite(metadata.videoStreamDurationSeconds) &&metadata.videoStreamDurationSeconds > 0- ? metadata.videoStreamDurationSeconds+ ? (metadata.videoStreamLeadSeconds ?? 0) + metadata.videoStreamDurationSeconds: metadata.durationSeconds;}@@ -1503,14 +1504,17 @@ export async function resolveFinalFrameExtractionWindow(): Promise<TimelineExtractionWindow> {if (!window.ensureFinalFrame) return window;const playableDuration = resolvePlayableVideoDuration(metadata);- const finalFrameTimestamp = await extractFinalVideoFrameTimestamp(- videoPath,- {- videoStreamDurationSeconds: playableDuration,- videoStreamStartSeconds: metadata.videoStreamStartSeconds,- },- signal,- );+ const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;+ const finalFrameTimestamp =+ videoStreamLead ++ (await extractFinalVideoFrameTimestamp(+ videoPath,+ {+ videoStreamDurationSeconds: playableDuration - videoStreamLead,+ videoStreamStartSeconds: metadata.videoStreamStartSeconds,+ },+ signal,+ ));if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;const sourceRemaining = playableDuration - video.mediaStart;@@ -2613,6 +2617,18 @@ export async function extractAllVideoFrames(};}+const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;++export function isVideoHiddenBeforeStreamStart(+ videoStreamLeadSeconds: number | undefined,+ mediaTime: number,+): boolean {+ return (+ (videoStreamLeadSeconds ?? 0) - mediaTime >=+ BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6+ );+}+function getFrameIndexAtTime(extracted: ExtractedFrames,globalTime: number,@@ -2646,6 +2662,14 @@ function getFrameIndexAtTime(// e.g. 0.28 * 25 === 6.999999999999999 instead of 7.const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);if (frameIndex < 0 || extracted.totalFrames <= 0) return null;+ if (+ isVideoHiddenBeforeStreamStart(+ extracted.metadata.videoStreamLeadSeconds,+ mediaStart + sourceTime,+ )+ ) {+ return null;+ }if (frameIndex >= extracted.totalFrames) {return loop || holdLastFrame ? extracted.totalFrames - 1 : null;}diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.tsindex ad63d91fff..76eb27acff 100644--- a/packages/engine/src/utils/ffprobe.ts+++ b/packages/engine/src/utils/ffprobe.ts@@ -184,6 +184,8 @@ export interface VideoMetadata {* timestamps are absolute, so callers crossing those APIs must normalize by* this value. Absent only in legacy/manually-constructed metadata. */videoStreamStartSeconds?: number;+ /** Seconds after the file's media time 0 (its earliest audio/video stream start) at which video begins. */+ videoStreamLeadSeconds?: number;width: number;height: number;fps: number;@@ -214,6 +216,8 @@ export interface AudioMetadata {channels: number;audioCodec: string;bitrate?: number;+ /** Seconds after the file's media time 0 at which its last audio or video stream starts. */+ latestStreamLeadSeconds?: number;}interface FFProbeStream {@@ -820,6 +824,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetaddurationSeconds: containerDuration,videoStreamDurationSeconds: effectiveStreamDuration,videoStreamStartSeconds: streamStart,+ videoStreamLeadSeconds: streamLeadSeconds(output, videoStream),width: videoStream.width || stillImage()?.width || 0,height: videoStream.height || stillImage()?.height || 0,fps,@@ -989,6 +994,7 @@ export async function extractAudioMetadata(channels: audioStream.channels || 2,audioCodec,bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,+ latestStreamLeadSeconds: latestStreamLeadSeconds(output),};})();@@ -997,6 +1003,28 @@ export async function extractAudioMetadata(return probePromise;}+/** Chrome's media time 0 is the earliest of these; a stream starting later plays that much in. */+function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {+ return (output?.streams ?? [])+ .filter(+ (s) =>+ (s.codec_type === "audio" || s.codec_type === "video") && s.disposition?.attached_pic !== 1,+ )+ .map((s) => parseFloat(s.start_time ?? ""))+ .filter(Number.isFinite);+}++function streamLeadSeconds(output: FFProbeOutput | null, stream: FFProbeStream): number {+ const starts = audioVideoStreamStarts(output);+ const start = parseFloat(stream.start_time ?? "");+ return Number.isFinite(start) && starts.length > 0 ? Math.max(0, start - Math.min(...starts)) : 0;+}++function latestStreamLeadSeconds(output: FFProbeOutput): number {+ const starts = audioVideoStreamStarts(output);+ return starts.length > 0 ? Math.max(...starts) - Math.min(...starts) : 0;+}+function audioStartSeconds(stream: FFProbeStream, format: FFProbeFormat): number {const start = parseFloat(stream.start_time ?? format.start_time ?? "");return Number.isFinite(start) ? start : 0;diff --git a/packages/producer/src/services/distributed/shared.ts b/packages/producer/src/services/distributed/shared.tsindex ef574f8d8f..6d5171bd0e 100644--- a/packages/producer/src/services/distributed/shared.ts+++ b/packages/producer/src/services/distributed/shared.ts@@ -239,6 +239,10 @@ function readVideoMetadata(record.videoStreamStartSeconds === undefined? 0: readFiniteNumber(record.videoStreamStartSeconds, `${field}.videoStreamStartSeconds`),+ videoStreamLeadSeconds:+ record.videoStreamLeadSeconds === undefined+ ? 0+ : readFiniteNumber(record.videoStreamLeadSeconds, `${field}.videoStreamLeadSeconds`),width: readPositiveInteger(record.width, `${field}.width`),height: readPositiveInteger(record.height, `${field}.height`),fps: readFiniteNumber(record.fps, `${field}.fps`),diff --git a/packages/producer/src/services/distributed/videoMetadata.test.ts b/packages/producer/src/services/distributed/videoMetadata.test.tsindex c62719e4c8..bf1fa104ee 100644--- a/packages/producer/src/services/distributed/videoMetadata.test.ts+++ b/packages/producer/src/services/distributed/videoMetadata.test.ts@@ -94,7 +94,7 @@ describe("distributed video metadata", () => {expect(sourceDerived.videos[0]?.mediaStart).toBe(1);});- it("round-trips a non-zero video stream start and defaults legacy plans to zero", () => {+ it("round-trips a non-zero video stream start and lead and defaults legacy plans to zero", () => {const withStart = buildPlanVideosJson({videos: [video()],extracted: [@@ -102,16 +102,22 @@ describe("distributed video metadata", () => {metadata: {...extractedMetadata().metadata,videoStreamStartSeconds: 5,+ videoStreamLeadSeconds: 7,},}),],compositionEnd: 8,});- expect(parsePlanVideosJson(withStart).extracted[0]?.metadata.videoStreamStartSeconds).toBe(5);+ const parsed = parsePlanVideosJson(withStart).extracted[0]?.metadata;+ expect(parsed?.videoStreamStartSeconds).toBe(5);+ expect(parsed?.videoStreamLeadSeconds).toBe(7);const legacy = structuredClone(withStart);delete legacy.extracted[0]?.metadata.videoStreamStartSeconds;- expect(parsePlanVideosJson(legacy).extracted[0]?.metadata.videoStreamStartSeconds).toBe(0);+ delete legacy.extracted[0]?.metadata.videoStreamLeadSeconds;+ const legacyParsed = parsePlanVideosJson(legacy).extracted[0]?.metadata;+ expect(legacyParsed?.videoStreamStartSeconds).toBe(0);+ expect(legacyParsed?.videoStreamLeadSeconds).toBe(0);});it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])(diff --git a/packages/producer/src/services/hdrCompositor.test.ts b/packages/producer/src/services/hdrCompositor.test.tsindex 774301c9cb..1680f9a90f 100644--- a/packages/producer/src/services/hdrCompositor.test.ts+++ b/packages/producer/src/services/hdrCompositor.test.ts@@ -1,6 +1,9 @@+import { mkdtempSync, openSync, closeSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";import { describe, expect, it } from "vitest";import type { ElementStackingInfo } from "@hyperframes/engine";-import { selectDomLayerShowIds } from "./hdrCompositor.js";+import { blitHdrVideoLayer, selectDomLayerShowIds } from "./hdrCompositor.js";function makeEl(id: string, overrides?: Partial<ElementStackingInfo>): ElementStackingInfo {return {@@ -62,3 +65,47 @@ describe("selectDomLayerShowIds", () => {).toEqual(["active-overlay"]);});});++describe("blitHdrVideoLayer", () => {+ it.each([+ { lead: undefined, mediaStart: 0, painted: true },+ { lead: 7, mediaStart: 0, painted: false },+ { lead: 7, mediaStart: 6.5, painted: true },+ ])(+ "paints at 1 s of a clip from $mediaStart s whose stream starts $lead s in: $painted",+ ({ lead, mediaStart, painted }) => {+ const dir = mkdtempSync(join(tmpdir(), "hf-hdr-lead-"));+ const rawPath = join(dir, "frames.rgb48le");+ const frameSize = 2 * 2 * 6;+ writeFileSync(rawPath, Buffer.alloc(frameSize * 2, 0x80));+ const fd = openSync(rawPath, "r");+ try {+ const canvas = Buffer.alloc(frameSize);+ const el = makeEl("v", {+ width: 2,+ height: 2,+ layoutWidth: 2,+ layoutHeight: 2,+ isHdr: true,+ });+ const source = {+ dir,+ rawPath,+ fd,+ width: 2,+ height: 2,+ frameSize,+ frameCount: 2,+ scratch: Buffer.alloc(frameSize),+ mediaStart,+ videoStreamLeadSeconds: lead,+ };+ blitHdrVideoLayer(canvas, el, 1, 1, new Map([["v", source]]), new Map([["v", 0]]), 2, 2);+ expect(canvas.some((byte) => byte !== 0)).toBe(painted);+ } finally {+ closeSync(fd);+ rmSync(dir, { recursive: true, force: true });+ }+ },+ );+});diff --git a/packages/producer/src/services/hdrCompositor.ts b/packages/producer/src/services/hdrCompositor.tsindex cff19df762..13e474dce7 100644--- a/packages/producer/src/services/hdrCompositor.ts+++ b/packages/producer/src/services/hdrCompositor.ts@@ -29,6 +29,7 @@ import {blitRgb48leAffine,parseTransformMatrix,convertTransfer,+ isVideoHiddenBeforeStreamStart,} from "@hyperframes/engine";import type { ProducerLogger } from "../logger.js";import { type HdrImageTransferCache } from "./hdrImageTransferCache.js";@@ -141,6 +142,8 @@ export interface HdrVideoFrameSource {scratch: Buffer;/** The raw file contains one playable source cycle and must wrap at EOF. */loop?: boolean;+ mediaStart?: number;+ videoStreamLeadSeconds?: number;}export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {@@ -198,6 +201,8 @@ export function blitHdrVideoLayer(frameSource.loop,);if (effectiveIndex === null) return;+ const mediaTime = (frameSource.mediaStart ?? 0) + effectiveIndex / fps;+ if (isVideoHiddenBeforeStreamStart(frameSource.videoStreamLeadSeconds, mediaTime)) return;const frameOffset = effectiveIndex * frameSource.frameSize;try {diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.tsindex 2634122b76..c53cf21f0f 100644--- a/packages/producer/src/services/render/stages/captureHdrResources.ts+++ b/packages/producer/src/services/render/stages/captureHdrResources.ts@@ -419,7 +419,10 @@ export async function extractHdrVideoFrames(args: {const out = new Map<string, HdrVideoFrameSource>();mkdirSync(framesDir, { recursive: true });const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = [];- const extractionWindows = new Map<string, HdrExtractionWindow>();+ const extractionWindows = new Map<+ string,+ { window: HdrExtractionWindow; videoStreamLeadSeconds?: number }+ >();for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {const video = composition.videos.find((v) => v.id === videoId);if (!video) continue;@@ -434,7 +437,10 @@ export async function extractHdrVideoFrames(args: {initialWindow,abortSignal,);- extractionWindows.set(videoId, window);+ extractionWindows.set(videoId, {+ window,+ videoStreamLeadSeconds: metadata.videoStreamLeadSeconds,+ });prep.hdrVideoStartTimes.set(videoId, window.compositionStart);plannedVideos.push({durationSeconds: window.durationSeconds,@@ -453,14 +459,16 @@ export async function extractHdrVideoFrames(args: {try {for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {const video = composition.videos.find((v) => v.id === videoId);- const window = extractionWindows.get(videoId);- if (!video || !window) continue;+ const planned = extractionWindows.get(videoId);+ if (!video || !planned) continue;+ const { window } = planned;mkdirSync(framesDir, { recursive: true });const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`));createdFrameDirs.add(frameDir);const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };const rawPath = join(frameDir, "frames.rgb48le");- const extractionStart = String(window.extractionMediaStart ?? window.mediaStart);+ const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;+ const extractionStart = String(extractionMediaStart);const ffmpegArgs: string[] = [];if (window.finalFrameOnly) {// Decode before seeking for the one-frame path. Input-side seeking can@@ -524,6 +532,8 @@ export async function extractHdrVideoFrames(args: {frameCount,scratch: Buffer.allocUnsafe(frameSize),loop: video.loop,+ mediaStart: extractionMediaStart,+ videoStreamLeadSeconds: planned.videoStreamLeadSeconds,});handedOff = true;} finally {