Skip to content

Write HTML. Render video. Built for agents.

fix(engine): MP4, MOV and TS exports keep sound and picture in sync when streams start apart (#5455)

Miguel Ángelb211fc7cb845
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 69d284b9cf..8a69146f7a 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -213,6 +213,7 @@ export {
extractionFrameCountForDuration,
resolveProjectRelativeSrc,
getFrameAtTime,
+ isVideoHiddenBeforeStreamStart,
createFrameLookupTable,
FrameLookupTable,
analyzeClipMediaFit,
diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts
index 1f97d2bf3f..ad1dc0edc2 100644
--- a/packages/engine/src/services/audioFxRender.ts
+++ b/packages/engine/src/services/audioFxRender.ts
@@ -176,14 +176,7 @@ export function writeWav(
writeFileSync(path, buf);
}
-/**
- * Split an interleaved buffer into one array per channel.
- *
- * The graph used to fold everything to mono, which collapsed a stereo bed's
- * width for the render only — and cost ~3 dB through the very mono-to-stereo
- * rematrix that `prepareAudioTrack`'s pan filter exists to avoid. Preview kept
- * the track stereo, so the two diverged the moment any effect was enabled.
- */
+/** Split an interleaved buffer into one array per channel, keeping stereo width as preview does. */
function deinterleave(samples: Float32Array, channels: number): Float32Array[] {
if (channels <= 1) return [samples];
const frames = Math.floor(samples.length / channels);
diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts
index a529255658..cdbdaf63e4 100644
--- a/packages/engine/src/services/audioMixer.ts
+++ b/packages/engine/src/services/audioMixer.ts
@@ -1,15 +1,9 @@
// fallow-ignore-file complexity code-duplication
-/**
- * Audio Mixer Service
- *
- * Processes and mixes audio tracks using FFmpeg.
- */
-
import { isSelfOrAncestorHidden, memberGroupKey, isMemberGroupHidden } from "./mediaHidden.js";
import { closeSync, existsSync, mkdirSync, mkdtempSync, openSync, rmSync, writeFileSync } from "fs";
import { join, dirname, isAbsolute, relative } from "path";
import { parseHTML } from "linkedom";
-import { extractAudioMetadata } from "../utils/ffprobe.js";
+import { extractAudioMetadata, type AudioMetadata } from "../utils/ffprobe.js";
import { isNotMediaPayload } from "../utils/notMediaPayload.js";
import { clampAudioGain } from "@hyperframes/core/audio-gain";
import { clampFadesToDuration, readElementFades } from "@hyperframes/core/audio-fade";
@@ -61,19 +55,8 @@ import type { AudioVolumeKeyframe } from "./audioMixer.types.js";
export type { AudioElement, MixResult } from "./audioMixer.types.js";
-/**
- * Filename every caller must use for the mixed-audio artifact.
- *
- * The extension is load-bearing, not cosmetic: FFmpeg picks the muxer from it,
- * and the mix is AAC-encoded. A raw ADTS `.aac` stream has nowhere to record
- * the encoder's priming delay, so those leading samples decode as real silence
- * and shift the whole track ~1024 samples (21.33 ms at 48 kHz) late against a
- * frame-accurate video track. An MP4-family container carries the delay as an
- * edit list, which every decoder then strips, so the mix lands on its authored
- * start. Keep the choice here rather than at each call site: the same file is
- * muxed into the video, shipped in a distributed plan, and handed to users as
- * the PNG-sequence sidecar, and all three have to agree.
- */
+/** An MP4-family container records AAC priming as an edit list; a raw .aac plays ~21 ms late.
+ * The video mux, the distributed plan and the PNG-sequence sidecar all read this one name. */
export const MIXED_AUDIO_FILENAME = "audio.m4a";
function clampVolume(volume: number): number {
@@ -129,12 +112,13 @@ const MAX_RAMP_SLICES = 240;
function buildRampFilterComplex(
lane: RateSpec & object,
duration: number,
+ headFilter: string | null,
tailFilter: string | null,
): string {
const slices = Math.min(MAX_RAMP_SLICES, Math.max(1, Math.ceil(duration / RAMP_SLICE_SECONDS)));
const step = duration / slices;
const split = Array.from({ length: slices }, (_, i) => `[s${i}]`).join("");
- const parts = [`[0:a]asplit=${slices}${split}`];
+ const parts = [`[0:a]${headFilter ? `${headFilter},` : ""}asplit=${slices}${split}`];
for (let i = 0; i < slices; i += 1) {
const from = sourceTimeAt(lane, i * step);
const to = sourceTimeAt(lane, (i + 1) * step);
@@ -156,27 +140,20 @@ function buildRampFilterComplex(
}
function preparedAudioOutputArgs(
- srcPath: string,
+ metadata: AudioMetadata | null,
playbackRate: RateSpec,
- duration = 0,
-): Promise<string[]> {
- return stereoOutputArgs(srcPath).then((channelArgs) => {
- const filters: string[] = [];
- const outputArgs: string[] = [];
- if (channelArgs[0] === "-af" && channelArgs[1]) {
- filters.push(channelArgs[1]);
- } else {
- outputArgs.push(...channelArgs);
- }
- if (typeof playbackRate === "object") {
- const graph = buildRampFilterComplex(playbackRate, duration, filters.join(",") || null);
- return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
- }
- const atempo = buildAtempoFilter(playbackRate);
- if (atempo) filters.push(atempo);
- if (filters.length > 0) outputArgs.push("-af", filters.join(","));
- return outputArgs;
- });
+ duration: number,
+ headFilter: string | null,
+): string[] {
+ const channelFilter = metadata?.channels === 1 ? STEREO_CHANNEL_FILTER : null;
+ const outputArgs = channelFilter ? [] : ["-ac", "2"];
+ if (typeof playbackRate === "object") {
+ const graph = buildRampFilterComplex(playbackRate, duration, headFilter, channelFilter);
+ return ["-filter_complex", graph, "-map", "[out]", ...outputArgs];
+ }
+ const filters = [headFilter, channelFilter, buildAtempoFilter(playbackRate)].filter(Boolean);
+ if (filters.length > 0) outputArgs.push("-af", filters.join(","));
+ return outputArgs;
}
function escapeExpressionCommas(expression: string): string {
@@ -217,16 +194,6 @@ const VOLUME_SIMPLIFY_EPSILON = 0.005;
// native stereo sources have FL/FR and pass through unchanged.
const STEREO_CHANNEL_FILTER = "pan=stereo|FL=FL+FC|FR=FR+FC";
-async function stereoOutputArgs(srcPath: string): Promise<string[]> {
- try {
- const { channels } = await extractAudioMetadata(srcPath);
- if (channels === 1) return ["-af", STEREO_CHANNEL_FILTER];
- } catch {
- // Preserve the previous FFmpeg conversion path when metadata probing fails.
- }
- return ["-ac", "2"];
-}
-
/**
* Reduce a sorted keyframe list to a perceptually-equivalent piecewise-linear
* envelope with a bounded segment count.
@@ -654,65 +621,12 @@ export function parseAudioElements(html: string): AudioElement[] {
return elements;
}
-async function extractAudioFromVideo(
- videoPath: string,
- outputPath: string,
- options?: { startTime?: number; duration?: number; playbackRate?: RateSpec },
- signal?: AbortSignal,
- config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
-): Promise<ExtractResult> {
- const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
- const outputDir = dirname(outputPath);
- if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
-
- const playbackRate = normalizeRateSpec(options?.playbackRate);
- const args: string[] = [];
- if (options?.startTime !== undefined) args.push("-ss", String(options.startTime));
- if (options?.duration !== undefined) {
- args.push("-t", String(sourceTimeAt(playbackRate, options.duration)));
- }
- args.push("-i", videoPath);
- const outputArgs = await preparedAudioOutputArgs(videoPath, playbackRate, options?.duration);
- args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
- if (playbackRate !== 1 && options?.duration !== undefined) {
- args.push("-t", String(options.duration));
- }
- args.push("-y", outputPath);
-
- const result = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout });
-
- if (signal?.aborted) {
- const failure: AudioProcessingFailure = {
- stage: "cancelled",
- reason: "cancelled",
- owner: "user",
- retryable: false,
- detail: "Audio extract cancelled",
- };
- return {
- success: false,
- outputPath,
- durationMs: result.durationMs,
- error: failure.detail,
- failure,
- };
- }
- if (!result.success) {
- const failure = ffmpegFailure("extract", result);
- return {
- success: false,
- outputPath,
- durationMs: result.durationMs,
- error: failure.detail,
- failure,
- };
- }
- return { success: true, outputPath, durationMs: result.durationMs };
-}
-
-async function prepareAudioTrack(
+/** Decode `duration` s of audio from `mediaStart` in Chrome's media time. Until every stream has
+ * started, an input seek snaps audio to the video's first keyframe, so read by timestamp instead. */
+async function extractAudioSegment(
srcPath: string,
outputPath: string,
+ stage: "extract" | "prepare",
mediaStart: number,
duration: number,
playbackRate: RateSpec = 1,
@@ -723,21 +637,25 @@ async function prepareAudioTrack(
const outputDir = dirname(outputPath);
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
- const outputArgs = await preparedAudioOutputArgs(srcPath, normalizedPlaybackRate, duration);
+ const sourceDuration = sourceTimeAt(normalizedPlaybackRate, duration);
+ // A failed probe keeps FFmpeg's default channel conversion and the input seek.
+ const metadata = await extractAudioMetadata(srcPath).catch(() => null);
+ const inputSeek = mediaStart >= (metadata?.latestStreamLeadSeconds ?? 0);
+ const start = formatFilterNumber(mediaStart);
+ const timestampTrim = inputSeek
+ ? null
+ : `atrim=start=${start}:end=${formatFilterNumber(mediaStart + sourceDuration)},asetpts=PTS-${start}/TB,aresample=async=1:first_pts=0`;
+ const outputArgs = preparedAudioOutputArgs(
+ metadata,
+ normalizedPlaybackRate,
+ duration,
+ timestampTrim,
+ );
- const args = [
- "-ss",
- String(mediaStart),
- "-t",
- String(sourceTimeAt(normalizedPlaybackRate, duration)),
- "-i",
- srcPath,
- "-acodec",
- "pcm_s16le",
- "-ar",
- "48000",
- ...outputArgs,
- ];
+ const args = inputSeek
+ ? ["-ss", String(mediaStart), "-t", String(sourceDuration), "-i", srcPath]
+ : ["-i", srcPath];
+ args.push("-vn", "-acodec", "pcm_s16le", "-ar", "48000", ...outputArgs);
if (normalizedPlaybackRate !== 1) args.push("-t", String(duration));
args.push("-y", outputPath);
@@ -749,7 +667,7 @@ async function prepareAudioTrack(
reason: "cancelled",
owner: "user",
retryable: false,
- detail: "Audio prepare cancelled",
+ detail: `Audio ${stage} cancelled`,
};
return {
success: false,
@@ -759,7 +677,7 @@ async function prepareAudioTrack(
failure,
};
}
- const failure = !result.success ? ffmpegFailure("prepare", result) : undefined;
+ const failure = !result.success ? ffmpegFailure(stage, result) : undefined;
return {
success: result.success,
outputPath,
@@ -1243,14 +1161,13 @@ export async function processCompositionAudio(
let audioSrcPath = srcPath;
if (element.type === "video") {
const extractedPath = join(workDir, `${element.id}-extracted.wav`);
- const extractResult = await extractAudioFromVideo(
+ const extractResult = await extractAudioSegment(
srcPath,
extractedPath,
- {
- startTime: element.mediaStart,
- duration: element.end - element.start,
- playbackRate: element.playbackRate,
- },
+ "extract",
+ element.mediaStart,
+ element.end - element.start,
+ element.playbackRate,
effectiveSignal,
config,
);
@@ -1272,9 +1189,10 @@ export async function processCompositionAudio(
audioSrcPath = extractedPath;
} else {
const trimmedPath = join(workDir, `${element.id}-trimmed.wav`);
- const prepResult = await prepareAudioTrack(
+ const prepResult = await extractAudioSegment(
srcPath,
trimmedPath,
+ "prepare",
element.mediaStart,
element.end - element.start,
element.playbackRate,
diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.ts
index f7459f2e25..c1d9f77845 100644
--- a/packages/engine/src/services/audioVolumeEnvelope.ts
+++ b/packages/engine/src/services/audioVolumeEnvelope.ts
@@ -11,7 +11,7 @@
* output (and the golden baselines) only change where a fade is actually applied.
*
* The prepared tracks are always `pcm_s16le`, 48 kHz, stereo (see
- * `prepareAudioTrack` / `extractAudioFromVideo`). Anything else is rejected so
+ * `extractAudioSegment`). Anything else is rejected so
* the caller can fall back to the expression path rather than corrupting audio.
*/
diff --git a/packages/engine/src/services/streamStartOffset.test.ts b/packages/engine/src/services/streamStartOffset.test.ts
new file mode 100644
index 0000000000..2d87f1f7c7
--- /dev/null
+++ b/packages/engine/src/services/streamStartOffset.test.ts
@@ -0,0 +1,246 @@
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { getFfmpegBinary } from "../utils/ffmpegBinaries.js";
+import { processCompositionAudio } from "./audioMixer.js";
+import { extractMediaMetadata } from "../utils/ffprobe.js";
+import {
+ extractVideoFramesRange,
+ getFrameAtTime,
+ resolveFinalFrameExtractionWindow,
+ resolveVideoExtractionWindow,
+} from "./videoFrameExtractor.js";
+
+const FFMPEG = getFfmpegBinary();
+const HAS_FFMPEG = spawnSync(FFMPEG, ["-version"]).status === 0;
+const SAMPLE_RATE = 48_000;
+const LEAD_SECONDS = 7;
+const CLIP_SECONDS = 12;
+const PICTURE_SECONDS = 20;
+
+function ffmpeg(args: string[]): void {
+ const result = spawnSync(FFMPEG, ["-nostdin", "-v", "error", "-y", ...args], {
+ encoding: "utf-8",
+ });
+ if (result.status !== 0) throw new Error(result.stderr);
+}
+
+/** Per output second, the source second heard (the tone is 200 + 100 * source second Hz), or null for silence. */
+function sourceSecondsHeard(path: string, seconds: number): Array<number | null> {
+ const decoded = spawnSync(
+ FFMPEG,
+ ["-v", "error", "-i", path, "-ac", "1", "-ar", String(SAMPLE_RATE), "-f", "s16le", "-"],
+ { maxBuffer: 1 << 28 },
+ );
+ if (decoded.status !== 0) throw new Error(decoded.stderr.toString());
+ const pcm = decoded.stdout;
+ const heard: Array<number | null> = [];
+ const edge = 2_000;
+ for (let second = 0; second < seconds; second += 1) {
+ let crossings = 0;
+ let energy = 0;
+ let previous = 0;
+ for (let i = second * SAMPLE_RATE + edge; i < (second + 1) * SAMPLE_RATE - edge; i += 1) {
+ const sample = pcm.readInt16LE(i * 2);
+ energy += sample * sample;
+ if (i > second * SAMPLE_RATE + edge && previous < 0 !== sample < 0) crossings += 1;
+ previous = sample;
+ }
+ const windowSeconds = (SAMPLE_RATE - 2 * edge) / SAMPLE_RATE;
+ const rms = Math.sqrt(energy / (SAMPLE_RATE - 2 * edge));
+ heard.push(rms < 30 ? null : Math.round((crossings / 2 / windowSeconds - 200) / 100) + 0);
+ }
+ return heard;
+}
+
+describe.skipIf(!HAS_FFMPEG)(
+ "sources whose audio and video streams start at different times",
+ () => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-stream-start-"));
+
+ beforeAll(() => {
+ ffmpeg([
+ "-f",
+ "lavfi",
+ "-i",
+ `testsrc=size=64x36:rate=30:duration=${PICTURE_SECONDS}`,
+ "-c:v",
+ "libx264",
+ "-g",
+ "300",
+ "-pix_fmt",
+ "yuv420p",
+ join(dir, "picture.mp4"),
+ ]);
+ ffmpeg([
+ "-f",
+ "lavfi",
+ "-i",
+ `aevalsrc='0.1*sin(2*PI*(200+100*floor(t))*t)':s=${SAMPLE_RATE}:d=20`,
+ "-c:a",
+ "aac",
+ join(dir, "steps.m4a"),
+ ]);
+ const lead = String(LEAD_SECONDS);
+ const picture = join(dir, "picture.mp4");
+ const steps = join(dir, "steps.m4a");
+ const mux = ["-map", "0:v", "-map", "1:a", "-c", "copy"];
+ ffmpeg(["-itsoffset", lead, "-i", picture, "-i", steps, ...mux, join(dir, "video-late.mp4")]);
+ ffmpeg(["-i", picture, "-itsoffset", lead, "-i", steps, ...mux, join(dir, "audio-late.mp4")]);
+ // MPEG-TS starts its clock at 1.4 s, so no stream starts at 0.
+ ffmpeg(["-i", join(dir, "video-late.mp4"), "-c", "copy", join(dir, "video-late.ts")]);
+ }, 60_000);
+
+ afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+ const flatLane = {
+ target: "rate",
+ points: [
+ { t: 0, v: 1 },
+ { t: CLIP_SECONDS, v: 1 },
+ ],
+ };
+ it.each([
+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 0 },
+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3 },
+ { source: "video-late.mp4", audioLead: 0, type: "audio", mediaStart: 0 },
+ { source: "video-late.ts", audioLead: 0, type: "video", mediaStart: 0 },
+ { source: "video-late.mp4", audioLead: 0, type: "video", mediaStart: 3, rate: flatLane },
+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 0 },
+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "video", mediaStart: 3 },
+ { source: "audio-late.mp4", audioLead: LEAD_SECONDS, type: "audio", mediaStart: 0 },
+ {
+ source: "audio-late.mp4",
+ audioLead: LEAD_SECONDS,
+ type: "video",
+ mediaStart: 0,
+ seconds: 5,
+ },
+ ] as const)(
+ "%#: $source as <$type> from $mediaStart s plays each second where the browser plays it",
+ async ({ source, audioLead, type, mediaStart, ...rest }) => {
+ const seconds = "seconds" in rest ? rest.seconds : CLIP_SECONDS;
+ const workDir = mkdtempSync(join(dir, "work-"));
+ const outputPath = `${workDir}.m4a`;
+ const result = await processCompositionAudio(
+ [
+ {
+ id: "clip",
+ src: source,
+ start: 0,
+ end: seconds,
+ mediaStart,
+ ...("rate" in rest ? { playbackRate: rest.rate } : {}),
+ layer: 0,
+ volume: 1,
+ type,
+ },
+ ],
+ dir,
+ workDir,
+ outputPath,
+ seconds,
+ );
+ expect(result.success, JSON.stringify(result)).toBe(true);
+ const expected = Array.from({ length: seconds }, (_, second) => {
+ const mediaTime = mediaStart + second;
+ return mediaTime < audioLead ? null : mediaTime - audioLead;
+ });
+ expect(sourceSecondsHeard(outputPath, seconds)).toEqual(expected);
+ },
+ 30_000,
+ );
+
+ // Chrome, seeking a late video stream, paints nothing until its first frame is under 1 s away.
+ it("shows no video frame while a late video stream's first frame is a second or more away", async () => {
+ const extracted = await extractVideoFramesRange(
+ join(dir, "video-late.mp4"),
+ "video-late",
+ 0,
+ CLIP_SECONDS,
+ { fps: 30, outputDir: join(dir, "frames") },
+ );
+ expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 1.1, 0)).toBeNull();
+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.9, 0)).not.toBeNull();
+ expect(getFrameAtTime(extracted, 8, 0)).not.toBeNull();
+ }, 30_000);
+
+ it("keeps every frame of a video stream that starts first", async () => {
+ const extracted = await extractVideoFramesRange(
+ join(dir, "audio-late.mp4"),
+ "audio-late",
+ 0,
+ CLIP_SECONDS,
+ { fps: 30, outputDir: join(dir, "frames") },
+ );
+ expect(getFrameAtTime(extracted, 0, 0)).not.toBeNull();
+ expect(getFrameAtTime(extracted, 1, 0)).not.toBeNull();
+ }, 30_000);
+
+ it("hides a late video stream's lead in a transport whose clock starts at 1.4 s", async () => {
+ const extracted = await extractVideoFramesRange(
+ join(dir, "video-late.ts"),
+ "video-late-ts",
+ 0,
+ CLIP_SECONDS,
+ { fps: 30, outputDir: join(dir, "frames") },
+ );
+ expect(getFrameAtTime(extracted, 1, 0)).toBeNull();
+ expect(getFrameAtTime(extracted, LEAD_SECONDS - 0.5, 0)).not.toBeNull();
+ }, 30_000);
+
+ it("plays a late video stream to its end on the media timeline and loops on its full length", async () => {
+ const source = join(dir, "video-late.mp4");
+ const mediaSeconds = LEAD_SECONDS + PICTURE_SECONDS;
+ const window = resolveVideoExtractionWindow(
+ { start: 0, end: mediaSeconds, mediaStart: 0, loop: false },
+ await extractMediaMetadata(source),
+ mediaSeconds,
+ );
+ expect(window.durationSeconds).toBeCloseTo(mediaSeconds, 1);
+
+ const extracted = await extractVideoFramesRange(source, "video-late-full", 0, mediaSeconds, {
+ fps: 10,
+ outputDir: join(dir, "frames"),
+ });
+ expect(getFrameAtTime(extracted, PICTURE_SECONDS + 1.5, 0, true)).not.toBeNull();
+ expect(getFrameAtTime(extracted, mediaSeconds + 1.5, 0, true)).toBeNull();
+ }, 30_000);
+
+ it.each([6.5, 10])(
+ "shows a late video stream from a trim at %s s on the media timeline",
+ async (mediaStart) => {
+ const extracted = await extractVideoFramesRange(
+ join(dir, "video-late.mp4"),
+ `video-late-from-${mediaStart}`,
+ mediaStart,
+ 5,
+ { fps: 10, outputDir: join(dir, "frames") },
+ );
+ expect(getFrameAtTime(extracted, 0.05, 0, false, mediaStart)).not.toBeNull();
+ },
+ 30_000,
+ );
+
+ it("holds a late video stream's last frame past its end", async () => {
+ const source = join(dir, "video-late.mp4");
+ const metadata = await extractMediaMetadata(source);
+ const video = {
+ start: 0,
+ end: 5,
+ mediaStart: LEAD_SECONDS + PICTURE_SECONDS + 0.5,
+ loop: false,
+ };
+ const window = await resolveFinalFrameExtractionWindow(
+ source,
+ video,
+ metadata,
+ resolveVideoExtractionWindow(video, metadata, 5),
+ );
+ expect(window.extractionMediaStart).toBeGreaterThan(LEAD_SECONDS + PICTURE_SECONDS - 0.1);
+ }, 30_000);
+ },
+);
diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts
index e5c9aac861..0a38fa3852 100644
--- a/packages/engine/src/services/videoFrameExtractor.ts
+++ b/packages/engine/src/services/videoFrameExtractor.ts
@@ -1229,9 +1229,10 @@ export async function extractVideoFramesRange(
if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {
const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {
- const correctedDuration = startTime + framePaths.size / fps;
- if (correctedDuration < metadata.videoStreamDurationSeconds) {
- metadata.videoStreamDurationSeconds = correctedDuration;
+ const correctedStreamDuration =
+ startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);
+ if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {
+ metadata.videoStreamDurationSeconds = correctedStreamDuration;
}
}
}
@@ -1302,7 +1303,7 @@ function resolveSegmentDuration(
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
return Number.isFinite(metadata.videoStreamDurationSeconds) &&
metadata.videoStreamDurationSeconds > 0
- ? metadata.videoStreamDurationSeconds
+ ? (metadata.videoStreamLeadSeconds ?? 0) + metadata.videoStreamDurationSeconds
: metadata.durationSeconds;
}
@@ -1503,14 +1504,17 @@ export async function resolveFinalFrameExtractionWindow(
): Promise<TimelineExtractionWindow> {
if (!window.ensureFinalFrame) return window;
const playableDuration = resolvePlayableVideoDuration(metadata);
- const finalFrameTimestamp = await extractFinalVideoFrameTimestamp(
- videoPath,
- {
- videoStreamDurationSeconds: playableDuration,
- videoStreamStartSeconds: metadata.videoStreamStartSeconds,
- },
- signal,
- );
+ const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
+ const finalFrameTimestamp =
+ videoStreamLead +
+ (await extractFinalVideoFrameTimestamp(
+ videoPath,
+ {
+ videoStreamDurationSeconds: playableDuration - videoStreamLead,
+ videoStreamStartSeconds: metadata.videoStreamStartSeconds,
+ },
+ signal,
+ ));
if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;
const sourceRemaining = playableDuration - video.mediaStart;
@@ -2613,6 +2617,18 @@ export async function extractAllVideoFrames(
};
}
+const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;
+
+export function isVideoHiddenBeforeStreamStart(
+ videoStreamLeadSeconds: number | undefined,
+ mediaTime: number,
+): boolean {
+ return (
+ (videoStreamLeadSeconds ?? 0) - mediaTime >=
+ BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
+ );
+}
+
function getFrameIndexAtTime(
extracted: ExtractedFrames,
globalTime: number,
@@ -2646,6 +2662,14 @@ function getFrameIndexAtTime(
// e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);
if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
+ if (
+ isVideoHiddenBeforeStreamStart(
+ extracted.metadata.videoStreamLeadSeconds,
+ mediaStart + sourceTime,
+ )
+ ) {
+ return null;
+ }
if (frameIndex >= extracted.totalFrames) {
return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
}
diff --git a/packages/engine/src/utils/ffprobe.ts b/packages/engine/src/utils/ffprobe.ts
index ad63d91fff..76eb27acff 100644
--- a/packages/engine/src/utils/ffprobe.ts
+++ b/packages/engine/src/utils/ffprobe.ts
@@ -184,6 +184,8 @@ export interface VideoMetadata {
* timestamps are absolute, so callers crossing those APIs must normalize by
* this value. Absent only in legacy/manually-constructed metadata. */
videoStreamStartSeconds?: number;
+ /** Seconds after the file's media time 0 (its earliest audio/video stream start) at which video begins. */
+ videoStreamLeadSeconds?: number;
width: number;
height: number;
fps: number;
@@ -214,6 +216,8 @@ export interface AudioMetadata {
channels: number;
audioCodec: string;
bitrate?: number;
+ /** Seconds after the file's media time 0 at which its last audio or video stream starts. */
+ latestStreamLeadSeconds?: number;
}
interface FFProbeStream {
@@ -820,6 +824,7 @@ export async function extractMediaMetadata(filePath: string): Promise<VideoMetad
durationSeconds: containerDuration,
videoStreamDurationSeconds: effectiveStreamDuration,
videoStreamStartSeconds: streamStart,
+ videoStreamLeadSeconds: streamLeadSeconds(output, videoStream),
width: videoStream.width || stillImage()?.width || 0,
height: videoStream.height || stillImage()?.height || 0,
fps,
@@ -989,6 +994,7 @@ export async function extractAudioMetadata(
channels: audioStream.channels || 2,
audioCodec,
bitrate: output.format.bit_rate ? parseInt(output.format.bit_rate) : undefined,
+ latestStreamLeadSeconds: latestStreamLeadSeconds(output),
};
})();
@@ -997,6 +1003,28 @@ export async function extractAudioMetadata(
return probePromise;
}
+/** Chrome's media time 0 is the earliest of these; a stream starting later plays that much in. */
+function audioVideoStreamStarts(output: FFProbeOutput | null): number[] {
+ return (output?.streams ?? [])
+ .filter(
+ (s) =>
+ (s.codec_type === "audio" || s.codec_type === "video") && s.disposition?.attached_pic !== 1,
+ )
+ .map((s) => parseFloat(s.start_time ?? ""))
+ .filter(Number.isFinite);
+}
+
+function streamLeadSeconds(output: FFProbeOutput | null, stream: FFProbeStream): number {
+ const starts = audioVideoStreamStarts(output);
+ const start = parseFloat(stream.start_time ?? "");
+ return Number.isFinite(start) && starts.length > 0 ? Math.max(0, start - Math.min(...starts)) : 0;
+}
+
+function latestStreamLeadSeconds(output: FFProbeOutput): number {
+ const starts = audioVideoStreamStarts(output);
+ return starts.length > 0 ? Math.max(...starts) - Math.min(...starts) : 0;
+}
+
function audioStartSeconds(stream: FFProbeStream, format: FFProbeFormat): number {
const start = parseFloat(stream.start_time ?? format.start_time ?? "");
return Number.isFinite(start) ? start : 0;
diff --git a/packages/producer/src/services/distributed/shared.ts b/packages/producer/src/services/distributed/shared.ts
index ef574f8d8f..6d5171bd0e 100644
--- a/packages/producer/src/services/distributed/shared.ts
+++ b/packages/producer/src/services/distributed/shared.ts
@@ -239,6 +239,10 @@ function readVideoMetadata(
record.videoStreamStartSeconds === undefined
? 0
: readFiniteNumber(record.videoStreamStartSeconds, `${field}.videoStreamStartSeconds`),
+ videoStreamLeadSeconds:
+ record.videoStreamLeadSeconds === undefined
+ ? 0
+ : readFiniteNumber(record.videoStreamLeadSeconds, `${field}.videoStreamLeadSeconds`),
width: readPositiveInteger(record.width, `${field}.width`),
height: readPositiveInteger(record.height, `${field}.height`),
fps: readFiniteNumber(record.fps, `${field}.fps`),
diff --git a/packages/producer/src/services/distributed/videoMetadata.test.ts b/packages/producer/src/services/distributed/videoMetadata.test.ts
index c62719e4c8..bf1fa104ee 100644
--- a/packages/producer/src/services/distributed/videoMetadata.test.ts
+++ b/packages/producer/src/services/distributed/videoMetadata.test.ts
@@ -94,7 +94,7 @@ describe("distributed video metadata", () => {
expect(sourceDerived.videos[0]?.mediaStart).toBe(1);
});
- it("round-trips a non-zero video stream start and defaults legacy plans to zero", () => {
+ it("round-trips a non-zero video stream start and lead and defaults legacy plans to zero", () => {
const withStart = buildPlanVideosJson({
videos: [video()],
extracted: [
@@ -102,16 +102,22 @@ describe("distributed video metadata", () => {
metadata: {
...extractedMetadata().metadata,
videoStreamStartSeconds: 5,
+ videoStreamLeadSeconds: 7,
},
}),
],
compositionEnd: 8,
});
- expect(parsePlanVideosJson(withStart).extracted[0]?.metadata.videoStreamStartSeconds).toBe(5);
+ const parsed = parsePlanVideosJson(withStart).extracted[0]?.metadata;
+ expect(parsed?.videoStreamStartSeconds).toBe(5);
+ expect(parsed?.videoStreamLeadSeconds).toBe(7);
const legacy = structuredClone(withStart);
delete legacy.extracted[0]?.metadata.videoStreamStartSeconds;
- expect(parsePlanVideosJson(legacy).extracted[0]?.metadata.videoStreamStartSeconds).toBe(0);
+ delete legacy.extracted[0]?.metadata.videoStreamLeadSeconds;
+ const legacyParsed = parsePlanVideosJson(legacy).extracted[0]?.metadata;
+ expect(legacyParsed?.videoStreamStartSeconds).toBe(0);
+ expect(legacyParsed?.videoStreamLeadSeconds).toBe(0);
});
it.each([Number.NaN, Number.POSITIVE_INFINITY, 0, 2])(
diff --git a/packages/producer/src/services/hdrCompositor.test.ts b/packages/producer/src/services/hdrCompositor.test.ts
index 774301c9cb..1680f9a90f 100644
--- a/packages/producer/src/services/hdrCompositor.test.ts
+++ b/packages/producer/src/services/hdrCompositor.test.ts
@@ -1,6 +1,9 @@
+import { mkdtempSync, openSync, closeSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
import { describe, expect, it } from "vitest";
import type { ElementStackingInfo } from "@hyperframes/engine";
-import { selectDomLayerShowIds } from "./hdrCompositor.js";
+import { blitHdrVideoLayer, selectDomLayerShowIds } from "./hdrCompositor.js";
function makeEl(id: string, overrides?: Partial<ElementStackingInfo>): ElementStackingInfo {
return {
@@ -62,3 +65,47 @@ describe("selectDomLayerShowIds", () => {
).toEqual(["active-overlay"]);
});
});
+
+describe("blitHdrVideoLayer", () => {
+ it.each([
+ { lead: undefined, mediaStart: 0, painted: true },
+ { lead: 7, mediaStart: 0, painted: false },
+ { lead: 7, mediaStart: 6.5, painted: true },
+ ])(
+ "paints at 1 s of a clip from $mediaStart s whose stream starts $lead s in: $painted",
+ ({ lead, mediaStart, painted }) => {
+ const dir = mkdtempSync(join(tmpdir(), "hf-hdr-lead-"));
+ const rawPath = join(dir, "frames.rgb48le");
+ const frameSize = 2 * 2 * 6;
+ writeFileSync(rawPath, Buffer.alloc(frameSize * 2, 0x80));
+ const fd = openSync(rawPath, "r");
+ try {
+ const canvas = Buffer.alloc(frameSize);
+ const el = makeEl("v", {
+ width: 2,
+ height: 2,
+ layoutWidth: 2,
+ layoutHeight: 2,
+ isHdr: true,
+ });
+ const source = {
+ dir,
+ rawPath,
+ fd,
+ width: 2,
+ height: 2,
+ frameSize,
+ frameCount: 2,
+ scratch: Buffer.alloc(frameSize),
+ mediaStart,
+ videoStreamLeadSeconds: lead,
+ };
+ blitHdrVideoLayer(canvas, el, 1, 1, new Map([["v", source]]), new Map([["v", 0]]), 2, 2);
+ expect(canvas.some((byte) => byte !== 0)).toBe(painted);
+ } finally {
+ closeSync(fd);
+ rmSync(dir, { recursive: true, force: true });
+ }
+ },
+ );
+});
diff --git a/packages/producer/src/services/hdrCompositor.ts b/packages/producer/src/services/hdrCompositor.ts
index cff19df762..13e474dce7 100644
--- a/packages/producer/src/services/hdrCompositor.ts
+++ b/packages/producer/src/services/hdrCompositor.ts
@@ -29,6 +29,7 @@ import {
blitRgb48leAffine,
parseTransformMatrix,
convertTransfer,
+ isVideoHiddenBeforeStreamStart,
} from "@hyperframes/engine";
import type { ProducerLogger } from "../logger.js";
import { type HdrImageTransferCache } from "./hdrImageTransferCache.js";
@@ -141,6 +142,8 @@ export interface HdrVideoFrameSource {
scratch: Buffer;
/** The raw file contains one playable source cycle and must wrap at EOF. */
loop?: boolean;
+ mediaStart?: number;
+ videoStreamLeadSeconds?: number;
}
export function closeHdrVideoFrameSource(source: HdrVideoFrameSource, log?: ProducerLogger): void {
@@ -198,6 +201,8 @@ export function blitHdrVideoLayer(
frameSource.loop,
);
if (effectiveIndex === null) return;
+ const mediaTime = (frameSource.mediaStart ?? 0) + effectiveIndex / fps;
+ if (isVideoHiddenBeforeStreamStart(frameSource.videoStreamLeadSeconds, mediaTime)) return;
const frameOffset = effectiveIndex * frameSource.frameSize;
try {
diff --git a/packages/producer/src/services/render/stages/captureHdrResources.ts b/packages/producer/src/services/render/stages/captureHdrResources.ts
index 2634122b76..c53cf21f0f 100644
--- a/packages/producer/src/services/render/stages/captureHdrResources.ts
+++ b/packages/producer/src/services/render/stages/captureHdrResources.ts
@@ -419,7 +419,10 @@ export async function extractHdrVideoFrames(args: {
const out = new Map<string, HdrVideoFrameSource>();
mkdirSync(framesDir, { recursive: true });
const plannedVideos: Array<{ durationSeconds: number; width: number; height: number }> = [];
- const extractionWindows = new Map<string, HdrExtractionWindow>();
+ const extractionWindows = new Map<
+ string,
+ { window: HdrExtractionWindow; videoStreamLeadSeconds?: number }
+ >();
for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
const video = composition.videos.find((v) => v.id === videoId);
if (!video) continue;
@@ -434,7 +437,10 @@ export async function extractHdrVideoFrames(args: {
initialWindow,
abortSignal,
);
- extractionWindows.set(videoId, window);
+ extractionWindows.set(videoId, {
+ window,
+ videoStreamLeadSeconds: metadata.videoStreamLeadSeconds,
+ });
prep.hdrVideoStartTimes.set(videoId, window.compositionStart);
plannedVideos.push({
durationSeconds: window.durationSeconds,
@@ -453,14 +459,16 @@ export async function extractHdrVideoFrames(args: {
try {
for (const [videoId, srcPath] of prep.hdrVideoSrcPaths) {
const video = composition.videos.find((v) => v.id === videoId);
- const window = extractionWindows.get(videoId);
- if (!video || !window) continue;
+ const planned = extractionWindows.get(videoId);
+ if (!video || !planned) continue;
+ const { window } = planned;
mkdirSync(framesDir, { recursive: true });
const frameDir = mkdtempSync(join(framesDir, `hdr_${tempDirSafePrefix(videoId)}-`));
createdFrameDirs.add(frameDir);
const dims = prep.hdrExtractionDims.get(videoId) ?? { width, height };
const rawPath = join(frameDir, "frames.rgb48le");
- const extractionStart = String(window.extractionMediaStart ?? window.mediaStart);
+ const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
+ const extractionStart = String(extractionMediaStart);
const ffmpegArgs: string[] = [];
if (window.finalFrameOnly) {
// Decode before seeking for the one-frame path. Input-side seeking can
@@ -524,6 +532,8 @@ export async function extractHdrVideoFrames(args: {
frameCount,
scratch: Buffer.allocUnsafe(frameSize),
loop: video.loop,
+ mediaStart: extractionMediaStart,
+ videoStreamLeadSeconds: planned.videoStreamLeadSeconds,
});
handedOff = true;
} finally {