Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
perf(media-local): reuse video frames that meet normalization requirements (#389)
Sharky Wangdc2b7a8f31b1
diff --git a/packages/media-local/README.md b/packages/media-local/README.mdindex 5fba56d3..c62dd19c 100644--- a/packages/media-local/README.md+++ b/packages/media-local/README.md@@ -70,6 +70,12 @@ The Endpoint:Normalize preserves alpha: transparent input becomes VP9 / `yuva420p` WebM, while opaque inputuses H.264 / `yuv420p` MP4. The input pixel format and WebM alpha metadata determine the choice.+Opaque H.264 / `yuv420p` video in Baseline, Main or High profiles up to level 4.1 can reuse its encoded+frames when every decoded frame has the exact+target timestamp and duration, the planned frame count, even dimensions, square pixels and zero+display rotation. A single-video-stream MP4 is reused directly; other matching inputs are remuxed+into a silent MP4 and checked before use. Reused video retains the source's decoded pixels.+Selected audio follows the same sample-alignment processing described above.Animated WebP is also normalized through an alpha-capable output. Transparent VP8/VP9 input uses libvpxdecoders so FFmpeg retains the alpha sidecar, including during PNG frame extraction.Transparent normalization requires an FFmpeg build with the `libvpx-vp9` encoder; the normaldiff --git a/packages/media-local/src/execute.ts b/packages/media-local/src/execute.tsindex 48b26e74..00e283b4 100644--- a/packages/media-local/src/execute.ts+++ b/packages/media-local/src/execute.ts@@ -32,6 +32,7 @@ import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocolimport { runBoundedMediaProcess } from "./process.js";import { parseMediaInspection } from "./probe.js";+import { normalizedVideoFrames, normalizedVideoSpan } from "./normalize-copy.js";import {compositeAnimatedWebpFrame,createAnimatedWebpCanvas,@@ -572,13 +573,14 @@ export async function executeInspectMedia(async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string, streamIndex: number) {const bytes = await runProcess({executable: env.ffprobePath,- argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", path],+ argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", "-show_format", path],timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),});const probe = JSON.parse(Buffer.from(bytes).toString("utf8")) as {- streams: { index: number; codec_name?: string; pix_fmt?: string; tags?: Record<string, string> }[];+ streams: { index: number; codec_name?: string; profile?: string; level?: number; pix_fmt?: string; time_base?: string; sample_aspect_ratio?: string; width?: number; height?: number; tags?: Record<string, string>; side_data_list?: { rotation?: number }[] }[];pixel_formats: { name: string; flags: { alpha: number } }[];+ format?: { nb_streams?: number; format_name?: string };};const stream = probe.streams.find((item) => item.index === streamIndex);assert(stream !== undefined, `Media source has no video stream ${streamIndex}`);@@ -591,10 +593,74 @@ async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string,: stream.codec_name === "vp9" ? "libvpx-vp9" : stream.codec_name === "vp8" ? "libvpx" : undefined;return {alpha,+ pixelFormat: stream.pix_fmt,+ stream,+ formatNames: probe.format?.format_name?.split(",") ?? [],+ streamCount: probe.format?.nb_streams,inputArgs: decoder === undefined ? [] : [`-c:${streamIndex}`, decoder],};}+async function copyNormalizedVisual(args: {+ env: MediaExecutionEnvironment;+ source: BlobRef;+ video: MediaVideoStream;+ frameRate: MediaRational;+ frameCount: number;+ encoding: Awaited<ReturnType<typeof sourceVideoEncoding>>;+ input: string;+ output: string;+}): Promise<BlobRef | undefined> {+ const { env, source, video, frameRate, frameCount, encoding, input, output } = args;+ const stream = encoding.stream;+ if (encoding.alpha || encoding.pixelFormat !== "yuv420p" || stream.codec_name !== "h264"+ || !["Constrained Baseline", "Baseline", "Main", "High"].includes(stream.profile ?? "")+ || stream.level === undefined || stream.level < 10 || stream.level > 41+ || video.codecName !== "h264" || video.width % 2 !== 0 || video.height % 2 !== 0+ || stream.width !== video.width || stream.height !== video.height+ || stream.time_base !== `${video.timeBase?.numerator}/${video.timeBase?.denominator}`+ || (stream.sample_aspect_ratio !== undefined && stream.sample_aspect_ratio !== "1:1")+ || Object.entries(stream.tags ?? {}).some(([name, value]) => name.toLowerCase() === "rotate" && Number(value) !== 0)+ || stream.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0)+ || video.rotationDegrees !== 0 || video.sampleAspectRatio.numerator !== video.sampleAspectRatio.denominator+ || !normalizedVideoSpan(video, frameRate, frameCount)) return undefined;+ let inspected: MediaInspection;+ try {+ const raw = await runProcess({+ executable: env.ffprobePath,+ argv: ["-v", "error", "-print_format", "json", "-select_streams", String(video.index),+ "-show_frames", "-show_entries", "frame=pts,duration,pkt_duration,width,height,pix_fmt,sample_aspect_ratio,interlaced_frame:frame_side_data=rotation", input],+ timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ const probe = JSON.parse(Buffer.from(raw).toString("utf8")) as {+ frames?: Parameters<typeof normalizedVideoFrames>[3];+ };+ if (!normalizedVideoFrames(video, frameRate, frameCount, probe.frames ?? [])) return undefined;+ if (source.mediaType === "video/mp4" && encoding.formatNames.includes("mp4") && encoding.streamCount === 1)+ return source;+ await runProcess({+ executable: env.ffmpegPath,+ argv: ["-y", "-copyts", "-i", input, "-map", `0:${video.index}`, "-an", "-sn", "-dn",+ "-map_metadata", "-1", "-c:v", "copy", "-avoid_negative_ts", "disabled", "-movflags", "+faststart", output],+ timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ inspected = await outputInspection({+ path: output, mediaType: "video/mp4", ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs, maxProbeOutputBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ } catch {+ return undefined;+ }+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");+ if (!visual || !normalizedVideoSpan(visual, frameRate, frameCount) || inspected.streams.length !== 1+ || visual.width !== video.width || visual.height !== video.height || visual.rotationDegrees !== 0+ || visual.sampleAspectRatio.numerator !== 1 || visual.sampleAspectRatio.denominator !== 1) return undefined;+ return await env.artifacts.putFile(output, "video/mp4");+}+export async function executeNormalizeMedia(env: MediaExecutionEnvironment,constraints: CanonicalValue,@@ -614,7 +680,7 @@ export async function executeNormalizeMedia(if (plan.video !== undefined) {const encoding = animation === undefined? await sourceVideoEncoding(env, input, plan.video.index)- : { alpha: true, inputArgs: [] };+ : { alpha: true, pixelFormat: undefined, stream: { index: plan.video.index }, formatNames: [], streamCount: undefined, inputArgs: [] };const outputType = encoding.alpha ? "video/webm" : "video/mp4";const output = join(work, encoding.alpha ? "visual.webm" : "visual.mp4");const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;@@ -642,31 +708,40 @@ export async function executeNormalizeMedia("-row-mt", "1", "-deadline", "realtime", "-cpu-used", "8",]: ["-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart"];- await runProcess({- executable: env.ffmpegPath,- argv: ["-y", ...visualInput, "-an", "-vf", filter,- "-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],- timeoutMs: env.processTimeoutMs,- maxStdoutBytes: 64 * 1024,- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),- });- const inspected = await outputInspection({- path: output,- mediaType: outputType,- ffprobePath: env.ffprobePath,- timeoutMs: env.processTimeoutMs,- maxProbeOutputBytes: env.maxProbeOutputBytes,- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),- });- const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");- assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",- "Normalized visual frame shape differs from its plan");- assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1- && visual.rotationDegrees === 0,- "Normalized visual retains non-square samples or display rotation");- visualArtifact = await env.artifacts.putFile(output, outputType);- visualWidth = visual.width;- visualHeight = visual.height;+ visualArtifact = animation === undefined+ ? await copyNormalizedVisual({ env, source: need.source, video: plan.video, frameRate: need.frameRate,+ frameCount: plan.frameCount, encoding, input, output })+ : undefined;+ if (visualArtifact) {+ visualWidth = plan.video.width;+ visualHeight = plan.video.height;+ } else {+ await runProcess({+ executable: env.ffmpegPath,+ argv: ["-y", ...visualInput, "-an", "-vf", filter,+ "-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],+ timeoutMs: env.processTimeoutMs,+ maxStdoutBytes: 64 * 1024,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ const inspected = await outputInspection({+ path: output,+ mediaType: outputType,+ ffprobePath: env.ffprobePath,+ timeoutMs: env.processTimeoutMs,+ maxProbeOutputBytes: env.maxProbeOutputBytes,+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),+ });+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");+ assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",+ "Normalized visual frame shape differs from its plan");+ assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1+ && visual.rotationDegrees === 0,+ "Normalized visual retains non-square samples or display rotation");+ visualArtifact = await env.artifacts.putFile(output, outputType);+ visualWidth = visual.width;+ visualHeight = visual.height;+ }}let audioArtifact: BlobRef | undefined;if (plan.audio !== undefined) {diff --git a/packages/media-local/src/normalize-copy.ts b/packages/media-local/src/normalize-copy.tsnew file mode 100644index 00000000..e18f73f2--- /dev/null+++ b/packages/media-local/src/normalize-copy.ts@@ -0,0 +1,53 @@+import type { MediaRational, MediaVideoStream } from "@hypit/media";++type VideoFrame = {+ readonly pts?: number | string;+ readonly duration?: number | string;+ readonly pkt_duration?: number | string;+ readonly width?: number;+ readonly height?: number;+ readonly pix_fmt?: string;+ readonly sample_aspect_ratio?: string;+ readonly interlaced_frame?: number;+ readonly side_data_list?: readonly { readonly rotation?: number }[];+};++function ticks(value: unknown): bigint | undefined {+ if (typeof value === "number" && Number.isSafeInteger(value)) return BigInt(value);+ if (typeof value === "string" && /^-?\d+$/u.test(value)) return BigInt(value);+ return undefined;+}++export function normalizedVideoSpan(+ video: MediaVideoStream,+ frameRate: MediaRational,+ frameCount: number,+): boolean {+ const end = video.endPts;+ return video.timingStatus === "admissible"+ && video.startPts !== undefined && ticks(video.startPts.ticks) === 0n+ && end !== undefined && video.decodedUnitCount === frameCount+ && BigInt(end.ticks) * BigInt(end.timeBase.numerator) * BigInt(frameRate.numerator)+ === BigInt(frameCount) * BigInt(frameRate.denominator) * BigInt(end.timeBase.denominator);+}++export function normalizedVideoFrames(+ video: MediaVideoStream,+ frameRate: MediaRational,+ frameCount: number,+ frames: readonly VideoFrame[],+): boolean {+ if (!normalizedVideoSpan(video, frameRate, frameCount) || !video.timeBase+ || frames.length !== frameCount) return false;+ const numerator = BigInt(video.timeBase.numerator) * BigInt(frameRate.numerator);+ const denominator = BigInt(video.timeBase.denominator) * BigInt(frameRate.denominator);+ return frames.every((frame, index) => {+ const pts = ticks(frame.pts);+ const duration = ticks(frame.duration ?? frame.pkt_duration);+ return pts !== undefined && pts * numerator === BigInt(index) * denominator+ && duration !== undefined && duration * numerator === denominator+ && frame.width === video.width && frame.height === video.height && frame.pix_fmt === "yuv420p"+ && frame.sample_aspect_ratio === "1:1" && frame.interlaced_frame === 0+ && !frame.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0);+ });+}