Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

perf(media-local): reuse video frames that meet normalization requirements (#389)

Sharky Wangdc2b7a8f31b1
diff --git a/packages/media-local/README.md b/packages/media-local/README.md
index 5fba56d3..c62dd19c 100644
--- a/packages/media-local/README.md
+++ b/packages/media-local/README.md
@@ -70,6 +70,12 @@ The Endpoint:
Normalize preserves alpha: transparent input becomes VP9 / `yuva420p` WebM, while opaque input
uses H.264 / `yuv420p` MP4. The input pixel format and WebM alpha metadata determine the choice.
+Opaque H.264 / `yuv420p` video in Baseline, Main or High profiles up to level 4.1 can reuse its encoded
+frames when every decoded frame has the exact
+target timestamp and duration, the planned frame count, even dimensions, square pixels and zero
+display rotation. A single-video-stream MP4 is reused directly; other matching inputs are remuxed
+into a silent MP4 and checked before use. Reused video retains the source's decoded pixels.
+Selected audio follows the same sample-alignment processing described above.
Animated WebP is also normalized through an alpha-capable output. Transparent VP8/VP9 input uses libvpx
decoders so FFmpeg retains the alpha sidecar, including during PNG frame extraction.
Transparent normalization requires an FFmpeg build with the `libvpx-vp9` encoder; the normal
diff --git a/packages/media-local/src/execute.ts b/packages/media-local/src/execute.ts
index 48b26e74..00e283b4 100644
--- a/packages/media-local/src/execute.ts
+++ b/packages/media-local/src/execute.ts
@@ -32,6 +32,7 @@ import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol
import { runBoundedMediaProcess } from "./process.js";
import { parseMediaInspection } from "./probe.js";
+import { normalizedVideoFrames, normalizedVideoSpan } from "./normalize-copy.js";
import {
compositeAnimatedWebpFrame,
createAnimatedWebpCanvas,
@@ -572,13 +573,14 @@ export async function executeInspectMedia(
async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string, streamIndex: number) {
const bytes = await runProcess({
executable: env.ffprobePath,
- argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", path],
+ argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", "-show_format", path],
timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const probe = JSON.parse(Buffer.from(bytes).toString("utf8")) as {
- streams: { index: number; codec_name?: string; pix_fmt?: string; tags?: Record<string, string> }[];
+ streams: { index: number; codec_name?: string; profile?: string; level?: number; pix_fmt?: string; time_base?: string; sample_aspect_ratio?: string; width?: number; height?: number; tags?: Record<string, string>; side_data_list?: { rotation?: number }[] }[];
pixel_formats: { name: string; flags: { alpha: number } }[];
+ format?: { nb_streams?: number; format_name?: string };
};
const stream = probe.streams.find((item) => item.index === streamIndex);
assert(stream !== undefined, `Media source has no video stream ${streamIndex}`);
@@ -591,10 +593,74 @@ async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string,
: stream.codec_name === "vp9" ? "libvpx-vp9" : stream.codec_name === "vp8" ? "libvpx" : undefined;
return {
alpha,
+ pixelFormat: stream.pix_fmt,
+ stream,
+ formatNames: probe.format?.format_name?.split(",") ?? [],
+ streamCount: probe.format?.nb_streams,
inputArgs: decoder === undefined ? [] : [`-c:${streamIndex}`, decoder],
};
}
+async function copyNormalizedVisual(args: {
+ env: MediaExecutionEnvironment;
+ source: BlobRef;
+ video: MediaVideoStream;
+ frameRate: MediaRational;
+ frameCount: number;
+ encoding: Awaited<ReturnType<typeof sourceVideoEncoding>>;
+ input: string;
+ output: string;
+}): Promise<BlobRef | undefined> {
+ const { env, source, video, frameRate, frameCount, encoding, input, output } = args;
+ const stream = encoding.stream;
+ if (encoding.alpha || encoding.pixelFormat !== "yuv420p" || stream.codec_name !== "h264"
+ || !["Constrained Baseline", "Baseline", "Main", "High"].includes(stream.profile ?? "")
+ || stream.level === undefined || stream.level < 10 || stream.level > 41
+ || video.codecName !== "h264" || video.width % 2 !== 0 || video.height % 2 !== 0
+ || stream.width !== video.width || stream.height !== video.height
+ || stream.time_base !== `${video.timeBase?.numerator}/${video.timeBase?.denominator}`
+ || (stream.sample_aspect_ratio !== undefined && stream.sample_aspect_ratio !== "1:1")
+ || Object.entries(stream.tags ?? {}).some(([name, value]) => name.toLowerCase() === "rotate" && Number(value) !== 0)
+ || stream.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0)
+ || video.rotationDegrees !== 0 || video.sampleAspectRatio.numerator !== video.sampleAspectRatio.denominator
+ || !normalizedVideoSpan(video, frameRate, frameCount)) return undefined;
+ let inspected: MediaInspection;
+ try {
+ const raw = await runProcess({
+ executable: env.ffprobePath,
+ argv: ["-v", "error", "-print_format", "json", "-select_streams", String(video.index),
+ "-show_frames", "-show_entries", "frame=pts,duration,pkt_duration,width,height,pix_fmt,sample_aspect_ratio,interlaced_frame:frame_side_data=rotation", input],
+ timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ const probe = JSON.parse(Buffer.from(raw).toString("utf8")) as {
+ frames?: Parameters<typeof normalizedVideoFrames>[3];
+ };
+ if (!normalizedVideoFrames(video, frameRate, frameCount, probe.frames ?? [])) return undefined;
+ if (source.mediaType === "video/mp4" && encoding.formatNames.includes("mp4") && encoding.streamCount === 1)
+ return source;
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: ["-y", "-copyts", "-i", input, "-map", `0:${video.index}`, "-an", "-sn", "-dn",
+ "-map_metadata", "-1", "-c:v", "copy", "-avoid_negative_ts", "disabled", "-movflags", "+faststart", output],
+ timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ inspected = await outputInspection({
+ path: output, mediaType: "video/mp4", ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs, maxProbeOutputBytes: env.maxProbeOutputBytes,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ } catch {
+ return undefined;
+ }
+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
+ if (!visual || !normalizedVideoSpan(visual, frameRate, frameCount) || inspected.streams.length !== 1
+ || visual.width !== video.width || visual.height !== video.height || visual.rotationDegrees !== 0
+ || visual.sampleAspectRatio.numerator !== 1 || visual.sampleAspectRatio.denominator !== 1) return undefined;
+ return await env.artifacts.putFile(output, "video/mp4");
+}
+
export async function executeNormalizeMedia(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
@@ -614,7 +680,7 @@ export async function executeNormalizeMedia(
if (plan.video !== undefined) {
const encoding = animation === undefined
? await sourceVideoEncoding(env, input, plan.video.index)
- : { alpha: true, inputArgs: [] };
+ : { alpha: true, pixelFormat: undefined, stream: { index: plan.video.index }, formatNames: [], streamCount: undefined, inputArgs: [] };
const outputType = encoding.alpha ? "video/webm" : "video/mp4";
const output = join(work, encoding.alpha ? "visual.webm" : "visual.mp4");
const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;
@@ -642,31 +708,40 @@ export async function executeNormalizeMedia(
"-row-mt", "1", "-deadline", "realtime", "-cpu-used", "8",
]
: ["-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart"];
- await runProcess({
- executable: env.ffmpegPath,
- argv: ["-y", ...visualInput, "-an", "-vf", filter,
- "-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],
- timeoutMs: env.processTimeoutMs,
- maxStdoutBytes: 64 * 1024,
- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
- });
- const inspected = await outputInspection({
- path: output,
- mediaType: outputType,
- ffprobePath: env.ffprobePath,
- timeoutMs: env.processTimeoutMs,
- maxProbeOutputBytes: env.maxProbeOutputBytes,
- ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
- });
- const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
- assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
- "Normalized visual frame shape differs from its plan");
- assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
- && visual.rotationDegrees === 0,
- "Normalized visual retains non-square samples or display rotation");
- visualArtifact = await env.artifacts.putFile(output, outputType);
- visualWidth = visual.width;
- visualHeight = visual.height;
+ visualArtifact = animation === undefined
+ ? await copyNormalizedVisual({ env, source: need.source, video: plan.video, frameRate: need.frameRate,
+ frameCount: plan.frameCount, encoding, input, output })
+ : undefined;
+ if (visualArtifact) {
+ visualWidth = plan.video.width;
+ visualHeight = plan.video.height;
+ } else {
+ await runProcess({
+ executable: env.ffmpegPath,
+ argv: ["-y", ...visualInput, "-an", "-vf", filter,
+ "-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],
+ timeoutMs: env.processTimeoutMs,
+ maxStdoutBytes: 64 * 1024,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ const inspected = await outputInspection({
+ path: output,
+ mediaType: outputType,
+ ffprobePath: env.ffprobePath,
+ timeoutMs: env.processTimeoutMs,
+ maxProbeOutputBytes: env.maxProbeOutputBytes,
+ ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
+ });
+ const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
+ assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
+ "Normalized visual frame shape differs from its plan");
+ assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
+ && visual.rotationDegrees === 0,
+ "Normalized visual retains non-square samples or display rotation");
+ visualArtifact = await env.artifacts.putFile(output, outputType);
+ visualWidth = visual.width;
+ visualHeight = visual.height;
+ }
}
let audioArtifact: BlobRef | undefined;
if (plan.audio !== undefined) {
diff --git a/packages/media-local/src/normalize-copy.ts b/packages/media-local/src/normalize-copy.ts
new file mode 100644
index 00000000..e18f73f2
--- /dev/null
+++ b/packages/media-local/src/normalize-copy.ts
@@ -0,0 +1,53 @@
+import type { MediaRational, MediaVideoStream } from "@hypit/media";
+
+type VideoFrame = {
+ readonly pts?: number | string;
+ readonly duration?: number | string;
+ readonly pkt_duration?: number | string;
+ readonly width?: number;
+ readonly height?: number;
+ readonly pix_fmt?: string;
+ readonly sample_aspect_ratio?: string;
+ readonly interlaced_frame?: number;
+ readonly side_data_list?: readonly { readonly rotation?: number }[];
+};
+
+function ticks(value: unknown): bigint | undefined {
+ if (typeof value === "number" && Number.isSafeInteger(value)) return BigInt(value);
+ if (typeof value === "string" && /^-?\d+$/u.test(value)) return BigInt(value);
+ return undefined;
+}
+
+export function normalizedVideoSpan(
+ video: MediaVideoStream,
+ frameRate: MediaRational,
+ frameCount: number,
+): boolean {
+ const end = video.endPts;
+ return video.timingStatus === "admissible"
+ && video.startPts !== undefined && ticks(video.startPts.ticks) === 0n
+ && end !== undefined && video.decodedUnitCount === frameCount
+ && BigInt(end.ticks) * BigInt(end.timeBase.numerator) * BigInt(frameRate.numerator)
+ === BigInt(frameCount) * BigInt(frameRate.denominator) * BigInt(end.timeBase.denominator);
+}
+
+export function normalizedVideoFrames(
+ video: MediaVideoStream,
+ frameRate: MediaRational,
+ frameCount: number,
+ frames: readonly VideoFrame[],
+): boolean {
+ if (!normalizedVideoSpan(video, frameRate, frameCount) || !video.timeBase
+ || frames.length !== frameCount) return false;
+ const numerator = BigInt(video.timeBase.numerator) * BigInt(frameRate.numerator);
+ const denominator = BigInt(video.timeBase.denominator) * BigInt(frameRate.denominator);
+ return frames.every((frame, index) => {
+ const pts = ticks(frame.pts);
+ const duration = ticks(frame.duration ?? frame.pkt_duration);
+ return pts !== undefined && pts * numerator === BigInt(index) * denominator
+ && duration !== undefined && duration * numerator === denominator
+ && frame.width === video.width && frame.height === video.height && frame.pix_fmt === "yuv420p"
+ && frame.sample_aspect_ratio === "1:1" && frame.interlaced_frame === 0
+ && !frame.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0);
+ });
+}