Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
import { audioLevelAutomationFilter } from "./audio-level-automation.js";
import { randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdtemp, open, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { verifyMediaFrameRange, mediaFrameRangeSamples, mediaTypes, sealMediaInspection, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyTimelineVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, TimelineVisual, SynchronizedMedia, TimelineAudio } from "@hypit/media";
import type { Timeline } from "@hypit/timeline";
import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "@hypit/speech-evidence";
import type { ProjectSpeechEvidenceAudioNeed, SpeechEvidenceAudio } from "@hypit/speech-evidence";
import {
verifyAudioExtractionRequest,
verifyAudioProgramPlan,
verifyFrameExtractionRequest,
verifyMediaTransformProgram,
type ExtractAudioNeed,
type ExtractFrameNeed,
type InspectMediaNeed,
type MuxMediaNeed,
type NormalizeMediaNeed,
type RenderAudioNeed,
type RenderStillVideoNeed,
type TransformMediaNeed,
verifyStillVideoRequest,
} from "@hypit/media-operations";
import type { AudioProgramClip, AudioProgramPlan, MediaTransformOperation } from "@hypit/media-operations";
import {
canonicalize,
} from "@hypit/hypit/protocol";
import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol";
import { runBoundedMediaProcess } from "./process.js";
import { parseMediaInspection } from "./probe.js";
import { normalizedVideoFrames, normalizedVideoSpan } from "./normalize-copy.js";
import {
compositeAnimatedWebpFrame,
createAnimatedWebpCanvas,
pamRgba,
parseAnimatedWebp,
} from "./webp.js";
import type { AnimatedWebp } from "./webp.js";
/**
* Where the bytes live and which binaries transform them.
*
* These byte operations are written once behind an Artifact gateway. The local Provider supplies
* the Build's ResourceStore and the ffmpeg on its PATH. Another execution owner may reuse the same
* algorithms with a different gateway and tool bundle; nothing here claims a Provider or deployment.
*/
export type MediaArtifactGateway = {
get(source: BlobRef): Promise<Uint8Array | undefined>;
open(source: BlobRef): Promise<AsyncIterable<Uint8Array> | undefined>;
put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
putFile(path: string, mediaType: string): Promise<BlobRef>;
};
export type MediaExecutionEnvironment = {
readonly artifacts: MediaArtifactGateway;
readonly ffmpegPath: string;
readonly ffprobePath: string;
/** Deployment fact for dynamically linked tool bundles; absent for ordinary local binaries. */
readonly sharedLibraryPath?: string;
readonly processTimeoutMs: number;
readonly maxProbeOutputBytes: number;
};
export type MediaOperationResult = {
readonly value: StoredValue;
};
function assert(condition: unknown, message: string): asserts condition {
if (!condition) throw new Error(message);
}
function positiveInteger(value: number, subject: string): number {
assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive integer`);
return value;
}
function object(value: unknown, subject: string): Record<string, unknown> {
assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);
return value as Record<string, unknown>;
}
async function runProcess(args: {
readonly executable: string;
readonly argv: readonly string[];
readonly timeoutMs: number;
readonly maxStdoutBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<Uint8Array> {
return await runBoundedMediaProcess(args);
}
async function version(executable: string, timeoutMs: number, sharedLibraryPath?: string): Promise<string> {
const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024,
...(sharedLibraryPath === undefined ? {} : { sharedLibraryPath }) });
const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();
assert(line !== undefined && line.length > 0, `${executable} returned no version`);
return line;
}
async function sourceBytes(env: MediaExecutionEnvironment, source: BlobRef): Promise<Uint8Array> {
const bytes = await env.artifacts.get(source);
assert(bytes !== undefined, `Media source ${source.resource} is unavailable`);
assert(bytes.byteLength === source.size, `Media source ${source.resource} size differs`);
return bytes;
}
async function stageArtifact(
env: MediaExecutionEnvironment,
source: BlobRef,
path: string,
): Promise<void> {
const chunks = await env.artifacts.open(source);
assert(chunks !== undefined, `Media source ${source.resource} is unavailable`);
const file = await open(path, "w");
let size = 0;
try {
for await (const chunk of chunks) {
await file.write(chunk);
size += chunk.byteLength;
}
} finally {
await file.close();
}
assert(size === source.size, `Media source ${source.resource} size differs`);
}
function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {
return {
numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),
denominator: BigInt(value.timeBase.denominator),
};
}
function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {
const a = timestampFraction(left);
const b = timestampFraction(right);
const x = a.numerator * b.denominator;
const y = b.numerator * a.denominator;
return x < y ? -1 : x > y ? 1 : 0;
}
function roundPositive(numerator: bigint, denominator: bigint): number {
assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
const value = (numerator * 2n + denominator) / (denominator * 2n);
assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
return Number(value);
}
function ceilPositive(numerator: bigint, denominator: bigint): number {
assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
const value = (numerator + denominator - 1n) / denominator;
assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
return Number(value);
}
function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {
assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");
const start = timestampFraction(value.startPts);
const end = timestampFraction(value.endPts);
const numerator = end.numerator * start.denominator - start.numerator * end.denominator;
const denominator = end.denominator * start.denominator;
assert(numerator > 0n, "Selected media stream has an empty presentation interval");
return { numerator, denominator };
}
function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {
const a = timestampFraction(origin);
const b = timestampFraction(value);
const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);
const denominator = b.denominator * a.denominator;
const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);
return numerator < 0n ? -magnitude : magnitude;
}
function stream<T extends MediaStream["kind"]>(
inspection: MediaInspection,
index: number | undefined,
kind: T,
): Extract<MediaStream, { readonly kind: T }> | undefined {
if (index === undefined) return undefined;
const result = inspection.streams.find((item) => item.index === index);
assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);
assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);
return result as Extract<MediaStream, { readonly kind: T }>;
}
type NormalizationPlan = {
readonly video?: MediaVideoStream;
readonly audio?: MediaAudioStream;
readonly frameCount: number;
readonly sampleFrames: number;
readonly audioTrimStartSamples: number;
readonly audioTrimEndSamples: number;
readonly audioHeadSamples: number;
readonly audioContentSamples: number;
readonly audioTailSamples: number;
};
function normalizationPlan(
inspection: MediaInspection,
selection: MediaStreamSelection,
frameRate: MediaRational,
): NormalizationPlan {
assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0
&& Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,
"Target frame rate is invalid");
const video = stream(inspection, selection.videoStreamIndex, "video");
const audio = stream(inspection, selection.audioStreamIndex, "audio");
if (video !== undefined && selection.spanAuthority !== "video") {
throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");
}
const authority = selection.spanAuthority === "video" ? video : audio;
assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");
const span = duration(authority);
const scaledNumerator = span.numerator * BigInt(frameRate.numerator);
const scaledDenominator = span.denominator * BigInt(frameRate.denominator);
const frameCount = selection.spanAuthority === "video"
? roundPositive(scaledNumerator, scaledDenominator)
: ceilPositive(scaledNumerator, scaledDenominator);
assert(frameCount > 0, "Normalized media would contain no frame interval");
const sampleFrames = roundPositive(
BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),
BigInt(frameRate.numerator),
);
if (audio === undefined) {
return {
...(video === undefined ? {} : { video }),
frameCount,
sampleFrames,
audioTrimStartSamples: 0,
audioTrimEndSamples: 0,
audioHeadSamples: 0,
audioContentSamples: 0,
audioTailSamples: 0,
};
}
assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");
const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);
const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);
const contentStart = Math.max(0, audioStart);
const contentEnd = Math.min(sampleFrames, audioEnd);
assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");
return {
...(video === undefined ? {} : { video }),
audio,
frameCount,
sampleFrames,
audioTrimStartSamples: Math.max(0, -audioStart),
audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),
audioHeadSamples: contentStart,
audioContentSamples: contentEnd - contentStart,
audioTailSamples: sampleFrames - contentEnd,
};
}
async function inspectFile(args: {
readonly source: BlobRef;
readonly input: string;
readonly ffprobePath: string;
readonly timeoutMs: number;
readonly maxProbeOutputBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<MediaInspection> {
const [raw, ffprobeVersion] = await Promise.all([
runProcess({
executable: args.ffprobePath,
argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],
timeoutMs: args.timeoutMs,
maxStdoutBytes: args.maxProbeOutputBytes,
...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }),
}),
version(args.ffprobePath, args.timeoutMs, args.sharedLibraryPath),
]);
let value: unknown;
try {
value = JSON.parse(Buffer.from(raw).toString("utf8"));
} catch {
throw new Error("ffprobe returned invalid JSON");
}
return parseMediaInspection({ source: args.source, ffprobeVersion, value });
}
function divisor(left: number, right: number): number {
let a = Math.abs(left);
let b = Math.abs(right);
while (b !== 0) [a, b] = [b, a % b];
return a;
}
function animatedWebpInspection(source: BlobRef, animation: AnimatedWebp): MediaInspection {
const durationMs = animation.frames.reduce((sum, frame) => sum + frame.durationMs, 0);
const numerator = animation.frames.length * 1_000;
const factor = divisor(numerator, durationMs);
return sealMediaInspection({
container: { formatNames: ["webp", "webp-animation"] },
streams: [{
kind: "video",
index: 0,
codecType: "video",
codecName: "webp",
disposition: { default: true, attachedPicture: false },
timingStatus: "admissible",
timeBase: { numerator: 1, denominator: 1_000 },
startPts: { ticks: "0", timeBase: { numerator: 1, denominator: 1_000 } },
endPts: { ticks: String(durationMs), timeBase: { numerator: 1, denominator: 1_000 } },
decodedUnitCount: animation.frames.length,
role: "moving",
width: animation.width,
height: animation.height,
sampleAspectRatio: { numerator: 1, denominator: 1 },
rotationDegrees: 0,
averageFrameRate: { numerator: numerator / factor, denominator: durationMs / factor },
}],
});
}
async function animatedWebpConcat(
env: MediaExecutionEnvironment,
animation: AnimatedWebp,
work: string,
): Promise<string> {
assert(animation.frames.length <= 10_000, "Animated WebP has too many frames");
const canvasBytes = animation.width * animation.height * 4;
assert(Number.isSafeInteger(canvasBytes) && canvasBytes > 0 && canvasBytes <= 512 * 1024 * 1024,
"Animated WebP canvas is outside the supported memory bound");
const canvas = createAnimatedWebpCanvas(animation);
const paths: string[] = [];
for (const [index, frame] of animation.frames.entries()) {
const imagePath = join(work, `webp-source-${String(index).padStart(6, "0")}.webp`);
await writeFile(imagePath, frame.image);
const decoded = await runProcess({
executable: env.ffmpegPath,
argv: ["-v", "error", "-i", imagePath, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgba", "-"],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: frame.width * frame.height * 4,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const snapshot = compositeAnimatedWebpFrame(canvas, animation, frame, decoded);
const path = join(work, `webp-frame-${String(index).padStart(6, "0")}.pam`);
await writeFile(path, pamRgba(animation.width, animation.height, snapshot));
paths.push(path);
}
const lines = ["ffconcat version 1.0"];
for (const [index, path] of paths.entries()) {
lines.push(`file '${path}'`, `duration ${(animation.frames[index]!.durationMs / 1_000).toFixed(6)}`);
}
lines.push(`file '${paths.at(-1)!}'`);
const manifest = join(work, "animated-webp.ffconcat");
await writeFile(manifest, `${lines.join("\n")}\n`, "utf8");
return manifest;
}
async function outputInspection(args: {
readonly path: string;
readonly mediaType: string;
readonly ffprobePath: string;
readonly timeoutMs: number;
readonly maxProbeOutputBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<MediaInspection> {
const size = (await stat(args.path)).size;
const source: BlobRef = {
kind: "blob",
resource: `res_${randomUUID()}`,
size,
mediaType: args.mediaType,
};
return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,
timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes,
...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }) });
}
function inlineResult(value: CanonicalValue): MediaOperationResult {
return { value: { kind: "inline", value } };
}
function artifactResult(value: BlobRef): MediaOperationResult {
return { value };
}
function inspectNeed(value: CanonicalValue): InspectMediaNeed {
const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;
assert(item.source?.kind === "blob", "InspectMediaNeed is invalid");
return item;
}
function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {
const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;
assert(item.source?.kind === "blob", "NormalizeMediaNeed is invalid");
verifyMediaInspection(item.inspection);
verifyMediaStreamSelection(item.selection);
assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2
&& item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",
"NormalizeMediaNeed audio profile is unsupported");
return item;
}
function transformNeed(value: CanonicalValue): TransformMediaNeed {
const item = object(value, "TransformMediaNeed") as unknown as TransformMediaNeed;
verifySynchronizedMedia(item.media);
verifyMediaTransformProgram(item.program);
assert(item.media.visual !== undefined, "TransformMediaNeed requires synchronized visual media");
return item;
}
function extractAudioNeed(value: CanonicalValue): ExtractAudioNeed {
const item = object(value, "ExtractAudioNeed") as unknown as ExtractAudioNeed;
assert(item.source?.kind === "blob", "ExtractAudioNeed is invalid");
assert(Number.isSafeInteger(item.streamIndex) && item.streamIndex >= 0,
"ExtractAudioNeed streamIndex is invalid");
verifyAudioExtractionRequest({
audio: { mode: "stream-index", streamIndex: item.streamIndex },
output: item.output,
});
return item;
}
function extractFrameNeed(value: CanonicalValue): ExtractFrameNeed {
const item = object(value, "ExtractFrameNeed") as unknown as ExtractFrameNeed;
assert(item.source?.kind === "blob", "ExtractFrameNeed is invalid");
assert(Number.isSafeInteger(item.streamIndex) && item.streamIndex >= 0
&& Number.isSafeInteger(item.sourceFrameCount) && item.sourceFrameCount > 0,
"ExtractFrameNeed stream domain is invalid");
verifyFrameExtractionRequest({
video: { mode: "stream-index", streamIndex: item.streamIndex },
at: item.at,
output: item.output,
});
if (item.at.kind === "frame") {
assert(item.at.index < item.sourceFrameCount, "ExtractFrameNeed frame lies outside the source stream");
}
return item;
}
function renderStillVideoNeed(value: CanonicalValue): RenderStillVideoNeed {
const item = object(value, "RenderStillVideoNeed") as unknown as RenderStillVideoNeed;
verifyStillVideoRequest(item.request);
assert(item.request.segments.every((segment) => segment.source?.kind === "blob" && segment.source.mediaType.startsWith("image/")),
"RenderStillVideoNeed needs an image Artifact for every segment");
return item;
}
function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {
const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;
assert(typeof item.domainId === "string" && item.domainId.length > 0
&& item.source?.kind === "blob", "ProjectSpeechEvidenceAudioNeed is invalid");
assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0
&& Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0
&& item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),
"Speech evidence audio sample projection is invalid");
return item;
}
function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {
const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;
verifyAudioProgramPlan(item.plan);
if (item.range !== undefined) verifyMediaFrameRange(item.range, item.plan.frameCount);
return item;
}
function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {
const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;
verifyTimelineVisual(item.visual);
verifyTimelineAudio(item.audio);
const expectedSamples = roundPositive(
BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),
BigInt(item.visual.frameRate.numerator),
);
assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");
return item;
}
function atempo(rate: number): string[] {
if (Math.abs(rate - 1) < 1e-12) return [];
const filters: string[] = [];
let remaining = rate;
while (remaining > 2) {
filters.push("atempo=2");
remaining /= 2;
}
while (remaining < 0.5) {
filters.push("atempo=0.5");
remaining /= 0.5;
}
if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);
return filters;
}
function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number,
window: { readonly startSample: number; readonly endSampleExclusive: number }): string {
const length = clip.targetEndSampleExclusive - clip.targetStartSample;
const sourceLength = clip.sourceEndSampleExclusive - clip.sourceStartSample;
const left = Math.max(window.startSample, clip.targetStartSample);
const right = Math.min(window.endSampleExclusive, clip.targetEndSampleExclusive);
const filters = [
`atrim=start_sample=${clip.sourceStartSample}:end_sample=${clip.sourceEndSampleExclusive}`,
"asetpts=PTS-STARTPTS",
...(clip.sourceLoop ? [`aloop=loop=-1:size=${sourceLength}:start=0`] : []),
...(clip.sourcePhaseSample === 0 ? [] : [
`atrim=start_sample=${clip.sourcePhaseSample}`,
"asetpts=PTS-STARTPTS",
]),
...atempo(clip.playbackRate),
`atrim=start_sample=0:end_sample=${length}`,
`apad=whole_len=${length}`,
`atrim=start_sample=0:end_sample=${length}`,
`volume=${clip.gain.toPrecision(15)}`,
...audioLevelAutomationFilter(clip),
// Crop after tempo, looping and envelopes, keeping their original phase.
`atrim=start_sample=${left - clip.targetStartSample}:end_sample=${right - clip.targetStartSample}`,
"asetpts=N/SR/TB",
`adelay=${left - window.startSample}S:all=1`,
];
return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;
}
async function assertCanonicalWav(args: {
readonly path: string;
readonly source: BlobRef;
readonly ffprobePath: string;
readonly timeoutMs: number;
readonly maxProbeOutputBytes: number;
readonly sharedLibraryPath?: string;
}): Promise<MediaAudioStream> {
const inspection = await inspectFile({
source: args.source,
input: args.path,
ffprobePath: args.ffprobePath,
timeoutMs: args.timeoutMs,
maxProbeOutputBytes: args.maxProbeOutputBytes,
...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }),
});
const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");
assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,
"Canonical audio must be 48 kHz stereo PCM s16");
assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");
return audio[0]!;
}
export async function executeInspectMedia(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = inspectNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-inspect-"));
try {
const input = join(work, "source.bin");
const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
if (animationBytes === undefined) await stageArtifact(env, need.source, input);
else await writeFile(input, animationBytes);
const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
const inspection = animation === undefined
? await inspectFile({
source: need.source,
input,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
})
: animatedWebpInspection(need.source, animation);
return inlineResult(canonicalize(inspection));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string, streamIndex: number) {
const bytes = await runProcess({
executable: env.ffprobePath,
argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", "-show_format", path],
timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const probe = JSON.parse(Buffer.from(bytes).toString("utf8")) as {
streams: { index: number; codec_name?: string; profile?: string; level?: number; pix_fmt?: string; time_base?: string; sample_aspect_ratio?: string; width?: number; height?: number; tags?: Record<string, string>; side_data_list?: { rotation?: number }[] }[];
pixel_formats: { name: string; flags: { alpha: number } }[];
format?: { nb_streams?: number; format_name?: string };
};
const stream = probe.streams.find((item) => item.index === streamIndex);
assert(stream !== undefined, `Media source has no video stream ${streamIndex}`);
const pixelFormat = probe.pixel_formats.find((item) => item.name === stream.pix_fmt);
assert(pixelFormat !== undefined, `Media source pixel format ${String(stream.pix_fmt)} is unknown`);
const alphaTag = Object.entries(stream.tags ?? {}).find(([key]) => key.toLowerCase() === "alpha_mode")?.[1];
const alpha = pixelFormat.flags.alpha === 1 || alphaTag === "1" || alphaTag === "straight";
// FFmpeg's native VP8/VP9 decoders discard the WebM alpha sidecar.
const decoder = !alpha ? undefined
: stream.codec_name === "vp9" ? "libvpx-vp9" : stream.codec_name === "vp8" ? "libvpx" : undefined;
return {
alpha,
pixelFormat: stream.pix_fmt,
stream,
formatNames: probe.format?.format_name?.split(",") ?? [],
streamCount: probe.format?.nb_streams,
inputArgs: decoder === undefined ? [] : [`-c:${streamIndex}`, decoder],
};
}
async function copyNormalizedVisual(args: {
env: MediaExecutionEnvironment;
source: BlobRef;
video: MediaVideoStream;
frameRate: MediaRational;
frameCount: number;
encoding: Awaited<ReturnType<typeof sourceVideoEncoding>>;
input: string;
output: string;
}): Promise<BlobRef | undefined> {
const { env, source, video, frameRate, frameCount, encoding, input, output } = args;
const stream = encoding.stream;
if (encoding.alpha || encoding.pixelFormat !== "yuv420p" || stream.codec_name !== "h264"
|| !["Constrained Baseline", "Baseline", "Main", "High"].includes(stream.profile ?? "")
|| stream.level === undefined || stream.level < 10 || stream.level > 41
|| video.codecName !== "h264" || video.width % 2 !== 0 || video.height % 2 !== 0
|| stream.width !== video.width || stream.height !== video.height
|| stream.time_base !== `${video.timeBase?.numerator}/${video.timeBase?.denominator}`
|| (stream.sample_aspect_ratio !== undefined && stream.sample_aspect_ratio !== "1:1")
|| Object.entries(stream.tags ?? {}).some(([name, value]) => name.toLowerCase() === "rotate" && Number(value) !== 0)
|| stream.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0)
|| video.rotationDegrees !== 0 || video.sampleAspectRatio.numerator !== video.sampleAspectRatio.denominator
|| !normalizedVideoSpan(video, frameRate, frameCount)) return undefined;
let inspected: MediaInspection;
try {
const raw = await runProcess({
executable: env.ffprobePath,
argv: ["-v", "error", "-print_format", "json", "-select_streams", String(video.index),
"-show_frames", "-show_entries", "frame=pts,duration,pkt_duration,width,height,pix_fmt,sample_aspect_ratio,interlaced_frame:frame_side_data=rotation", input],
timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const probe = JSON.parse(Buffer.from(raw).toString("utf8")) as {
frames?: Parameters<typeof normalizedVideoFrames>[3];
};
if (!normalizedVideoFrames(video, frameRate, frameCount, probe.frames ?? [])) return undefined;
if (source.mediaType === "video/mp4" && encoding.formatNames.includes("mp4") && encoding.streamCount === 1)
return source;
await runProcess({
executable: env.ffmpegPath,
argv: ["-y", "-copyts", "-i", input, "-map", `0:${video.index}`, "-an", "-sn", "-dn",
"-map_metadata", "-1", "-c:v", "copy", "-avoid_negative_ts", "disabled", "-movflags", "+faststart", output],
timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
inspected = await outputInspection({
path: output, mediaType: "video/mp4", ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs, maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
} catch {
return undefined;
}
const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
if (!visual || !normalizedVideoSpan(visual, frameRate, frameCount) || inspected.streams.length !== 1
|| visual.width !== video.width || visual.height !== video.height || visual.rotationDegrees !== 0
|| visual.sampleAspectRatio.numerator !== 1 || visual.sampleAspectRatio.denominator !== 1) return undefined;
return await env.artifacts.putFile(output, "video/mp4");
}
export async function executeNormalizeMedia(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = normalizeNeed(constraints);
const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);
const work = await mkdtemp(join(tmpdir(), "hypit-media-normalize-"));
try {
const input = join(work, "source.bin");
const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
if (animationBytes === undefined) await stageArtifact(env, need.source, input);
else await writeFile(input, animationBytes);
const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
let visualArtifact: BlobRef | undefined;
let visualWidth: number | undefined;
let visualHeight: number | undefined;
if (plan.video !== undefined) {
const encoding = animation === undefined
? await sourceVideoEncoding(env, input, plan.video.index)
: { alpha: true, pixelFormat: undefined, stream: { index: plan.video.index }, formatNames: [], streamCount: undefined, inputArgs: [] };
const outputType = encoding.alpha ? "video/webm" : "video/mp4";
const output = join(work, encoding.alpha ? "visual.webm" : "visual.mp4");
const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;
const filter = [
"setpts=PTS-STARTPTS",
`fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
`trim=start_frame=0:end_frame=${plan.frameCount}`,
`setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,
// Autorotation runs before this filter. Expand, never shrink, the axis
// carrying non-square samples, then erase SAR so downstream Spatial
// receives only truthful square-pixel display dimensions.
"scale=trunc(iw*max(sar\\,1)/2)*2:trunc(ih*max(1/sar\\,1)/2)*2",
"setsar=1",
].join(",");
// FFmpeg enables autorotation by default. Its positive boolean option changed
// syntax across releases; no override is needed to materialize source rotation.
const visualInput = animation === undefined
? [...encoding.inputArgs, "-i", input, "-map", `0:${plan.video.index}`]
: ["-f", "concat", "-safe", "0", "-i", await animatedWebpConcat(env, animation, work), "-map", "0:v:0"];
// The execution format must retain the source's alpha while materializing
// the program clock. Both encodings publish the same SynchronizedMedia type.
const encoderArgs = encoding.alpha
? [
"-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-lossless", "1", "-auto-alt-ref", "0",
"-row-mt", "1", "-deadline", "realtime", "-cpu-used", "8",
]
: ["-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart"];
visualArtifact = animation === undefined
? await copyNormalizedVisual({ env, source: need.source, video: plan.video, frameRate: need.frameRate,
frameCount: plan.frameCount, encoding, input, output })
: undefined;
if (visualArtifact) {
visualWidth = plan.video.width;
visualHeight = plan.video.height;
} else {
await runProcess({
executable: env.ffmpegPath,
argv: ["-y", ...visualInput, "-an", "-vf", filter,
"-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const inspected = await outputInspection({
path: output,
mediaType: outputType,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
"Normalized visual frame shape differs from its plan");
assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
&& visual.rotationDegrees === 0,
"Normalized visual retains non-square samples or display rotation");
visualArtifact = await env.artifacts.putFile(output, outputType);
visualWidth = visual.width;
visualHeight = visual.height;
}
}
let audioArtifact: BlobRef | undefined;
if (plan.audio !== undefined) {
const output = join(work, "audio.wav");
const filter = [
"asetpts=PTS-STARTPTS",
"aresample=48000:async=0:first_pts=0",
"aformat=sample_rates=48000:channel_layouts=stereo",
`atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,
"asetpts=N/SR/TB",
`adelay=${plan.audioHeadSamples}S:all=1`,
`apad=whole_len=${plan.sampleFrames}`,
`atrim=start_sample=0:end_sample=${plan.sampleFrames}`,
"asetpts=N/SR/TB",
].join(",");
await runProcess({
executable: env.ffmpegPath,
argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,
"-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const inspected = await outputInspection({
path: output,
mediaType: "audio/wav",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");
assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,
"Normalized audio sample shape differs from its plan");
audioArtifact = await env.artifacts.putFile(output, "audio/wav");
}
const media = sealSynchronizedMedia({
frameDomain: {
frameRate: need.frameRate,
frameCount: plan.frameCount,
},
...(visualArtifact === undefined ? {} : {
visual: {
artifact: visualArtifact,
width: visualWidth!,
height: visualHeight!,
},
}),
...(audioArtifact === undefined ? {} : {
audio: {
artifact: audioArtifact,
},
}),
});
return inlineResult(canonicalize(media));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
/**
* Encode authored images into an exact video-only CFR clip. One picture is held for the whole frame
* count; several are each held for their planned segment, fitted into the first picture's frame
* and letterboxed on black, then concatenated in order.
*/
export async function executeRenderStillVideo(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = renderStillVideoNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-still-"));
try {
const output = join(work, "still.mp4");
const staged = new Map<string, string>();
const inputs: string[] = [];
for (const segment of need.request.segments) {
const source = segment.source!;
let path = staged.get(source.resource);
if (path === undefined) {
path = join(work, `picture-${staged.size}.image`);
await stageArtifact(env, source, path);
staged.set(source.resource, path);
}
inputs.push(path);
}
const { numerator, denominator } = need.request.frameRate;
const fps = `${numerator}/${denominator}`;
const hold = (frames: number): string => [
"select=eq(n\\,0)",
"loop=loop=-1:size=1:start=0",
`trim=start_frame=0:end_frame=${frames}`,
`settb=expr=${denominator}/${numerator}`,
`setpts=N*${denominator}/(${numerator}*TB)`,
].join(",");
const bt709Output = [
"scale=out_color_matrix=bt709:out_range=tv",
"format=yuv420p",
"setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
].join(",");
let argv: string[];
if (need.request.segments.length === 1) {
const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,${bt709Output}`;
argv = ["-y", "-i", inputs[0]!, "-map", "0:v:0", "-an", "-vf", filter];
} else {
const first = await outputInspection({
path: inputs[0]!,
mediaType: need.request.segments[0]!.source!.mediaType,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const picture = first.streams.find((item): item is MediaVideoStream => item.kind === "video");
assert(picture !== undefined, "Still video first picture has no decodable image");
const width = Math.ceil(picture.width / 2) * 2;
const height = Math.ceil(picture.height / 2) * 2;
const chains = need.request.segments.map((segment, index) =>
`[${index}:v]${hold(segment.endFrameExclusive - segment.startFrame)},`
+ `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[s${index}]`);
const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[joined];[joined]${bt709Output}[v]`;
argv = ["-y", ...inputs.flatMap((path) => ["-i", path]), "-filter_complex", `${chains.join(";")};${concat}`, "-map", "[v]", "-an"];
}
await runProcess({
executable: env.ffmpegPath,
argv: [
...argv,
"-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr",
"-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
"-movflags", "+faststart", output,
],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const inspected = await outputInspection({
path: output,
mediaType: "video/mp4",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const videos = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
assert(videos.length === 1 && inspected.streams.length === 1
&& videos[0]!.decodedUnitCount === need.request.frameCount,
"Still media output differs from its requested frame domain");
const visual = videos[0]!;
assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
&& visual.rotationDegrees === 0,
"Still media output retains non-square samples or display rotation");
assert(visual.averageFrameRate !== undefined
&& visual.averageFrameRate.numerator * need.request.frameRate.denominator
=== need.request.frameRate.numerator * visual.averageFrameRate.denominator,
"Still media output differs from its requested frame rate");
return artifactResult(await env.artifacts.putFile(output, "video/mp4"));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
type TransformPlan = {
readonly videoFilters: readonly string[];
readonly audioFilters: readonly string[];
readonly frameCount: number;
readonly sampleFrames: number;
readonly durationSec: number;
};
function decimal(value: number): string {
assert(Number.isFinite(value), "Media transform produced a non-finite time");
return value.toPrecision(15);
}
function compileTransformPlan(media: SynchronizedMedia, operations: readonly MediaTransformOperation[]): TransformPlan {
const rate = media.frameDomain.frameRate.numerator / media.frameDomain.frameRate.denominator;
let durationSec = media.frameDomain.frameCount / rate;
const videoFilters: string[] = ["setpts=PTS-STARTPTS"];
const audioFilters: string[] = ["asetpts=N/SR/TB"];
for (const [index, operation] of operations.entries()) {
if (operation.kind === "trim") {
const start = operation.startSec ?? 0;
const end = operation.endSec ?? (operation.tailSec === undefined
? durationSec
: durationSec - operation.tailSec);
assert(start >= 0 && start < durationSec,
`Media transform trim ${index} starts outside its current ${durationSec}s timeline`);
assert(end > start && end <= durationSec + 1e-9,
`Media transform trim ${index} ends outside its current ${durationSec}s timeline`);
const boundedEnd = Math.min(end, durationSec);
videoFilters.push(`trim=start=${decimal(start)}:end=${decimal(boundedEnd)}`, "setpts=PTS-STARTPTS");
audioFilters.push(`atrim=start=${decimal(start)}:end=${decimal(boundedEnd)}`, "asetpts=N/SR/TB");
durationSec = boundedEnd - start;
continue;
}
videoFilters.push(`setpts=PTS/${decimal(operation.rate)}`);
audioFilters.push(...atempo(operation.rate));
durationSec /= operation.rate;
}
const frameCount = Math.max(1, Math.round(durationSec * rate));
assert(Number.isSafeInteger(frameCount), "Media transform output frame count exceeds safe arithmetic");
const sampleFrames = roundPositive(
BigInt(frameCount) * 48_000n * BigInt(media.frameDomain.frameRate.denominator),
BigInt(media.frameDomain.frameRate.numerator),
);
const fps = `${media.frameDomain.frameRate.numerator}/${media.frameDomain.frameRate.denominator}`;
videoFilters.push(
`fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
`trim=start_frame=0:end_frame=${frameCount}`,
`setpts=N*${media.frameDomain.frameRate.denominator}/(${media.frameDomain.frameRate.numerator}*TB)`,
);
audioFilters.push(
`apad=whole_len=${sampleFrames}`,
`atrim=start_sample=0:end_sample=${sampleFrames}`,
"asetpts=N/SR/TB",
);
return { videoFilters, audioFilters, frameCount, sampleFrames, durationSec };
}
/** Ordered trim/retime over one exact synchronized A/V value. */
export async function executeTransformMedia(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = transformNeed(constraints);
const media = need.media;
const plan = compileTransformPlan(media, need.program.operations);
const work = await mkdtemp(join(tmpdir(), "hypit-media-transform-"));
try {
const visualPath = join(work, "visual.mp4");
const audioPath = join(work, "audio.wav");
const output = join(work, "transformed.mp4");
await stageArtifact(env, media.visual!.artifact, visualPath);
if (media.audio !== undefined) await stageArtifact(env, media.audio.artifact, audioPath);
const argv = ["-y", "-i", visualPath];
if (media.audio !== undefined) {
argv.push(
"-i", audioPath,
"-filter_complex",
`[0:v:0]${plan.videoFilters.join(",")}[video];[1:a:0]${plan.audioFilters.join(",")}[audio]`,
"-map", "[video]", "-map", "[audio]",
"-c:a", "aac", "-ar", "48000", "-ac", "2",
);
} else {
argv.push("-map", "0:v:0", "-an", "-vf", plan.videoFilters.join(","));
}
argv.push(
"-frames:v", String(plan.frameCount),
"-r", `${media.frameDomain.frameRate.numerator}/${media.frameDomain.frameRate.denominator}`, "-fps_mode", "cfr",
"-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
"-movflags", "+faststart", output,
);
await runProcess({
executable: env.ffmpegPath,
argv,
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const inspected = await outputInspection({
path: output,
mediaType: "video/mp4",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const videos = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
const audios = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
assert(videos.length === 1 && videos[0]!.role === "moving"
&& videos[0]!.decodedUnitCount === plan.frameCount,
"Transformed media video differs from its compiled frame domain");
assert(audios.length === (media.audio === undefined ? 0 : 1),
"Transformed media audio presence differs from its synchronized input");
if (audios[0] !== undefined) {
assert(audios[0].sampleRate === 48_000 && audios[0].channels === 2,
"Transformed media audio is not 48 kHz stereo");
}
const artifact = await env.artifacts.putFile(output, "video/mp4");
return artifactResult(artifact);
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
/** Generic model-reference audio: no Narrative or alignment claim is introduced. */
export async function executeExtractAudio(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = extractAudioNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-extract-audio-"));
try {
const input = join(work, "source.bin");
const output = join(work, "audio.wav");
await stageArtifact(env, need.source, input);
await runProcess({
executable: env.ffmpegPath,
argv: [
"-y", "-i", input, "-map", `0:${need.streamIndex}`, "-vn",
"-af", "asetpts=PTS-STARTPTS,aresample=48000:async=0:first_pts=0,aformat=sample_rates=48000:channel_layouts=stereo",
"-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output,
],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const artifact = await env.artifacts.putFile(output, "audio/wav");
await assertCanonicalWav({
path: output,
source: artifact,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
return artifactResult(artifact);
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
export async function executeExtractFrame(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = extractFrameNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-extract-frame-"));
try {
const input = join(work, "source.bin");
const output = join(work, "frame.png");
await stageArtifact(env, need.source, input);
const encoding = await sourceVideoEncoding(env, input, need.streamIndex);
const selection = need.at.kind === "first"
? "eq(n\\,0)"
: need.at.kind === "last"
? `eq(n\\,${need.sourceFrameCount - 1})`
: need.at.kind === "frame"
? `eq(n\\,${need.at.index})`
: `gte(t\\,${decimal(need.at.seconds)})`;
await runProcess({
executable: env.ffmpegPath,
argv: [
"-y", ...encoding.inputArgs, "-i", input, "-map", `0:${need.streamIndex}`, "-an",
"-vf", `setpts=PTS-STARTPTS,select=${selection}`,
"-frames:v", "1", "-fps_mode", "vfr", "-c:v", "png", output,
],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const bytes = await readFile(output);
assert(bytes.byteLength > 0, "Frame extraction produced no image");
const inspected = await outputInspection({
path: output,
mediaType: "image/png",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const images = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
assert(images.length === 1 && images[0]!.decodedUnitCount === 1,
"Frame extraction output must decode to exactly one image");
const artifact = await env.artifacts.put(bytes, "image/png");
return artifactResult(artifact);
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
export async function executeProjectSpeechEvidenceAudio(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = evidenceAudioNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-speech-evidence-"));
try {
const input = join(work, "speech-master.wav");
const output = join(work, "alignment-evidence.wav");
await stageArtifact(env, need.source, input);
const source = await assertCanonicalWav({
path: input,
source: need.source,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
assert(source.decodedSampleFrames === need.sourceSampleFrames,
"Speech master sample count differs from its Timeline");
const filter = [
"asetpts=N/SR/TB",
"aresample=16000:async=0:first_pts=0",
"aformat=sample_rates=16000:channel_layouts=mono",
`apad=whole_len=${need.evidenceSampleFrames}`,
`atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,
"asetpts=N/SR/TB",
].join(",");
await runProcess({
executable: env.ffmpegPath,
argv: ["-y", "-i", input, "-vn", "-af", filter,
"-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const inspected = await outputInspection({
path: output,
mediaType: "audio/wav",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
assert(streams.length === 1 && inspected.streams.length === 1
&& streams[0]!.codecName === "pcm_s16le"
&& streams[0]!.sampleRate === 16_000
&& streams[0]!.channels === 1
&& streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,
"Alignment evidence must be exact 16 kHz mono PCM s16");
const artifact = await env.artifacts.putFile(output, "audio/wav");
const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({
domainId: need.domainId,
artifact,
sampleFrames: need.evidenceSampleFrames,
});
assertSpeechEvidenceAudioIdentity(evidence);
return inlineResult(canonicalize(evidence));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
export async function executeRenderTimelineAudio(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = renderAudioNeed(constraints);
const plan: AudioProgramPlan = need.plan;
const window = need.range === undefined
? { startSample: 0, endSampleExclusive: plan.sampleFrames, sampleFrames: plan.sampleFrames }
: mediaFrameRangeSamples(need.range, plan.frameRate);
const clips = plan.clips.filter((clip) => clip.targetStartSample < window.endSampleExclusive
&& clip.targetEndSampleExclusive > window.startSample);
const work = await mkdtemp(join(tmpdir(), "hypit-media-audio-"));
try {
const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number; sampleFrames: number }>();
for (const clip of clips) {
const existing = artifacts.get(clip.artifact.resource);
if (existing !== undefined) {
assert(existing.sampleFrames === clip.sourceSampleFrames,
`Audio input ${clip.artifact.resource} has conflicting sample counts in one plan`);
continue;
}
const inputIndex = artifacts.size;
const path = join(work, `input-${inputIndex}.wav`);
await stageArtifact(env, clip.artifact, path);
const audio = await assertCanonicalWav({
path,
source: clip.artifact,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
assert(audio.decodedSampleFrames === clip.sourceSampleFrames,
`Audio input ${clip.artifact.resource} sample count differs from its plan`);
artifacts.set(clip.artifact.resource, {
source: clip.artifact,
path,
inputIndex,
sampleFrames: audio.decodedSampleFrames,
});
}
const output = join(work, "program.wav");
const argv = ["-y"];
for (const item of artifacts.values()) argv.push("-i", item.path);
if (clips.length === 0) {
argv.push(
"-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",
"-af", `atrim=start_sample=0:end_sample=${window.sampleFrames},asetpts=N/SR/TB`,
);
} else {
const chains = clips.map((clip, index) => {
const inputIndex = artifacts.get(clip.artifact.resource)!.inputIndex;
return audioClipFilter(clip, inputIndex, index, window);
});
const labels = clips.map((_clip, index) => `[clip${index}]`).join("");
chains.push(
`${labels}amix=inputs=${clips.length}:duration=longest:dropout_transition=0:normalize=0,`
+ `apad=whole_len=${window.sampleFrames},atrim=start_sample=0:end_sample=${window.sampleFrames},`
+ "asetpts=N/SR/TB[out]",
);
argv.push("-filter_complex", chains.join(";"), "-map", "[out]");
}
argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);
await runProcess({
executable: env.ffmpegPath,
argv,
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const artifact = await env.artifacts.putFile(output, "audio/wav");
const audio = await assertCanonicalWav({
path: output,
source: artifact,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
assert(audio.decodedSampleFrames === window.sampleFrames,
"Rendered TimelineAudio sample count differs from its plan");
const value: TimelineAudio = sealTimelineAudio({
artifact,
sampleFrames: window.sampleFrames,
});
return inlineResult(canonicalize(value));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}
export async function executeMuxProgramMedia(
env: MediaExecutionEnvironment,
constraints: CanonicalValue,
): Promise<MediaOperationResult> {
const need = muxMediaNeed(constraints);
const work = await mkdtemp(join(tmpdir(), "hypit-media-mux-"));
try {
const visualPath = join(work, "visual.mp4");
const audioPath = join(work, "audio.wav");
const output = join(work, "final.mp4");
await Promise.all([
stageArtifact(env, need.visual.artifact, visualPath),
stageArtifact(env, need.audio.artifact, audioPath),
]);
const [visualInspection, audio] = await Promise.all([
inspectFile({
source: need.visual.artifact,
input: visualPath,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
}),
assertCanonicalWav({
path: audioPath,
source: need.audio.artifact,
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
}),
]);
const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
assert(visualStreams.length === 1 && visualInspection.streams.length === 1,
"TimelineVisual Artifact must contain exactly one silent video stream");
const visual = visualStreams[0]!;
assert(visual.role === "moving" && visual.timingStatus === "admissible"
&& visual.decodedUnitCount === need.visual.frameCount
&& visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,
"TimelineVisual bytes differ from their declared frame domain");
assert(audio.decodedSampleFrames === need.audio.sampleFrames,
"TimelineAudio bytes differ from their declared sample domain");
await runProcess({
executable: env.ffmpegPath,
argv: [
"-y", "-i", visualPath, "-i", audioPath,
"-map", `0:${visual.index}`, "-map", "1:0",
"-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",
"-movflags", "+faststart", output,
],
timeoutMs: env.processTimeoutMs,
maxStdoutBytes: 64 * 1024,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const finalInspection = await outputInspection({
path: output,
mediaType: "video/mp4",
ffprobePath: env.ffprobePath,
timeoutMs: env.processTimeoutMs,
maxProbeOutputBytes: env.maxProbeOutputBytes,
...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
});
const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,
"Final mux must contain exactly one video and one audio stream");
assert(finalVideo[0]!.timingStatus === "admissible"
&& finalVideo[0]!.decodedUnitCount === need.visual.frameCount,
"Final mux video frame count differs from TimelineVisual");
assert(finalAudio[0]!.timingStatus === "admissible"
&& finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,
"Final mux audio shape differs from TimelineAudio");
assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined
&& compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,
"Final mux audio and video do not share one presentation origin");
const finalAudioSpan = duration(finalAudio[0]!);
/*
* The staged WAV is asserted sample-exact above; this checks what survived AAC.
*
* AAC cannot carry an arbitrary sample count: the encoder emits 1024-sample frames and
* reports a priming delay that the MP4 muxer compensates with an edit list, so the
* presented span lands a few samples away from the input. Measured here with ffmpeg 6.1
* on synthetic tone (i.e. independent of any project's material): a 4 249 600-sample input
* presents as 4 249 584 (-16), and 4 236 800 presents as 4 236 768 (-32). Padding the
* timeline to a whole number of AAC frames does not remove it — the delay is not a
* frame-alignment artifact.
*
* Demanding exact equality therefore rejects every correct mux whose length is not a
* fixed point of that round trip. The real invariant AAC can hold is that no whole frame
* of audio went missing, so the tolerance is one AAC frame rather than an arbitrary epsilon;
* anything larger still means genuine desync and still fails.
*/
const AAC_FRAME_SAMPLES = 1_024;
const finalAudioSamples = roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator);
assert(Math.abs(finalAudioSamples - need.audio.sampleFrames) < AAC_FRAME_SAMPLES,
`Final mux audio presentation span differs from TimelineAudio by ${
String(finalAudioSamples - need.audio.sampleFrames)} samples`);
const artifact = await env.artifacts.putFile(output, "video/mp4");
const value: MuxedMedia = sealMuxedMedia({
frameRate: need.visual.frameRate,
frameCount: need.visual.frameCount,
canvas: need.visual.canvas,
presentationSampleFrames: need.audio.sampleFrames,
artifact,
});
return inlineResult(canonicalize(value));
} finally {
await rm(work, { recursive: true, force: true }).catch(() => {});
}
}