Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

62.6 KB
Raw Blame
import { audioLevelAutomationFilter } from "./audio-level-automation.js";
import { randomUUID } from "node:crypto";
import { createReadStream } from "node:fs";
import { mkdtemp, open, readFile, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { verifyMediaFrameRange, mediaFrameRangeSamples, mediaTypes, sealMediaInspection, sealMuxedMedia, sealSynchronizedMedia, sealTimelineAudio, verifyMediaInspection, verifyMediaStreamSelection, verifyTimelineVisual, verifySynchronizedMedia, verifyTimelineAudio } from "@hypit/media";
import type { MediaAudioStream, MediaInspection, MediaRational, MediaStream, MediaStreamSelection, MediaTimestamp, MediaVideoStream, MuxedMedia, TimelineVisual, SynchronizedMedia, TimelineAudio } from "@hypit/media";
import type { Timeline } from "@hypit/timeline";
import { assertSpeechEvidenceAudioIdentity, sealSpeechEvidenceAudio, speechEvidenceSampleBoundary } from "@hypit/speech-evidence";
import type { ProjectSpeechEvidenceAudioNeed, SpeechEvidenceAudio } from "@hypit/speech-evidence";
import {
  verifyAudioExtractionRequest,
  verifyAudioProgramPlan,
  verifyFrameExtractionRequest,
  verifyMediaTransformProgram,
  type ExtractAudioNeed,
  type ExtractFrameNeed,
  type InspectMediaNeed,
  type MuxMediaNeed,
  type NormalizeMediaNeed,
  type RenderAudioNeed,
  type RenderStillVideoNeed,
  type TransformMediaNeed,
  verifyStillVideoRequest,
} from "@hypit/media-operations";
import type { AudioProgramClip, AudioProgramPlan, MediaTransformOperation } from "@hypit/media-operations";
import {
  canonicalize,
  } from "@hypit/hypit/protocol";
import type { BlobRef, CanonicalValue, StoredValue } from "@hypit/hypit/protocol";

import { runBoundedMediaProcess } from "./process.js";
import { parseMediaInspection } from "./probe.js";
import { normalizedVideoFrames, normalizedVideoSpan } from "./normalize-copy.js";
import {
  compositeAnimatedWebpFrame,
  createAnimatedWebpCanvas,
  pamRgba,
  parseAnimatedWebp,
} from "./webp.js";
import type { AnimatedWebp } from "./webp.js";

/**
 * Where the bytes live and which binaries transform them.
 *
 * These byte operations are written once behind an Artifact gateway. The local Provider supplies
 * the Build's ResourceStore and the ffmpeg on its PATH. Another execution owner may reuse the same
 * algorithms with a different gateway and tool bundle; nothing here claims a Provider or deployment.
 */
export type MediaArtifactGateway = {
  get(source: BlobRef): Promise<Uint8Array | undefined>;
  open(source: BlobRef): Promise<AsyncIterable<Uint8Array> | undefined>;
  put(bytes: Uint8Array, mediaType: string): Promise<BlobRef>;
  putFile(path: string, mediaType: string): Promise<BlobRef>;
};

export type MediaExecutionEnvironment = {
  readonly artifacts: MediaArtifactGateway;
  readonly ffmpegPath: string;
  readonly ffprobePath: string;
  /** Deployment fact for dynamically linked tool bundles; absent for ordinary local binaries. */
  readonly sharedLibraryPath?: string;
  readonly processTimeoutMs: number;
  readonly maxProbeOutputBytes: number;
};

export type MediaOperationResult = {
  readonly value: StoredValue;
};

function assert(condition: unknown, message: string): asserts condition {
  if (!condition) throw new Error(message);
}

function positiveInteger(value: number, subject: string): number {
  assert(Number.isSafeInteger(value) && value > 0, `${subject} must be a positive integer`);
  return value;
}

function object(value: unknown, subject: string): Record<string, unknown> {
  assert(value !== null && typeof value === "object" && !Array.isArray(value), `${subject} must be an object`);
  return value as Record<string, unknown>;
}

async function runProcess(args: {
  readonly executable: string;
  readonly argv: readonly string[];
  readonly timeoutMs: number;
  readonly maxStdoutBytes: number;
  readonly sharedLibraryPath?: string;
}): Promise<Uint8Array> {
  return await runBoundedMediaProcess(args);
}

async function version(executable: string, timeoutMs: number, sharedLibraryPath?: string): Promise<string> {
  const bytes = await runProcess({ executable, argv: ["-version"], timeoutMs, maxStdoutBytes: 64 * 1024,
    ...(sharedLibraryPath === undefined ? {} : { sharedLibraryPath }) });
  const line = Buffer.from(bytes).toString("utf8").split(/\r?\n/u, 1)[0]?.trim();
  assert(line !== undefined && line.length > 0, `${executable} returned no version`);
  return line;
}

async function sourceBytes(env: MediaExecutionEnvironment, source: BlobRef): Promise<Uint8Array> {
  const bytes = await env.artifacts.get(source);
  assert(bytes !== undefined, `Media source ${source.resource} is unavailable`);
  assert(bytes.byteLength === source.size, `Media source ${source.resource} size differs`);
  return bytes;
}

async function stageArtifact(
  env: MediaExecutionEnvironment,
  source: BlobRef,
  path: string,
): Promise<void> {
  const chunks = await env.artifacts.open(source);
  assert(chunks !== undefined, `Media source ${source.resource} is unavailable`);
  const file = await open(path, "w");
  let size = 0;
  try {
    for await (const chunk of chunks) {
      await file.write(chunk);
      size += chunk.byteLength;
    }
  } finally {
    await file.close();
  }
  assert(size === source.size, `Media source ${source.resource} size differs`);
}

function timestampFraction(value: MediaTimestamp): { numerator: bigint; denominator: bigint } {
  return {
    numerator: BigInt(value.ticks) * BigInt(value.timeBase.numerator),
    denominator: BigInt(value.timeBase.denominator),
  };
}

function compareTimestamp(left: MediaTimestamp, right: MediaTimestamp): number {
  const a = timestampFraction(left);
  const b = timestampFraction(right);
  const x = a.numerator * b.denominator;
  const y = b.numerator * a.denominator;
  return x < y ? -1 : x > y ? 1 : 0;
}

function roundPositive(numerator: bigint, denominator: bigint): number {
  assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
  const value = (numerator * 2n + denominator) / (denominator * 2n);
  assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
  return Number(value);
}

function ceilPositive(numerator: bigint, denominator: bigint): number {
  assert(numerator >= 0n && denominator > 0n, "Media duration is invalid");
  const value = (numerator + denominator - 1n) / denominator;
  assert(value <= BigInt(Number.MAX_SAFE_INTEGER), "Media duration exceeds safe arithmetic");
  return Number(value);
}

function duration(value: { startPts?: MediaTimestamp; endPts?: MediaTimestamp }): { numerator: bigint; denominator: bigint } {
  assert(value.startPts !== undefined && value.endPts !== undefined, "Selected media stream has no presentation interval");
  const start = timestampFraction(value.startPts);
  const end = timestampFraction(value.endPts);
  const numerator = end.numerator * start.denominator - start.numerator * end.denominator;
  const denominator = end.denominator * start.denominator;
  assert(numerator > 0n, "Selected media stream has an empty presentation interval");
  return { numerator, denominator };
}

function signedOffset(origin: MediaTimestamp, value: MediaTimestamp, rate: number): number {
  const a = timestampFraction(origin);
  const b = timestampFraction(value);
  const numerator = (b.numerator * a.denominator - a.numerator * b.denominator) * BigInt(rate);
  const denominator = b.denominator * a.denominator;
  const magnitude = roundPositive(numerator < 0n ? -numerator : numerator, denominator);
  return numerator < 0n ? -magnitude : magnitude;
}

function stream<T extends MediaStream["kind"]>(
  inspection: MediaInspection,
  index: number | undefined,
  kind: T,
): Extract<MediaStream, { readonly kind: T }> | undefined {
  if (index === undefined) return undefined;
  const result = inspection.streams.find((item) => item.index === index);
  assert(result?.kind === kind, `Selected ${kind} stream ${index} is absent`);
  assert(result.timingStatus === "admissible", `Selected ${kind} stream ${index} timing is ${result.timingStatus}`);
  return result as Extract<MediaStream, { readonly kind: T }>;
}

type NormalizationPlan = {
  readonly video?: MediaVideoStream;
  readonly audio?: MediaAudioStream;
  readonly frameCount: number;
  readonly sampleFrames: number;
  readonly audioTrimStartSamples: number;
  readonly audioTrimEndSamples: number;
  readonly audioHeadSamples: number;
  readonly audioContentSamples: number;
  readonly audioTailSamples: number;
};

function normalizationPlan(
  inspection: MediaInspection,
  selection: MediaStreamSelection,
  frameRate: MediaRational,
): NormalizationPlan {
  assert(Number.isSafeInteger(frameRate.numerator) && frameRate.numerator > 0
    && Number.isSafeInteger(frameRate.denominator) && frameRate.denominator > 0,
  "Target frame rate is invalid");
  const video = stream(inspection, selection.videoStreamIndex, "video");
  const audio = stream(inspection, selection.audioStreamIndex, "audio");
  if (video !== undefined && selection.spanAuthority !== "video") {
    throw new Error("A synchronized visual currently requires video span authority; use audio authority only for audio-only media");
  }
  const authority = selection.spanAuthority === "video" ? video : audio;
  assert(authority?.startPts !== undefined && authority.endPts !== undefined, "Span authority stream is absent");
  const span = duration(authority);
  const scaledNumerator = span.numerator * BigInt(frameRate.numerator);
  const scaledDenominator = span.denominator * BigInt(frameRate.denominator);
  const frameCount = selection.spanAuthority === "video"
    ? roundPositive(scaledNumerator, scaledDenominator)
    : ceilPositive(scaledNumerator, scaledDenominator);
  assert(frameCount > 0, "Normalized media would contain no frame interval");
  const sampleFrames = roundPositive(
    BigInt(frameCount) * 48_000n * BigInt(frameRate.denominator),
    BigInt(frameRate.numerator),
  );
  if (audio === undefined) {
    return {
      ...(video === undefined ? {} : { video }),
      frameCount,
      sampleFrames,
      audioTrimStartSamples: 0,
      audioTrimEndSamples: 0,
      audioHeadSamples: 0,
      audioContentSamples: 0,
      audioTailSamples: 0,
    };
  }
  assert(audio.startPts !== undefined && audio.endPts !== undefined, "Selected audio has no presentation interval");
  const audioStart = signedOffset(authority.startPts, audio.startPts, 48_000);
  const audioEnd = signedOffset(authority.startPts, audio.endPts, 48_000);
  const contentStart = Math.max(0, audioStart);
  const contentEnd = Math.min(sampleFrames, audioEnd);
  assert(audioEnd > audioStart && contentEnd > contentStart, "Selected audio is outside the authoritative media span");
  return {
    ...(video === undefined ? {} : { video }),
    audio,
    frameCount,
    sampleFrames,
    audioTrimStartSamples: Math.max(0, -audioStart),
    audioTrimEndSamples: Math.max(0, audioEnd - sampleFrames),
    audioHeadSamples: contentStart,
    audioContentSamples: contentEnd - contentStart,
    audioTailSamples: sampleFrames - contentEnd,
  };
}

async function inspectFile(args: {
  readonly source: BlobRef;
  readonly input: string;
  readonly ffprobePath: string;
  readonly timeoutMs: number;
  readonly maxProbeOutputBytes: number;
  readonly sharedLibraryPath?: string;
}): Promise<MediaInspection> {
  const [raw, ffprobeVersion] = await Promise.all([
    runProcess({
      executable: args.ffprobePath,
      argv: ["-v", "error", "-print_format", "json", "-show_format", "-show_streams", "-show_frames", args.input],
      timeoutMs: args.timeoutMs,
      maxStdoutBytes: args.maxProbeOutputBytes,
      ...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }),
    }),
    version(args.ffprobePath, args.timeoutMs, args.sharedLibraryPath),
  ]);
  let value: unknown;
  try {
    value = JSON.parse(Buffer.from(raw).toString("utf8"));
  } catch {
    throw new Error("ffprobe returned invalid JSON");
  }
  return parseMediaInspection({ source: args.source, ffprobeVersion, value });
}

function divisor(left: number, right: number): number {
  let a = Math.abs(left);
  let b = Math.abs(right);
  while (b !== 0) [a, b] = [b, a % b];
  return a;
}

function animatedWebpInspection(source: BlobRef, animation: AnimatedWebp): MediaInspection {
  const durationMs = animation.frames.reduce((sum, frame) => sum + frame.durationMs, 0);
  const numerator = animation.frames.length * 1_000;
  const factor = divisor(numerator, durationMs);
  return sealMediaInspection({
    container: { formatNames: ["webp", "webp-animation"] },
    streams: [{
      kind: "video",
      index: 0,
      codecType: "video",
      codecName: "webp",
      disposition: { default: true, attachedPicture: false },
      timingStatus: "admissible",
      timeBase: { numerator: 1, denominator: 1_000 },
      startPts: { ticks: "0", timeBase: { numerator: 1, denominator: 1_000 } },
      endPts: { ticks: String(durationMs), timeBase: { numerator: 1, denominator: 1_000 } },
      decodedUnitCount: animation.frames.length,
      role: "moving",
      width: animation.width,
      height: animation.height,
      sampleAspectRatio: { numerator: 1, denominator: 1 },
      rotationDegrees: 0,
      averageFrameRate: { numerator: numerator / factor, denominator: durationMs / factor },
    }],
  });
}

async function animatedWebpConcat(
  env: MediaExecutionEnvironment,
  animation: AnimatedWebp,
  work: string,
): Promise<string> {
  assert(animation.frames.length <= 10_000, "Animated WebP has too many frames");
  const canvasBytes = animation.width * animation.height * 4;
  assert(Number.isSafeInteger(canvasBytes) && canvasBytes > 0 && canvasBytes <= 512 * 1024 * 1024,
    "Animated WebP canvas is outside the supported memory bound");
  const canvas = createAnimatedWebpCanvas(animation);
  const paths: string[] = [];
  for (const [index, frame] of animation.frames.entries()) {
    const imagePath = join(work, `webp-source-${String(index).padStart(6, "0")}.webp`);
    await writeFile(imagePath, frame.image);
    const decoded = await runProcess({
      executable: env.ffmpegPath,
      argv: ["-v", "error", "-i", imagePath, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgba", "-"],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: frame.width * frame.height * 4,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const snapshot = compositeAnimatedWebpFrame(canvas, animation, frame, decoded);
    const path = join(work, `webp-frame-${String(index).padStart(6, "0")}.pam`);
    await writeFile(path, pamRgba(animation.width, animation.height, snapshot));
    paths.push(path);
  }
  const lines = ["ffconcat version 1.0"];
  for (const [index, path] of paths.entries()) {
    lines.push(`file '${path}'`, `duration ${(animation.frames[index]!.durationMs / 1_000).toFixed(6)}`);
  }
  lines.push(`file '${paths.at(-1)!}'`);
  const manifest = join(work, "animated-webp.ffconcat");
  await writeFile(manifest, `${lines.join("\n")}\n`, "utf8");
  return manifest;
}

async function outputInspection(args: {
  readonly path: string;
  readonly mediaType: string;
  readonly ffprobePath: string;
  readonly timeoutMs: number;
  readonly maxProbeOutputBytes: number;
  readonly sharedLibraryPath?: string;
}): Promise<MediaInspection> {
  const size = (await stat(args.path)).size;
  const source: BlobRef = {
    kind: "blob",
    resource: `res_${randomUUID()}`,
    size,
    mediaType: args.mediaType,
  };
  return await inspectFile({ source, input: args.path, ffprobePath: args.ffprobePath,
    timeoutMs: args.timeoutMs, maxProbeOutputBytes: args.maxProbeOutputBytes,
    ...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }) });
}

function inlineResult(value: CanonicalValue): MediaOperationResult {
  return { value: { kind: "inline", value } };
}

function artifactResult(value: BlobRef): MediaOperationResult {
  return { value };
}

function inspectNeed(value: CanonicalValue): InspectMediaNeed {
  const item = object(value, "InspectMediaNeed") as unknown as InspectMediaNeed;
  assert(item.source?.kind === "blob", "InspectMediaNeed is invalid");
  return item;
}

function normalizeNeed(value: CanonicalValue): NormalizeMediaNeed {
  const item = object(value, "NormalizeMediaNeed") as unknown as NormalizeMediaNeed;
  assert(item.source?.kind === "blob", "NormalizeMediaNeed is invalid");
  verifyMediaInspection(item.inspection);
  verifyMediaStreamSelection(item.selection);
  assert(item.audio.sampleRate === 48_000 && item.audio.channels === 2
    && item.audio.codec === "pcm_s16le" && item.audio.loudness === "preserve",
  "NormalizeMediaNeed audio profile is unsupported");
  return item;
}

function transformNeed(value: CanonicalValue): TransformMediaNeed {
  const item = object(value, "TransformMediaNeed") as unknown as TransformMediaNeed;
  verifySynchronizedMedia(item.media);
  verifyMediaTransformProgram(item.program);
  assert(item.media.visual !== undefined, "TransformMediaNeed requires synchronized visual media");
  return item;
}

function extractAudioNeed(value: CanonicalValue): ExtractAudioNeed {
  const item = object(value, "ExtractAudioNeed") as unknown as ExtractAudioNeed;
  assert(item.source?.kind === "blob", "ExtractAudioNeed is invalid");
  assert(Number.isSafeInteger(item.streamIndex) && item.streamIndex >= 0,
    "ExtractAudioNeed streamIndex is invalid");
  verifyAudioExtractionRequest({
    audio: { mode: "stream-index", streamIndex: item.streamIndex },
    output: item.output,
  });
  return item;
}

function extractFrameNeed(value: CanonicalValue): ExtractFrameNeed {
  const item = object(value, "ExtractFrameNeed") as unknown as ExtractFrameNeed;
  assert(item.source?.kind === "blob", "ExtractFrameNeed is invalid");
  assert(Number.isSafeInteger(item.streamIndex) && item.streamIndex >= 0
    && Number.isSafeInteger(item.sourceFrameCount) && item.sourceFrameCount > 0,
  "ExtractFrameNeed stream domain is invalid");
  verifyFrameExtractionRequest({
    video: { mode: "stream-index", streamIndex: item.streamIndex },
    at: item.at,
    output: item.output,
  });
  if (item.at.kind === "frame") {
    assert(item.at.index < item.sourceFrameCount, "ExtractFrameNeed frame lies outside the source stream");
  }
  return item;
}

function renderStillVideoNeed(value: CanonicalValue): RenderStillVideoNeed {
  const item = object(value, "RenderStillVideoNeed") as unknown as RenderStillVideoNeed;
  verifyStillVideoRequest(item.request);
  assert(item.request.segments.every((segment) => segment.source?.kind === "blob" && segment.source.mediaType.startsWith("image/")),
    "RenderStillVideoNeed needs an image Artifact for every segment");
  return item;
}

function evidenceAudioNeed(value: CanonicalValue): ProjectSpeechEvidenceAudioNeed {
  const item = object(value, "ProjectSpeechEvidenceAudioNeed") as unknown as ProjectSpeechEvidenceAudioNeed;
  assert(typeof item.domainId === "string" && item.domainId.length > 0
    && item.source?.kind === "blob", "ProjectSpeechEvidenceAudioNeed is invalid");
  assert(Number.isSafeInteger(item.sourceSampleFrames) && item.sourceSampleFrames > 0
    && Number.isSafeInteger(item.evidenceSampleFrames) && item.evidenceSampleFrames > 0
    && item.evidenceSampleFrames === speechEvidenceSampleBoundary(item.sourceSampleFrames),
  "Speech evidence audio sample projection is invalid");
  return item;
}

function renderAudioNeed(value: CanonicalValue): RenderAudioNeed {
  const item = object(value, "RenderAudioNeed") as unknown as RenderAudioNeed;
  verifyAudioProgramPlan(item.plan);
  if (item.range !== undefined) verifyMediaFrameRange(item.range, item.plan.frameCount);
  return item;
}

function muxMediaNeed(value: CanonicalValue): MuxMediaNeed {
  const item = object(value, "MuxMediaNeed") as unknown as MuxMediaNeed;
  verifyTimelineVisual(item.visual);
  verifyTimelineAudio(item.audio);
  const expectedSamples = roundPositive(
    BigInt(item.visual.frameCount) * 48_000n * BigInt(item.visual.frameRate.denominator),
    BigInt(item.visual.frameRate.numerator),
  );
  assert(item.audio.sampleFrames === expectedSamples, "MuxMediaNeed audio differs from the visual frame domain");
  return item;
}

function atempo(rate: number): string[] {
  if (Math.abs(rate - 1) < 1e-12) return [];
  const filters: string[] = [];
  let remaining = rate;
  while (remaining > 2) {
    filters.push("atempo=2");
    remaining /= 2;
  }
  while (remaining < 0.5) {
    filters.push("atempo=0.5");
    remaining /= 0.5;
  }
  if (Math.abs(remaining - 1) >= 1e-12) filters.push(`atempo=${remaining.toPrecision(15)}`);
  return filters;
}

function audioClipFilter(clip: AudioProgramClip, inputIndex: number, outputIndex: number,
  window: { readonly startSample: number; readonly endSampleExclusive: number }): string {
  const length = clip.targetEndSampleExclusive - clip.targetStartSample;
  const sourceLength = clip.sourceEndSampleExclusive - clip.sourceStartSample;
  const left = Math.max(window.startSample, clip.targetStartSample);
  const right = Math.min(window.endSampleExclusive, clip.targetEndSampleExclusive);
  const filters = [
    `atrim=start_sample=${clip.sourceStartSample}:end_sample=${clip.sourceEndSampleExclusive}`,
    "asetpts=PTS-STARTPTS",
    ...(clip.sourceLoop ? [`aloop=loop=-1:size=${sourceLength}:start=0`] : []),
    ...(clip.sourcePhaseSample === 0 ? [] : [
      `atrim=start_sample=${clip.sourcePhaseSample}`,
      "asetpts=PTS-STARTPTS",
    ]),
    ...atempo(clip.playbackRate),
    `atrim=start_sample=0:end_sample=${length}`,
    `apad=whole_len=${length}`,
    `atrim=start_sample=0:end_sample=${length}`,
    `volume=${clip.gain.toPrecision(15)}`,
    ...audioLevelAutomationFilter(clip),
    // Crop after tempo, looping and envelopes, keeping their original phase.
    `atrim=start_sample=${left - clip.targetStartSample}:end_sample=${right - clip.targetStartSample}`,
    "asetpts=N/SR/TB",
    `adelay=${left - window.startSample}S:all=1`,
  ];
  return `[${inputIndex}:a:0]${filters.join(",")}[clip${outputIndex}]`;
}

async function assertCanonicalWav(args: {
  readonly path: string;
  readonly source: BlobRef;
  readonly ffprobePath: string;
  readonly timeoutMs: number;
  readonly maxProbeOutputBytes: number;
  readonly sharedLibraryPath?: string;
}): Promise<MediaAudioStream> {
  const inspection = await inspectFile({
    source: args.source,
    input: args.path,
    ffprobePath: args.ffprobePath,
    timeoutMs: args.timeoutMs,
    maxProbeOutputBytes: args.maxProbeOutputBytes,
    ...(args.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: args.sharedLibraryPath }),
  });
  const audio = inspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
  assert(audio.length === 1 && inspection.streams.length === 1, "Canonical audio must contain exactly one stream");
  assert(audio[0]!.codecName === "pcm_s16le" && audio[0]!.sampleRate === 48_000 && audio[0]!.channels === 2,
    "Canonical audio must be 48 kHz stereo PCM s16");
  assert(audio[0]!.timingStatus === "admissible", "Canonical audio timing is not admissible");
  return audio[0]!;
}

export async function executeInspectMedia(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = inspectNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-inspect-"));
  try {
    const input = join(work, "source.bin");
    const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
    if (animationBytes === undefined) await stageArtifact(env, need.source, input);
    else await writeFile(input, animationBytes);
    const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
    const inspection = animation === undefined
      ? await inspectFile({
        source: need.source,
        input,
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      })
      : animatedWebpInspection(need.source, animation);
    return inlineResult(canonicalize(inspection));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

async function sourceVideoEncoding(env: MediaExecutionEnvironment, path: string, streamIndex: number) {
  const bytes = await runProcess({
    executable: env.ffprobePath,
    argv: ["-v", "error", "-print_format", "json", "-show_streams", "-show_pixel_formats", "-show_format", path],
    timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
    ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
  });
  const probe = JSON.parse(Buffer.from(bytes).toString("utf8")) as {
    streams: { index: number; codec_name?: string; profile?: string; level?: number; pix_fmt?: string; time_base?: string; sample_aspect_ratio?: string; width?: number; height?: number; tags?: Record<string, string>; side_data_list?: { rotation?: number }[] }[];
    pixel_formats: { name: string; flags: { alpha: number } }[];
    format?: { nb_streams?: number; format_name?: string };
  };
  const stream = probe.streams.find((item) => item.index === streamIndex);
  assert(stream !== undefined, `Media source has no video stream ${streamIndex}`);
  const pixelFormat = probe.pixel_formats.find((item) => item.name === stream.pix_fmt);
  assert(pixelFormat !== undefined, `Media source pixel format ${String(stream.pix_fmt)} is unknown`);
  const alphaTag = Object.entries(stream.tags ?? {}).find(([key]) => key.toLowerCase() === "alpha_mode")?.[1];
  const alpha = pixelFormat.flags.alpha === 1 || alphaTag === "1" || alphaTag === "straight";
  // FFmpeg's native VP8/VP9 decoders discard the WebM alpha sidecar.
  const decoder = !alpha ? undefined
    : stream.codec_name === "vp9" ? "libvpx-vp9" : stream.codec_name === "vp8" ? "libvpx" : undefined;
  return {
    alpha,
    pixelFormat: stream.pix_fmt,
    stream,
    formatNames: probe.format?.format_name?.split(",") ?? [],
    streamCount: probe.format?.nb_streams,
    inputArgs: decoder === undefined ? [] : [`-c:${streamIndex}`, decoder],
  };
}

async function copyNormalizedVisual(args: {
  env: MediaExecutionEnvironment;
  source: BlobRef;
  video: MediaVideoStream;
  frameRate: MediaRational;
  frameCount: number;
  encoding: Awaited<ReturnType<typeof sourceVideoEncoding>>;
  input: string;
  output: string;
}): Promise<BlobRef | undefined> {
  const { env, source, video, frameRate, frameCount, encoding, input, output } = args;
  const stream = encoding.stream;
  if (encoding.alpha || encoding.pixelFormat !== "yuv420p" || stream.codec_name !== "h264"
    || !["Constrained Baseline", "Baseline", "Main", "High"].includes(stream.profile ?? "")
    || stream.level === undefined || stream.level < 10 || stream.level > 41
    || video.codecName !== "h264" || video.width % 2 !== 0 || video.height % 2 !== 0
    || stream.width !== video.width || stream.height !== video.height
    || stream.time_base !== `${video.timeBase?.numerator}/${video.timeBase?.denominator}`
    || (stream.sample_aspect_ratio !== undefined && stream.sample_aspect_ratio !== "1:1")
    || Object.entries(stream.tags ?? {}).some(([name, value]) => name.toLowerCase() === "rotate" && Number(value) !== 0)
    || stream.side_data_list?.some((side) => side.rotation !== undefined && side.rotation !== 0)
    || video.rotationDegrees !== 0 || video.sampleAspectRatio.numerator !== video.sampleAspectRatio.denominator
    || !normalizedVideoSpan(video, frameRate, frameCount)) return undefined;
  let inspected: MediaInspection;
  try {
    const raw = await runProcess({
      executable: env.ffprobePath,
      argv: ["-v", "error", "-print_format", "json", "-select_streams", String(video.index),
        "-show_frames", "-show_entries", "frame=pts,duration,pkt_duration,width,height,pix_fmt,sample_aspect_ratio,interlaced_frame:frame_side_data=rotation", input],
      timeoutMs: env.processTimeoutMs, maxStdoutBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const probe = JSON.parse(Buffer.from(raw).toString("utf8")) as {
      frames?: Parameters<typeof normalizedVideoFrames>[3];
    };
    if (!normalizedVideoFrames(video, frameRate, frameCount, probe.frames ?? [])) return undefined;
    if (source.mediaType === "video/mp4" && encoding.formatNames.includes("mp4") && encoding.streamCount === 1)
      return source;
    await runProcess({
      executable: env.ffmpegPath,
      argv: ["-y", "-copyts", "-i", input, "-map", `0:${video.index}`, "-an", "-sn", "-dn",
        "-map_metadata", "-1", "-c:v", "copy", "-avoid_negative_ts", "disabled", "-movflags", "+faststart", output],
      timeoutMs: env.processTimeoutMs, maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    inspected = await outputInspection({
      path: output, mediaType: "video/mp4", ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs, maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
  } catch {
    return undefined;
  }
  const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
  if (!visual || !normalizedVideoSpan(visual, frameRate, frameCount) || inspected.streams.length !== 1
    || visual.width !== video.width || visual.height !== video.height || visual.rotationDegrees !== 0
    || visual.sampleAspectRatio.numerator !== 1 || visual.sampleAspectRatio.denominator !== 1) return undefined;
  return await env.artifacts.putFile(output, "video/mp4");
}

export async function executeNormalizeMedia(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = normalizeNeed(constraints);
  const plan = normalizationPlan(need.inspection, need.selection, need.frameRate);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-normalize-"));
  try {
    const input = join(work, "source.bin");
    const animationBytes = need.source.mediaType === "image/webp" ? await sourceBytes(env, need.source) : undefined;
    if (animationBytes === undefined) await stageArtifact(env, need.source, input);
    else await writeFile(input, animationBytes);
    const animation = animationBytes === undefined ? undefined : parseAnimatedWebp(animationBytes);
    let visualArtifact: BlobRef | undefined;
    let visualWidth: number | undefined;
    let visualHeight: number | undefined;
    if (plan.video !== undefined) {
      const encoding = animation === undefined
        ? await sourceVideoEncoding(env, input, plan.video.index)
        : { alpha: true, pixelFormat: undefined, stream: { index: plan.video.index }, formatNames: [], streamCount: undefined, inputArgs: [] };
      const outputType = encoding.alpha ? "video/webm" : "video/mp4";
      const output = join(work, encoding.alpha ? "visual.webm" : "visual.mp4");
      const fps = `${need.frameRate.numerator}/${need.frameRate.denominator}`;
      const filter = [
        "setpts=PTS-STARTPTS",
        `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
        `trim=start_frame=0:end_frame=${plan.frameCount}`,
        `setpts=N*${need.frameRate.denominator}/(${need.frameRate.numerator}*TB)`,
        // Autorotation runs before this filter. Expand, never shrink, the axis
        // carrying non-square samples, then erase SAR so downstream Spatial
        // receives only truthful square-pixel display dimensions.
        "scale=trunc(iw*max(sar\\,1)/2)*2:trunc(ih*max(1/sar\\,1)/2)*2",
        "setsar=1",
      ].join(",");
      // FFmpeg enables autorotation by default. Its positive boolean option changed
      // syntax across releases; no override is needed to materialize source rotation.
      const visualInput = animation === undefined
        ? [...encoding.inputArgs, "-i", input, "-map", `0:${plan.video.index}`]
        : ["-f", "concat", "-safe", "0", "-i", await animatedWebpConcat(env, animation, work), "-map", "0:v:0"];
      // The execution format must retain the source's alpha while materializing
      // the program clock. Both encodings publish the same SynchronizedMedia type.
      const encoderArgs = encoding.alpha
        ? [
          "-c:v", "libvpx-vp9", "-pix_fmt", "yuva420p", "-lossless", "1", "-auto-alt-ref", "0",
          "-row-mt", "1", "-deadline", "realtime", "-cpu-used", "8",
        ]
        : ["-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p", "-movflags", "+faststart"];
      visualArtifact = animation === undefined
        ? await copyNormalizedVisual({ env, source: need.source, video: plan.video, frameRate: need.frameRate,
          frameCount: plan.frameCount, encoding, input, output })
        : undefined;
      if (visualArtifact) {
        visualWidth = plan.video.width;
        visualHeight = plan.video.height;
      } else {
        await runProcess({
          executable: env.ffmpegPath,
          argv: ["-y", ...visualInput, "-an", "-vf", filter,
            "-frames:v", String(plan.frameCount), "-r", fps, "-fps_mode", "cfr", ...encoderArgs, output],
          timeoutMs: env.processTimeoutMs,
          maxStdoutBytes: 64 * 1024,
          ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
        });
        const inspected = await outputInspection({
          path: output,
          mediaType: outputType,
          ffprobePath: env.ffprobePath,
          timeoutMs: env.processTimeoutMs,
          maxProbeOutputBytes: env.maxProbeOutputBytes,
          ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
        });
        const visual = inspected.streams.find((item): item is MediaVideoStream => item.kind === "video");
        assert(visual?.decodedUnitCount === plan.frameCount && visual.role === "moving",
          "Normalized visual frame shape differs from its plan");
        assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
          && visual.rotationDegrees === 0,
        "Normalized visual retains non-square samples or display rotation");
        visualArtifact = await env.artifacts.putFile(output, outputType);
        visualWidth = visual.width;
        visualHeight = visual.height;
      }
    }
    let audioArtifact: BlobRef | undefined;
    if (plan.audio !== undefined) {
      const output = join(work, "audio.wav");
      const filter = [
        "asetpts=PTS-STARTPTS",
        "aresample=48000:async=0:first_pts=0",
        "aformat=sample_rates=48000:channel_layouts=stereo",
        `atrim=start_sample=${plan.audioTrimStartSamples}:end_sample=${plan.audioTrimStartSamples + plan.audioContentSamples}`,
        "asetpts=N/SR/TB",
        `adelay=${plan.audioHeadSamples}S:all=1`,
        `apad=whole_len=${plan.sampleFrames}`,
        `atrim=start_sample=0:end_sample=${plan.sampleFrames}`,
        "asetpts=N/SR/TB",
      ].join(",");
      await runProcess({
        executable: env.ffmpegPath,
        argv: ["-y", "-i", input, "-map", `0:${plan.audio.index}`, "-vn", "-af", filter,
          "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output],
        timeoutMs: env.processTimeoutMs,
        maxStdoutBytes: 64 * 1024,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      });
      const inspected = await outputInspection({
        path: output,
        mediaType: "audio/wav",
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      });
      const audio = inspected.streams.find((item): item is MediaAudioStream => item.kind === "audio");
      assert(audio?.decodedSampleFrames === plan.sampleFrames && audio.sampleRate === 48_000 && audio.channels === 2,
        "Normalized audio sample shape differs from its plan");
      audioArtifact = await env.artifacts.putFile(output, "audio/wav");
    }
    const media = sealSynchronizedMedia({
      frameDomain: {
        frameRate: need.frameRate,
        frameCount: plan.frameCount,
      },
      ...(visualArtifact === undefined ? {} : {
        visual: {
          artifact: visualArtifact,
          width: visualWidth!,
          height: visualHeight!,
        },
      }),
      ...(audioArtifact === undefined ? {} : {
        audio: {
          artifact: audioArtifact,
        },
      }),
    });
    return inlineResult(canonicalize(media));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

/**
 * Encode authored images into an exact video-only CFR clip. One picture is held for the whole frame
 * count; several are each held for their planned segment, fitted into the first picture's frame
 * and letterboxed on black, then concatenated in order.
 */
export async function executeRenderStillVideo(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = renderStillVideoNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-still-"));
  try {
    const output = join(work, "still.mp4");
    const staged = new Map<string, string>();
    const inputs: string[] = [];
    for (const segment of need.request.segments) {
      const source = segment.source!;
      let path = staged.get(source.resource);
      if (path === undefined) {
        path = join(work, `picture-${staged.size}.image`);
        await stageArtifact(env, source, path);
        staged.set(source.resource, path);
      }
      inputs.push(path);
    }
    const { numerator, denominator } = need.request.frameRate;
    const fps = `${numerator}/${denominator}`;
    const hold = (frames: number): string => [
      "select=eq(n\\,0)",
      "loop=loop=-1:size=1:start=0",
      `trim=start_frame=0:end_frame=${frames}`,
      `settb=expr=${denominator}/${numerator}`,
      `setpts=N*${denominator}/(${numerator}*TB)`,
    ].join(",");
    const bt709Output = [
      "scale=out_color_matrix=bt709:out_range=tv",
      "format=yuv420p",
      "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
    ].join(",");
    let argv: string[];
    if (need.request.segments.length === 1) {
      const filter = `${hold(need.request.frameCount)},pad=ceil(iw/2)*2:ceil(ih/2)*2:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,${bt709Output}`;
      argv = ["-y", "-i", inputs[0]!, "-map", "0:v:0", "-an", "-vf", filter];
    } else {
      const first = await outputInspection({
        path: inputs[0]!,
        mediaType: need.request.segments[0]!.source!.mediaType,
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      });
      const picture = first.streams.find((item): item is MediaVideoStream => item.kind === "video");
      assert(picture !== undefined, "Still video first picture has no decodable image");
      const width = Math.ceil(picture.width / 2) * 2;
      const height = Math.ceil(picture.height / 2) * 2;
      const chains = need.request.segments.map((segment, index) =>
        `[${index}:v]${hold(segment.endFrameExclusive - segment.startFrame)},`
        + `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[s${index}]`);
      const concat = `${need.request.segments.map((_, index) => `[s${index}]`).join("")}concat=n=${need.request.segments.length}:v=1:a=0[joined];[joined]${bt709Output}[v]`;
      argv = ["-y", ...inputs.flatMap((path) => ["-i", path]), "-filter_complex", `${chains.join(";")};${concat}`, "-map", "[v]", "-an"];
    }
    await runProcess({
      executable: env.ffmpegPath,
      argv: [
        ...argv,
        "-frames:v", String(need.request.frameCount), "-r", fps, "-fps_mode", "cfr",
        "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
        "-movflags", "+faststart", output,
      ],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const inspected = await outputInspection({
      path: output,
      mediaType: "video/mp4",
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const videos = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
    assert(videos.length === 1 && inspected.streams.length === 1
      && videos[0]!.decodedUnitCount === need.request.frameCount,
    "Still media output differs from its requested frame domain");
    const visual = videos[0]!;
    assert(visual.sampleAspectRatio.numerator === 1 && visual.sampleAspectRatio.denominator === 1
      && visual.rotationDegrees === 0,
    "Still media output retains non-square samples or display rotation");
    assert(visual.averageFrameRate !== undefined
      && visual.averageFrameRate.numerator * need.request.frameRate.denominator
        === need.request.frameRate.numerator * visual.averageFrameRate.denominator,
    "Still media output differs from its requested frame rate");
    return artifactResult(await env.artifacts.putFile(output, "video/mp4"));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

type TransformPlan = {
  readonly videoFilters: readonly string[];
  readonly audioFilters: readonly string[];
  readonly frameCount: number;
  readonly sampleFrames: number;
  readonly durationSec: number;
};

function decimal(value: number): string {
  assert(Number.isFinite(value), "Media transform produced a non-finite time");
  return value.toPrecision(15);
}

function compileTransformPlan(media: SynchronizedMedia, operations: readonly MediaTransformOperation[]): TransformPlan {
  const rate = media.frameDomain.frameRate.numerator / media.frameDomain.frameRate.denominator;
  let durationSec = media.frameDomain.frameCount / rate;
  const videoFilters: string[] = ["setpts=PTS-STARTPTS"];
  const audioFilters: string[] = ["asetpts=N/SR/TB"];
  for (const [index, operation] of operations.entries()) {
    if (operation.kind === "trim") {
      const start = operation.startSec ?? 0;
      const end = operation.endSec ?? (operation.tailSec === undefined
        ? durationSec
        : durationSec - operation.tailSec);
      assert(start >= 0 && start < durationSec,
        `Media transform trim ${index} starts outside its current ${durationSec}s timeline`);
      assert(end > start && end <= durationSec + 1e-9,
        `Media transform trim ${index} ends outside its current ${durationSec}s timeline`);
      const boundedEnd = Math.min(end, durationSec);
      videoFilters.push(`trim=start=${decimal(start)}:end=${decimal(boundedEnd)}`, "setpts=PTS-STARTPTS");
      audioFilters.push(`atrim=start=${decimal(start)}:end=${decimal(boundedEnd)}`, "asetpts=N/SR/TB");
      durationSec = boundedEnd - start;
      continue;
    }
    videoFilters.push(`setpts=PTS/${decimal(operation.rate)}`);
    audioFilters.push(...atempo(operation.rate));
    durationSec /= operation.rate;
  }
  const frameCount = Math.max(1, Math.round(durationSec * rate));
  assert(Number.isSafeInteger(frameCount), "Media transform output frame count exceeds safe arithmetic");
  const sampleFrames = roundPositive(
    BigInt(frameCount) * 48_000n * BigInt(media.frameDomain.frameRate.denominator),
    BigInt(media.frameDomain.frameRate.numerator),
  );
  const fps = `${media.frameDomain.frameRate.numerator}/${media.frameDomain.frameRate.denominator}`;
  videoFilters.push(
    `fps=fps=${fps}:round=near:start_time=0:eof_action=round`,
    `trim=start_frame=0:end_frame=${frameCount}`,
    `setpts=N*${media.frameDomain.frameRate.denominator}/(${media.frameDomain.frameRate.numerator}*TB)`,
  );
  audioFilters.push(
    `apad=whole_len=${sampleFrames}`,
    `atrim=start_sample=0:end_sample=${sampleFrames}`,
    "asetpts=N/SR/TB",
  );
  return { videoFilters, audioFilters, frameCount, sampleFrames, durationSec };
}

/** Ordered trim/retime over one exact synchronized A/V value. */
export async function executeTransformMedia(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = transformNeed(constraints);
  const media = need.media;
  const plan = compileTransformPlan(media, need.program.operations);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-transform-"));
  try {
    const visualPath = join(work, "visual.mp4");
    const audioPath = join(work, "audio.wav");
    const output = join(work, "transformed.mp4");
    await stageArtifact(env, media.visual!.artifact, visualPath);
    if (media.audio !== undefined) await stageArtifact(env, media.audio.artifact, audioPath);
    const argv = ["-y", "-i", visualPath];
    if (media.audio !== undefined) {
      argv.push(
        "-i", audioPath,
        "-filter_complex",
        `[0:v:0]${plan.videoFilters.join(",")}[video];[1:a:0]${plan.audioFilters.join(",")}[audio]`,
        "-map", "[video]", "-map", "[audio]",
        "-c:a", "aac", "-ar", "48000", "-ac", "2",
      );
    } else {
      argv.push("-map", "0:v:0", "-an", "-vf", plan.videoFilters.join(","));
    }
    argv.push(
      "-frames:v", String(plan.frameCount),
      "-r", `${media.frameDomain.frameRate.numerator}/${media.frameDomain.frameRate.denominator}`, "-fps_mode", "cfr",
      "-c:v", "libx264", "-preset", "veryfast", "-pix_fmt", "yuv420p",
      "-movflags", "+faststart", output,
    );
    await runProcess({
      executable: env.ffmpegPath,
      argv,
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const inspected = await outputInspection({
      path: output,
      mediaType: "video/mp4",
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const videos = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
    const audios = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
    assert(videos.length === 1 && videos[0]!.role === "moving"
      && videos[0]!.decodedUnitCount === plan.frameCount,
    "Transformed media video differs from its compiled frame domain");
    assert(audios.length === (media.audio === undefined ? 0 : 1),
      "Transformed media audio presence differs from its synchronized input");
    if (audios[0] !== undefined) {
      assert(audios[0].sampleRate === 48_000 && audios[0].channels === 2,
        "Transformed media audio is not 48 kHz stereo");
    }
    const artifact = await env.artifacts.putFile(output, "video/mp4");
    return artifactResult(artifact);
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

/** Generic model-reference audio: no Narrative or alignment claim is introduced. */
export async function executeExtractAudio(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = extractAudioNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-extract-audio-"));
  try {
    const input = join(work, "source.bin");
    const output = join(work, "audio.wav");
    await stageArtifact(env, need.source, input);
    await runProcess({
      executable: env.ffmpegPath,
      argv: [
        "-y", "-i", input, "-map", `0:${need.streamIndex}`, "-vn",
        "-af", "asetpts=PTS-STARTPTS,aresample=48000:async=0:first_pts=0,aformat=sample_rates=48000:channel_layouts=stereo",
        "-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output,
      ],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const artifact = await env.artifacts.putFile(output, "audio/wav");
    await assertCanonicalWav({
      path: output,
      source: artifact,
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    return artifactResult(artifact);
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

export async function executeExtractFrame(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = extractFrameNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-extract-frame-"));
  try {
    const input = join(work, "source.bin");
    const output = join(work, "frame.png");
    await stageArtifact(env, need.source, input);
    const encoding = await sourceVideoEncoding(env, input, need.streamIndex);
    const selection = need.at.kind === "first"
      ? "eq(n\\,0)"
      : need.at.kind === "last"
        ? `eq(n\\,${need.sourceFrameCount - 1})`
        : need.at.kind === "frame"
          ? `eq(n\\,${need.at.index})`
          : `gte(t\\,${decimal(need.at.seconds)})`;
    await runProcess({
      executable: env.ffmpegPath,
      argv: [
        "-y", ...encoding.inputArgs, "-i", input, "-map", `0:${need.streamIndex}`, "-an",
        "-vf", `setpts=PTS-STARTPTS,select=${selection}`,
        "-frames:v", "1", "-fps_mode", "vfr", "-c:v", "png", output,
      ],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const bytes = await readFile(output);
    assert(bytes.byteLength > 0, "Frame extraction produced no image");
    const inspected = await outputInspection({
      path: output,
      mediaType: "image/png",
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const images = inspected.streams.filter((item): item is MediaVideoStream => item.kind === "video");
    assert(images.length === 1 && images[0]!.decodedUnitCount === 1,
      "Frame extraction output must decode to exactly one image");
    const artifact = await env.artifacts.put(bytes, "image/png");
    return artifactResult(artifact);
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

export async function executeProjectSpeechEvidenceAudio(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = evidenceAudioNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-speech-evidence-"));
  try {
    const input = join(work, "speech-master.wav");
    const output = join(work, "alignment-evidence.wav");
    await stageArtifact(env, need.source, input);
    const source = await assertCanonicalWav({
      path: input,
      source: need.source,
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    assert(source.decodedSampleFrames === need.sourceSampleFrames,
      "Speech master sample count differs from its Timeline");
    const filter = [
      "asetpts=N/SR/TB",
      "aresample=16000:async=0:first_pts=0",
      "aformat=sample_rates=16000:channel_layouts=mono",
      `apad=whole_len=${need.evidenceSampleFrames}`,
      `atrim=start_sample=0:end_sample=${need.evidenceSampleFrames}`,
      "asetpts=N/SR/TB",
    ].join(",");
    await runProcess({
      executable: env.ffmpegPath,
      argv: ["-y", "-i", input, "-vn", "-af", filter,
        "-c:a", "pcm_s16le", "-ar", "16000", "-ac", "1", output],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const inspected = await outputInspection({
      path: output,
      mediaType: "audio/wav",
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const streams = inspected.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
    assert(streams.length === 1 && inspected.streams.length === 1
      && streams[0]!.codecName === "pcm_s16le"
      && streams[0]!.sampleRate === 16_000
      && streams[0]!.channels === 1
      && streams[0]!.decodedSampleFrames === need.evidenceSampleFrames,
    "Alignment evidence must be exact 16 kHz mono PCM s16");
    const artifact = await env.artifacts.putFile(output, "audio/wav");
    const evidence: SpeechEvidenceAudio = sealSpeechEvidenceAudio({
      domainId: need.domainId,
      artifact,
      sampleFrames: need.evidenceSampleFrames,
    });
    assertSpeechEvidenceAudioIdentity(evidence);
    return inlineResult(canonicalize(evidence));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

export async function executeRenderTimelineAudio(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = renderAudioNeed(constraints);
  const plan: AudioProgramPlan = need.plan;
  const window = need.range === undefined
    ? { startSample: 0, endSampleExclusive: plan.sampleFrames, sampleFrames: plan.sampleFrames }
    : mediaFrameRangeSamples(need.range, plan.frameRate);
  const clips = plan.clips.filter((clip) => clip.targetStartSample < window.endSampleExclusive
    && clip.targetEndSampleExclusive > window.startSample);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-audio-"));
  try {
    const artifacts = new Map<string, { source: BlobRef; path: string; inputIndex: number; sampleFrames: number }>();
    for (const clip of clips) {
      const existing = artifacts.get(clip.artifact.resource);
      if (existing !== undefined) {
        assert(existing.sampleFrames === clip.sourceSampleFrames,
          `Audio input ${clip.artifact.resource} has conflicting sample counts in one plan`);
        continue;
      }
      const inputIndex = artifacts.size;
      const path = join(work, `input-${inputIndex}.wav`);
      await stageArtifact(env, clip.artifact, path);
      const audio = await assertCanonicalWav({
        path,
        source: clip.artifact,
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      });
      assert(audio.decodedSampleFrames === clip.sourceSampleFrames,
        `Audio input ${clip.artifact.resource} sample count differs from its plan`);
      artifacts.set(clip.artifact.resource, {
        source: clip.artifact,
        path,
        inputIndex,
        sampleFrames: audio.decodedSampleFrames,
      });
    }

    const output = join(work, "program.wav");
    const argv = ["-y"];
    for (const item of artifacts.values()) argv.push("-i", item.path);
    if (clips.length === 0) {
      argv.push(
        "-f", "lavfi", "-i", "anullsrc=r=48000:cl=stereo",
        "-af", `atrim=start_sample=0:end_sample=${window.sampleFrames},asetpts=N/SR/TB`,
      );
    } else {
      const chains = clips.map((clip, index) => {
        const inputIndex = artifacts.get(clip.artifact.resource)!.inputIndex;
        return audioClipFilter(clip, inputIndex, index, window);
      });
      const labels = clips.map((_clip, index) => `[clip${index}]`).join("");
      chains.push(
        `${labels}amix=inputs=${clips.length}:duration=longest:dropout_transition=0:normalize=0,`
        + `apad=whole_len=${window.sampleFrames},atrim=start_sample=0:end_sample=${window.sampleFrames},`
        + "asetpts=N/SR/TB[out]",
      );
      argv.push("-filter_complex", chains.join(";"), "-map", "[out]");
    }
    argv.push("-c:a", "pcm_s16le", "-ar", "48000", "-ac", "2", output);
    await runProcess({
      executable: env.ffmpegPath,
      argv,
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const artifact = await env.artifacts.putFile(output, "audio/wav");
    const audio = await assertCanonicalWav({
      path: output,
      source: artifact,
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    assert(audio.decodedSampleFrames === window.sampleFrames,
      "Rendered TimelineAudio sample count differs from its plan");
    const value: TimelineAudio = sealTimelineAudio({
      artifact,
      sampleFrames: window.sampleFrames,
    });
    return inlineResult(canonicalize(value));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

export async function executeMuxProgramMedia(
  env: MediaExecutionEnvironment,
  constraints: CanonicalValue,
): Promise<MediaOperationResult> {
  const need = muxMediaNeed(constraints);
  const work = await mkdtemp(join(tmpdir(), "hypit-media-mux-"));
  try {
    const visualPath = join(work, "visual.mp4");
    const audioPath = join(work, "audio.wav");
    const output = join(work, "final.mp4");
    await Promise.all([
      stageArtifact(env, need.visual.artifact, visualPath),
      stageArtifact(env, need.audio.artifact, audioPath),
    ]);
    const [visualInspection, audio] = await Promise.all([
      inspectFile({
        source: need.visual.artifact,
        input: visualPath,
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      }),
      assertCanonicalWav({
        path: audioPath,
        source: need.audio.artifact,
        ffprobePath: env.ffprobePath,
        timeoutMs: env.processTimeoutMs,
        maxProbeOutputBytes: env.maxProbeOutputBytes,
        ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
      }),
    ]);
    const visualStreams = visualInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
    assert(visualStreams.length === 1 && visualInspection.streams.length === 1,
      "TimelineVisual Artifact must contain exactly one silent video stream");
    const visual = visualStreams[0]!;
    assert(visual.role === "moving" && visual.timingStatus === "admissible"
      && visual.decodedUnitCount === need.visual.frameCount
      && visual.width === need.visual.canvas.width && visual.height === need.visual.canvas.height,
    "TimelineVisual bytes differ from their declared frame domain");
    assert(audio.decodedSampleFrames === need.audio.sampleFrames,
      "TimelineAudio bytes differ from their declared sample domain");
    await runProcess({
      executable: env.ffmpegPath,
      argv: [
        "-y", "-i", visualPath, "-i", audioPath,
        "-map", `0:${visual.index}`, "-map", "1:0",
        "-c:v", "copy", "-c:a", "aac", "-ar", "48000", "-ac", "2",
        "-movflags", "+faststart", output,
      ],
      timeoutMs: env.processTimeoutMs,
      maxStdoutBytes: 64 * 1024,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const finalInspection = await outputInspection({
      path: output,
      mediaType: "video/mp4",
      ffprobePath: env.ffprobePath,
      timeoutMs: env.processTimeoutMs,
      maxProbeOutputBytes: env.maxProbeOutputBytes,
      ...(env.sharedLibraryPath === undefined ? {} : { sharedLibraryPath: env.sharedLibraryPath }),
    });
    const finalVideo = finalInspection.streams.filter((item): item is MediaVideoStream => item.kind === "video");
    const finalAudio = finalInspection.streams.filter((item): item is MediaAudioStream => item.kind === "audio");
    assert(finalVideo.length === 1 && finalAudio.length === 1 && finalInspection.streams.length === 2,
      "Final mux must contain exactly one video and one audio stream");
    assert(finalVideo[0]!.timingStatus === "admissible"
      && finalVideo[0]!.decodedUnitCount === need.visual.frameCount,
      "Final mux video frame count differs from TimelineVisual");
    assert(finalAudio[0]!.timingStatus === "admissible"
      && finalAudio[0]!.sampleRate === 48_000 && finalAudio[0]!.channels === 2,
      "Final mux audio shape differs from TimelineAudio");
    assert(finalVideo[0]!.startPts !== undefined && finalAudio[0]!.startPts !== undefined
      && compareTimestamp(finalVideo[0]!.startPts, finalAudio[0]!.startPts) === 0,
    "Final mux audio and video do not share one presentation origin");
    const finalAudioSpan = duration(finalAudio[0]!);
    /*
     * The staged WAV is asserted sample-exact above; this checks what survived AAC.
     *
     * AAC cannot carry an arbitrary sample count: the encoder emits 1024-sample frames and
     * reports a priming delay that the MP4 muxer compensates with an edit list, so the
     * presented span lands a few samples away from the input. Measured here with ffmpeg 6.1
     * on synthetic tone (i.e. independent of any project's material): a 4 249 600-sample input
     * presents as 4 249 584 (-16), and 4 236 800 presents as 4 236 768 (-32). Padding the
     * timeline to a whole number of AAC frames does not remove it — the delay is not a
     * frame-alignment artifact.
     *
     * Demanding exact equality therefore rejects every correct mux whose length is not a
     * fixed point of that round trip. The real invariant AAC can hold is that no whole frame
     * of audio went missing, so the tolerance is one AAC frame rather than an arbitrary epsilon;
     * anything larger still means genuine desync and still fails.
     */
    const AAC_FRAME_SAMPLES = 1_024;
    const finalAudioSamples = roundPositive(finalAudioSpan.numerator * 48_000n, finalAudioSpan.denominator);
    assert(Math.abs(finalAudioSamples - need.audio.sampleFrames) < AAC_FRAME_SAMPLES,
      `Final mux audio presentation span differs from TimelineAudio by ${
        String(finalAudioSamples - need.audio.sampleFrames)} samples`);
    const artifact = await env.artifacts.putFile(output, "video/mp4");
    const value: MuxedMedia = sealMuxedMedia({
      frameRate: need.visual.frameRate,
      frameCount: need.visual.frameCount,
      canvas: need.visual.canvas,
      presentationSampleFrames: need.audio.sampleFrames,
      artifact,
    });
    return inlineResult(canonicalize(value));
  } finally {
    await rm(work, { recursive: true, force: true }).catch(() => {});
  }
}