Skip to content

Write HTML. Render video. Built for agents.

103 KB
Raw Blame
// fallow-ignore-file unused-class-member code-duplication complexity
/**
 * Video Frame Extractor Service
 *
 * Pre-extracts video frames using FFmpeg for frame-accurate rendering.
 * Videos are replaced with <img> elements during capture.
 */

import { audioGroupsById, isMemberGroupHidden, isSelfOrAncestorHidden } from "./mediaHidden.js";
import { copyFileSync, existsSync, linkSync, mkdirSync, rmSync } from "fs";
import { join } from "path";
import { cpus } from "os";
import { createConcurrencyLimit } from "../utils/concurrencyLimit.js";
import { parseHTML } from "linkedom";
import { isPathInside } from "@hyperframes/parsers/asset-paths";
import { resolveProjectRelativeSrc } from "@hyperframes/parsers/asset-resolution";
import { alphaDecoderFor } from "@hyperframes/parsers/media-alpha";
import {
  MEDIA_RENDER_ID_ATTR,
  isAudibleVideoElement,
  fpsToFfmpegArg,
  fpsToNumber,
  MEDIA_DURATION_CLAMP_EPSILON_SECONDS,
  normalizeRateSpec,
  readElementRateSpec,
  shiftRateLane,
  sourceTimeAt,
  timeAtSourceTime,
  type RateSpec,
  exportClipWindow,
  hasClipStarted,
  isInClipWindow,
  parseStrictFiniteTimingNumber,
  readMediaStart,
  toFps,
  type FpsInput,
  firstFrameColourArgs,
  hdrToSdrToneMapFilter,
  parseFirstFrameColour,
} from "@hyperframes/core";
import { resolveReferencedStart, type RefResolverEl } from "./referenceResolver.js";
import { isKnownInactiveTimelineWindow } from "./mediaTimelineWindow.js";
import {
  extractFinalVideoFrameTimestamp,
  extractMediaMetadata,
  mediaFileIdentity,
  readMetadataCache,
  writeMetadataCache,
  type MediaProbeCacheEntry,
  type VideoMetadata,
} from "../utils/ffprobe.js";
import { inputAlphaOpaqueWarning, probeInputAlphaPlane } from "../utils/alphaPlaneProbe.js";
import { redactUrlQueriesInError, urlInputArgs } from "../utils/urlInput.js";
import {
  analyzeCompositionHdr,
  isHdrColorSpace as isHdrColorSpaceUtil,
  type HdrTransfer,
} from "../utils/hdr.js";
import {
  downloadToTemp,
  isHttpUrl,
  safeDownloadUrlIdentity,
  UrlDownloadError,
  writeUrlDownloadTelemetry,
} from "../utils/urlDownloader.js";
import { runFfmpeg, type RunFfmpegResult } from "../utils/runFfmpeg.js";
import { isFfmpegFilterAvailable } from "../utils/psnrFilterAvailability.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { unwrapTemplate } from "../utils/htmlTemplate.js";
import {
  FRAME_FILENAME_PREFIX,
  gcExtractionCache,
  gcSweepDue,
  lookupCacheEntry,
  partialCacheEntryDir,
  publishCacheEntry,
  readContentSha256,
  readKeyStat,
  rehydrateCacheEntry,
  touchCacheEntry,
  type CacheEntry,
  type CacheFrameFormat,
} from "./extractionCache.js";
import { framePathsFromDirectory, frameSequencePath } from "./extractedFrameIndex.js";

export { resolveProjectRelativeSrc };

export interface VideoElement {
  id: string;
  src: string;
  start: number;
  end: number;
  mediaStart: number;
  playbackRate?: RateSpec;
  loop: boolean;
  hasAudio: boolean;
  hidden?: boolean;
}

export interface ExtractedFrames {
  videoId: string;
  srcPath: string;
  outputDir: string;
  framePattern: string;
  fps: number;
  totalFrames: number;
  metadata: VideoMetadata;
  framePaths: Map<number, string>;
  deferredRange?: DeferredFrameRange;
  /**
   * True when the extractor owns `outputDir` and cleanup should rm it when
   * the render ends. Cache hits set this to false so the shared entry isn't
   * deleted by a single render's cleanup — the cache dir is owned by the
   * caller's gc policy, not any one render.
   */
  ownedByLookup?: boolean;
}

/**
 * The single source of truth for the source-video frame-extraction allow-list.
 * The CLI flag parser, the producer HTTP server, and the distributed-config
 * validator all validate against this same set via {@link isVideoFrameFormat}
 * so the boundaries can't drift when a new format is added.
 */
export const VIDEO_FRAME_FORMATS = ["auto", "jpg", "png"] as const;
export type VideoFrameFormat = (typeof VIDEO_FRAME_FORMATS)[number];

/** Runtime guard for {@link VideoFrameFormat} over an untrusted value. */
export function isVideoFrameFormat(value: unknown): value is VideoFrameFormat {
  return typeof value === "string" && (VIDEO_FRAME_FORMATS as readonly string[]).includes(value);
}

/**
 * Resolve the frame count produced for a requested extraction duration.
 *
 * Both paths sample with FFmpeg's fps filter. A CFR window ends at the nearest
 * frame boundary; a VFR window rounds up, keeping the slot a trailing still
 * covers. Keep this calculation shared by superset slicing and
 * producer coverage accounting so a complete VFR extraction cannot be
 * rejected because the two paths disagree by one frame.
 */
export function extractionFrameCountForDuration(
  durationSeconds: number,
  fps: FpsInput,
  isVFR: boolean,
): number {
  if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return 0;
  // FFmpeg receives `String(durationSeconds)` and parses at microsecond
  // precision. Derive the integer microseconds from that same decimal text:
  // multiplying the binary float first is not equivalent (`2.05 * 1e6` is
  // 2049999.9999999998 in JS and would incorrectly truncate one microsecond).
  const serialized = String(durationSeconds).toLowerCase();
  const decimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serialized);
  if (!decimal) return 0;
  const whole = decimal[1] ?? "0";
  const fraction = decimal[2] ?? "";
  const exponent = Number.parseInt(decimal[3] ?? "0", 10);
  const digits = BigInt(`${whole}${fraction}`);
  const microsecondScale = exponent + 6 - fraction.length;
  const microseconds =
    microsecondScale >= 0
      ? digits * 10n ** BigInt(microsecondScale)
      : digits / 10n ** BigInt(-microsecondScale);

  // Keep the frame-boundary calculation rational too. Converting the exact
  // microseconds back to a binary float recreates the same problem at .5-frame
  // boundaries (`2.05 * 30` is 61.49999999999999 in JS).
  let fpsNumerator: bigint;
  let fpsDenominator: bigint;
  if (typeof fps === "object") {
    if (
      !Number.isSafeInteger(fps.num) ||
      !Number.isSafeInteger(fps.den) ||
      fps.num <= 0 ||
      fps.den <= 0
    ) {
      return 0;
    }
    fpsNumerator = BigInt(fps.num);
    fpsDenominator = BigInt(fps.den);
  } else {
    if (!Number.isFinite(fps) || fps <= 0) return 0;
    // Number-only callers retain their decimal FFmpeg argument exactly. The
    // production render path supplies Fps, so NTSC rates never round-trip
    // through `String(30000 / 1001)` here.
    const serializedFps = String(fps).toLowerCase();
    const fpsDecimal = /^(\d+)(?:\.(\d+))?(?:e([+-]?\d+))?$/.exec(serializedFps);
    if (!fpsDecimal) return 0;
    const fpsWhole = fpsDecimal[1] ?? "0";
    const fpsFraction = fpsDecimal[2] ?? "";
    const fpsExponent = Number.parseInt(fpsDecimal[3] ?? "0", 10);
    const fpsDigits = BigInt(`${fpsWhole}${fpsFraction}`);
    const fpsScale = fpsExponent - fpsFraction.length;
    fpsNumerator = fpsScale >= 0 ? fpsDigits * 10n ** BigInt(fpsScale) : fpsDigits;
    fpsDenominator = fpsScale >= 0 ? 1n : 10n ** BigInt(-fpsScale);
  }

  const frameNumerator = microseconds * fpsNumerator;
  const frameDenominator = 1_000_000n * fpsDenominator;
  const frameCount = isVFR
    ? (frameNumerator + frameDenominator - 1n) / frameDenominator
    : (2n * frameNumerator + frameDenominator) / (2n * frameDenominator);
  const frames = Number(frameCount);
  return Math.max(1, Number.isSafeInteger(frames) ? frames : Number.MAX_SAFE_INTEGER);
}

export interface FrameRange {
  firstFrame: number;
  frames: number;
}

/** The window a deferred extraction samples; frame k is the source frame on screen at `startTime + k / fps`. */
export interface DeferredFrameRange {
  startTime: number;
  durationSeconds: number;
  format: "jpg" | "png";
}

export interface ExtractionOptions {
  /** Exact configured rate. Rational rates are passed to FFmpeg verbatim. */
  fps: FpsInput;
  outputDir: string;
  quality?: number;
  format?: VideoFrameFormat;
  sdrToHdrTransfer?: HdrTransfer;
  toneMapHdrToSdr?: boolean;
  /** Extract exactly one frame at `startTime`. Used only after ffprobe has
   *  resolved the actual final decoded-frame timestamp for a held tail. */
  finalFrameOnly?: boolean;
  /**
   * Absolute composition/timeline end in seconds. Applied only after source
   * metadata resolves open-ended/natural-duration media. Invisible negative
   * preroll is trimmed while advancing mediaStart to preserve source alignment.
   */
  timelineEnd?: number;
  /**
   * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior;
   * the producer may canary at most one retry after observing typed failures.
   */
  maxTransientRetries?: number;
  /**
   * Collect metadata-probe failures into `ExtractionResult.errors` instead
   * of preserving the legacy Promise rejection. Default false; only the
   * candidate enforce lane may opt into typed aggregation.
   */
  collectProbeFailures?: boolean;
  /** Extract only these runs of output frames, named as a full extraction names them. CFR only. */
  frameRanges?: readonly FrameRange[];
  /**
   * Leave CFR windows longer than a second unextracted, returning `deferredRange` and an exact
   * `totalFrames` so another process can extract just the frames it needs with `frameRanges`.
   */
  deferRangeExtraction?: boolean;
  /** Directories the caller rewrites on every render (downloaded media); their sources are cached by content. */
  contentKeyedDirs?: readonly string[];
  urlByAbsentLocalPath?: ReadonlyMap<string, string>;
  shownBox?: { width: number; height: number };
}

const DOWNLOAD_SUBDIR = "_downloads";

export const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000;
const GC_STALENESS_MS = 24 * 60 * 60 * 1000;
const SDR_TO_HDR_COLORSPACE_FILTER = "colorspace=all=bt2020:iall=bt709:range=tv";
const HDR_TO_SDR_ZSCALE_TRANSFORM_KEY = "hdr2sdr-hable-srgb";
const HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY = "hdr2sdr-videotoolbox-hdr";
const SDR_CANVAS_PASSTHROUGH_FILTER = "setparams=color_primaries=bt709:color_trc=iec61966-2-1";
function halveToCoverScaleFilter(box: { width: number; height: number }): string {
  const fit = `max(${Math.ceil(box.width)}/iw\\,${Math.ceil(box.height)}/ih)`;
  const halvings = `floor(log(1/${fit})/log(2))`;
  const side = (dim: "iw" | "ih") =>
    `if(lt(${fit}\\,0.5)\\,2*ceil(${dim}/pow(2\\,${halvings})/2)\\,${dim})`;
  return `scale=w=${side("iw")}:h=${side("ih")}:flags=area+accurate_rnd`;
}

const SDR_JPEG_AS_BT601_FULL_RANGE_FILTER =
  "scale=flags=neighbor+accurate_rnd,format=gbrp,scale=out_color_matrix=bt601:out_range=pc:flags=neighbor,format=yuv420p";

let warnedMissingZscale = false;

async function toneMapsWithZscale(): Promise<boolean> {
  return process.platform !== "darwin" || isFfmpegFilterAvailable("zscale");
}

async function hdrToSdrTransformKey(): Promise<string> {
  if (await toneMapsWithZscale()) return HDR_TO_SDR_ZSCALE_TRANSFORM_KEY;
  if (!warnedMissingZscale) {
    warnedMissingZscale = true;
    process.stderr.write(
      "[hyperframes:render] WARNING: this ffmpeg has no zscale filter, so forced-SDR HDR footage " +
        "is tone-mapped by Chrome instead of the hable curve used on Linux and in Studio, and " +
        "renders darker. Install an ffmpeg built with libzimg to match.\n",
    );
  }
  return HDR_TO_SDR_VIDEOTOOLBOX_TRANSFORM_KEY;
}

function sdrToHdrTransformKey(transfer: HdrTransfer): string {
  return `sdr2hdr-${transfer}`;
}

/**
 * Per-phase timings and counters emitted by `extractAllVideoFrames`.
 *
 * Used by the producer to surface `perfSummary.videoExtractBreakdown` — without
 * this breakdown, a single `videoExtractMs` stage timing hides where cost lives
 * (HDR preflight, VFR classification, per-video ffmpeg extract) when tuning renders.
 *
 * Field semantics:
 *   - *Ms fields are wall-clock durations inside each phase.
 *   - *Count fields report how many sources triggered that phase.
 *   - extractMs wraps the parallel `extractVideoFramesRange` calls; it
 *     reflects max-across-parallel-workers, not sum.
 *   - hdrPreflightMs includes its probe-time sibling (hdrProbeMs); the
 *     probe-only field is a finer decomposition, not a separate carve-out.
 *   - vfrPreflightCount reports sources classified as VFR and extracted with
 *     a rounded-up frame count. DEFINITION CHANGE:
 *     before the one-pass refactor, vfrPreflightMs timed a per-source
 *     VFR-to-CFR re-encode and could reach seconds; it now times only the
 *     (promise-cached) classification probe and is expected to be ~0.
 *     Dashboards alerting on vfrPreflightMs thresholds should key on
 *     vfrPreflightCount or extractMs instead.
 */
export interface ExtractionPhaseBreakdown {
  resolveMs: number;
  /** Publishes that could not land atomically — the render still succeeded
   *  from the partial dir, but future renders re-extract. A rising rate is
   *  the first signal that warm renders are silently going cold. */
  cachePublishFailures: number;
  /** Entries evicted by the post-extraction LRU sweep. */
  cacheGcEvictions: number;
  /** Bytes reclaimed by the LRU sweep. */
  cacheGcBytesFreed: number;
  /** Aged .partial-* dirs (crashed writers) removed by the sweep. */
  cacheAgedPartialsCleared: number;
  hdrProbeMs: number;
  hdrPreflightMs: number;
  hdrPreflightCount: number;
  vfrProbeMs: number;
  vfrPreflightMs: number;
  vfrPreflightCount: number;
  extractMs: number;
  cacheHits: number;
  cacheMisses: number;
  /** Number of per-source transient failures retried inside this extraction. */
  transientRetries?: number;
}

export type VideoExtractionFailureKind =
  | "cancelled"
  | "external_interruption"
  | "source_missing"
  | "source_rejected"
  | "download_not_found"
  | "download_transient"
  | "invalid_media"
  | "media_start_out_of_range"
  | "ffmpeg_unavailable"
  | "ffmpeg_timeout"
  | "ffmpeg_transient"
  | "ffmpeg_failed"
  | "zero_output"
  | "internal";

export interface VideoExtractionFailure {
  videoId: string;
  /** Always populated by this engine version; optional for source compatibility with older consumers. */
  kind?: VideoExtractionFailureKind;
  /** Always populated by this engine version; absent legacy values fail closed. */
  retryable?: boolean;
  /** Bounded, path-free grouping data safe for producer-owned error metadata. */
  group?: VideoExtractionFailureGroupDetails;
  /**
   * Operator diagnostic retained inside the engine result. Producer-facing
   * errors must summarize `kind`/counts and must not forward this field: it
   * can contain a local path or a signed source URL.
   */
  error: string;
}

export type VideoExtractionFailureStatusClass =
  | "http_4xx"
  | "http_5xx"
  | "timeout"
  | "network"
  | "other";

export interface VideoExtractionFailureRetry {
  phase: "download";
  used: 0 | 1;
  budget: 1;
}

export interface VideoExtractionFailureGroupDetails {
  sourceFingerprint?: string;
  host?: string;
  statusClass?: VideoExtractionFailureStatusClass;
  retry?: VideoExtractionFailureRetry;
}

export interface SafeVideoExtractionSourceIdentity {
  sourceFingerprint: string;
  host: string;
}

/** Query/fragment-free remote identity safe for extraction logs and wire metadata. */
export function safeVideoExtractionSourceIdentity(
  source: string,
): SafeVideoExtractionSourceIdentity | null {
  if (!isHttpUrl(source)) return null;
  const identity = safeDownloadUrlIdentity(source);
  return {
    sourceFingerprint: `sha256:${identity.urlFingerprint}`,
    host: identity.host ?? "other",
  };
}

function downloadStatusClass(error: UrlDownloadError): VideoExtractionFailureStatusClass {
  const status = error.status ?? error.telemetry?.status;
  if (typeof status === "number" && status >= 400 && status < 500) return "http_4xx";
  if (typeof status === "number" && status >= 500 && status < 600) return "http_5xx";
  if (error.kind === "timeout") return "timeout";
  if (error.kind === "network") return "network";
  return "other";
}

function downloadFailureGroup(
  source: string,
  error: UrlDownloadError,
): VideoExtractionFailureGroupDetails {
  const sourceIdentity = safeVideoExtractionSourceIdentity(source);
  const failureHost = error.telemetry?.finalHost ?? error.telemetry?.initialHost;
  const attempt = error.telemetry?.attempt;
  return {
    ...(sourceIdentity ? { sourceFingerprint: sourceIdentity.sourceFingerprint } : {}),
    ...((failureHost ?? sourceIdentity?.host) ? { host: failureHost ?? sourceIdentity?.host } : {}),
    statusClass: downloadStatusClass(error),
    retry: {
      phase: "download",
      used: typeof attempt === "number" && attempt >= 2 ? 1 : 0,
      budget: 1,
    },
  };
}

export class VideoSourceExtractionError extends Error {
  readonly hyperframesVideoSourceExtractionError = true as const;

  constructor(
    readonly kind: VideoExtractionFailureKind,
    readonly retryable: boolean,
    message: string,
    readonly diagnostic: string = message,
  ) {
    super(message);
    this.name = "VideoSourceExtractionError";
  }
}

export function isVideoSourceExtractionError(error: unknown): error is VideoSourceExtractionError {
  return (
    typeof error === "object" &&
    error !== null &&
    "hyperframesVideoSourceExtractionError" in error &&
    error.hyperframesVideoSourceExtractionError === true
  );
}

function boundedTransientRetryBudget(value: number | undefined): 0 | 1 {
  return Number.isFinite(value) && (value ?? 0) >= 1 ? 1 : 0;
}

function errorText(error: unknown): string {
  return error instanceof Error ? error.message : String(error);
}

/**
 * Convert legacy/raw downloader and filesystem errors into the bounded
 * extraction taxonomy. New extraction code should throw
 * `VideoSourceExtractionError` directly; this classifier keeps older utility
 * boundaries safe while they migrate.
 */
export function classifyVideoExtractionError(error: unknown): VideoSourceExtractionError {
  if (isVideoSourceExtractionError(error)) return error;
  const diagnostic = errorText(error);
  const lowered = diagnostic.toLowerCase();

  if (error instanceof UrlDownloadError) {
    if (error.kind === "cancelled") {
      return new VideoSourceExtractionError(
        "cancelled",
        false,
        "Video extraction cancelled",
        diagnostic,
      );
    }
    if (error.kind === "http_not_found") {
      return new VideoSourceExtractionError(
        "download_not_found",
        false,
        "Video source was not found",
        diagnostic,
      );
    }
    if (error.kind === "http_rejected") {
      return new VideoSourceExtractionError(
        "source_rejected",
        false,
        "Video source download was rejected",
        diagnostic,
      );
    }
    if (error.kind === "invalid_payload") {
      return new VideoSourceExtractionError(
        "invalid_media",
        false,
        "Video source download returned a non-media payload",
        diagnostic,
      );
    }
    if (error.retryable) {
      return new VideoSourceExtractionError(
        "download_transient",
        true,
        "Video source download failed transiently",
        diagnostic,
      );
    }
    return new VideoSourceExtractionError(
      "internal",
      false,
      "Video source download failed internally",
      diagnostic,
    );
  }
  if (lowered.includes("cancelled") || lowered.includes("aborted")) {
    return new VideoSourceExtractionError(
      "cancelled",
      false,
      "Video extraction cancelled",
      diagnostic,
    );
  }
  if (lowered.includes("video file not found")) {
    return new VideoSourceExtractionError(
      "source_missing",
      false,
      "Video source is missing",
      diagnostic,
    );
  }
  if (
    lowered.includes("only https urls are permitted") ||
    lowered.includes("private/reserved address") ||
    lowered.includes("invalid url")
  ) {
    return new VideoSourceExtractionError(
      "source_rejected",
      false,
      "Video source URL is not permitted",
      diagnostic,
    );
  }
  const httpStatus = diagnostic.match(/\bHTTP\s+(\d{3})\b/i)?.[1];
  if (httpStatus) {
    const status = Number(httpStatus);
    if (status === 404 || status === 410) {
      return new VideoSourceExtractionError(
        "download_not_found",
        false,
        "Video source was not found",
        diagnostic,
      );
    }
    if (status === 408 || status === 429 || status >= 500) {
      return new VideoSourceExtractionError(
        "download_transient",
        true,
        "Video source download failed transiently",
        diagnostic,
      );
    }
    return new VideoSourceExtractionError(
      "source_rejected",
      false,
      "Video source download was rejected",
      diagnostic,
    );
  }
  if (
    lowered.includes("[urldownloader] download timeout") ||
    lowered.includes("[urldownloader] download failed") ||
    lowered.includes("fetch failed") ||
    lowered.includes("network")
  ) {
    return new VideoSourceExtractionError(
      "download_transient",
      true,
      "Video source download failed transiently",
      diagnostic,
    );
  }
  if (lowered.includes("ffprobe not found")) {
    return new VideoSourceExtractionError(
      "ffmpeg_unavailable",
      false,
      "FFprobe is unavailable",
      diagnostic,
    );
  }
  if (lowered.includes("ffprobe deadline")) {
    return new VideoSourceExtractionError(
      "ffmpeg_timeout",
      true,
      "Video inspection timed out",
      diagnostic,
    );
  }
  if (
    lowered.includes("ffprobe") ||
    lowered.includes("failed to parse ffprobe output") ||
    lowered.includes("no video stream found")
  ) {
    return new VideoSourceExtractionError(
      "invalid_media",
      false,
      "Video source could not be inspected",
      diagnostic,
    );
  }
  return new VideoSourceExtractionError(
    "internal",
    false,
    "Video extraction failed internally",
    diagnostic,
  );
}

export async function runVideoExtractionWithRetry<T>(
  operation: () => Promise<T>,
  options: {
    signal?: AbortSignal;
    onRetry?: () => Promise<void> | void;
    maxTransientRetries?: number;
  } = {},
): Promise<{ result: T; retries: number }> {
  const maxTransientRetries = boundedTransientRetryBudget(options.maxTransientRetries);
  let retries = 0;
  for (;;) {
    if (options.signal?.aborted) {
      throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
    }
    try {
      return { result: await operation(), retries };
    } catch (error) {
      const classified = classifyVideoExtractionError(error);
      if (options.signal?.aborted) {
        throw new VideoSourceExtractionError(
          "cancelled",
          false,
          "Video extraction cancelled",
          classified.diagnostic,
        );
      }
      if (
        classified.kind === "cancelled" ||
        !classified.retryable ||
        retries >= maxTransientRetries
      ) {
        throw classified;
      }
      retries += 1;
      await options.onRetry?.();
    }
  }
}

export interface ExtractionResult {
  success: boolean;
  extracted: ExtractedFrames[];
  errors: VideoExtractionFailure[];
  totalFramesExtracted: number;
  durationMs: number;
  phaseBreakdown: ExtractionPhaseBreakdown;
}

/** Minimal structural shape for resolving parent/`<source>` media `src`. */
interface MediaSrcEl {
  getAttribute(name: string): string | null;
  querySelectorAll(selectors: string): Iterable<{ getAttribute(name: string): string | null }>;
}

/**
 * Parent `src`, else a `<source src>`. Prefer local paths over http(s) so a
 * localized sibling wins when another `<source>` failed to download.
 */
export function resolveMediaElementSrc(el: MediaSrcEl): string | null {
  const direct = el.getAttribute("src");
  if (direct) return direct;
  let remote: string | null = null;
  for (const source of el.querySelectorAll("source")) {
    const src = source.getAttribute("src");
    if (!src) continue;
    if (!/^https?:\/\//i.test(src)) return src;
    remote ??= src;
  }
  return remote;
}

export function parseVideoElements(html: string): VideoElement[] {
  const videos: VideoElement[] = [];
  const { document } = parseHTML(unwrapTemplate(html));
  const startCache = new Map<RefResolverEl, number>();
  const visiting = new Set<RefResolverEl>();

  const groupsById = audioGroupsById(document);
  const videoEls = document.querySelectorAll("video");
  let autoIdCounter = 0;
  for (const el of videoEls) {
    const src = resolveMediaElementSrc(el);
    if (!src) continue;
    // Generate a stable ID for videos without one — the producer needs IDs
    // to track extracted frames and composite them during encoding.
    // A compiled render document stamps a document-unique render id; prefer it,
    // because element ids are only unique within one composition file and the
    // render document inlines many.
    const id =
      el.getAttribute(MEDIA_RENDER_ID_ATTR) ||
      el.getAttribute("id") ||
      `hf-video-${autoIdCounter++}`;
    if (!el.getAttribute("id")) {
      el.setAttribute("id", id);
    }

    const startAttr = el.getAttribute("data-start");
    const endAttr = el.getAttribute("data-end");
    const durationAttr = el.getAttribute("data-duration");
    const hasAudioAttr = el.getAttribute("data-has-audio");

    // Resolve data-start, including relative references ("intro", "intro + 2")
    // to another clip's end — the browser runtime resolves these but a raw
    // parseFloat here would yield NaN, placing the clip at NaN so it composites
    // blank in the final render. `startAttr` may be a plain number or a
    // reference; the resolver handles both.
    const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
    if (isKnownInactiveTimelineWindow(el, start)) continue;
    // Derive end from data-end → data-start+data-duration → Infinity (natural duration).
    // Static compilation cannot always clamp root media because GSAP may supply
    // the root duration at runtime. The producer passes the resolved timeline
    // end into frame extraction, which caps the source duration only after the
    // natural duration is known without rewriting authored timing metadata.
    let end = 0;
    const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
    const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
    if (authoredEnd != null) {
      end = authoredEnd;
    } else if (authoredDuration != null) {
      end = start + authoredDuration;
    } else {
      end = Infinity; // no explicit bounds — play for the full natural video duration
    }

    videos.push({
      id,
      src,
      start,
      end,
      mediaStart: readMediaStart(el),
      playbackRate: readElementRateSpec(el),
      loop: el.hasAttribute("loop"),
      hasAudio: hasAudioAttr === "true" && isAudibleVideoElement(el),
      ...(isSelfOrAncestorHidden(el) ||
      (isAudibleVideoElement(el) && isMemberGroupHidden(groupsById, el))
        ? { hidden: true }
        : {}),
    });
  }

  return videos;
}

export interface ImageElement {
  id: string;
  src: string;
  start: number;
  end: number;
}

export function parseImageElements(html: string): ImageElement[] {
  const images: ImageElement[] = [];
  const { document } = parseHTML(unwrapTemplate(html));
  const startCache = new Map<RefResolverEl, number>();
  const visiting = new Set<RefResolverEl>();

  const imgEls = document.querySelectorAll("img[src]");
  let autoIdCounter = 0;
  for (const el of imgEls) {
    const src = el.getAttribute("src");
    if (!src) continue;

    // See parseVideoElements: the stamped render id wins over the authored id.
    const id =
      el.getAttribute(MEDIA_RENDER_ID_ATTR) || el.getAttribute("id") || `hf-img-${autoIdCounter++}`;
    if (!el.getAttribute("id")) {
      el.setAttribute("id", id);
    }

    const startAttr = el.getAttribute("data-start");
    const endAttr = el.getAttribute("data-end");
    const durationAttr = el.getAttribute("data-duration");

    // Resolve relative data-start references (see parseVideoElements) so a
    // referenced image start doesn't become NaN and drop the image from the render.
    const start = startAttr ? resolveReferencedStart(document, el, startCache, visiting) : 0;
    let end = 0;
    const authoredEnd = parseStrictFiniteTimingNumber(endAttr);
    const authoredDuration = parseStrictFiniteTimingNumber(durationAttr);
    if (authoredEnd != null) {
      end = authoredEnd;
    } else if (authoredDuration != null) {
      end = start + authoredDuration;
    } else {
      end = Infinity;
    }

    images.push({ id, src, start, end });
  }

  return images;
}

const firstFrameColourCache = new Map<
  string,
  MediaProbeCacheEntry<ReturnType<typeof parseFirstFrameColour>>
>();

/** The first shown frame's colour tags (see {@link firstFrameColourArgs}), probed once per source file. */
function readFirstFrameColour(videoPath: string) {
  const identity = mediaFileIdentity(videoPath);
  const cached = readMetadataCache(firstFrameColourCache, videoPath, identity);
  if (cached) return cached;
  // In an extraction slot, so it never adds a decoder; unsignalled, so one cancel cannot cut it short for others.
  const probe = inExtractionSlot(() =>
    runFfmpeg([...urlInputArgs(videoPath), ...firstFrameColourArgs(videoPath)]),
  ).then((result) => {
    // A failed probe answers this cut only; the next one probes again.
    if (!result.success && firstFrameColourCache.get(videoPath)?.promise === probe) {
      firstFrameColourCache.delete(videoPath);
    }
    return parseFirstFrameColour(result.stderr);
  });
  writeMetadataCache(firstFrameColourCache, videoPath, identity, probe);
  return probe;
}

/** Chrome plays untagged VP9 and AV1 as BT.601, H.264 and VP8 as BT.709 from 720 coded lines (assumed for the rest). */
const CHROME_BT601_UNTAGGED_CODECS = new Set(["vp9", "av1"]);

function chromeGuessForUntaggedMatrix(metadata: VideoMetadata): string[] {
  const matrix = metadata.colorSpace?.colorSpace;
  if (matrix && matrix !== "unknown") return [];
  const hd = metadata.height >= 720 && !CHROME_BT601_UNTAGGED_CODECS.has(metadata.videoCodec);
  return [`setparams=colorspace=${hd ? "bt709" : "smpte170m"}`];
}

/**
 * Longest stretch of output one ffmpeg process extracts; longer ranges split into segments
 * that run side by side. Segments sample the same absolute times as a single pass, so their
 * frames are byte-identical to it.
 */
const EXTRACTION_SEGMENT_SECONDS = 120;
/** Frame times since file start are `PTS*TB + seek`; snapping them to `lead + k/FR` gives every process one grid. */
function undoTickRoundingOnTheSourceFrameGrid(seek: number, lead: number): string[] {
  const sinceFileStart = `(PTS*TB+${seek}-${lead})`;
  return [
    "settb=AVTB",
    `setpts=if(isnan(FR)\\,PTS\\,(${lead}+round(${sinceFileStart}*FR)/FR-${seek})/TB)`,
  ];
}

// Caps extraction ffmpeg runs across all clips; uncapped, dozens of clips starved each other into timeouts.
const EXTRACTION_SLOTS = Math.max(1, Math.floor(cpus().length / 2));
const inExtractionSlot = createConcurrencyLimit(EXTRACTION_SLOTS);

interface SegmentedExtraction {
  decodeArgs: string[];
  filterAndEncodeArgs: (seek: number) => string[];
  videoPath: string;
  startTime: number;
  fps: number;
  segments: readonly FrameRange[];
  runOptions: { signal?: AbortSignal; stallTimeout: number };
}

/** Split runs so no single ffmpeg process extracts more than `segmentFrames` frames. */
function splitFrameRanges(ranges: readonly FrameRange[], segmentFrames: number): FrameRange[] {
  const segments: FrameRange[] = [];
  for (const range of ranges) {
    for (let offset = 0; offset < range.frames; offset += segmentFrames) {
      segments.push({
        firstFrame: range.firstFrame + offset,
        frames: Math.min(segmentFrames, range.frames - offset),
      });
    }
  }
  return segments;
}

async function runSegmentedExtraction(job: SegmentedExtraction): Promise<RunFfmpegResult> {
  const { decodeArgs, filterAndEncodeArgs, videoPath, startTime, fps, segments } = job;
  const segmentCount = segments.length;
  // One failed segment fails the range, so stop the others instead of finishing them.
  const failed = new AbortController();
  const signal = job.runOptions.signal
    ? AbortSignal.any([job.runOptions.signal, failed.signal])
    : failed.signal;
  const results: RunFfmpegResult[] = [];
  let next = 0;
  const worker = async () => {
    while (next < segmentCount && !signal.aborted) {
      const index = next++;
      const { firstFrame, frames } = segments[index]!;
      const seek = startTime + firstFrame / fps;
      const result = await inExtractionSlot(
        () =>
          runFfmpeg(
            [
              ...decodeArgs,
              "-noaccurate_seek",
              "-ss",
              String(seek),
              "-i",
              videoPath,
              // Read one frame past the segment and keep exactly its own frames.
              "-t",
              String((frames + 1) / fps),
              "-frames:v",
              String(frames),
              "-start_number",
              String(firstFrame + 1),
              ...filterAndEncodeArgs(seek),
            ],
            { ...job.runOptions, signal },
          ),
        signal,
      );
      results[index] = result;
      if (!result.success) failed.abort();
    }
  };
  const workers = Math.min(segmentCount, EXTRACTION_SLOTS);
  await Promise.all(Array.from({ length: workers }, worker));
  const ran = results.filter(Boolean);
  // Report the segment that failed, not the ones stopped because of it.
  const failure =
    ran.find((r) => !r.success && r.terminationReason !== "abort") ?? ran.find((r) => !r.success);
  if (failure) return failure;
  // A cancel between segments leaves later ones unrun; their frames are missing, not done.
  if (ran.length < segmentCount) {
    return {
      success: false,
      exitCode: null,
      stderr: "",
      durationMs: 0,
      terminationReason: "abort",
    };
  }
  return ran[ran.length - 1]!;
}

export async function extractVideoFramesRange(
  ...args: Parameters<typeof extractVideoFramesRangeUnredacted>
): Promise<ExtractedFrames> {
  try {
    return await extractVideoFramesRangeUnredacted(...args);
  } catch (error) {
    throw redactUrlQueriesInError(error);
  }
}

async function extractVideoFramesRangeUnredacted(
  videoPath: string,
  videoId: string,
  startTime: number,
  duration: number,
  options: ExtractionOptions,
  signal?: AbortSignal,
  config?: Partial<Pick<EngineConfig, "ffmpegProcessTimeout">>,
  /** Frames go straight here, with no per-videoId subdir (the cache layer's keyed entry). */
  outputDirOverride?: string,
): Promise<ExtractedFrames> {
  const ffmpegProcessTimeout = config?.ffmpegProcessTimeout ?? DEFAULT_CONFIG.ffmpegProcessTimeout;
  const { outputDir, quality = 95 } = options;
  const normalizedFps = toFps(options.fps);
  const fps = fpsToNumber(normalizedFps);
  const ffmpegFps = fpsToFfmpegArg(normalizedFps);

  const videoOutputDir = outputDirOverride ?? join(outputDir, videoId);
  if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true });

  let metadata: VideoMetadata;
  try {
    metadata = await extractMediaMetadata(videoPath);
  } catch (error) {
    throw classifyVideoExtractionError(error);
  }
  const playableDuration = resolvePlayableVideoDuration(metadata);
  if (!(playableDuration > 0)) {
    throw new VideoSourceExtractionError(
      "invalid_media",
      false,
      "Video source has no positive duration",
      `Playable video stream duration is ${playableDuration}s`,
    );
  }
  if (startTime >= playableDuration) {
    throw new VideoSourceExtractionError(
      "media_start_out_of_range",
      false,
      "Video media start is outside the source duration",
      `Video media start ${startTime}s is outside playable video duration ${playableDuration}s`,
    );
  }
  const format = resolveFrameFormat(metadata, options.format);
  const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`;
  const outputPattern = frameSequencePath(videoOutputDir, framePattern);

  // Forced-SDR extraction tone-maps HDR with zscale, the Studio proxy policy. A macOS ffmpeg
  // without zscale falls back to VideoToolbox, whose frames stay HDR and are tone-mapped by
  // Chrome instead (darker), and the render warns once.
  const isHdr = isHdrColorSpaceUtil(metadata.colorSpace);
  const isMacOS = process.platform === "darwin";
  const toneMappedToSdr = options.toneMapHdrToSdr === true && isHdr && (await toneMapsWithZscale());
  const decodeWithVideoToolbox = isHdr && isMacOS && !toneMappedToSdr;

  const sampleAtOutputFps = !options.finalFrameOnly;
  if (options.frameRanges && (!sampleAtOutputFps || metadata.isVFR)) {
    throw new Error("frameRanges extraction needs a constant-frame-rate source");
  }

  const args: string[] = urlInputArgs(videoPath);
  if (decodeWithVideoToolbox) {
    args.push("-hwaccel", "videotoolbox");
  }
  // Always force the alpha-aware decoder on codecs that can carry alpha. The
  // alternative — gating on `metadata.hasAlpha` — relies on tag detection that
  // has at least three known failure modes: case-sensitivity across ffmpeg
  // versions (`alpha_mode` vs `ALPHA_MODE`), missing tags from older muxers,
  // and mp4-as-webm rewraps that drop the sidecar. A wrong negative there
  // silently strips alpha during decode and the bug doesn't surface until
  // the rendered video is missing layers. Codec-based default has no such
  // ambiguity: libvpx-vp9 reads the alpha sidecar when present and decodes
  // normally when it isn't.
  if (codecMayHaveAlpha(metadata.videoCodec)) {
    args.push("-c:v", decoderForCodec(metadata.videoCodec));
  }
  const decodeArgs = [...args];
  if (options.finalFrameOnly) {
    // Output-side seek decodes from the start before selecting the final
    // sample. This is intentionally reserved for the one-frame path: input
    // seeking is faster, but valid unindexed transports (notably MPEG-TS with
    // a negative timestamp base) can seek to EOF and emit zero frames.
    args.push("-i", videoPath, "-ss", String(startTime), "-frames:v", "1");
  } else {
    // An accurate seek drops the frame on screen at startTime; the fps sampling below needs it.
    if (sampleAtOutputFps) args.push("-noaccurate_seek");
    args.push("-ss", String(startTime), "-i", videoPath, "-t", String(duration));
  }

  const vfFilters: string[] = [];
  if (decodeWithVideoToolbox) {
    vfFilters.push("format=nv12");
  }
  const samplingAt = vfFilters.length;
  // Each slot takes the last frame starting at or before its time. Ticks of 2 ms or finer
  // first shift half a tick, absorbing container rounding (WebM/MKV use whole ms).
  const samplingFilters = (seek: number): string[] =>
    sampleAtOutputFps
      ? [
          ...(metadata.isVFR
            ? []
            : undoTickRoundingOnTheSourceFrameGrid(seek, metadata.videoStreamLeadSeconds ?? 0)),
          "settb=intb/2",
          "setpts=PTS-lte(TB\\,0.001)",
          `fps=${ffmpegFps}:start_time=0:round=up`,
        ]
      : [];
  if (options.sdrToHdrTransfer) {
    // Ordering intent: fps sampling runs BEFORE the colorspace remap so only
    // kept frames are converted. The remap is pointwise per-frame, so the
    // output is identical either way for the SDR (BT.709, 8-bit) inputs this
    // flag is set for. If format=nv12 (macOS HDR-source decode) ever combines
    // with this flag, revisit: nv12 subsampling before a BT.2020 remap is an
    // untested interaction (today the flags are mutually exclusive — the
    // remap only applies to SDR sources, nv12 only to HDR sources).
    vfFilters.push(SDR_TO_HDR_COLORSPACE_FILTER);
  }
  if (toneMappedToSdr) {
    const frameColour = await readFirstFrameColour(videoPath);
    vfFilters.push(hdrToSdrToneMapFilter(metadata.colorSpace ?? {}, frameColour));
  }
  if (toneMappedToSdr || (!isHdr && !options.sdrToHdrTransfer)) {
    vfFilters.push(
      ...(isHdr ? [] : chromeGuessForUntaggedMatrix(metadata)),
      SDR_CANVAS_PASSTHROUGH_FILTER,
    );
    if (format === "jpg") {
      if (options.shownBox) vfFilters.push(halveToCoverScaleFilter(options.shownBox));
      vfFilters.push(SDR_JPEG_AS_BT601_FULL_RANGE_FILTER);
    }
  }
  const encodeArgs = ["-q:v", format === "jpg" ? String(Math.ceil((100 - quality) / 3)) : "0"];
  // Render-scoped temp frames are read once; level 1 measured 3-5x faster for ~14% larger files.
  if (format === "png") encodeArgs.push("-compression_level", "1");
  encodeArgs.push("-y", outputPattern);

  // A long 4K run can take far longer than the timeout and still be healthy; only a stall is a hang.
  const runOptions = { signal, stallTimeout: ffmpegProcessTimeout };
  const filterArgsFor = (seek: number): string[] => {
    const filters = [...vfFilters];
    filters.splice(samplingAt, 0, ...samplingFilters(seek));
    return filters.length > 0 ? ["-vf", filters.join(",")] : [];
  };
  const segmentFrames = Math.round(EXTRACTION_SEGMENT_SECONDS * fps);
  const totalFrames = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
  // A VFR window rounds its frame count up, and only the segments' `-frames:v` bound
  // emits that last slot: the fps filter ends a `-t` window at the nearest slot. A
  // decimal rate counts zero frames here and keeps the `-t` window.
  const boundVfrFrameCount = metadata.isVFR && totalFrames > 0;
  const processResult =
    sampleAtOutputFps && (options.frameRanges || boundVfrFrameCount || totalFrames > segmentFrames)
      ? await runSegmentedExtraction({
          decodeArgs,
          filterAndEncodeArgs: (seek) => [...filterArgsFor(seek), ...encodeArgs],
          videoPath,
          startTime,
          fps,
          segments: splitFrameRanges(
            options.frameRanges ?? [{ firstFrame: 0, frames: totalFrames }],
            segmentFrames,
          ),
          runOptions,
        })
      : await inExtractionSlot(
          () => runFfmpeg([...args, ...filterArgsFor(startTime), ...encodeArgs], runOptions),
          signal,
        );
  if (processResult.failureReason === "external_interruption") {
    throw new VideoSourceExtractionError(
      "external_interruption",
      true,
      "Video frame extraction interrupted by host lifecycle",
      `FFmpeg exited with code ${processResult.exitCode}: ${processResult.stderr.slice(-500)}`,
    );
  }
  if (processResult.terminationReason === "abort") {
    throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled");
  }
  if (processResult.terminationReason === "spawn_error") {
    throw classifyFfmpegSpawnError(processResult.error, processResult.stderr);
  }
  if (!processResult.success) {
    // With the SDR-to-HDR remap folded into this pass, a filter failure
    // (e.g. an ffmpeg built without the colorspace filter) would otherwise
    // surface as a generic extract error and the operator has to grep the
    // filter chain to learn it was the HDR conversion. Attribute it.
    const hdrPrefix = options.sdrToHdrTransfer
      ? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): `
      : "";
    const timedOut = processResult.terminationReason === "inactivity";
    const timeoutSuffix = timedOut ? ` (no new frame for ${ffmpegProcessTimeout} ms)` : "";
    const diagnostic =
      `${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` +
      processResult.stderr.slice(-500);
    if (timedOut) {
      throw new VideoSourceExtractionError(
        "ffmpeg_timeout",
        true,
        "Video frame extraction timed out",
        diagnostic,
      );
    }
    const transientIo =
      /resource temporarily unavailable|device or resource busy|input\/output error/i.test(
        processResult.stderr,
      );
    throw new VideoSourceExtractionError(
      transientIo ? "ffmpeg_transient" : "ffmpeg_failed",
      transientIo,
      transientIo
        ? "Video frame extraction hit a transient I/O failure"
        : "Video source could not be decoded",
      diagnostic,
    );
  }

  const framePaths = framePathsFromDirectory(videoOutputDir, format, !!options.frameRanges);
  if (framePaths.size === 0 && duration > 0) {
    throw new VideoSourceExtractionError(
      "zero_output",
      false,
      "Video source produced no decodable frames",
      `FFmpeg exited successfully but produced no frames (start=${startTime}, duration=${duration})`,
    );
  }

  // Post-extraction reconciliation: when FFmpeg delivers <=60% of the
  // frames predicted by the probed duration, the duration is likely
  // inflated (container duration includes a longer audio track). Correct
  // it so downstream coverage accounting uses the actual video extent.
  if (framePaths.size > 0 && duration > 0 && !options.finalFrameOnly && !options.frameRanges) {
    const expectedAtFps = extractionFrameCountForDuration(duration, normalizedFps, metadata.isVFR);
    if (expectedAtFps > 0 && framePaths.size <= expectedAtFps * 0.6) {
      const correctedStreamDuration =
        startTime + framePaths.size / fps - (metadata.videoStreamLeadSeconds ?? 0);
      if (correctedStreamDuration < metadata.videoStreamDurationSeconds) {
        metadata.videoStreamDurationSeconds = correctedStreamDuration;
      }
    }
  }

  return {
    videoId,
    srcPath: videoPath,
    outputDir: videoOutputDir,
    framePattern,
    fps,
    totalFrames: framePaths.size,
    metadata,
    framePaths,
  };
}

const TRANSIENT_FFMPEG_SPAWN_CODES = new Set(["EAGAIN", "EMFILE", "ENFILE"]);

export function classifyFfmpegSpawnError(error: unknown, stderr = ""): VideoSourceExtractionError {
  const code =
    typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
      ? error.code
      : "";
  if (code === "ENOENT") {
    return new VideoSourceExtractionError(
      "ffmpeg_unavailable",
      false,
      "FFmpeg is unavailable",
      "[FFmpeg] ffmpeg not found",
    );
  }
  const diagnostic = error instanceof Error ? error.message : stderr;
  const retryable = TRANSIENT_FFMPEG_SPAWN_CODES.has(code);
  return new VideoSourceExtractionError(
    retryable ? "ffmpeg_transient" : "ffmpeg_failed",
    retryable,
    retryable
      ? "FFmpeg could not be started due to transient resource pressure"
      : "FFmpeg could not be started",
    diagnostic,
  );
}

/**
 * Resolve the used-segment duration for a video, falling back to the source's
 * natural duration when the caller hasn't specified bounds (end=Infinity) or
 * the bounds are nonsensical (end<=start).
 */
function resolveSegmentDuration(
  requested: number,
  mediaStart: number,
  sourceDuration: number,
): number {
  if (Number.isFinite(requested) && requested > 0) return requested;
  const sourceRemaining = sourceDuration - mediaStart;
  return sourceRemaining > 0 ? sourceRemaining : sourceDuration;
}

/**
 * Return the range that can actually produce video frames.
 *
 * Container duration may include a longer audio stream or mux padding. Using
 * it for video extraction planning can reserve raw-frame scratch for seconds
 * where no video frames exist. `extractMediaMetadata` already falls back to
 * the container duration when ffprobe omits the stream duration; keep the
 * explicit fallback here for callers supplying older/manual metadata.
 */
export function resolvePlayableVideoDuration(metadata: VideoMetadata): number {
  return Number.isFinite(metadata.videoStreamDurationSeconds) &&
    metadata.videoStreamDurationSeconds > 0
    ? (metadata.videoStreamLeadSeconds ?? 0) + metadata.videoStreamDurationSeconds
    : metadata.durationSeconds;
}

export interface TimelineExtractionWindow {
  compositionStart: number;
  mediaStart: number;
  durationSeconds: number;
  /** Visible duration on the authored composition timeline after retiming. */
  timelineDurationSeconds?: number;
  /**
   * Preserve the authored timeline origin and mediaStart for lookup. This is
   * required when a looped visible interval crosses a source boundary and
   * still needs modulo phase against the complete extracted source cycle.
   */
  preserveTimelinePhase?: boolean;
  /**
   * Keep the authored end while rebasing start/mediaStart to the extracted
   * source suffix. Non-looping lookup then holds the suffix's final frame
   * through the remainder of the authored slot.
   */
  preserveTimelineEnd?: boolean;
  /** This window reaches a held tail and must be checked against the actual
   *  final decoded-frame timestamp before extraction. */
  ensureFinalFrame?: boolean;
  /** Source timestamp used by FFmpeg when it differs from the logical lookup
   *  mediaStart (the one-frame held-tail representation). */
  extractionMediaStart?: number;
  /** FFmpeg emits one decoded frame; lookup then holds that frame. */
  finalFrameOnly?: boolean;
}

type TimelineWindowVideo = Pick<VideoElement, "start" | "end" | "mediaStart"> &
  Partial<Pick<VideoElement, "playbackRate">> &
  Partial<Pick<VideoElement, "loop">>;

function canHoldFinalFramePastEof(video: TimelineWindowVideo): boolean {
  const timelineDuration = video.end - video.start;
  return !video.loop && Number.isFinite(timelineDuration) && timelineDuration > 0;
}

// Logical duration assigned to a one-frame held-tail representation. This is
// deliberately below any supported output frame interval: coverage expects
// one frame, while FFmpeg seeks to the separately probed real frame timestamp.
const FINAL_FRAME_LOGICAL_DURATION_SECONDS = 1e-6;

export function isVideoPastTimelineEnd(video: Pick<VideoElement, "start">, timelineEnd: number) {
  return video.start >= timelineEnd;
}

/** Move a clip to its extraction window; a rate lane is shifted so it keeps integrating from the new origin. */
export function rebaseVideoToWindow(video: VideoElement, window: TimelineExtractionWindow): void {
  if (window.preserveTimelinePhase) return;
  // A trim landing on the timeline origin can come out as 1e-16 and drop the first frame.
  const start = Math.abs(window.compositionStart) < 1e-9 ? 0 : window.compositionStart;
  video.playbackRate = shiftRateLane(normalizeRateSpec(video.playbackRate), start - video.start);
  video.start = start;
  if (!window.preserveTimelineEnd) {
    video.end = start + (window.timelineDurationSeconds ?? window.durationSeconds);
  }
  video.mediaStart = window.mediaStart;
}

/**
 * Intersect an authored slot with the render timeline, then select the
 * smallest playable source range that preserves timeline lookup semantics.
 *
 * A finite authored slot can outlive the source. In that case FFmpeg should
 * still extract at most one source range: lookup either wraps that range for
 * loops or holds its final frame for non-looping video. Keeping the authored
 * timeline origin separate from the extracted range is what makes both
 * behaviours survive the source-duration cap.
 */
export function resolveTimelineExtractionWindow(
  video: TimelineWindowVideo,
  resolvedDuration: number,
  timelineEnd?: number,
  sourceDuration?: number,
): TimelineExtractionWindow {
  const playbackRate = normalizeRateSpec(video.playbackRate);
  const withTimelineDuration = (
    window: TimelineExtractionWindow,
    timelineDurationSeconds: number,
  ): TimelineExtractionWindow =>
    playbackRate === 1 ? window : { ...window, timelineDurationSeconds };
  if (timelineEnd === undefined) {
    return withTimelineDuration(
      {
        compositionStart: video.start,
        mediaStart: video.mediaStart,
        durationSeconds: sourceTimeAt(playbackRate, resolvedDuration),
      },
      resolvedDuration,
    );
  }
  if (!Number.isFinite(timelineEnd)) {
    throw new Error(`Video extraction timelineEnd must be finite; got ${String(timelineEnd)}`);
  }
  const compositionStart = Math.max(0, video.start);
  const trimmedPreroll = compositionStart - video.start;
  const trimmedSourcePreroll = sourceTimeAt(playbackRate, trimmedPreroll);
  const timelineDuration = Math.max(0, timelineEnd - compositionStart);
  // Infinity means "natural source duration", not an authored infinite slot.
  // Explicit finite slots may outlive the source (loop or held tail), while an
  // omitted duration remains source-bounded exactly like the browser runtime.
  const resolvedVisibleDuration = resolvedDuration - trimmedPreroll;
  const visibleDuration = Math.max(0, Math.min(resolvedVisibleDuration, timelineDuration));
  const visibleSourceDuration =
    typeof playbackRate === "number"
      ? visibleDuration * playbackRate
      : sourceTimeAt(playbackRate, trimmedPreroll + visibleDuration) - trimmedSourcePreroll;
  let mediaStart = video.mediaStart + trimmedSourcePreroll;
  if (visibleDuration > 0 && sourceDuration !== undefined) {
    const sourceRemaining = Math.max(0, sourceDuration - video.mediaStart);
    if (sourceRemaining > 0 && video.loop && Number.isFinite(video.end)) {
      const phaseOffset = trimmedSourcePreroll % sourceRemaining;
      const phaseRemaining = sourceRemaining - phaseOffset;
      // Keep a complete cycle on equality too, otherwise a rebased suffix would
      // wrap to its own first frame instead of the source cycle's first frame.
      if (visibleSourceDuration >= phaseRemaining) {
        return withTimelineDuration(
          {
            compositionStart: video.start,
            mediaStart: video.mediaStart,
            durationSeconds: sourceRemaining,
            preserveTimelinePhase: true,
          },
          visibleDuration,
        );
      }
      mediaStart = video.mediaStart + phaseOffset;
    } else if (sourceRemaining > 0) {
      const sourceVisibleAfterPreroll = Math.max(0, sourceRemaining - trimmedSourcePreroll);
      if (visibleSourceDuration <= sourceVisibleAfterPreroll) {
        return withTimelineDuration(
          {
            compositionStart,
            mediaStart,
            durationSeconds: visibleSourceDuration,
          },
          visibleDuration,
        );
      }

      // The visible interval enters (or is entirely inside) the held tail.
      // Extract the visible source suffix. If preroll is already at/past the
      // final decoded timestamp, the async resolver below replaces this tiny
      // provisional suffix with one exact final frame.
      const extractionDuration = Math.min(
        sourceRemaining,
        Math.max(sourceVisibleAfterPreroll, FINAL_FRAME_LOGICAL_DURATION_SECONDS),
      );
      const extractionOffset = sourceRemaining - extractionDuration;
      return withTimelineDuration(
        {
          compositionStart: video.start + timeAtSourceTime(playbackRate, extractionOffset),
          mediaStart: video.mediaStart + extractionOffset,
          durationSeconds: extractionDuration,
          preserveTimelineEnd: true,
          ensureFinalFrame: true,
        },
        visibleDuration,
      );
    } else if (canHoldFinalFramePastEof(video)) {
      const logicalDuration = Math.min(sourceDuration, FINAL_FRAME_LOGICAL_DURATION_SECONDS);
      return withTimelineDuration(
        {
          compositionStart: video.start,
          mediaStart: sourceDuration - logicalDuration,
          durationSeconds: logicalDuration,
          preserveTimelineEnd: true,
          ensureFinalFrame: true,
        },
        visibleDuration,
      );
    }
  }
  return withTimelineDuration(
    {
      compositionStart,
      mediaStart,
      durationSeconds: visibleSourceDuration,
    },
    visibleDuration,
  );
}

/**
 * Replace a held-tail suffix that starts at/after the final decoded timestamp
 * with one exact frame. This keeps raw HDR scratch O(one frame) without
 * assuming a one-second seek window contains a CFR/VFR timestamp.
 */
export async function resolveFinalFrameExtractionWindow(
  videoPath: string,
  video: TimelineWindowVideo,
  metadata: VideoMetadata,
  window: TimelineExtractionWindow,
  signal?: AbortSignal,
): Promise<TimelineExtractionWindow> {
  if (!window.ensureFinalFrame) return window;
  const playableDuration = resolvePlayableVideoDuration(metadata);
  const videoStreamLead = metadata.videoStreamLeadSeconds ?? 0;
  const finalFrameTimestamp =
    videoStreamLead +
    (await extractFinalVideoFrameTimestamp(
      videoPath,
      {
        videoStreamDurationSeconds: playableDuration - videoStreamLead,
        videoStreamStartSeconds: metadata.videoStreamStartSeconds,
      },
      signal,
    ));
  if (window.mediaStart < finalFrameTimestamp - 1e-9) return window;

  const sourceRemaining = playableDuration - video.mediaStart;
  const logicalDuration = Math.min(
    Math.max(sourceRemaining, window.durationSeconds),
    FINAL_FRAME_LOGICAL_DURATION_SECONDS,
  );
  return {
    compositionStart: Math.max(0, video.start),
    mediaStart: playableDuration - logicalDuration,
    extractionMediaStart: finalFrameTimestamp,
    durationSeconds: logicalDuration,
    ...(window.timelineDurationSeconds !== undefined
      ? { timelineDurationSeconds: window.timelineDurationSeconds }
      : {}),
    preserveTimelineEnd: true,
    finalFrameOnly: true,
  };
}

/** Resolve source duration first, then intersect it with the render timeline. */
export function resolveVideoExtractionWindow(
  video: TimelineWindowVideo,
  metadata: VideoMetadata,
  timelineEnd?: number,
): TimelineExtractionWindow {
  const playableDuration = resolvePlayableVideoDuration(metadata);
  if (!(playableDuration > 0)) {
    throw new VideoSourceExtractionError(
      "invalid_media",
      false,
      "Video source has no positive duration",
      `Playable video stream duration is ${playableDuration}s`,
    );
  }
  const requestedTimelineDuration = video.end - video.start;
  const heldPastEof = video.mediaStart >= playableDuration && canHoldFinalFramePastEof(video);
  if (video.mediaStart >= playableDuration && !heldPastEof) {
    throw new VideoSourceExtractionError(
      "media_start_out_of_range",
      false,
      "Video media start is outside the source duration",
      `Video media start ${video.mediaStart}s is outside playable video duration ${playableDuration}s`,
    );
  }
  const playbackRate = normalizeRateSpec(video.playbackRate);
  const resolvedDuration =
    Number.isFinite(requestedTimelineDuration) && requestedTimelineDuration > 0
      ? requestedTimelineDuration
      : timeAtSourceTime(
          playbackRate,
          resolveSegmentDuration(requestedTimelineDuration, video.mediaStart, playableDuration),
        );
  return resolveTimelineExtractionWindow(
    video,
    resolvedDuration,
    timelineEnd ?? (heldPastEof ? video.end : undefined),
    playableDuration,
  );
}

export function resolveVideoExtractionDuration(
  video: TimelineWindowVideo,
  metadata: VideoMetadata,
  timelineEnd?: number,
): number {
  return resolveVideoExtractionWindow(video, metadata, timelineEnd).durationSeconds;
}

/**
 * Codecs whose bitstream is allowed to carry an alpha channel. Default the
 * extraction path to PNG output for these regardless of `metadata.hasAlpha`
 * so a missed sidecar tag doesn't silently strip transparency. Opaque content
 * encoded in one of these codecs pays a small file-size cost on the cached
 * frames but stays correct on the rare case where alpha IS present and the
 * tag was missed.
 */
const ALPHA_CAPABLE_CODECS = new Set(["vp9", "vp8", "prores"]);

export function codecMayHaveAlpha(codec: string | undefined): boolean {
  return ALPHA_CAPABLE_CODECS.has((codec ?? "").toLowerCase());
}

export function decoderForCodec(codec: string | undefined): string {
  return alphaDecoderFor(codec) ?? (codec ?? "").toLowerCase();
}

export function resolveFrameFormat(
  metadata: VideoMetadata,
  requested?: VideoFrameFormat,
): CacheFrameFormat {
  if (metadata.hasAlpha || codecMayHaveAlpha(metadata.videoCodec)) return "png";
  if (requested === "png" || requested === "jpg") return requested;
  return "jpg";
}

type PreparedExtraction = {
  video: VideoElement;
  videoPath: string;
  index: number;
  metadata: VideoMetadata;
  videoDuration: number;
  extractionMediaStart: number;
  finalFrameOnly: boolean;
  format: CacheFrameFormat;
  sdrToHdrTransfer?: HdrTransfer;
  hdrToSdrTransformKey?: string;
  dedupeKey: string;
};

type CacheMissTarget = {
  entry: CacheEntry;
  srcPath: string;
};

type UniqueExtractionMiss = {
  work: PreparedExtraction;
  cacheTarget?: CacheMissTarget;
};

type SupersetMemberPlan = {
  miss: UniqueExtractionMiss;
  offsetFrames: number;
};

type SupersetGroupPlan = {
  groupId: string;
  baseStart: number;
  unionDuration: number;
  members: SupersetMemberPlan[];
};

function extractedFramesFromDirectory(
  work: PreparedExtraction,
  outputDir: string,
  srcPath: string,
  fps: number,
): ExtractedFrames {
  const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${work.format}`;
  const framePaths = framePathsFromDirectory(outputDir, work.format);
  return {
    videoId: work.video.id,
    srcPath,
    outputDir,
    framePattern,
    fps,
    totalFrames: framePaths.size,
    metadata: work.metadata,
    framePaths,
  };
}

function frameFileName(frameNumber: number, format: CacheFrameFormat): string {
  return `${FRAME_FILENAME_PREFIX}${String(frameNumber).padStart(5, "0")}.${format}`;
}

function linkOrCopyFrame(src: string, dest: string): void {
  try {
    linkSync(src, dest);
  } catch {
    copyFileSync(src, dest);
  }
}

function supersetGroupingKey(work: PreparedExtraction, fps: number): string {
  return [
    work.videoPath,
    String(fps),
    work.format,
    work.sdrToHdrTransfer ?? "",
    work.hdrToSdrTransformKey ?? "",
    work.finalFrameOnly ? "final" : "range",
  ].join("\0");
}

function isIntegralFrameOffset(offsetSeconds: number, fps: number): boolean {
  const frames = offsetSeconds * fps;
  return Math.abs(frames - Math.round(frames)) <= 1e-4;
}

function windowsOverlapOrTouch(misses: UniqueExtractionMiss[], baseStart: number): boolean {
  const unionEnd = Math.max(
    ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
  );
  const unionDuration = unionEnd - baseStart;
  const summedDuration = misses.reduce((sum, { work }) => sum + work.videoDuration, 0);
  return unionDuration > 0 && unionDuration <= summedDuration + 1e-9;
}

function buildSupersetGroup(
  groupId: string,
  misses: UniqueExtractionMiss[],
  fps: number,
): SupersetGroupPlan | null {
  if (misses.length < 2) return null;
  if (misses.some(({ work }) => work.finalFrameOnly)) return null;
  // VFR ranges round their frame counts up, so a union extraction's slices
  // are not proven to match independently sought member ranges. Keep them direct.
  if (misses.some(({ work }) => work.metadata.isVFR)) return null;
  const baseStart = Math.min(...misses.map(({ work }) => work.video.mediaStart));
  if (!misses.every(({ work }) => isIntegralFrameOffset(work.video.mediaStart - baseStart, fps))) {
    return null;
  }
  if (!windowsOverlapOrTouch(misses, baseStart)) return null;

  const unionEnd = Math.max(
    ...misses.map(({ work }) => work.video.mediaStart + work.videoDuration),
  );
  return {
    groupId,
    baseStart,
    unionDuration: unionEnd - baseStart,
    members: misses.map((miss) => ({
      miss,
      offsetFrames: Math.round((miss.work.video.mediaStart - baseStart) * fps),
    })),
  };
}

/**
 * Partition one source's misses into overlap-connected components: sort by
 * window start and cut wherever the next window starts past the running end.
 * Without this, one disjoint outlier trim (e.g. [100..105] next to three
 * overlapping trims at [0..11]) fails the union<=sum check for the whole
 * bucket and every trim falls back to direct extraction.
 */
function overlapClusters(misses: UniqueExtractionMiss[]): UniqueExtractionMiss[][] {
  const sorted = [...misses].sort((a, b) => a.work.video.mediaStart - b.work.video.mediaStart);
  const clusters: UniqueExtractionMiss[][] = [];
  let current: UniqueExtractionMiss[] = [];
  let currentEnd = -Infinity;
  for (const miss of sorted) {
    const start = miss.work.video.mediaStart;
    const end = start + miss.work.videoDuration;
    if (current.length > 0 && start > currentEnd + 1e-9) {
      clusters.push(current);
      current = [];
      currentEnd = -Infinity;
    }
    current.push(miss);
    currentEnd = Math.max(currentEnd, end);
  }
  if (current.length > 0) clusters.push(current);
  return clusters;
}

function planSupersetGroups(
  misses: UniqueExtractionMiss[],
  fps: number,
): { groups: SupersetGroupPlan[]; direct: UniqueExtractionMiss[] } {
  const bySource = new Map<string, UniqueExtractionMiss[]>();
  for (const miss of misses) {
    const key = supersetGroupingKey(miss.work, fps);
    bySource.set(key, [...(bySource.get(key) ?? []), miss]);
  }

  const groups: SupersetGroupPlan[] = [];
  const direct: UniqueExtractionMiss[] = [];
  let groupIndex = 0;
  for (const groupMisses of bySource.values()) {
    for (const cluster of overlapClusters(groupMisses)) {
      const group = buildSupersetGroup(`__superset-${groupIndex}`, cluster, fps);
      if (group) {
        groups.push(group);
        groupIndex += 1;
      } else {
        direct.push(...cluster);
      }
    }
  }
  return { groups, direct };
}

function sliceSupersetMember(
  member: SupersetMemberPlan,
  superset: ExtractedFrames,
  outputDir: string,
  fps: number,
  configuredFps: FpsInput,
): ExtractedFrames {
  const { work } = member.miss;
  rmSync(outputDir, { recursive: true, force: true });
  mkdirSync(outputDir, { recursive: true });

  // Sample-time correctness: member frame k uses superset frame
  // offset_i + k, so its source time is
  // baseStart + (offset_i + k) / fps = mediaStart_i + k / fps.
  // The frame-alignment precondition is what makes offset_i integral.
  const requestedFrames = extractionFrameCountForDuration(
    work.videoDuration,
    configuredFps,
    work.metadata.isVFR,
  );
  const availableFrames = Math.max(0, superset.totalFrames - member.offsetFrames);
  const frameCount = Math.min(requestedFrames, availableFrames);
  for (let i = 0; i < frameCount; i += 1) {
    const sourceFrame = superset.framePaths.get(member.offsetFrames + i);
    if (!sourceFrame) throw new Error(`superset frame ${member.offsetFrames + i} missing`);
    linkOrCopyFrame(sourceFrame, join(outputDir, frameFileName(i + 1, work.format)));
  }

  return extractedFramesFromDirectory(work, outputDir, work.videoPath, fps);
}

export async function extractAllVideoFrames(
  videos: VideoElement[],
  baseDir: string,
  options: ExtractionOptions,
  signal?: AbortSignal,
  config?: Partial<
    Pick<EngineConfig, "ffmpegProcessTimeout" | "extractCacheDir" | "extractCacheMaxBytes">
  >,
  compiledDir?: string,
): Promise<ExtractionResult> {
  if (options.timelineEnd !== undefined && !Number.isFinite(options.timelineEnd)) {
    throw new Error(
      `Video extraction timelineEnd must be finite; got ${String(options.timelineEnd)}`,
    );
  }
  const configuredFps = toFps(options.fps);
  const fps = fpsToNumber(configuredFps);
  const fpsKey = fpsToFfmpegArg(configuredFps);
  const startTime = Date.now();
  const extracted: ExtractedFrames[] = [];
  const errors: VideoExtractionFailure[] = [];
  let totalFramesExtracted = 0;
  const breakdown: ExtractionPhaseBreakdown = {
    resolveMs: 0,
    cachePublishFailures: 0,
    cacheGcEvictions: 0,
    cacheGcBytesFreed: 0,
    cacheAgedPartialsCleared: 0,
    hdrProbeMs: 0,
    hdrPreflightMs: 0,
    hdrPreflightCount: 0,
    vfrProbeMs: 0,
    vfrPreflightMs: 0,
    vfrPreflightCount: 0,
    extractMs: 0,
    cacheHits: 0,
    cacheMisses: 0,
    transientRetries: 0,
  };
  const recordTransientRetries = (count: number): void => {
    breakdown.transientRetries = (breakdown.transientRetries ?? 0) + count;
  };

  // Phase 1: Resolve paths and download remote videos
  const phase1Start = Date.now();
  const resolvedVideos: Array<{ video: VideoElement; videoPath: string }> = [];
  // Dedupe missing-src warnings: a composition with N <video> elements all
  // pointing at the same broken src should only print one warning, not N.
  const warnedSrcs = new Set<string>();
  for (const video of videos) {
    if (signal?.aborted) break;
    if (options.timelineEnd !== undefined && isVideoPastTimelineEnd(video, options.timelineEnd)) {
      continue;
    }
    try {
      let videoPath = video.src;
      if (!isHttpUrl(videoPath)) {
        videoPath = resolveProjectRelativeSrc(video.src, baseDir, compiledDir);
      }
      const remoteUrl = options.urlByAbsentLocalPath?.get(videoPath);
      if (remoteUrl) {
        resolvedVideos.push({ video, videoPath: remoteUrl });
        continue;
      }

      if (isHttpUrl(videoPath)) {
        const downloadDir = join(options.outputDir, DOWNLOAD_SUBDIR);
        mkdirSync(downloadDir, { recursive: true });
        videoPath = await downloadToTemp(
          videoPath,
          downloadDir,
          undefined,
          signal,
          () => recordTransientRetries(1),
          { onTelemetry: writeUrlDownloadTelemetry },
        );
      }

      if (!existsSync(videoPath)) {
        // Loud: silent miss leaves the rendered video frozen at frame 0 with
        // no error in stdout — extremely confusing for authors. Dedupe by
        // src so 50 broken videos pointing at the same path don't spam.
        if (!warnedSrcs.has(video.src)) {
          warnedSrcs.add(video.src);
          process.stderr.write(
            `[hyperframes:render] WARNING: video src="${video.src}" ` +
              `could not be resolved on disk (looked for ${videoPath}). ` +
              `The rendered output will show this video's first frame for the entire clip duration. ` +
              `If your <video> lives inside a sub-composition, prefer project-root-relative paths ` +
              `(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`,
          );
        }
        errors.push({
          videoId: video.id,
          kind: "source_missing",
          retryable: false,
          error: `Video file not found: ${videoPath}`,
        });
        continue;
      }
      resolvedVideos.push({ video, videoPath });
    } catch (err) {
      const classified = classifyVideoExtractionError(err);
      errors.push({
        videoId: video.id,
        kind: classified.kind,
        retryable: classified.retryable,
        ...(err instanceof UrlDownloadError ? { group: downloadFailureGroup(video.src, err) } : {}),
        error: classified.diagnostic,
      });
    }
  }

  breakdown.resolveMs = Date.now() - phase1Start;

  // Snapshot the pre-preflight key inputs so the extraction cache keys on the
  // user-visible source path rather than the
  // workDir-local normalized file produced by the
  // HDR preflight. Without this, every render would write a new
  // normalized file with a fresh mtime → fresh cache key → perpetual misses.
  // Phase 3 updates mediaStart after trimming any invisible negative preroll.
  // Downloads land at a new path and mtime every render, so they are keyed by content instead.
  const contentKeyedDirs = [
    join(options.outputDir, DOWNLOAD_SUBDIR),
    ...(options.contentKeyedDirs ?? []),
  ];
  const cacheKeyInputs = resolvedVideos.map(({ video, videoPath }) => {
    const stat = readKeyStat(videoPath);
    // Missing files return null — skip the cache path for that entry. The
    // extractor will surface the real file-not-found error downstream, and we
    // avoid polluting the cache with a `(mtimeMs: 0, size: 0)` tuple that two
    // unrelated missing paths would otherwise share.
    if (!stat) return null;
    return {
      videoPath,
      mtimeMs: stat.mtimeMs,
      size: stat.size,
      mediaStart: video.mediaStart,
      contentKeyed: contentKeyedDirs.some((dir) => isPathInside(videoPath, dir)),
    };
  });

  // Phase 2: Probe color spaces and normalize if mixed HDR/SDR
  const phase2ProbeStart = Date.now();
  const metadataResults = await Promise.all(
    resolvedVideos.map(async ({ video, videoPath }, index) => {
      try {
        // Keep the default/off path byte-for-byte compatible with the legacy
        // Promise.all rejection. Classification is introduced only when a
        // bounded retry or explicit typed aggregation is enabled.
        const attempted =
          !options.collectProbeFailures &&
          boundedTransientRetryBudget(options.maxTransientRetries) === 0
            ? { result: await extractMediaMetadata(videoPath), retries: 0 }
            : await runVideoExtractionWithRetry(() => extractMediaMetadata(videoPath), {
                signal,
                maxTransientRetries: options.maxTransientRetries,
                onRetry: () => recordTransientRetries(1),
              });
        return {
          video,
          videoPath,
          metadata: attempted.result,
          cacheKeyInput: cacheKeyInputs[index] ?? null,
        };
      } catch (error) {
        if (!options.collectProbeFailures) throw error;
        errors.push(extractionError(video.id, error));
        return null;
      }
    }),
  );
  const probedVideos = metadataResults.filter((entry) => entry !== null);
  resolvedVideos.splice(
    0,
    resolvedVideos.length,
    ...probedVideos.map(({ video, videoPath }) => ({ video, videoPath })),
  );
  cacheKeyInputs.splice(
    0,
    cacheKeyInputs.length,
    ...probedVideos.map(({ cacheKeyInput }) => cacheKeyInput),
  );
  const videoMetadata = probedVideos.map(({ metadata }) => metadata);
  const videoColorSpaces = videoMetadata.map((m) => m.colorSpace);
  // Canonical per-index record of the SDR-to-HDR transform decision. BOTH the
  // cache key (transform discriminator) and the extraction options read from
  // this array via the prepared work items — never set one side independently
  // or cache lookups and written frames drift apart (the poisoning bug this
  // field exists to fix).
  const sdrToHdrTransfers: Array<HdrTransfer | undefined> = resolvedVideos.map(() => undefined);
  breakdown.hdrProbeMs = Date.now() - phase2ProbeStart;

  // Warning only: an opaque video used as a full-frame background is legitimate.
  const alphaWarnedSrcs = new Set<string>();
  if (resolvedVideos.length > 0) {
    await Promise.all(
      resolvedVideos.map(async ({ video, videoPath }, index) => {
        if (signal?.aborted) return;
        const metadata = videoMetadata[index];
        if (!metadata?.hasAlpha || !codecMayHaveAlpha(metadata.videoCodec)) return;
        if (alphaWarnedSrcs.has(video.src)) return;
        alphaWarnedSrcs.add(video.src);
        const decoder = decoderForCodec(metadata.videoCodec);
        if ((await probeInputAlphaPlane(videoPath, decoder, signal)) !== true) return;
        process.stderr.write(inputAlphaOpaqueWarning(video.src));
      }),
    );
  }

  const hdrPreflightStart = Date.now();
  const hdrInfo = analyzeCompositionHdr(videoColorSpaces);
  // Track entries the HDR preflight validated as non-extractable so they can
  // be removed from every parallel array before Phase 2b and Phase 3 see them.
  // Without this, `errors.push({...}); continue;` only short-circuits the
  // normalization step — the invalid entry stays in `resolvedVideos` and
  // Phase 3 still calls `extractVideoFramesRange` on the same past-EOF
  // mediaStart, surfacing a second raw FFmpeg error for the same clip.
  const hdrSkippedIndices = new Set<number>();
  if (hdrInfo.hasHdr && hdrInfo.dominantTransfer) {
    // dominantTransfer is "majority wins" — if a composition mixes PQ and HLG
    // sources (rare but legal), the minority transfer's videos get converted
    // with the wrong curve. We treat this as caller-error: a single composition
    // should not mix PQ and HLG sources, the orchestrator picks one transfer
    // for the whole render, and any source not on that curve is normalized to
    // it. If you need both transfers, render two separate compositions.
    const targetTransfer = hdrInfo.dominantTransfer;

    for (let i = 0; i < resolvedVideos.length; i++) {
      if (signal?.aborted) break;
      const cs = videoColorSpaces[i] ?? null;
      if (!isHdrColorSpaceUtil(cs)) {
        // SDR video in a mixed timeline — extract through a BT.709→BT.2020
        // colorspace filter so the encoder tags the final video correctly
        // (PQ vs HLG) without a separate normalized intermediate.
        const entry = resolvedVideos[i];
        const metadata = videoMetadata[i];
        if (!entry || !metadata) continue;

        // Guard past-EOF windows that cannot use the non-looping held-tail plan.
        // FFmpeg's `-ss` otherwise silently produces a 0-byte intermediate.
        const playableDuration = resolvePlayableVideoDuration(metadata);
        if (entry.video.mediaStart >= playableDuration && !canHoldFinalFramePastEof(entry.video)) {
          errors.push({
            videoId: entry.video.id,
            kind: "media_start_out_of_range",
            retryable: false,
            error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ playable video duration (${playableDuration}s)`,
          });
          hdrSkippedIndices.add(i);
          continue;
        }

        sdrToHdrTransfers[i] = targetTransfer;
        breakdown.hdrPreflightCount += 1;
      }
    }
  }
  breakdown.hdrPreflightMs = Date.now() - hdrPreflightStart;

  // Remove HDR-preflight-skipped entries from every parallel array so Phase 2b
  // (VFR classification) and Phase 3 (extract) don't re-process them. Iterate
  // backwards to keep indices stable while splicing.
  if (hdrSkippedIndices.size > 0) {
    for (let i = resolvedVideos.length - 1; i >= 0; i--) {
      if (hdrSkippedIndices.has(i)) {
        resolvedVideos.splice(i, 1);
        videoMetadata.splice(i, 1);
        videoColorSpaces.splice(i, 1);
        // Added by the extraction-cache commit: keep cacheKeyInputs aligned
        // with the other parallel arrays so Phase 3's `cacheKeyInputs[i]`
        // lookup doesn't point at a stale slot after the splice.
        cacheKeyInputs.splice(i, 1);
        sdrToHdrTransfers.splice(i, 1);
      }
    }
  }

  // Phase 2b: Keep VFR observability while routing VFR inputs through the
  // one-pass CFR extraction path in Phase 3.
  const vfrPreflightStart = Date.now();
  for (let i = 0; i < resolvedVideos.length; i++) {
    if (signal?.aborted) break;
    const vfrProbeStart = Date.now();
    const metadata = videoMetadata[i];
    breakdown.vfrProbeMs += Date.now() - vfrProbeStart;
    if (metadata?.isVFR) breakdown.vfrPreflightCount += 1;
  }
  breakdown.vfrPreflightMs = Date.now() - vfrPreflightStart;

  const phase3Start = Date.now();
  const configuredCacheRootDir = config?.extractCacheDir;
  let cacheRootDir: string | undefined;
  if (configuredCacheRootDir) {
    try {
      mkdirSync(configuredCacheRootDir, { recursive: true });
      cacheRootDir = configuredCacheRootDir;
    } catch {
      process.stderr.write(
        `[hyperframes:render] WARNING: extraction cache dir ${configuredCacheRootDir} is not writable; caching disabled for this render\n`,
      );
    }
  }

  function extractionError(videoId: string, err: unknown): VideoExtractionFailure {
    const classified = classifyVideoExtractionError(err);
    return {
      videoId,
      kind: classified.kind,
      retryable: classified.retryable,
      error: classified.diagnostic,
    };
  }

  type PreparedExtractionResult =
    | { work: PreparedExtraction }
    | { error: VideoExtractionFailure }
    | { skipped: true };

  type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure };

  function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions {
    return {
      ...options,
      format: work.format,
      sdrToHdrTransfer: work.sdrToHdrTransfer,
      toneMapHdrToSdr: work.hdrToSdrTransformKey !== undefined,
      finalFrameOnly: work.finalFrameOnly,
    };
  }

  function rehydratePublishedCache(work: PreparedExtraction, target: CacheMissTarget) {
    const rehydrated = rehydrateCacheEntry(target.entry, {
      videoId: work.video.id,
      srcPath: target.srcPath,
      fps,
      format: work.format,
      metadata: work.metadata,
    });
    return { ...rehydrated, ownedByLookup: true };
  }

  // Hashed only for works that reach a lookup (deferred ranges never do), once per file.
  const contentSha256ByPath = new Map<string, string | null>();
  async function hashContentKeyedSources(works: PreparedExtraction[]): Promise<void> {
    if (!cacheRootDir) return;
    const paths = new Set<string>();
    for (const work of works) {
      const keyInput = cacheKeyInputs[work.index];
      if (keyInput?.contentKeyed) paths.add(keyInput.videoPath);
    }
    await Promise.all(
      [...paths].map(async (path) =>
        contentSha256ByPath.set(path, await readContentSha256(path, signal)),
      ),
    );
  }

  function lookupCacheFor(work: PreparedExtraction): ExtractionOutcome | UniqueExtractionMiss {
    if (!cacheRootDir) return { work };
    const keyInput = cacheKeyInputs[work.index];
    if (!keyInput) return { work };
    const contentSha256 = keyInput.contentKeyed
      ? (contentSha256ByPath.get(keyInput.videoPath) ?? null)
      : undefined;
    if (contentSha256 === null) return { work };
    const transformParts = [
      work.sdrToHdrTransfer ? sdrToHdrTransformKey(work.sdrToHdrTransfer) : undefined,
      work.hdrToSdrTransformKey,
      work.finalFrameOnly ? "final-frame" : undefined,
    ].filter((part): part is string => part !== undefined);
    const transform = transformParts.length > 0 ? transformParts.join("+") : undefined;

    const lookup = lookupCacheEntry(cacheRootDir, {
      videoPath: keyInput.videoPath,
      mtimeMs: keyInput.mtimeMs,
      size: keyInput.size,
      mediaStart: keyInput.mediaStart,
      contentSha256,
      duration: work.videoDuration,
      fps: fpsKey,
      format: work.format,
      transform,
    });

    if (!lookup.hit) {
      breakdown.cacheMisses += 1;
      return { work, cacheTarget: { entry: lookup.entry, srcPath: keyInput.videoPath } };
    }

    breakdown.cacheHits += 1;
    touchCacheEntry(lookup.entry);
    return {
      result: rehydratePublishedCache(work, { entry: lookup.entry, srcPath: keyInput.videoPath }),
    };
  }

  async function extractDirectMiss(
    miss: UniqueExtractionMiss,
    maxTransientRetries = options.maxTransientRetries ?? 0,
  ): Promise<ExtractedFrames> {
    const { work, cacheTarget } = miss;
    if (!cacheTarget) {
      const outputDir = join(options.outputDir, work.video.id);
      const attempted = await runVideoExtractionWithRetry(
        () =>
          extractVideoFramesRange(
            work.videoPath,
            work.video.id,
            work.extractionMediaStart,
            work.videoDuration,
            scopedExtractionOptions(work),
            signal,
            config,
          ),
        {
          signal,
          maxTransientRetries,
          onRetry: () => {
            recordTransientRetries(1);
            rmSync(outputDir, { recursive: true, force: true });
          },
        },
      );
      return attempted.result;
    }

    const partialDir = partialCacheEntryDir(cacheTarget.entry);
    rmSync(partialDir, { recursive: true, force: true });
    mkdirSync(partialDir, { recursive: true });
    const attempted = await runVideoExtractionWithRetry(
      () =>
        extractVideoFramesRange(
          work.videoPath,
          work.video.id,
          work.extractionMediaStart,
          work.videoDuration,
          scopedExtractionOptions(work),
          signal,
          config,
          partialDir,
        ),
      {
        signal,
        maxTransientRetries,
        onRetry: () => {
          recordTransientRetries(1);
          rmSync(partialDir, { recursive: true, force: true });
          mkdirSync(partialDir, { recursive: true });
        },
      },
    );
    const result = attempted.result;
    const published = publishCacheEntry(cacheTarget.entry, partialDir);
    if (!published.published) {
      breakdown.cachePublishFailures += 1;
      return { ...result, ownedByLookup: false };
    }
    return rehydratePublishedCache(work, cacheTarget);
  }

  /**
   * Every output slot before the source's end holds a frame, so decoding the last second gives
   * the exact count. Anything unusual returns null and takes the normal path.
   */
  async function deferExtraction(work: PreparedExtraction): Promise<ExtractedFrames | null> {
    if (
      work.finalFrameOnly ||
      work.metadata.isVFR ||
      work.sdrToHdrTransfer !== undefined ||
      work.hdrToSdrTransformKey !== undefined
    ) {
      return null;
    }
    const expectedFrames = extractionFrameCountForDuration(
      work.videoDuration,
      configuredFps,
      false,
    );
    const tailFrames = Math.min(expectedFrames, Math.ceil(fps));
    if (expectedFrames <= tailFrames) return null;
    const probeDir = join(options.outputDir, `${work.video.id}.tail-probe`);
    try {
      const tail = await extractVideoFramesRange(
        work.videoPath,
        work.video.id,
        work.extractionMediaStart,
        work.videoDuration,
        {
          ...scopedExtractionOptions(work),
          frameRanges: [{ firstFrame: expectedFrames - tailFrames, frames: tailFrames }],
        },
        signal,
        config,
        probeDir,
      );
      const totalFrames = expectedFrames - tailFrames + tail.totalFrames;
      // A full extraction this short rewrites the probed duration; let it.
      if (totalFrames <= expectedFrames * 0.6) return null;
      const outputDir = join(options.outputDir, work.video.id);
      mkdirSync(outputDir, { recursive: true });
      return {
        videoId: work.video.id,
        srcPath: work.videoPath,
        outputDir,
        framePattern: `${FRAME_FILENAME_PREFIX}%05d.${work.format}`,
        fps,
        totalFrames,
        metadata: work.metadata,
        framePaths: new Map(),
        deferredRange: {
          startTime: work.extractionMediaStart,
          durationSeconds: work.videoDuration,
          format: work.format,
        },
      };
    } catch {
      return null;
    } finally {
      rmSync(probeDir, { recursive: true, force: true });
    }
  }

  async function executeDirectMiss(
    miss: UniqueExtractionMiss,
    maxTransientRetries = options.maxTransientRetries ?? 0,
  ): Promise<ExtractionOutcome> {
    try {
      return { result: await extractDirectMiss(miss, maxTransientRetries) };
    } catch (err) {
      return { error: extractionError(miss.work.video.id, err) };
    }
  }

  function materializeSupersetMember(
    member: SupersetMemberPlan,
    superset: ExtractedFrames,
  ): ExtractedFrames {
    const { miss } = member;
    const { work, cacheTarget } = miss;
    if (!cacheTarget) {
      return sliceSupersetMember(
        member,
        superset,
        join(options.outputDir, work.video.id),
        fps,
        configuredFps,
      );
    }

    const partialDir = partialCacheEntryDir(cacheTarget.entry);
    const sliced = sliceSupersetMember(member, superset, partialDir, fps, configuredFps);
    const published = publishCacheEntry(cacheTarget.entry, partialDir);
    if (!published.published) {
      breakdown.cachePublishFailures += 1;
      return { ...sliced, ownedByLookup: false };
    }
    return rehydratePublishedCache(work, cacheTarget);
  }

  async function executeSupersetGroup(
    group: SupersetGroupPlan,
  ): Promise<Array<[string, ExtractionOutcome]>> {
    const first = group.members[0]?.miss.work;
    if (!first) return [];
    // Hardlinks require source and destination on ONE filesystem. Cache-bound
    // members link into partial dirs under cacheRootDir, which is commonly a
    // different mount than the render's outputDir — extracting the superset
    // next to the cache keeps linkSync viable there (the EXDEV copyFileSync
    // fallback would silently multiply disk usage per member). The
    // `.partial-` name puts crashed leftovers under the GC's aged-partial
    // sweep.
    const tempDir = cacheRootDir
      ? join(cacheRootDir, `${group.groupId}.partial-${process.pid}`)
      : join(options.outputDir, group.groupId);

    try {
      rmSync(tempDir, { recursive: true, force: true });
      // A long union can fail even when each shorter
      // member range succeeds. Do not retry the optimization itself; preserve
      // the established grouped→direct fallback and apply bounded retries only
      // to the individual source ranges below.
      const superset = await extractVideoFramesRange(
        first.videoPath,
        group.groupId,
        group.baseStart,
        group.unionDuration,
        scopedExtractionOptions(first),
        signal,
        config,
        tempDir,
      );
      const outcomes: Array<[string, ExtractionOutcome]> = [];
      for (const member of group.members) {
        outcomes.push([
          member.miss.work.dedupeKey,
          { result: materializeSupersetMember(member, superset) },
        ]);
      }
      return outcomes;
    } catch (err) {
      // On abort, the union failure is the cancellation itself — re-running
      // every member through direct extraction would spawn N doomed ffmpeg
      // processes. Surface the cancellation per member instead.
      if (signal?.aborted) {
        return group.members.map((member) => [
          member.miss.work.dedupeKey,
          { error: extractionError(member.miss.work.video.id, err) },
        ]);
      }
      const fallback = await Promise.all(
        group.members.map(
          async (member) =>
            [member.miss.work.dedupeKey, await executeDirectMiss(member.miss)] as [
              string,
              ExtractionOutcome,
            ],
        ),
      );
      return fallback;
    } finally {
      rmSync(tempDir, { recursive: true, force: true });
    }
  }

  const preparedExtractions: PreparedExtractionResult[] = await Promise.all(
    resolvedVideos.map(async ({ video, videoPath }, index) => {
      if (signal?.aborted) {
        throw new Error("Video frame extraction cancelled");
      }
      try {
        const metadata = videoMetadata[index] ?? (await extractMediaMetadata(videoPath));
        const initialWindow = resolveVideoExtractionWindow(video, metadata, options.timelineEnd);
        const window = await resolveFinalFrameExtractionWindow(
          videoPath,
          video,
          metadata,
          initialWindow,
          signal,
        );
        const videoDuration = window.durationSeconds;
        if (videoDuration <= 0) {
          return { skipped: true };
        }
        rebaseVideoToWindow(video, window);
        const keyInput = cacheKeyInputs[index];
        const extractionMediaStart = window.extractionMediaStart ?? window.mediaStart;
        if (keyInput) keyInput.mediaStart = extractionMediaStart;

        const format = resolveFrameFormat(metadata, options.format);
        const sdrToHdrTransfer = sdrToHdrTransfers[index];
        const hdrToSdrTransform =
          options.toneMapHdrToSdr === true && isHdrColorSpaceUtil(metadata.colorSpace)
            ? await hdrToSdrTransformKey()
            : undefined;
        const finalFrameOnly = window.finalFrameOnly === true;
        const dedupeKey = `${videoPath}\0${extractionMediaStart}\0${videoDuration}\0${fpsKey}\0${format}\0${sdrToHdrTransfer ?? ""}\0${hdrToSdrTransform ?? ""}\0${finalFrameOnly ? "final" : "range"}`;

        return {
          work: {
            video,
            videoPath,
            index,
            metadata,
            videoDuration,
            extractionMediaStart,
            finalFrameOnly,
            format,
            sdrToHdrTransfer,
            hdrToSdrTransformKey: hdrToSdrTransform,
            dedupeKey,
          },
        };
      } catch (err) {
        return { error: extractionError(video.id, err) };
      }
    }),
  );

  const uniqueWorks = new Map<string, PreparedExtraction>();
  for (const prepared of preparedExtractions) {
    if ("work" in prepared && !uniqueWorks.has(prepared.work.dedupeKey)) {
      uniqueWorks.set(prepared.work.dedupeKey, prepared.work);
    }
  }

  const uniqueOutcomes = new Map<string, ExtractionOutcome>();
  if (options.deferRangeExtraction) {
    const works = [...uniqueWorks.values()];
    const deferred = await Promise.all(works.map(deferExtraction));
    works.forEach((work, i) => {
      const result = deferred[i];
      if (result) uniqueOutcomes.set(work.dedupeKey, { result });
    });
  }
  const cacheMisses: UniqueExtractionMiss[] = [];
  const worksToLookUp = [...uniqueWorks.values()].filter(
    (work) => !uniqueOutcomes.has(work.dedupeKey),
  );
  await hashContentKeyedSources(worksToLookUp);
  for (const work of worksToLookUp) {
    const lookup = lookupCacheFor(work);
    if ("work" in lookup) {
      cacheMisses.push(lookup);
    } else {
      uniqueOutcomes.set(work.dedupeKey, lookup);
    }
  }

  const supersetPlan = planSupersetGroups(cacheMisses, fps);
  const directOutcomes = await Promise.all(
    supersetPlan.direct.map(
      async (miss) =>
        [miss.work.dedupeKey, await executeDirectMiss(miss)] as [string, ExtractionOutcome],
    ),
  );
  for (const [key, outcome] of directOutcomes) uniqueOutcomes.set(key, outcome);

  const supersetOutcomes = await Promise.all(
    supersetPlan.groups.map((group) => executeSupersetGroup(group)),
  );
  for (const groupOutcomes of supersetOutcomes) {
    for (const [key, outcome] of groupOutcomes) uniqueOutcomes.set(key, outcome);
  }

  const results: ExtractionOutcome[] = [];
  for (const prepared of preparedExtractions) {
    if ("skipped" in prepared) continue;
    if ("error" in prepared) {
      results.push(prepared);
      continue;
    }
    const outcome = uniqueOutcomes.get(prepared.work.dedupeKey);
    if (!outcome) {
      results.push({
        error: extractionError(prepared.work.video.id, "missing extraction result"),
      });
      continue;
    }
    if ("error" in outcome) {
      // A shared (deduped/superset) failure fans out to every element with the
      // same key; annotate followers with the leader's videoId so N copies of
      // one root failure are traceable to a single extraction in traces.
      const isFollower = outcome.error.videoId !== prepared.work.video.id;
      const message = isFollower
        ? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}`
        : outcome.error.error;
      results.push({
        error: {
          videoId: prepared.work.video.id,
          kind: outcome.error.kind,
          retryable: outcome.error.retryable,
          error: message,
        },
      });
      continue;
    }
    results.push({ result: { ...outcome.result, videoId: prepared.work.video.id } });
  }

  breakdown.extractMs = Date.now() - phase3Start;

  // Collect results and errors
  for (const item of results) {
    if ("error" in item && item.error) {
      errors.push(item.error);
    } else if ("result" in item) {
      extracted.push(item.result);
      if (!item.result.deferredRange) totalFramesExtracted += item.result.totalFrames;
    }
  }

  // Sweep when this render wrote something, plus a staleness fallback so a
  // 100%-warm workload (misses never > 0) still reclaims space once a day.
  const sweepDue =
    breakdown.cacheMisses > 0 ||
    (cacheRootDir !== undefined && gcSweepDue(cacheRootDir, GC_STALENESS_MS));
  if (cacheRootDir && sweepDue) {
    const gcStats = gcExtractionCache(cacheRootDir, {
      maxBytes: config?.extractCacheMaxBytes ?? DEFAULT_CONFIG.extractCacheMaxBytes,
      minAgeMs: EXTRACT_CACHE_MIN_AGE_MS,
    });
    breakdown.cacheGcEvictions = gcStats.evictedEntries;
    breakdown.cacheGcBytesFreed = gcStats.evictedBytes;
    breakdown.cacheAgedPartialsCleared = gcStats.agedPartialsRemoved;
  }

  return {
    success: errors.length === 0,
    extracted,
    errors,
    totalFramesExtracted,
    durationMs: Date.now() - startTime,
    phaseBreakdown: breakdown,
  };
}

const BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS = 1;

export function isVideoHiddenBeforeStreamStart(
  videoStreamLeadSeconds: number | undefined,
  mediaTime: number,
): boolean {
  return (
    (videoStreamLeadSeconds ?? 0) - mediaTime >=
    BROWSER_SHOWS_UPCOMING_FIRST_FRAME_WITHIN_SECONDS - 1e-6
  );
}

function getFrameIndexAtTime(
  extracted: ExtractedFrames,
  globalTime: number,
  videoStart: number,
  loop = false,
  mediaStart = 0,
  holdLastFrame = false,
  playbackRate: RateSpec = 1,
): number | null {
  let localTime = Math.max(0, globalTime - videoStart);
  const normalizedPlaybackRate = normalizeRateSpec(playbackRate);
  const loopDuration = timeAtSourceTime(
    normalizedPlaybackRate,
    Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart),
  );
  if (
    typeof normalizedPlaybackRate === "number" &&
    loop &&
    loopDuration > 0 &&
    localTime >= loopDuration
  ) {
    localTime %= loopDuration;
  }
  let sourceTime = sourceTimeAt(normalizedPlaybackRate, localTime);
  if (typeof normalizedPlaybackRate === "object" && loop) {
    // A ramped loop wraps in source space, so the lane keeps running across cycles as in the preview.
    const cycle = Math.max(0, resolvePlayableVideoDuration(extracted.metadata) - mediaStart);
    if (cycle > 0 && sourceTime >= cycle) sourceTime %= cycle;
  }
  // Add epsilon before flooring to avoid IEEE 754 boundary errors where
  // e.g. 0.28 * 25 === 6.999999999999999 instead of 7.
  const frameIndex = Math.floor(sourceTime * extracted.fps + 1e-9);
  if (frameIndex < 0 || extracted.totalFrames <= 0) return null;
  if (
    isVideoHiddenBeforeStreamStart(
      extracted.metadata.videoStreamLeadSeconds,
      mediaStart + sourceTime,
    )
  ) {
    return null;
  }
  if (frameIndex >= extracted.totalFrames) {
    return loop || holdLastFrame ? extracted.totalFrames - 1 : null;
  }
  return frameIndex;
}

export function getFrameAtTime(
  extracted: ExtractedFrames,
  globalTime: number,
  videoStart: number,
  loop = false,
  mediaStart = 0,
): string | null {
  if (!hasClipStarted(globalTime, videoStart)) return null;
  const frameIndex = getFrameIndexAtTime(extracted, globalTime, videoStart, loop, mediaStart);
  return frameIndex == null ? null : extracted.framePaths.get(frameIndex) || null;
}

/**
 * Whether a media source is shorter than its `data-duration` slot by more than
 * the compiler tolerance. The calculation stays tag-agnostic; current in-repo
 * warnings call it for audio only because video slots may intentionally outlive
 * their source and hold the final frame.
 */
export function analyzeClipMediaFit(params: {
  /** Timeline slot length in seconds — `end - start` (a.k.a. data-duration). */
  slotSeconds: number;
  /** Playable source media after the trim offset — `duration - mediaStart`. */
  mediaSeconds: number;
  /** Looping clips repeat to fill the slot, so they never fall short. */
  loop?: boolean;
}): { shortfallSeconds: number; toleranceSeconds: number } | null {
  const { slotSeconds, mediaSeconds, loop } = params;
  if (loop) return null;
  if (!(slotSeconds > 0) || !Number.isFinite(mediaSeconds) || mediaSeconds < 0) return null;
  const toleranceSeconds = MEDIA_DURATION_CLAMP_EPSILON_SECONDS;
  const shortfallSeconds = slotSeconds - mediaSeconds;
  if (shortfallSeconds <= toleranceSeconds) return null;
  return { shortfallSeconds, toleranceSeconds };
}

export class FrameLookupTable {
  private videos: Map<
    string,
    {
      extracted: ExtractedFrames;
      start: number;
      clipWindow: { start: number; end: number };
      mediaStart: number;
      loop: boolean;
      playbackRate: RateSpec;
    }
  > = new Map();
  private orderedVideos: Array<{
    videoId: string;
    extracted: ExtractedFrames;
    start: number;
    clipWindow: { start: number; end: number };
    mediaStart: number;
    loop: boolean;
    playbackRate: RateSpec;
  }> = [];
  private activeVideoIds: Set<string> = new Set();
  private startCursor = 0;
  private lastTime: number | null = null;

  /** The render's `fps`, so a clip shows in exactly export visibility's frames; omit for authored times. */
  constructor(private readonly fps?: number) {}

  addVideo(
    extracted: ExtractedFrames,
    start: number,
    end: number,
    mediaStart: number,
    loop = false,
    playbackRate: RateSpec = 1,
  ): void {
    this.videos.set(extracted.videoId, {
      extracted,
      start,
      clipWindow: this.fps ? exportClipWindow(start, end, this.fps) : { start, end },
      mediaStart,
      loop,
      playbackRate: normalizeRateSpec(playbackRate),
    });
    this.orderedVideos = Array.from(this.videos.entries())
      .map(([videoId, video]) => ({ videoId, ...video }))
      .sort((a, b) => a.clipWindow.start - b.clipWindow.start);
    this.resetActiveState();
  }

  getFrame(videoId: string, globalTime: number): string | null {
    const video = this.videos.get(videoId);
    if (!video) return null;
    if (!isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) return null;
    const frameIndex = getFrameIndexAtTime(
      video.extracted,
      globalTime,
      video.start,
      video.loop,
      video.mediaStart,
      true,
      video.playbackRate,
    );
    return frameIndex == null ? null : video.extracted.framePaths.get(frameIndex) || null;
  }

  frameDirs(): string[] {
    return [...new Set(Array.from(this.videos.values(), (video) => video.extracted.outputDir))];
  }

  private resetActiveState(): void {
    this.activeVideoIds.clear();
    this.startCursor = 0;
    this.lastTime = null;
  }

  private refreshActiveSet(globalTime: number): void {
    // Half-open exportClipWindow; rendered times stay below the composition end, so no terminal hold here.
    if (this.lastTime == null || globalTime < this.lastTime) {
      this.activeVideoIds.clear();
      this.startCursor = 0;
      for (const entry of this.orderedVideos) {
        if (!hasClipStarted(globalTime, entry.clipWindow.start)) break;
        if (isInClipWindow(globalTime, entry.clipWindow.start, entry.clipWindow.end)) {
          this.activeVideoIds.add(entry.videoId);
        }
        this.startCursor += 1;
      }
      this.lastTime = globalTime;
      return;
    }

    while (this.startCursor < this.orderedVideos.length) {
      const candidate = this.orderedVideos[this.startCursor];
      if (!candidate) break;
      if (!hasClipStarted(globalTime, candidate.clipWindow.start)) {
        break;
      }
      if (isInClipWindow(globalTime, candidate.clipWindow.start, candidate.clipWindow.end)) {
        this.activeVideoIds.add(candidate.videoId);
      }
      this.startCursor += 1;
    }

    for (const videoId of Array.from(this.activeVideoIds)) {
      const video = this.videos.get(videoId);
      if (!video || !isInClipWindow(globalTime, video.clipWindow.start, video.clipWindow.end)) {
        this.activeVideoIds.delete(videoId);
      }
    }
    this.lastTime = globalTime;
  }

  getActiveFramePayloads(
    globalTime: number,
  ): Map<string, { framePath: string; frameIndex: number }> {
    const frames = new Map<string, { framePath: string; frameIndex: number }>();
    this.refreshActiveSet(globalTime);
    for (const videoId of this.activeVideoIds) {
      const video = this.videos.get(videoId);
      if (!video) continue;
      const frameIndex = getFrameIndexAtTime(
        video.extracted,
        globalTime,
        video.start,
        video.loop,
        video.mediaStart,
        true,
        video.playbackRate,
      );
      if (frameIndex == null) continue;
      const framePath = video.extracted.framePaths.get(frameIndex);
      if (!framePath) continue;
      frames.set(videoId, { framePath, frameIndex });
    }
    return frames;
  }

  getActiveFrames(globalTime: number): Map<string, string> {
    const payloads = this.getActiveFramePayloads(globalTime);
    const frames = new Map<string, string>();
    for (const [videoId, payload] of payloads) {
      frames.set(videoId, payload.framePath);
    }
    return frames;
  }

  cleanup(): void {
    for (const video of this.videos.values()) {
      // Cache-hit / cache-write entries are owned by the extraction cache —
      // a single render must not delete them, or the next render's lookup
      // would miss and re-extract unnecessarily.
      if (video.extracted.ownedByLookup) continue;
      if (existsSync(video.extracted.outputDir)) {
        rmSync(video.extracted.outputDir, { recursive: true, force: true });
      }
    }
    this.videos.clear();
    this.orderedVideos = [];
    this.resetActiveState();
  }
}

export function createFrameLookupTable(
  videos: VideoElement[],
  extracted: ExtractedFrames[],
  fps?: number,
): FrameLookupTable {
  const table = new FrameLookupTable(fps);
  const extractedMap = new Map<string, ExtractedFrames>();
  for (const ext of extracted) extractedMap.set(ext.videoId, ext);

  for (const video of videos) {
    const ext = extractedMap.get(video.id);
    if (ext) {
      table.addVideo(ext, video.start, video.end, video.mediaStart, video.loop, video.playbackRate);
    }
  }

  return table;
}