Skip to content

Write HTML. Render video. Built for agents.

10.9 KB
Raw Blame
/**
 * Per-clip render-time frame-coverage accounting + threshold fail-loud gate.
 *
 * Sibling to #2474's `hasRuntimeMediaChanges` probe: that PR guarantees the
 * DISCOVERY of runtime-inserted media (so the browser probe launches and
 * reconciles element identity). This module owns the DELIVERY side —
 * for each authored/discovered video clip on the timeline, did the
 * extractor actually produce enough source-video frames to composite the
 * clip's authored `[data-start,data-end]` window? Any clip whose
 * `capturedFrames / expectedFrames` ratio falls below a configurable
 * threshold aborts the render with a `VideoFrameCoverageError` at extract
 * finalization, BEFORE encode produces an MP4 that silently drops the
 * clip's pixels to black.
 *
 * Two field signals defined the failure surface this exists to close
 * (both `#hyperframes-cli-feedback`, both `check`/`snapshot` pass /
 * final MP4 wrong):
 *
 *   • ts=1784139267 · win32/x64 CLI 0.7.58 156s render, 15 injected
 *     videos: several later-injected video clips render BLANK in the
 *     encoded MP4. Injection-count-scaled — 12 injections fail, 4
 *     succeed (workaround: pre-compose into one base timeline). Points
 *     at injector scheduling / worker-seek saturation / extractor
 *     concurrency. Directly covered here — the per-video capture
 *     shortfall lands with `capturedFrames << expectedFrames`.
 *
 *   • ts=1784144554 · darwin/arm64 CLI 0.7.59 3/10 147-clip
 *     composition (130 word-level caption divs): producer left a
 *     subset of authored `[data-start]` div clips permanently visible
 *     in the rendered MP4 while preview/snapshot showed correct
 *     visibility. Authored-clip-count-scaled — 147 clips fails, 3
 *     succeeds. This case runs through `syncTimedElementVisibility`
 *     at runtime and is NOT source-video-frame-shaped; the coverage
 *     gate cannot directly observe it. We surface an
 *     `authoredTimedClipCount` gauge so a 147-clip composition is
 *     visible in telemetry, and leave the per-tick visibility parity
 *     check as follow-up work (a separate runtime observability
 *     channel is required — the extractor doesn't see div visibility).
 *
 * Threshold: default 0.95 (a 5% capture-frame drop is loud); override
 * via `HF_VIDEO_COVERAGE_THRESHOLD` env; disable entirely by setting
 * the env to `0` or a negative number. A threshold of `1` requires
 * exact coverage (no slack for the ffmpeg ±1-frame boundary rounding
 * that legitimately happens on a 29.97 fps timeline).
 */

import { parseHTML } from "linkedom";
import {
  fpsToNumber,
  normalizeRateSpec,
  sourceTimeAt,
  toFps,
  type FpsInput,
} from "@hyperframes/core";
import {
  extractionFrameCountForDuration,
  resolvePlayableVideoDuration,
  type ExtractedFrames,
  type VideoElement,
} from "@hyperframes/engine";

const SHORT_CLIP_ONE_FRAME_TOLERANCE_MIN_EXPECTED_FRAMES = 14;
const SHORT_CLIP_ONE_FRAME_TOLERANCE_MAX_EXPECTED_FRAMES = 20;

export interface VideoFrameCoverageReport {
  videoId: string;
  clipStart: number;
  clipEnd: number;
  expectedFrames: number;
  capturedFrames: number;
  /** `capturedFrames / expectedFrames`; `1` when `expectedFrames === 0` (nothing to cover). */
  ratio: number;
}

/**
 * Discriminant-based error so callers cross-module (producer server,
 * distributed worker) can identify a coverage-gate failure without
 * `instanceof` (which is fragile across duplicated module instances,
 * see `DrawElementVerificationError` for the same problem).
 */
export interface VideoFrameCoverageErrorDetails {
  readonly hyperframesVideoFrameCoverageError: true;
  readonly threshold: number;
  readonly worst: VideoFrameCoverageReport;
  readonly failedReports: VideoFrameCoverageReport[];
}

export class VideoFrameCoverageError extends Error {
  // Read structurally by isVideoFrameCoverageError and cross-module callers.
  // fallow-ignore-next-line unused-class-member
  readonly hyperframesVideoFrameCoverageError = true as const;
  readonly threshold: number;
  readonly worst: VideoFrameCoverageReport;
  readonly failedReports: VideoFrameCoverageReport[];

  constructor(
    message: string,
    details: Omit<VideoFrameCoverageErrorDetails, "hyperframesVideoFrameCoverageError">,
  ) {
    super(message);
    this.name = "VideoFrameCoverageError";
    this.threshold = details.threshold;
    this.worst = details.worst;
    this.failedReports = details.failedReports;
  }
}

export function isVideoFrameCoverageError(err: unknown): err is VideoFrameCoverageError {
  return (
    typeof err === "object" &&
    err !== null &&
    (err as { hyperframesVideoFrameCoverageError?: unknown }).hyperframesVideoFrameCoverageError ===
      true
  );
}

/**
 * Resolve the coverage threshold from `HF_VIDEO_COVERAGE_THRESHOLD` env.
 *
 * Defaults to `0.95`. Values outside `(0, 1]` disable the gate: `0` or
 * negative → off (return `null`); `>1` clamps to `1`. Non-numeric env
 * values fall back to the default (with a caller-side warning if a log
 * is available).
 */
export function resolveVideoCoverageThreshold(
  envValue: string | undefined = process.env.HF_VIDEO_COVERAGE_THRESHOLD,
): number | null {
  if (envValue === undefined) return 0.95;
  const parsed = Number(envValue);
  if (!Number.isFinite(parsed)) return 0.95;
  if (parsed <= 0) return null;
  if (parsed > 1) return 1;
  return parsed;
}

/**
 * Count frames in a clip's authored `[start,end)` window. CFR extraction
 * rounds to the nearest output-frame boundary and VFR extraction rounds up,
 * as `extractionFrameCountForDuration` does. Missing entries
 * retain the fail-closed `ceil` default. Positive sub-frame clips emit one.
 */
export function expectedFramesForClip(
  start: number,
  end: number,
  fps: FpsInput,
  rounding: "ceil" | "nearest" = "ceil",
): number {
  const fpsValue = fpsToNumber(toFps(fps));
  if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(fpsValue)) return 0;
  if (fpsValue <= 0) return 0;
  const duration = Math.max(0, end - start);
  if (duration === 0) return 0;
  return extractionFrameCountForDuration(duration, fps, rounding === "ceil");
}

function isToleratedShortClipBoundaryMiss(
  report: VideoFrameCoverageReport,
  threshold: number,
): boolean {
  return (
    threshold < 1 &&
    report.capturedFrames > 0 &&
    report.expectedFrames >= SHORT_CLIP_ONE_FRAME_TOLERANCE_MIN_EXPECTED_FRAMES &&
    report.expectedFrames <= SHORT_CLIP_ONE_FRAME_TOLERANCE_MAX_EXPECTED_FRAMES &&
    report.expectedFrames - report.capturedFrames === 1
  );
}

function expectedFramesForVideo(
  video: VideoElement,
  entry: ExtractedFrames | undefined,
  fps: FpsInput,
): number {
  const rounding = entry && !entry.metadata.isVFR ? "nearest" : "ceil";
  const slotSourceDuration = sourceTimeAt(
    normalizeRateSpec(video.playbackRate),
    Math.max(0, video.end - video.start),
  );
  const slotFrames = expectedFramesForClip(0, slotSourceDuration, fps, rounding);
  if (!entry) return slotFrames;

  // A short source in a longer slot has a legitimate delivery ceiling of
  // the source portion, not the full slot: a non-looping clip holds its
  // final decoded frame across the tail (#2516/#2606), and a looping clip
  // reuses its full source frame set per repeat (#2665). In both cases the
  // full source *has* been delivered — the same 90 unique source frames
  // cover the 300-frame slot — so coverage must measure source-source, not
  // slot-source.
  const sourceDuration = resolvePlayableVideoDuration(entry.metadata) - video.mediaStart;
  if (!Number.isFinite(sourceDuration) || sourceDuration <= 0) return slotFrames;

  const sourceFrames = expectedFramesForClip(0, sourceDuration, fps, rounding);
  return Math.min(slotFrames, sourceFrames);
}

export function computeVideoFrameCoverage(
  videos: readonly VideoElement[],
  extracted: readonly ExtractedFrames[],
  fps: FpsInput,
): VideoFrameCoverageReport[] {
  const byId = new Map<string, ExtractedFrames>();
  for (const entry of extracted) byId.set(entry.videoId, entry);

  const reports: VideoFrameCoverageReport[] = [];
  for (const video of videos) {
    const entry = byId.get(video.id);
    const expectedFrames = expectedFramesForVideo(video, entry, fps);
    // framePaths is a Map — `size` is the number of distinct captured frames
    // delivered to the runtime injector, which is the load-bearing count
    // (some extractors report a total that includes cache-hit-skipped frames
    // via a stale `totalFrames`, so we trust the delivered-path count).
    const capturedFrames = entry ? entry.framePaths.size : 0;
    const ratio = expectedFrames === 0 ? 1 : capturedFrames / expectedFrames;
    reports.push({
      videoId: video.id,
      clipStart: video.start,
      clipEnd: video.end,
      expectedFrames,
      capturedFrames,
      ratio,
    });
  }
  return reports;
}

/**
 * Throws `VideoFrameCoverageError` when any per-clip ratio is below
 * `threshold`. A `null` threshold disables the gate (env opt-out).
 * A clip with `expectedFrames === 0` (0-duration or non-authored) is
 * unconditionally passing so the gate never fires on a degenerate window.
 */
export function assertVideoFrameCoverage(
  reports: readonly VideoFrameCoverageReport[],
  threshold: number | null,
): void {
  if (threshold === null) return;
  const failed = reports.filter(
    (report) =>
      report.expectedFrames > 0 &&
      report.ratio < threshold &&
      !isToleratedShortClipBoundaryMiss(report, threshold),
  );
  if (failed.length === 0) return;
  // Sort ascending by ratio so the "worst" is first — that's what we cite
  // in the message and pin on the error details for telemetry.
  const sorted = [...failed].sort((a, b) => a.ratio - b.ratio);
  const worst = sorted[0]!;
  const pct = (worst.ratio * 100).toFixed(1);
  const thresholdPct = (threshold * 100).toFixed(1);
  const suffix = sorted.length > 1 ? ` (+${sorted.length - 1} more clip(s) below threshold)` : "";
  throw new VideoFrameCoverageError(
    `Video "${worst.videoId}" captured ${worst.capturedFrames} of expected ${worst.expectedFrames} frames ` +
      `(coverage ${pct}%, threshold ${thresholdPct}%). ` +
      `check/snapshot may pass while the encoded MP4 renders this clip blank, so the render is aborted ` +
      `to prevent shipping a wrong MP4.${suffix} ` +
      `Set HF_VIDEO_COVERAGE_THRESHOLD=0 to disable this gate.`,
    { threshold, worst, failedReports: sorted },
  );
}

/**
 * Count authored `[data-start]` clip windows in the compiled HTML.
 *
 * Not a fail-loud gate — a raw counter that lands in
 * `RenderExtractionObservability.authoredTimedClipCount` so a 147-clip
 * composition is queryable in telemetry (the ts=1784144554 field signal
 * shape). Runtime `syncTimedElementVisibility` iterates the same set at
 * render time; counting statically here is a coarse proxy — dynamic
 * script-inserted `[data-start]` divs land in `hasRuntimeMediaChanges`'s
 * probe path (PR #2474), not this static scan.
 */
export function countAuthoredTimedClips(html: string): number {
  const { document } = parseHTML(html);
  return document.querySelectorAll("[data-start]").length;
}