Skip to content

Write HTML. Render video. Built for agents.

18.4 KB
Raw Blame
/**
 * @hyperframes/engine — Protocol Types
 *
 * The engine's page contract. Any web page that wants to be rendered
 * as video must expose `window.__hf` implementing the HfProtocol interface.
 */
import type { AudioFailureDiagnostic } from "./services/audioMixer.types.js";
import type { Fps } from "@hyperframes/core";
import type { ChromeMemoryStats } from "./services/chromeMemorySampler.js";
import type { MotionBlurOptions } from "./services/motionBlur.js";

export type PageMotionBlurOff = "off" | "hdr-render" | "layered-composite" | "distributed-render";

/**
 * Outcome of waiting for a sub-composition's GSAP timelines to register.
 * Threaded string-typed through `CapturePerfSummary` / `RenderPerfSummary` /
 * telemetry so a single alias keeps the values in sync end-to-end.
 */
export type SubTimelineWaitOutcome = "ready" | "timeout" | "script_failure";

/**
 * Shared by every capture session of one render: composition ids whose timeline
 * wait already timed out, so later sessions report the timeout without waiting again.
 */
export interface SubTimelineWaitMemo {
  unregisteredIds?: readonly string[];
}

export type CaptureWarningCode =
  | "media_readiness_timeout"
  | "media_load_failed"
  | "audio_processing_failed"
  | "sub_timeline_readiness_timeout"
  | "sub_timeline_script_failure"
  | "vfx_failure"
  | "live_map_detected";

/** Structured correctness warning produced while preparing a capture session. */
export interface CaptureWarning {
  code: CaptureWarningCode;
  message: string;
  details?: {
    mediaType?: "image" | "video" | "audio";
    sources?: string[];
    pendingCompositionIds?: string[];
    timeoutMs?: number;
    audioFailures?: AudioFailureDiagnostic[];
    failureReasons?: string[];
    failureStages?: string[];
    failureOwner?: "user" | "system";
    retryable?: boolean;
  };
}

// ── Seek Protocol ──────────────────────────────────────────────────────────────

/**
 * Declares a media element the engine should handle.
 *
 * Headless Chrome in BeginFrame mode cannot play <video> or produce audio.
 * The engine pre-extracts video frames and audio tracks from declared media
 * elements and handles injection/mixing automatically.
 */
export interface HfMediaElement {
  /** DOM id of the <video> or <audio> element */
  elementId: string;
  /** Source file path or URL */
  src: string;
  /** When in the composition this element appears (seconds) */
  startTime: number;
  /** When in the composition this element disappears (seconds) */
  endTime: number;
  /** Offset into the source file (seconds, default: 0) */
  mediaOffset?: number;
  /** Audio volume 0-1 (default: 1) */
  volume?: number;
  /** Whether this element has audio that should be extracted */
  hasAudio?: boolean;
}

/**
 * Metadata for a shader transition between two scenes.
 *
 * Compositions using @hyperframes/shader-transitions populate
 * `window.__hf.transitions` with one entry per transition so the
 * producer can pre-compute scene ranges, capture per-scene buffers,
 * and apply the transition in HDR-aware compositing.
 */
export interface HfTransitionMeta {
  /** Time the transition starts (seconds) */
  time: number;
  /** Transition duration (seconds) */
  duration: number;
  /** Shader identifier. Undefined when the transition is a CSS crossfade. */
  shader?: string;
  /** GSAP easing string (e.g. "power2.inOut") */
  ease: string;
  /** Scene id the transition starts from */
  fromScene: string;
  /** Scene id the transition ends on */
  toScene: string;
}

/**
 * The seek protocol. The only contract between the engine and a page.
 *
 * The engine reads `duration` to calculate total frames, calls `seek(time)`
 * before each frame capture, and uses `media` (if provided) to handle
 * video frame injection and audio mixing.
 *
 * The engine does NOT care what animation framework drives the page.
 * GSAP, Framer Motion, CSS animations, Three.js — anything works as long
 * as `seek()` produces deterministic visual output for a given time.
 */
/**
 * Per-seek controls the page honours. Both default off, which is an ordinary frame seek.
 *
 * `suppressEvents` stops a composition's own timeline callbacks from firing, and
 * `subFrameDivisions` refines the grid the page quantizes onto so a fractional time is
 * not floored back onto the output frame. Motion-blur sampling sets both.
 */
export interface HfSeekOptions {
  suppressEvents?: boolean;
  subFrameDivisions?: number;
}

export interface HfProtocol {
  /** Total duration of the composition in seconds */
  duration: number;
  /** Seek to a specific time. Must produce deterministic visual output. */
  seek(time: number, options?: HfSeekOptions): void;
  /** Optional: media elements the engine should handle */
  media?: HfMediaElement[];
  /** Optional: shader transition metadata, populated by @hyperframes/shader-transitions */
  transitions?: HfTransitionMeta[];
}

// ── Capture Types ──────────────────────────────────────────────────────────────

export interface CaptureOptions {
  width: number;
  height: number;
  /**
   * Producer-resolved composition duration (seconds) — the data-duration
   * clamp actually rendered, which can differ from the page's raw
   * `__hf.duration` (infinite-repeat GSAP timelines report a huge sentinel;
   * timelines can outrun their declared duration). Consumers that derive
   * frame indices meant to be drained by the producer (drawElement
   * self-verification) MUST prefer this over `__hf.duration`.
   */
  compositionDurationSeconds?: number;
  /** The composition declares `data-requires-webgpu`; skips createCaptureSession's own fetch. */
  requiresWebGpu?: boolean;
  /**
   * Live Chrome memory samples during capture (browser/renderer RSS peaks,
   * last total, GPU process presence). Invoked from an unref'd interval; must
   * not throw. The producer forwards these to capture observability so a
   * crash mid-render still reports the last known memory state.
   */
  onMemorySample?: (stats: ChromeMemoryStats) => void;
  /**
   * Frame rate as an exact rational. Integer fps is `{ num: 30, den: 1 }`;
   * NTSC is `{ num: 30000, den: 1001 }`. Captures are scheduled by the
   * decimal interval (1000 * den / num ms) but FFmpeg arg builders emit the
   * rational form verbatim — see `fpsToFfmpegArg`.
   */
  fps: Fps;
  format?: "jpeg" | "png";
  quality?: number;
  /**
   * Opt into sub-frame multi-sample motion blur (issue #4010). Absent means off and the
   * capture path is byte-identical to a render without it. Requires `format: "png"` and
   * screenshot capture mode; see `services/motionBlur.ts`.
   */
  motionBlur?: MotionBlurOptions;
  /** Why the page's capture motion blur stands down: the preview could not blur, or the route cannot host it. */
  pageMotionBlur?: PageMotionBlurOff;
  deviceScaleFactor?: number;
  /**
   * Opt into Chrome's capture-beyond-viewport screenshot path. Leave undefined
   * to let the engine pick the safe browser-specific default. Pass false only
   * when the caller explicitly wants Chrome's faster viewport-bound path.
   * Enable for known compositor edge cases such as native video surfaces in
   * tall portrait renders.
   */
  captureBeyondViewport?: boolean;
  /**
   * FFmpeg-probed intrinsic dimensions for videos whose frames are injected
   * out-of-band. Applied before the readiness wait so layout that depends on
   * video aspect ratio (e.g. `height:auto`) stays stable even if Chromium never
   * loads native metadata.
   */
  videoMetadataHints?: readonly CaptureVideoMetadataHint[];
  /**
   * Video element IDs to exclude from the in-page readiness check that waits
   * for `video.readyState >= 1` before capture starts.
   *
   * Use for videos whose frames are supplied out-of-band, including standard
   * FFmpeg frame injection and native HDR extraction. Pair with
   * `videoMetadataHints` for any skipped video whose CSS layout may depend on
   * intrinsic media dimensions.
   */
  skipReadinessVideoIds?: readonly string[];
  /** Pass one object to every session of a render; see `SubTimelineWaitMemo`. */
  subTimelineWaitMemo?: SubTimelineWaitMemo;
  /**
   * Render-time variable overrides for the composition. The engine injects
   * these as `window.__hfVariables` via `evaluateOnNewDocument` before any
   * page script runs, so the runtime helper `getVariables()` returns the
   * merged result of declared defaults (`data-composition-variables`) and
   * these overrides on its first call.
   *
   * The CLI populates this from `--variables '<json>'` /
   * `--variables-file <path>`. Must be a JSON-serializable plain object.
   */
  variables?: Record<string, unknown>;
  /**
   * When `true`, the BeginFrame warmup loop driven during page navigation
   * runs exactly `LOCKED_WARMUP_TICKS` (60) iterations regardless of how
   * long the page load takes, making `session.beginFrameTimeTicks`
   * deterministic across machines with different page-load latencies.
   *
   * Default `false`: wall-clock-bounded driver — ticks until page-readiness
   * completes, accumulating whatever count the host CPU manages. Preserves
   * the in-process renderer's BeginFrame timing baselines.
   *
   * Has no effect outside BeginFrame mode (screenshot capture never runs a
   * warmup loop).
   */
  lockWarmupTicks?: boolean;
  /**
   * drawElement self-verify ground-truth sample count for this session.
   * Overrides the HF_DE_VERIFY default (4). The parallel coordinator raises
   * it for multi-worker drawElement capture — N concurrent hardware-GPU
   * browsers widen the damage surface (compositor tile eviction under
   * GPU/memory pressure), and each worker only drains ~1/N of the shared
   * sample grid, thinning effective coverage exactly when risk peaks.
   * Clamped to 0..8 like the env knob; HF_DE_VERIFY, when set, still wins.
   */
  deVerifySamples?: number;
}

export interface CaptureVideoMetadataHint {
  id: string;
  width: number;
  height: number;
}

export interface CaptureResult {
  frameIndex: number;
  time: number;
  path: string;
  captureTimeMs: number;
}

export interface CaptureBufferResult {
  buffer: Buffer;
  captureTimeMs: number;
}

export interface CapturePerfSummary {
  frames: number;
  avgTotalMs: number;
  avgSeekMs: number;
  avgBeforeCaptureMs: number;
  avgScreenshotMs: number;
  /**
   * Median per-frame capture time — warmup-robust, unlike `avgTotalMs`
   * (first frames pay font/image decode + GC that swamps short renders'
   * averages). Basis for in-the-wild speedup estimates. 0 when no frames.
   */
  p50TotalMs: number;
  /**
   * 95th-percentile per-frame capture time (nearest-rank). Emitted alongside
   * p50 so the fast-capture-fallback-profile diagnostic (opt-in via
   * `HF_PROFILE_FALLBACK_CAPTURE=true`) can distinguish "steady-state slow"
   * from "long-tail spikes"; the p50 alone hides the tail on any sample set
   * with a heavy right-skew (typical of screenshot capture on GC pauses or
   * paint-heavy frames). 0 when no frames.
   */
  p95TotalMs: number;
  /**
   * 99th-percentile per-frame capture time (nearest-rank). Same purpose as
   * `p95TotalMs` — the extreme-tail counterpart, useful for characterizing
   * the WORST frame you're likely to encounter on the fallback path. 0 when
   * no frames.
   */
  p99TotalMs: number;
  /** Sub-composition timeline wait outcome (absent pre-init). */
  subTimelineWaitOutcome?: SubTimelineWaitOutcome;
  /**
   * Session init telemetry, mirrored from the `[FrameCapture:INIT]` console
   * line so PARALLEL workers report it too: worker sessions' console buffers
   * only propagate to the orchestrator on failure, which left the
   * multi-worker path — the short-comp band's entire population — with 0%
   * coverage of the motion axis (`observability_init_tween_count`) in fleet
   * telemetry. Riding the perf summary reuses the one channel that already
   * flows back per worker on success.
   */
  initDurationMs?: number;
  /** GSAP tween count at init — the motion-axis signal for capture routing analysis. */
  initTweenCount?: number;
  /**
   * Live DOM element count at end of init; undefined when the measurement
   * failed (never 0 — see collectSessionInitTelemetry).
   *
   * WHICH FIELD TO QUERY — two element counts exist and they answer
   * different questions:
   *   • `composition_element_count` (+ `_source`) is the ROUTING-RELEVANT
   *     one. Measured from the PROBE session before the routing decision,
   *     falling back to a static source scan. That is what the short-comp
   *     band actually gates on.
   *   • `observability_init_element_count` (this field) is the
   *     OBSERVATIONAL counterpart. Measured from the capture session's own
   *     DOM at end of init, on every surviving render — including the ~83%
   *     with no probe, where the routing signal is a blind static scan.
   * They agree for most comps and diverge for one that mutates its DOM
   * between probe launch and capture init. Use this for distribution and
   * tail questions; use `composition_element_count` for anything about what
   * the router did (review finding).
   */
  initElementCount?: number;
  /** Correctness warnings observed before or during capture. */
  warnings?: CaptureWarning[];
  /**
   * Frames served from the static-dedup cache instead of a real seek+screenshot
   * (opt-out HF_STATIC_DEDUP=false). 0 when dedup was off or never armed. NOT counted
   * in `frames` (reuses are excluded so they don't dilute the per-frame
   * averages) — the captured total this session is `frames + staticDedupReused`.
   */
  staticDedupReused: number;
  /** `HF_STATIC_DEDUP=true` was set for this render (adoption signal). */
  staticDedupEnabled: boolean;
  /** Dedup passed every gate + verification and was active. */
  staticDedupArmed: boolean;
  /** Original predicted reusable frame count before profitability/budget filtering. */
  staticDedupPredicted: number;
  /** Frames retained after complete-run verification. */
  staticDedupVerified?: number;
  /** Bounded verifier result taxonomy. */
  staticDedupVerificationOutcome?:
    | "verified"
    | "unprofitable"
    | "time_budget"
    | "count_budget"
    | "mismatch"
    | "infrastructure";
  staticDedupVerificationPlannedRuns?: number;
  staticDedupVerificationCompletedRuns?: number;
  staticDedupVerificationScreenshots?: number;
  staticDedupVerificationSeeks?: number;
  staticDedupVerificationComparisons?: number;
  staticDedupVerificationElapsedMs?: number;
  /**
   * Low-cardinality reason dedup did not arm: `capture_mode` | `video_injection`
   * | `page_composite` | `ineligible` | `unprofitable` | `verification_failed` | `verification_budget`.
   * Undefined when armed or when dedup was disabled. (Render-level aggregation may
   * `|`-join distinct reasons when parallel workers diverge.)
   */
  staticDedupSkipReason?: string;
  // ── BeginFrame no-damage reuse (Linux/Docker lastFrameCache visibility) ──
  /**
   * BeginFrame frames where Chrome reported `hasDamage=false` and the previous
   * buffer was reused from the per-page lastFrameCache (screenshotService.ts) —
   * the BF counterpart of `staticDedupReused` (predictive dedup never arms
   * under beginframe). Undefined/0 outside beginframe capture mode.
   */
  beginFrameNoDamage?: number;
  /** BeginFrame frames where Chrome reported damage (fresh screenshot encoded). */
  beginFrameHasDamage?: number;
  // ── drawElement fast-capture outcome (default-on release visibility) ──
  /** Final capture mode this session used: "drawelement" | "screenshot" | "beginframe". */
  captureMode: string;
  /**
   * Low-cardinality GPU bucket from DE session init: `<backend>/<vendor>`
   * (e.g. `metal/apple`, `d3d11/nvidia`). Undefined when drawElement was
   * never attempted. Lets telemetry cluster backend-specific damage now that
   * DE engages on both Metal (darwin) and D3D11 (win32). Bucketed, not raw —
   * see `classifyGpuRenderer`.
   */
  gpuRenderer?: string;
  // ── Chrome process memory (spec: long-form render capture, Phase −1).
  // Undefined when the sampler was disabled (HF_CHROME_MEMORY_SAMPLER=false)
  // or never produced a successful sample. ──
  chromeBrowserRssPeakMb?: number;
  chromeRendererRssPeakMb?: number;
  chromeRssLastMb?: number;
  chromeGpuProcessSeenLastSample?: boolean;
  chromeMemorySamples?: number;
  /**
   * Low-cardinality init-time gate that routed a drawElement-eligible session
   * to the baseline: `swiftshader` | `css_effect:<fx>` | `at_risk_timeline` |
   * `3d_init_failed` | `supersampling` | `render_mode_hint`. Undefined when
   * drawElement ran or was never attempted.
   */
  deGateReason?: string;
  /**
   * Full-fidelity fallback trigger — the specific reason (CSS FX + property,
   * e.g. `filter:blur`, `filter:drop-shadow`, `backdrop-filter`, `clip-path`,
   * or an unsanitized gate name like `at_risk_timeline`, `swiftshader`,
   * `unsupported_chrome`, `render_mode_hint`, `supersampling`, `3d_init_failed`)
   * that gated drawElement off. Complementary to {@link deGateReason}, which
   * sanitizes down to a low-cardinality bucket for aggregation; this field
   * keeps the specific CSS FX so the `capture_fallback_profile` observability
   * checkpoint can characterize per-frame perf by the exact trigger. Undefined
   * when drawElement ran (no fallback) or was never attempted.
   */
  deFallbackTrigger?: string;
  /** Worker-encode pipeline active (the drain that runs self-verification). */
  deWorkerEncode: boolean;
  /** Self-verification ground-truth samples armed at init (0 = verification off/skipped). */
  deVerifyArmed: number;
  /** Wall-clock cost of capturing the ground-truth samples at init. */
  deVerifyInitMs: number;
  /** Clip-cut boundary frames routed to per-frame screenshot (Lim 6). */
  deBoundaryFrames: number;
  /** Per-frame "No cached paint record" screenshot fallbacks during capture. */
  deNcprFallbacks: number;
  /**
   * Per-frame drawElement captures that blew the `HF_DE_FRAME_TIMEOUT_MS`
   * deadline (renderer stopped scheduling after drawElementImage returned —
   * PRINFRA-488). Each timeout aborts that attempt so the producer can retry the
   * whole render on a fresh screenshot session.
   */
  deFrameTimeouts: number;
}

// ── Global Augmentation ────────────────────────────────────────────────────────

declare global {
  interface Window {
    __hf?: HfProtocol;
  }
}