Skip to content

Write HTML. Render video. Built for agents.

18.6 KB
Raw Blame
import { postRuntimeMessage } from "./bridge";
import { swallow } from "./diagnostics";
import { evictMediaSyncState, HOLD_CAP_MS } from "./media";
import { findInjectedRenderFrame } from "./renderFrameSibling";
import type { RuntimeJson } from "./types";
import { isVideoElement } from "./domRealm";
import { swappedElements } from "./proxySrc";
import { waitForServedProxy } from "./proxyWait";
import { registerSeekCompletion } from "./adapters/seek-dispatch";
import {
  formatPreviewProxyBox,
  PREVIEW_PROXY_BOX_PARAM,
  quantizePreviewProxyBox,
  type PreviewProxyBox,
} from "../previewProxyBox";

/**
 * One entry per project-root-relative asset pathname, injected by the
 * server as `window.__HF_MEDIA_CODEC_MAP__` (see the plan's server-contract
 * KTD). `representativeMime` is a coarse per-codec-family MIME string (e.g.
 * `video/mp4; codecs="hvc1.1.6.L120.B0"` for hevc) fed to `canPlayType`;
 * `null` when ffprobe couldn't produce one, in which case the browser check
 * is skipped and `browserHostile` alone decides.
 */
export type MediaCodecMapEntry = {
  codecName: string;
  browserHostile: boolean;
  representativeMime: string | null;
  /** Source carries an alpha channel and therefore needs a VP8/WebM proxy.
   * Optional so pre-alpha-aware maps stay assignable; absent means "no alpha detected". */
  hasAlpha?: boolean;
};

declare global {
  interface Window {
    __HF_MEDIA_CODEC_MAP__?: Record<string, MediaCodecMapEntry>;
  }
}

const PROXY_QUERY_PARAM = "hf-proxy";

/** Fired whenever an element is swapped to its authoring proxy (any trigger). */
const DIAGNOSTIC_FALLBACK_CODE = "runtime_media_proxy_fallback";
/** Fired when the runtime detects an undecodable video but cannot (or already
 *  did) proxy it — a remote asset, or the proxy URL itself failing. */
const DIAGNOSTIC_UNAVAILABLE_CODE = "runtime_media_proxy_unavailable";

type ProxyTrigger = "proactive" | "reactive" | "tertiary";

// `swappedElements` gates every trigger, so a second undecodable-video signal never re-swaps.

// Elements that already got the "can't help you" diagnostic (cross-origin,
// or the proxy itself failing) — one-shot per element, independent of
// `swappedElements` so the proxy-failed case (which fires AFTER a real swap)
// still gets its own single diagnostic.
const unavailableDiagnosedElements = new WeakSet<HTMLMediaElement>();

// Elements whose swap has started, with the src it started for. Until the copy is served they
// keep that src, so nothing that reads it (timeline, player mirror) loads a copy still being made.
const proxyRequested = new WeakMap<HTMLMediaElement, string | null>();

function currentSrcValue(el: HTMLMediaElement): string {
  return el.currentSrc || el.src;
}

// How large the host shows this document, in its CSS pixels per ours (player fit x page zoom).
// A top-level page shows itself at 1; a framed one waits for its host to say.
let displayScale: number | null = window.parent === window ? 1 : null;
let displayScaleArrived: () => void = () => {};
const displayScaleKnown = new Promise<void>((arrived) => (displayScaleArrived = arrived));
// A host that never says (an older player) gets the source-size copy after this wait.
const DISPLAY_SCALE_WAIT_MS = 1000;

// Elements on a size-bounded copy: the source, the copy's URL and the box it was made for.
type BoxedCopy = { originalSrc: string; variant: string; href: string; box: PreviewProxyBox };
const boxedElements = new Map<HTMLMediaElement, BoxedCopy>();
// The larger copy each element waits for; a newer ask replaces it and the older wait stops.
const pendingUpgrades = new WeakMap<HTMLMediaElement, string>();

function shownScale(): Promise<number | null> {
  if (displayScale !== null) return Promise.resolve(displayScale);
  return Promise.race([
    displayScaleKnown.then(() => displayScale),
    new Promise<null>((give) => setTimeout(() => give(null), DISPLAY_SCALE_WAIT_MS)),
  ]);
}

/** The video's box in device pixels, at least the stage so a later zoom or a clip still hidden fits. */
function shownBox(el: HTMLMediaElement, scale: number): PreviewProxyBox | null {
  // object-fit: none shows the source at its own size.
  if (getComputedStyle(el).objectFit === "none") return null;
  const rect = el.getBoundingClientRect();
  const stage = document.documentElement;
  const pixels = scale * (window.devicePixelRatio || 1);
  return quantizePreviewProxyBox(
    Math.max(rect.width, stage.clientWidth) * pixels,
    Math.max(rect.height, stage.clientHeight) * pixels,
  );
}

/** The host reports how large it shows this document; copies too small for it are replaced. */
export function setProxyDisplayScale(scale: number): void {
  if (!(scale > 0) || !Number.isFinite(scale)) return;
  displayScale = scale;
  displayScaleArrived();
  for (const el of boxedElements.keys()) keepSized(el, scale);
}

function keepSized(el: HTMLMediaElement, scale: number): void {
  const copy = boxedElements.get(el);
  if (!copy) return;
  if (!el.isConnected || el.getAttribute("src") !== copy.href) {
    boxedElements.delete(el);
    return;
  }
  const needed = shownBox(el, scale);
  if (needed && needed.width <= copy.box.width && needed.height <= copy.box.height) return;
  // Never smaller on either side than the copy it replaces.
  const box = needed && {
    width: Math.max(needed.width, copy.box.width),
    height: Math.max(needed.height, copy.box.height),
  };
  const href = appendProxyParam(copy.originalSrc, copy.variant, box);
  if (pendingUpgrades.get(el) === href) return;
  pendingUpgrades.set(el, href);
  const wanted = () =>
    pendingUpgrades.get(el) === href && el.isConnected && el.getAttribute("src") === copy.href;
  void waitForServedProxy(href, wanted).then((served) => {
    if (served && wanted()) loadProxy(el, href, copy.originalSrc, copy.variant, box);
    if (pendingUpgrades.get(el) === href) pendingUpgrades.delete(el);
  });
}

function loadProxy(
  el: HTMLMediaElement,
  href: string,
  originalSrc: string,
  variant: string,
  box: PreviewProxyBox | null,
): void {
  if (box) boxedElements.set(el, { originalSrc, variant, href, box });
  else boxedElements.delete(el);
  // Sync state measured on the previous file would read the new file's buffering as drift.
  evictMediaSyncState(el);
  el.src = href;
  el.load();
}

/**
 * Render mode never proxies: the codec map arrives by injection (not a
 * fetch, so determinism holds) but render always decodes the original via
 * FFmpeg frame extraction, never a browser-side element. Mirrors the two
 * render-mode signals already used elsewhere in the runtime — the global
 * export-seek config (init.ts) and the per-element `__render_frame_<id>__`
 * sibling image the producer's frame-injection pipeline creates during
 * render (the same check `syncRuntimeMedia`'s `skipForInjectedVideo` makes
 * in media.ts).
 */
export function isRenderMode(el: HTMLMediaElement): boolean {
  if (window.__HF_EXPORT_RENDER_SEEK_CONFIG) return true;
  return isVideoElement(el) && !!findInjectedRenderFrame(el);
}

/**
 * Resolve an element's codec-map key: the served-root-relative,
 * percent-decoded, query-string-stripped URL pathname (per the server
 * contract KTD). Returns null when there's no usable src, or the src is
 * cross-origin — the server can't have scanned (or proxy) a file it doesn't
 * host, so there is never a map entry to look up.
 */
export function deriveCodecMapKey(el: HTMLMediaElement): string | null {
  const raw = currentSrcValue(el);
  if (!raw) return null;
  let url: URL;
  try {
    url = new URL(raw, document.baseURI);
  } catch {
    return null;
  }
  if (url.origin !== window.location.origin) return null;
  try {
    return decodeURIComponent(url.pathname);
  } catch {
    return url.pathname;
  }
}

/**
 * The composition is served under a route prefix on some surfaces —
 * studio-server's preview route injects `<base href="/api/projects/:id/preview/">`
 * (packages/studio-server/src/routes/preview.ts) so every relative asset
 * resolves through that prefix — while the codec map (per the server
 * contract) is keyed by project-root-relative pathnames with no prefix.
 * `play` and the CLI's static project server serve from the root with no
 * such prefix, where an exact match already succeeds. So: try exact first
 * (covers unprefixed servers), then fall back to the longest map key that
 * is a suffix of the pathname. No separate segment-boundary check is
 * needed: every key is contractually root-relative and leading-slash
 * (`"/assets/x.mp4"`, never `"assets/x.mp4"`), so a suffix match's boundary
 * is always that leading `/` itself — `pathname.endsWith(key)` can't match
 * a partial segment (e.g. key `"/foo.mp4"` can never match a pathname
 * ending in `"/notfoo.mp4"`, since that would require the literal substring
 * `"/foo.mp4"` to appear where the last `/` already fell a segment later).
 */
function lookupLongestExactSuffix(
  pathname: string,
  map: Record<string, MediaCodecMapEntry>,
): MediaCodecMapEntry | null {
  let bestKey: string | null = null;
  for (const key of Object.keys(map)) {
    if (!pathname.endsWith(key)) continue;
    if (bestKey === null || key.length > bestKey.length) bestKey = key;
  }
  return bestKey ? (map[bestKey] ?? null) : null;
}

function lookupNormalizedSuffix(
  pathname: string,
  map: Record<string, MediaCodecMapEntry>,
): MediaCodecMapEntry | null {
  // Case-insensitive filesystems and Unicode-normalizing filesystems can
  // serve an authored URL whose spelling differs from the canonical path
  // ffprobe returned (Clip.mp4 vs clip.mp4, NFC vs NFD). Exact matching stays
  // authoritative; this normalized fallback is used only when it finds one
  // unambiguous longest suffix, so case-sensitive projects containing both
  // spellings never select the wrong asset.
  const normalizedPathname = pathname.normalize("NFC").toLowerCase();
  let normalizedBestLength = -1;
  let normalizedBest: MediaCodecMapEntry | null = null;
  let ambiguous = false;
  for (const [key, entry] of Object.entries(map)) {
    const normalizedKey = key.normalize("NFC").toLowerCase();
    if (!normalizedPathname.endsWith(normalizedKey)) continue;
    if (normalizedKey.length > normalizedBestLength) {
      normalizedBestLength = normalizedKey.length;
      normalizedBest = entry;
      ambiguous = false;
    } else if (normalizedKey.length === normalizedBestLength) {
      ambiguous = true;
    }
  }
  return ambiguous ? null : normalizedBest;
}

function lookupCodecMapEntry(
  pathname: string,
  map: Record<string, MediaCodecMapEntry>,
): MediaCodecMapEntry | null {
  return (
    map[pathname] ??
    lookupLongestExactSuffix(pathname, map) ??
    lookupNormalizedSuffix(pathname, map)
  );
}

function appendProxyParam(src: string, variant: string, box: PreviewProxyBox | null): string {
  const url = new URL(src, document.baseURI);
  url.searchParams.set(PROXY_QUERY_PARAM, variant);
  if (box) url.searchParams.set(PREVIEW_PROXY_BOX_PARAM, formatPreviewProxyBox(box));
  return url.href;
}

type UnavailableReason =
  | "cross_origin"
  | "proxy_playback_failed"
  | "browser_safe_codec"
  | "invalid_source_url";

const UNAVAILABLE_NOTES: Record<UnavailableReason, string> = {
  cross_origin:
    "video reports zero decodable width but its source is cross-origin; no local proxy can be served for it",
  proxy_playback_failed: "the authoring proxy itself failed to decode; render output is unaffected",
  browser_safe_codec:
    "the file errored but its codec is browser-decodable; a proxy cannot help (the file itself is likely corrupt)",
  invalid_source_url: "the media source URL is malformed and cannot be proxied",
};

function emitUnavailableDiagnostic(
  el: HTMLMediaElement,
  reason: UnavailableReason,
  asset: string,
): void {
  if (unavailableDiagnosedElements.has(el)) return;
  unavailableDiagnosedElements.add(el);
  const note = UNAVAILABLE_NOTES[reason];
  const details: Record<string, RuntimeJson> = {
    asset,
    codecName: null,
    reason,
    note,
  };
  postRuntimeMessage({
    source: "hf-preview",
    type: "diagnostic",
    code: DIAGNOSTIC_UNAVAILABLE_CODE,
    details,
  });
  // Mirrors swapToProxy's fallback line: the stable diagnostic code is in the
  // text so checkBrowser.ts's console scraper can match a token, not prose.
  console.info(`[hyperframes] ${DIAGNOSTIC_UNAVAILABLE_CODE}: "${asset}" (${reason}): ${note}`);
}

/**
 * Swap `el` to its alpha-aware proxy URL, evict stale per-source sync state, and
 * emit the one-time diagnostic + console line. Safe to call from any of the
 * three triggers (proactive/reactive/tertiary); a no-op if already swapped.
 */
export function swapToProxy(
  el: HTMLMediaElement,
  entry: MediaCodecMapEntry | null = null,
  trigger: ProxyTrigger = "reactive",
): void {
  if (swappedElements.has(el)) return;
  if (proxyRequested.has(el) && proxyRequested.get(el) === el.getAttribute("src")) return;
  const originalSrc = currentSrcValue(el);
  const variant = entry ? (entry.hasAlpha ? "vp8" : "h264") : "auto";
  try {
    new URL(originalSrc, document.baseURI);
  } catch (err) {
    swallow("runtime.mediaProxy.swap", err);
    emitUnavailableDiagnostic(el, "invalid_source_url", originalSrc);
    return;
  }
  const originalAttr = el.getAttribute("src");
  proxyRequested.set(el, originalAttr);
  const live = () => el.isConnected && el.getAttribute("src") === originalAttr;
  const swap = shownScale().then(async (scale) => {
    // Asked once the shown size is known, so the copy is made at that size and only once.
    const box = scale === null ? null : shownBox(el, scale);
    const proxiedSrc = appendProxyParam(originalSrc, variant, box);
    const served = await waitForServedProxy(proxiedSrc, live);
    if (!live()) {
      if (proxyRequested.get(el) === originalAttr) proxyRequested.delete(el);
      return;
    }
    if (!served) {
      emitUnavailableDiagnostic(el, "proxy_playback_failed", originalSrc);
      return;
    }
    swappedElements.set(el, originalAttr);
    loadProxy(el, proxiedSrc, originalSrc, variant, box);
    // The shown size may have grown while this copy was being made.
    if (displayScale !== null) keepSized(el, displayScale);
    return new Promise((landed) => {
      el.addEventListener("loadeddata", landed, { once: true });
      el.addEventListener("error", landed, { once: true });
    });
  });
  // A frame capture holds for the copy as it held for a loading video before, up to the same cap.
  registerSeekCompletion(
    Promise.race([swap, new Promise((cap) => setTimeout(cap, HOLD_CAP_MS))]),
    "video-frame",
  );
  const codecName = entry?.codecName ?? null;
  const details: Record<string, RuntimeJson> = {
    asset: originalSrc,
    codecName,
    trigger,
    note: "render output is unaffected; only this preview element was swapped to an authoring proxy",
  };
  postRuntimeMessage({
    source: "hf-preview",
    type: "diagnostic",
    code: DIAGNOSTIC_FALLBACK_CODE,
    details,
  });
  // The diagnostic code doubles as the stable token check's console scraper
  // matches on (packages/cli/src/utils/checkBrowser.ts); keep it in the text.
  console.info(
    `[hyperframes] ${DIAGNOSTIC_FALLBACK_CODE}: "${originalSrc}" uses a codec (${codecName ?? "unknown"}) this browser can't decode; ` +
      "auto-swapped to an authoring proxy for this preview only. Render output is unaffected.",
  );
}

/**
 * Proactive trigger: consult the codec map before an element's first
 * `load()` and swap ahead of time when the browser is known-unlikely to
 * decode it, avoiding an error flash. `<audio>` is never proxied (per the
 * plan's KTD — an HEVC container's AAC track demuxes fine regardless of the
 * browser's video codec support).
 */
export function maybeProxyProactively(el: HTMLMediaElement): void {
  if (isRenderMode(el)) return;
  if (!isVideoElement(el)) return;
  if (swappedElements.has(el)) return;
  const map = window.__HF_MEDIA_CODEC_MAP__;
  if (!map) return;
  const key = deriveCodecMapKey(el);
  if (key === null) return;
  const entry = lookupCodecMapEntry(key, map);
  if (!entry || !entry.browserHostile) return;
  const canPlay = entry.representativeMime ? el.canPlayType(entry.representativeMime) : "";
  if (canPlay === "probably" || canPlay === "maybe") return;
  swapToProxy(el, entry, "proactive");
}

/**
 * Reactive trigger (the primary catch-all per the plan's KTD): a `<video>`
 * reporting `videoWidth === 0` at `loadedmetadata` has an undecodable (or
 * absent) video track — the common case for an HEVC+AAC file, which fires
 * no error event because the AAC track satisfies the demuxer. Swaps once
 * per element; if the element is already on its proxy src, the proxy
 * itself is the one failing, so this only emits the failure diagnostic.
 *
 * No map, no swaps: the codec map global is only injected on surfaces where
 * auto-proxying is enabled and served, so its absence means a `?hf-proxy=`
 * request would 404 — never swap there. When the map is present but has no
 * entry for this key, swapping stays allowed (unlisted-asset rescue). A
 * mapped alpha entries select the VP8/WebM proxy variant.
 */
export function handleMetadataForProxy(el: HTMLMediaElement): void {
  if (isRenderMode(el)) return;
  if (!isVideoElement(el)) return;
  if (el.videoWidth !== 0) return;
  const src = currentSrcValue(el);
  if (swappedElements.has(el)) {
    emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
    return;
  }
  const map = window.__HF_MEDIA_CODEC_MAP__;
  if (!map) return;
  const key = deriveCodecMapKey(el);
  if (key === null) {
    emitUnavailableDiagnostic(el, "cross_origin", src);
    return;
  }
  const entry = lookupCodecMapEntry(key, map);
  if (entry && !entry.browserHostile) {
    emitUnavailableDiagnostic(el, "browser_safe_codec", src);
    return;
  }
  swapToProxy(el, entry, "reactive");
}

/**
 * Tertiary trigger: the `error` event, for the rarer zero-decodable-stream
 * file (video-only hostile codec) where the demuxer has nothing to satisfy
 * it and `loadedmetadata` never fires. Same guards and once-per-element
 * behavior as the reactive path, plus one extra skip: an entry the scan
 * mapped as browser-SAFE that still errors is a corrupt-but-safe file — an
 * proxy of a broken source can't help, so only diagnose.
 */
export function handleErrorForProxy(el: HTMLMediaElement): void {
  if (isRenderMode(el)) return;
  if (!isVideoElement(el)) return;
  const src = currentSrcValue(el);
  if (swappedElements.has(el)) {
    emitUnavailableDiagnostic(el, "proxy_playback_failed", src);
    return;
  }
  const map = window.__HF_MEDIA_CODEC_MAP__;
  if (!map) return;
  const key = deriveCodecMapKey(el);
  if (key === null) {
    emitUnavailableDiagnostic(el, "cross_origin", src);
    return;
  }
  const entry = lookupCodecMapEntry(key, map);
  if (entry && !entry.browserHostile) {
    emitUnavailableDiagnostic(el, "browser_safe_codec", src);
    return;
  }
  swapToProxy(el, entry, "tertiary");
}