Skip to content

Write HTML. Render video. Built for agents.

14.0 KB
Raw Blame
import { HF_AUDIO_AUTOMATION_ATTR } from "../audioAutomation.js";
import { HF_AUDIO_FX_ATTR } from "../audioFx.js";
import { HF_AUDIO_GROUP_ATTR } from "../audioGroups.js";
import { postRuntimeMessage } from "./bridge";
import type { RuntimeJson } from "./types";

/**
 * Which transport may claim an `<audio>` or audible `<video>` element's output.
 *
 * `createMediaElementSource()` is a one-way door, and over a resource that fails
 * the CORS-cross-origin check it outputs SILENCE without throwing (#3458). So the
 * verdict is reached BEFORE the call by this pure classifier, shared by media
 * discovery (diagnostic) and scheduling (withholding the node). Callers that build
 * their own `AudioContext` (e.g. `AudioRow.tsx`) must route through it too.
 * Known gaps: `srcObject` elements always read `web-audio`, and the RAW url is
 * judged, not a redirect chain's final target.
 */
export type WebAudioMediaRoute =
  /** Same-origin, CORS-opted-in, or a scheme the check doesn't apply to. */
  | { kind: "web-audio" }
  /**
   * Cross-origin without a `crossorigin` opt-in. MediaElementSource would be
   * silent, but `fetch` + `decodeAudioData` may still succeed — a CDN that
   * sends `Access-Control-Allow-Origin` while the author simply never wrote the
   * attribute is the common shape of this bug — and that route keeps the whole
   * FX graph. So: withhold the node, still let the decode path try.
   */
  | { kind: "decode-only"; reason: WebAudioSilenceReason; asset: string };

/**
 * Why capture was withheld. The two cases need DIFFERENT remediation, which is
 * the whole reason they are separate values: a foreign asset can be proxied to
 * a same-origin URL, but an opaque document has no same-origin URL to proxy to.
 */
export type WebAudioSilenceReason =
  /** The asset's origin differs from ours, and no `crossorigin` opt-in is set. */
  | "cross_origin_no_cors"
  /** OUR document's security origin is opaque, so nothing is same-origin with it. */
  | "opaque_document_origin";

/** Fired when the runtime withholds Web Audio capture from a media element. */
const DIAGNOSTIC_BYPASS_CODE = "runtime_web_audio_bypass";

function getAttr(el: HTMLMediaElement, name: string): string | null {
  return typeof el.getAttribute === "function" ? el.getAttribute(name) : null;
}

function hasAttr(el: HTMLMediaElement, name: string): boolean {
  return getAttr(el, name) !== null;
}

/**
 * `crossorigin` is an enumerated attribute whose invalid-value default is
 * `anonymous`, so PRESENCE is the opt-in — `crossorigin=""` and even
 * `crossorigin="garbage"` both make the fetch a CORS request. Comparing the
 * value against `"anonymous"` would wrongly block those.
 *
 * Two independent reads, because a spec-faithful host and a permissive one
 * disagree about where the truth lives:
 *  - `getAttribute` is the primary read and covers every real browser: the
 *    markup is unambiguous regardless of what the IDL getter does with it.
 *  - `el.crossOrigin` is a secondary read for a host that sets the IDL
 *    property without reflecting it back to the attribute — some
 *    jsdom-style test/preview hosts do this. The check is `!= null`
 *    (covers both `null` and `undefined`), not a truthiness check, ON
 *    PURPOSE: `crossorigin=""` is a valid, common opt-in (see above), and
 *    its IDL fallback value is the empty string — a falsy value that
 *    `Boolean(el.crossOrigin)` would silently misread as "not opted in",
 *    reintroducing the exact silent-audio bug this module exists to close.
 */
function hasCorsOptIn(el: HTMLMediaElement): boolean {
  if (hasAttr(el, "crossorigin")) return true;
  return el.crossOrigin != null;
}

function baseUri(el: HTMLMediaElement): string {
  if (typeof el.baseURI === "string" && el.baseURI) return el.baseURI;
  return typeof document !== "undefined" ? document.baseURI : "";
}

/**
 * The URLs whose origin could decide this element's route.
 *
 * Order matters and mirrors the HTML resource selection algorithm. Once
 * `currentSrc` is set the browser has COMMITTED to that resource, so it is the
 * only candidate that can matter; a `<source>` sibling it passed over must not
 * cost a same-origin element its Web Audio graph. A `src` attribute is equally
 * definitive — the spec has it win outright over `<source>` children. Only
 * before selection settles (no `currentSrc`, no `src`) do the `<source>`
 * candidates matter, and there the conservative read is right: any of them
 * could be the one that gets picked.
 */
function routeCandidates(el: HTMLMediaElement): string[] {
  const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
  if (current) return [current];
  const srcAttr = getAttr(el, "src");
  if (srcAttr) return [srcAttr];
  if (typeof el.querySelectorAll !== "function") return [];
  const sources: string[] = [];
  for (const source of Array.from(el.querySelectorAll("source"))) {
    const value = source.getAttribute("src");
    if (value) sources.push(value);
  }
  return sources;
}

/**
 * Whether this URL would make MediaElementSource silent. Only http(s) is
 * judged: `blob:` and `data:` are same-origin by construction, and a `file:`
 * page's opaque origin can't be compared meaningfully — so those keep the
 * pre-existing behaviour rather than losing Web Audio on a guess. The guard
 * changes behaviour ONLY in the case that is already known-broken.
 */
/**
 * The candidate as an absolute URL, or null when it is not a fetch this check
 * governs: unparseable, or a scheme (`data:`, `blob:`, `file:`) where origin
 * comparison means nothing.
 */
function httpMediaUrl(rawUrl: string, el: HTMLMediaElement): URL | null {
  let url: URL;
  try {
    url = new URL(rawUrl, baseUri(el));
  } catch {
    return null;
  }
  if (url.protocol !== "http:" && url.protocol !== "https:") return null;
  return url;
}

/**
 * Our OWN security origin, which is `"null"` for an opaque document.
 *
 * Not `location.origin`: that is the URL's origin, so a sandboxed iframe
 * without `allow-same-origin` still reports the parent-looking URL there
 * while `window.origin` reads `"null"`. An opaque document cannot prove
 * same-origin with anything, including an asset served from its own host.
 */
function documentSecurityOrigin(): string {
  return typeof window.origin === "string" ? window.origin : window.location.origin;
}

function corsSilenceReason(rawUrl: string, el: HTMLMediaElement): WebAudioSilenceReason | null {
  if (typeof window === "undefined") return null;
  const url = httpMediaUrl(rawUrl, el);
  if (!url) return null;
  const selfOrigin = documentSecurityOrigin();
  const opaqueDocument = selfOrigin === "null";
  if (!opaqueDocument && url.origin === selfOrigin) return null;
  if (hasCorsOptIn(el)) return null;
  // Opacity wins when both hold. It is the blocking condition, and it is the
  // one the author cannot fix by moving the asset.
  return opaqueDocument ? "opaque_document_origin" : "cross_origin_no_cors";
}

/**
 * Whether resource selection has settled enough for a verdict to be a FACT
 * rather than a guess. `currentSrc`/`src` are both definitive per the HTML
 * resource-selection algorithm (see `routeCandidates` above); before either
 * is set, a verdict can only be built from `<source>` children, any of which
 * the browser may still pass over before committing.
 *
 * `classifyWebAudioMediaRoute` itself stays unsettled-tolerant on purpose —
 * the schedule path needs *a* verdict even before selection settles, and
 * conservatively withholding the node there costs nothing but a decode-only
 * fallback. This predicate exists for the one caller that must NOT act on a
 * guess: the discovery-time diagnostic, which drops a message in a human's
 * lap and only gets to say it once (see `reportWebAudioMediaRoute`'s latch).
 */
export function isRouteSelectionSettled(el: HTMLMediaElement): boolean {
  const current = typeof el.currentSrc === "string" ? el.currentSrc : "";
  if (current) return true;
  return hasAttr(el, "src");
}

/**
 * Pure — no node creation, no diagnostics, no element mutation. Called from
 * both the schedule path (where it withholds the node) and the discovery path
 * (where it only reports), which is the point: `hyperframes check` never calls
 * `play()`, so a verdict reachable only from the transport would be invisible
 * to the very gate meant to surface it.
 */
export function classifyWebAudioMediaRoute(el: HTMLMediaElement): WebAudioMediaRoute {
  for (const candidate of routeCandidates(el)) {
    const reason = corsSilenceReason(candidate, el);
    if (reason) {
      return { kind: "decode-only", reason, asset: candidate };
    }
  }
  return { kind: "web-audio" };
}

/**
 * Processing the native HTMLMediaElement fallback cannot reproduce. The track
 * stays AUDIBLE either way — silence is the bug being fixed, so failing closed
 * would just reinstate it — but these authored intentions are quietly dropped,
 * which is worth saying out loud.
 *
 * Wider than the FX/automation pair the schedule path already tests: group
 * membership carries a whole shared bus (chain, fader, its own automation
 * clock), and an above-unity `data-volume` cannot survive a route whose only
 * gain stage is `el.volume`, which the spec pins to [0,1].
 */
export function nativeUnexpressibleProcessing(el: HTMLMediaElement): string[] {
  const lost: string[] = [];
  if (hasAttr(el, HF_AUDIO_FX_ATTR)) lost.push("fx-chain");
  if (hasAttr(el, HF_AUDIO_AUTOMATION_ATTR)) lost.push("automation");
  if (hasAttr(el, HF_AUDIO_GROUP_ATTR)) lost.push("audio-group");
  const volume = Number.parseFloat(getAttr(el, "data-volume") ?? "");
  if (Number.isFinite(volume) && volume > 1) lost.push("above-unity-gain");
  return lost;
}

/**
 * Render never plays through Web Audio at all: the producer mixes offline from
 * the source files, and that mix applies the FX chain itself (see
 * `applyAudioFxChain` in `packages/engine/src/services/audioMixer.ts`). So a
 * bypass is not a fact about the render, and `lostProcessing` would be an
 * outright false claim there — the offline mix DOES reproduce the chain the
 * line says native output cannot. The engine forwards every console line to
 * producer stdout, so an ungated report would also land in every render log.
 *
 * Reporting is all that is gated: `classifyWebAudioMediaRoute` stays pure and
 * the routing decision is unchanged, which costs nothing in render because the
 * element is not the audio source there in the first place.
 *
 * Same signal `mediaProxy.ts`'s `isRenderMode` gates on. The `<video>` half of
 * that check (the injected render-frame sibling) is not mirrored: the
 * export-seek config alone gates reporting.
 */
function isRenderMode(): boolean {
  // Read through an inline cast rather than the ambient `Window` augmentation
  // in `window.d.ts`: that augmentation is only in scope for programs that
  // include it (core's own tsconfig does), and this module is also exported
  // as `./runtime/web-audio-route` for non-runtime consumers (e.g. the studio
  // asset sidebar's preview player, `AudioRow.tsx`) whose tsconfig doesn't
  // pull it in. This is a plain existence check, so the cast costs nothing.
  return (
    typeof window !== "undefined" &&
    !!(window as unknown as { __HF_EXPORT_RENDER_SEEK_CONFIG?: unknown })
      .__HF_EXPORT_RENDER_SEEK_CONFIG
  );
}

// One diagnostic per element. Latched only when something is actually emitted,
// so an early "web-audio" verdict taken before the resource selection settled
// can't suppress the real one at `loadedmetadata`.
const diagnosedElements = new WeakSet<HTMLMediaElement>();

/**
 * Emit the one-time diagnostic for a non-Web-Audio verdict.
 *
 * The bypass always reports: it is an accident by definition, and the whole
 * complaint in #3458 is that nothing was said.
 */
/**
 * What was actually wrong, per reason. The opaque case is NOT "cross-origin
 * media": the asset can be same-origin by URL and still unreadable, because it
 * is OUR document that cannot prove an origin.
 */
const NOTE_BY_REASON: Record<WebAudioSilenceReason, string> = {
  cross_origin_no_cors:
    "cross-origin media without a `crossorigin` opt-in is silent through createMediaElementSource (Web Audio spec); using native playback instead",
  opaque_document_origin:
    "this document's security origin is opaque (a sandbox without `allow-same-origin`), so no media is same-origin with it and createMediaElementSource would be silent (Web Audio spec); using native playback instead",
};

/**
 * How to get the processing back, per reason. Keeping these apart is the point:
 * telling an opaque document to "proxy the asset to a same-origin URL" names a
 * fix that cannot exist, since nothing is same-origin with an opaque origin.
 */
const REMEDY_BY_REASON: Record<WebAudioSilenceReason, string> = {
  cross_origin_no_cors: "proxy or download the asset to a same-origin URL to keep it.",
  opaque_document_origin:
    "add `allow-same-origin` to the sandbox, or serve the asset with a `crossorigin` attribute and an `Access-Control-Allow-Origin` of `null` or `*`, to keep it.",
};

export function reportWebAudioMediaRoute(el: HTMLMediaElement, route: WebAudioMediaRoute): void {
  if (route.kind === "web-audio") return;
  if (isRenderMode()) return;
  if (diagnosedElements.has(el)) return;
  const lost = nativeUnexpressibleProcessing(el);
  diagnosedElements.add(el);

  const details: Record<string, RuntimeJson> = {
    asset: route.asset,
    reason: route.reason,
    lostProcessing: lost,
    note: NOTE_BY_REASON[route.reason],
  };
  postRuntimeMessage({
    source: "hf-preview",
    type: "diagnostic",
    code: DIAGNOSTIC_BYPASS_CODE,
    details,
  });
  // The stable code lives in the console text so the CLI's scraper
  // (packages/cli/src/utils/checkBrowser.ts) can match a token, not prose —
  // same contract mediaProxy.ts's diagnostics use.
  const lostNote =
    lost.length > 0
      ? ` Native playback cannot reproduce: ${lost.join(", ")}. Fix: ${REMEDY_BY_REASON[route.reason]}`
      : "";
  console.info(
    `[hyperframes] ${DIAGNOSTIC_BYPASS_CODE}: "${route.asset}" (${route.reason}): ` +
      `Web Audio capture withheld; the track plays through native HTMLMediaElement output.${lostNote}`,
  );
}