Skip to content

Write HTML. Render video. Built for agents.

9.6 KB
Raw Blame
/**
 * Collect the render pipeline's media list from the fully inlined document.
 *
 * Ids are unique per composition file, not per render document (see mediaRenderIds.ts), so media is
 * read from the inlined document: each element once, keyed by its render id or else its author id,
 * and placed by the composition hosts it sits inside.
 */

import { parseHTML } from "linkedom";
import {
  MEDIA_RENDER_ID_ATTR,
  normalizeRateSpec,
  resolveAuthoredTimingWindow,
  shiftRateLane,
  sourceTimeAt,
} from "@hyperframes/core";
import {
  MEDIA_START_BASIS_ATTR,
  compositionOriginSeconds,
  cutToHostSlots,
  hostInPointSeconds,
  type HostSlot,
  readMediaStartBasis,
  resolveAbsoluteMediaStartSeconds,
  type MediaStartBasis,
} from "@hyperframes/core/media-timing";
import {
  parseVideoElements,
  parseImageElements,
  parseAudioElements,
  resolveReferencedStart,
  type RefResolverEl,
  type RefResolverDoc,
  type VideoElement,
  type ImageElement,
  type AudioElement,
} from "@hyperframes/engine";

/**
 * Where a composition host closes in its parent's time, or null when unbounded.
 * Delegates to `resolveAuthoredTimingWindow`, the same window the runtime uses to hide descendants.
 */
function resolveHostEnd(host: Element, hostStart: number): number | null {
  return (
    resolveAuthoredTimingWindow({
      start: hostStart,
      duration: host.getAttribute("data-duration"),
      end: host.getAttribute("data-end"),
    })?.end ?? null
  );
}

interface HostWindow {
  /** Seconds to add to a descendant's authored, scene-relative start. */
  offset: number;
  /** Absolute time past which a descendant is outside its host, or Infinity. */
  limit: number;
  /** Whether authored media time is composition-local or legacy root-global. */
  basis: MediaStartBasis;
  /** Root-time slots of the enclosing hosts with an in-point. */
  slots: HostSlot[];
}

const ROOT_WINDOW: HostWindow = { offset: 0, limit: Infinity, basis: "local", slots: [] };

// Every composition host shifts what it holds, inline or inlined from a file, as in core's start resolver.
const COMPOSITION_HOST_SELECTOR =
  "[data-composition-id], [data-composition-src], [data-composition-file]";

/** Composition hosts that place `element`, root first. */
function enclosingHosts(element: Element): Element[] {
  const hosts: Element[] = [];
  for (let ancestor = element.parentElement; ancestor; ancestor = ancestor.parentElement) {
    if (ancestor.matches(COMPOSITION_HOST_SELECTOR)) {
      hosts.unshift(ancestor);
    }
  }
  return hosts;
}

/**
 * Fold a media element's chain of composition hosts into one window.
 *
 * Host `data-start` is resolved the same way media is (`resolveReferencedStart`):
 * numeric literals, or an id / `data-composition-id` ref to a sibling slot's
 * end (`data-start="hook"`). `parseFloat("hook")` is 0, which stacked every
 * chained scene at 0–2s. Each host's end comes from `resolveHostEnd`, which
 * shares the runtime's timing resolver, so the planner's media windows match
 * the window in which the runtime shows the host's descendants.
 */
function resolveHostWindow(
  element: Element,
  document: RefResolverDoc,
  startCache: Map<RefResolverEl, number>,
  visiting: Set<RefResolverEl>,
): HostWindow {
  const hosts = enclosingHosts(element);
  if (hosts.length === 0) return ROOT_WINDOW;

  let offset = 0;
  let limit = Infinity;
  const slots: HostSlot[] = [];
  for (const [index, host] of hosts.entries()) {
    const hostStart = resolveReferencedStart(document, host, startCache, visiting);
    const hostEnd = resolveHostEnd(host, hostStart);
    if (hostEnd != null && !isRootComposition(host, index)) {
      limit = Math.min(limit, offset + hostEnd);
    }
    if (hostInPointSeconds(host) > 0) {
      slots.push({ start: offset + hostStart, end: offset + (hostEnd ?? Infinity) });
    }
    offset += compositionOriginSeconds(hostStart, host);
  }
  return { offset, limit, basis: mediaStartBasis(element), slots };
}

// The root composition's end is the render's length, not a cut: a cut would move clip fade-outs.
const isRootComposition = (host: Element, depth: number): boolean =>
  depth === 0 && !host.matches("[data-composition-src], [data-composition-file]");

function mediaStartBasis(element: Element): MediaStartBasis {
  const tag = element.tagName.toLowerCase();
  return tag === "video" || tag === "audio"
    ? readMediaStartBasis(element.getAttribute(MEDIA_START_BASIS_ATTR))
    : "local";
}

/**
 * Map each media id the parsers report (render id, else author id: render ids are only stamped
 * when files are inlined) to its hosts' window, keyed on id so two parses need not walk in step.
 */
function collectHostWindows(html: string): Map<string, HostWindow> {
  const { document } = parseHTML(html);
  const windows = new Map<string, HostWindow>();
  const startCache = new Map<RefResolverEl, number>();
  const visiting = new Set<RefResolverEl>();
  // What the parsers report; an author id never displaces a render id, or another author id.
  for (const element of document.querySelectorAll("video, audio, img[src]")) {
    const renderId = element.getAttribute(MEDIA_RENDER_ID_ATTR);
    const id = renderId || element.getAttribute("id");
    if (!id || (!renderId && windows.has(id))) continue;
    windows.set(
      id,
      resolveHostWindow(element as unknown as Element, document, startCache, visiting),
    );
  }
  return windows;
}

/**
 * Shift a scene-relative window onto the root timeline; null when the clip starts after its host
 * ends (the file-tree walk's `start < absoluteEnd` drop) or its in-point hosts cut all of it.
 * `start` stays the clip's origin; `played` is the part the in-point hosts' slots keep.
 */
function toAbsoluteWindow(
  start: number,
  end: number,
  window: HostWindow,
): { start: number; end: number; played: { start: number; end: number } } | null {
  const absoluteStart = resolveAbsoluteMediaStartSeconds({
    authoredStart: start,
    hostStart: window.offset,
    basis: window.basis,
  });
  if (absoluteStart >= window.limit) return null;
  const absoluteEnd = resolveAbsoluteMediaStartSeconds({
    authoredStart: end,
    hostStart: window.offset,
    basis: window.basis,
  });
  const absolute = { start: absoluteStart, end: Math.min(absoluteEnd, window.limit) };
  const played = cutToHostSlots(absolute, window.slots);
  if (played.end === played.start && absolute.end > absolute.start) return null;
  return { start: absolute.start, end: played.end, played };
}

/** Start a clip where its in-point hosts let it play, that much further into its source. */
function startAtPlayed<T extends AudioElement | VideoElement>(
  clip: T,
  played: { start: number },
): T {
  const head = played.start - clip.start;
  if (head <= 0) return clip;
  // ponytail: a loop runs from mediaStart, so it only moves to its slot and keeps main's phase error;
  // an exact cut needs a visible start separate from the loop origin in the engine.
  if ("loop" in clip && clip.loop) return { ...clip, start: played.start };
  const rate = normalizeRateSpec(clip.playbackRate);
  return {
    ...clip,
    start: played.start,
    mediaStart: clip.mediaStart + sourceTimeAt(rate, head),
    playbackRate: shiftRateLane(rate, head),
  };
}

export interface RenderMedia {
  videos: VideoElement[];
  audios: AudioElement[];
  images: ImageElement[];
  /** Ids of clips in the document that never play; the browser probe must not add them back. */
  droppedMediaIds: Set<string>;
}

/**
 * Parse every media element in the inlined render document, with each clip's
 * window resolved onto the root timeline.
 *
 * Expects `assignMediaRenderIds` to have run: the parsers report the stamped
 * render id as each element's `id`, which is what the rest of the pipeline
 * keys on and what the engine resolves back to a DOM node.
 */
export function collectRenderMedia(html: string): RenderMedia {
  const windows = collectHostWindows(html);
  const windowFor = (id: string): HostWindow => windows.get(id) ?? ROOT_WINDOW;
  const droppedMediaIds = new Set<string>();

  const videos: VideoElement[] = [];
  for (const video of parseVideoElements(html)) {
    const absolute = toAbsoluteWindow(video.start, video.end, windowFor(video.id));
    if (!absolute) {
      droppedMediaIds.add(video.id);
      continue;
    }
    videos.push(
      startAtPlayed({ ...video, start: absolute.start, end: absolute.end }, absolute.played),
    );
  }

  const images: ImageElement[] = [];
  for (const image of parseImageElements(html)) {
    const absolute = toAbsoluteWindow(image.start, image.end, windowFor(image.id));
    if (absolute) images.push({ ...image, start: absolute.start, end: absolute.end });
    else droppedMediaIds.add(image.id);
  }

  // A <video data-has-audio> track is reported as "<renderId>-audio"; strip the
  // suffix to look the element's host window back up.
  const audios: AudioElement[] = [];
  for (const audio of parseAudioElements(html)) {
    const elementId = audio.type === "video" ? audio.id.replace(/-audio$/, "") : audio.id;
    const placed = placeAudio(audio, windowFor(elementId));
    if (placed) audios.push(placed);
    else droppedMediaIds.add(elementId);
  }

  return { videos, audios, images, droppedMediaIds };
}

function placeAudio(audio: AudioElement, window: HostWindow): AudioElement | null {
  // The mixer reads end === 0 as "run to the natural media length", so an
  // unbounded track must stay unbounded rather than collapse onto its start.
  const authoredEnd = audio.end > 0 ? audio.end : Infinity;
  const absolute = toAbsoluteWindow(audio.start, authoredEnd, window);
  if (!absolute) return null;
  const end = Number.isFinite(absolute.end) ? absolute.end : 0;
  return startAtPlayed({ ...audio, start: absolute.start, end }, absolute.played);
}