Skip to content

Write HTML. Render video. Built for agents.

11.6 KB
Raw Blame
import { parseHTML } from "linkedom";
import { topLevelElements, trackKindOf } from "@hyperframes/parsers";
import { toDomNode } from "../timeline/domStructure.js";
import {
  resolveSnapshotCompositionRange,
  snapshotStartResolver,
  type SnapshotRange,
} from "./snapshotSampling.js";

/** A beat's frame lands this long after its start, once the entrance has settled. */
const REVIEW_SETTLE_SEC = 0.3;
const REVIEW_MID_GAP_SEC = 0.5;
/** Seam strip offsets around each cut. */
export const REVIEW_SEAM_OFFSETS = [-0.2, -0.1, 0, 0.1, 0.2] as const;
/** Phone sheet tile width: the frame as wide as a phone held upright. */
export const REVIEW_PHONE_WIDTH = 360;
/** Starts and ends closer than this are one boundary. */
const SAME_TIME_SEC = 0.05;
/** Without two scene starts there is nothing to cut between, so the beats are spread evenly. */
const EVEN_BEATS = 5;

export interface ReviewScene {
  id: string;
  start: number;
  end: number;
}

export interface ReviewBeat {
  time: number;
  /** Scene id, `beat:<data-beat>` for an authored beat, or `t=<time>` when sampled evenly. */
  label: string;
}

export interface ReviewSeam {
  time: number;
  from: string | null;
  to: string | null;
  times: number[];
}

export interface ReviewPlan {
  window: SnapshotRange;
  scenes: ReviewScene[];
  beats: ReviewBeat[];
  seams: ReviewSeam[];
  notes: string[];
}

function roundMs(value: number): number {
  return Math.round(value * 1000) / 1000;
}

function sceneId(el: Element, index: number): string {
  return (
    el.id ||
    el.getAttribute("data-composition-id") ||
    el.getAttribute("data-layout-name") ||
    `${el.tagName.toLowerCase()}-${index + 1}`
  );
}

/**
 * The review's scenes and authored beats, read like `timeline` reads rows: the composition's
 * top-level timed elements and sub-composition hosts (captions and audio tracks excluded), timed by
 * the runtime's own start resolver over the live page's markup. `durations` are the registered
 * GSAP timeline lengths, which time a host that declares no data-duration.
 */
export function resolveReviewTimeline(
  html: string,
  durations: Record<string, number>,
  duration: number,
  compositionId?: string,
): { window: SnapshotRange; scenes: ReviewScene[]; beats: ReviewBeat[]; notes: string[] } {
  const bounds: SnapshotRange = compositionId
    ? resolveSnapshotCompositionRange(html, durations, compositionId, duration)
    : [0, duration];
  const { document } = parseHTML(html);
  const target = compositionId
    ? Array.from(document.querySelectorAll("[data-composition-id]")).find(
        (el) => el.getAttribute("data-composition-id") === compositionId,
      )
    : document.querySelector("[data-composition-id]");
  if (!target) {
    return { window: bounds, scenes: [], beats: [], notes: ["No composition root found."] };
  }
  const resolver = snapshotStartResolver(document as unknown as Document, durations);
  const span = (el: Element): [number, number] | null => {
    // The fallback reaches every untimed ancestor, the root included, so it must be the root's 0.
    const start = resolver.resolveStartForElement(el, 0);
    if (!Number.isFinite(start)) return null;
    const length = resolver.resolveDurationForElement(el);
    const from = Math.max(bounds[0], start);
    const to = Math.min(bounds[1], length != null ? start + length : bounds[1]);
    return to > from ? [roundMs(from), roundMs(to)] : null;
  };
  const scenes: ReviewScene[] = [];
  const rows = topLevelElements(toDomNode(target as unknown as Element)).filter((node) => {
    const kind = trackKindOf(node).kind;
    return kind === "video" || kind === "graphics";
  });
  rows.forEach((node, index) => {
    const interval = span(node.el);
    if (interval)
      scenes.push({ id: sceneId(node.el, index), start: interval[0], end: interval[1] });
  });
  const beats: ReviewBeat[] = [];
  const notes: string[] = [];
  for (const el of Array.from((target as unknown as Element).querySelectorAll("[data-beat]"))) {
    const name = el.getAttribute("data-beat")?.trim() || sceneId(el, beats.length);
    const timed = el.hasAttribute("data-start") || el.hasAttribute("data-composition-id");
    const interval = timed ? span(el) : null;
    if (!interval) {
      notes.push(`data-beat="${name}" was skipped: give it a data-start so the beat has a time.`);
      continue;
    }
    beats.push({ time: settledTime(interval[0], interval[1]), label: `beat:${name}` });
  }
  scenes.sort((a, b) => a.start - b.start || a.end - b.end);
  return { window: bounds, scenes, beats, notes };
}

/** Just after a scene starts, and its middle once that is far enough on to show what the scene builds. */
function sceneBeats(scene: ReviewScene): ReviewBeat[] {
  const settled = settledTime(scene.start, scene.end);
  const middle = roundMs((scene.start + scene.end) / 2);
  const beats = [{ time: settled, label: scene.id }];
  return middle - settled >= REVIEW_MID_GAP_SEC
    ? [...beats, { time: middle, label: `${scene.id} mid` }]
    : beats;
}

/** start + 0.3 s, or the middle of a span too short for that. */
function settledTime(start: number, end: number): number {
  return roundMs(Math.min(start + REVIEW_SETTLE_SEC, (start + end) / 2));
}

/** Beats, seams and their frame times. `lastFrame` is the latest seek that still shows the film. */
export function planReview(
  timeline: { window: SnapshotRange; scenes: ReviewScene[]; beats: ReviewBeat[]; notes: string[] },
  lastFrame: number,
): ReviewPlan {
  const { window: bounds, scenes } = timeline;
  const notes = [...timeline.notes];
  const clamp = (time: number) => roundMs(Math.min(Math.max(time, bounds[0]), lastFrame));
  const distinctStarts = mergeBeats(
    scenes.map((scene) => ({ time: scene.start, label: scene.id })),
  );
  const beats = mergeBeats([
    ...(distinctStarts.length >= 2 ? scenes.flatMap(sceneBeats) : evenBeats(bounds, notes)),
    ...timeline.beats,
  ]).map((beat) => ({ ...beat, time: clamp(beat.time) }));
  const seams = seamTimes(scenes, bounds).map((time) => ({
    time,
    ...seamSides(scenes, time),
    times: REVIEW_SEAM_OFFSETS.map((offset) => clamp(time + offset)),
  }));
  return { window: bounds, scenes, beats, seams, notes };
}

function evenBeats(bounds: SnapshotRange, notes: string[]): ReviewBeat[] {
  notes.push(
    `Fewer than two scene starts; ${EVEN_BEATS} beats are spread evenly instead. Time scenes as sub-composition hosts or clips to review each one.`,
  );
  const [start, end] = bounds;
  return Array.from({ length: EVEN_BEATS }, (_, index) => {
    const time = roundMs(start + ((index + 0.5) / EVEN_BEATS) * (end - start));
    return { time, label: `t=${time}` };
  });
}

function mergeBeats(beats: ReviewBeat[]): ReviewBeat[] {
  const merged: ReviewBeat[] = [];
  for (const beat of [...beats].sort((a, b) => a.time - b.time)) {
    const same = merged.find((entry) => Math.abs(entry.time - beat.time) < SAME_TIME_SEC);
    if (same) same.label = `${same.label} + ${beat.label}`;
    else merged.push({ ...beat });
  }
  return merged;
}

/** Every scene start after the window opens, and every scene end no other scene starts at. */
export function seamTimes(scenes: ReviewScene[], bounds: SnapshotRange): number[] {
  const inside = (time: number) =>
    time > bounds[0] + SAME_TIME_SEC && time < bounds[1] - SAME_TIME_SEC;
  const starts = scenes.map((scene) => scene.start).filter(inside);
  const ends = scenes
    .map((scene) => scene.end)
    .filter(inside)
    .filter((end) => !starts.some((start) => Math.abs(start - end) < SAME_TIME_SEC));
  const seams: number[] = [];
  for (const time of [...starts, ...ends].sort((a, b) => a - b)) {
    if (!seams.some((seam) => Math.abs(seam - time) < SAME_TIME_SEC)) seams.push(time);
  }
  return seams;
}

function seamSides(
  scenes: ReviewScene[],
  time: number,
): { from: string | null; to: string | null } {
  const before = scenes
    .filter((scene) => scene.start < time - SAME_TIME_SEC && scene.end >= time - SAME_TIME_SEC)
    .sort((a, b) => b.start - a.start)[0];
  const after = scenes
    .filter((scene) => Math.abs(scene.start - time) < SAME_TIME_SEC)
    .sort((a, b) => a.end - b.end)[0];
  const next = after ?? scenes.find((scene) => scene.start > time);
  return { from: before?.id ?? null, to: next?.id ?? null };
}

export interface ReviewFrame {
  role: "beat" | "seam";
  /** 1-based beat or seam number. */
  index: number;
  time: number;
  label: string;
  file: string;
}

function pad(index: number): string {
  return String(index).padStart(2, "0");
}

function stamp(time: number): string {
  return `${Number(time.toFixed(3))}s`;
}

const SEAM_TILE_LABELS = ["cut -0.2", "-0.1", "cut", "+0.1", "+0.2"] as const;

/** Every still a review captures, in capture order, with its file name. */
export function reviewFrames(plan: ReviewPlan): ReviewFrame[] {
  const beats = plan.beats.map((beat, index) => ({
    role: "beat" as const,
    index: index + 1,
    time: beat.time,
    label: beat.label,
    file: `beat-${pad(index + 1)}-at-${stamp(beat.time)}.png`,
  }));
  const seams = plan.seams.flatMap((seam, index) =>
    seam.times.map((time, slot) => ({
      role: "seam" as const,
      index: index + 1,
      time,
      label:
        slot === 2
          ? `cut ${seam.from ?? "start"} -> ${seam.to ?? "end"}`
          : (SEAM_TILE_LABELS[slot] ?? ""),
      file: `seam-${pad(index + 1)}-${slot + 1}-at-${stamp(time)}.png`,
    })),
  );
  return [...beats, ...seams];
}

export interface ReviewSheets {
  beats: string[];
  seams: string[];
  phone: string[];
}

const BEAT_SHEET_PAGE = 16;

/** The beat sheet, one strip per seam, and the beat frames again at phone width. */
export async function writeReviewSheets(
  dir: string,
  frames: ReviewFrame[],
  written: (frame: ReviewFrame) => string | null,
): Promise<ReviewSheets> {
  const { createContactSheet } = await import("../capture/contactSheet.js");
  const { join } = await import("node:path");
  const captured = frames.flatMap((frame) => {
    const path = written(frame);
    return path ? [{ frame, path }] : [];
  });
  const beats = captured.filter((entry) => entry.frame.role === "beat");
  const beatLabel = ({ frame }: (typeof captured)[number]) => `${stamp(frame.time)} ${frame.label}`;
  const pages = async (name: string, cellWidth: number, colsFor: (count: number) => number) => {
    const sheets: string[] = [];
    for (let start = 0; start < beats.length; start += BEAT_SHEET_PAGE) {
      const page = beats.slice(start, start + BEAT_SHEET_PAGE);
      const suffix = beats.length > BEAT_SHEET_PAGE ? `-${start / BEAT_SHEET_PAGE + 1}` : "";
      const sheet = await createContactSheet(
        page.map((entry) => entry.path),
        join(dir, `${name}${suffix}.jpg`),
        {
          cols: colsFor(page.length),
          maxImages: page.length,
          cellWidth,
          labelMode: "custom",
          labels: page.map(beatLabel),
        },
      );
      if (sheet) sheets.push(sheet);
    }
    return sheets;
  };
  const seams: string[] = [];
  const seamNumbers = [
    ...new Set(captured.filter((e) => e.frame.role === "seam").map((e) => e.frame.index)),
  ];
  for (const index of seamNumbers) {
    const strip = captured.filter((e) => e.frame.role === "seam" && e.frame.index === index);
    const sheet = await createContactSheet(
      strip.map((entry) => entry.path),
      join(dir, `review-seam-${pad(index)}.jpg`),
      {
        cols: strip.length,
        maxImages: strip.length,
        cellWidth: 384,
        labelMode: "custom",
        labels: strip.map(({ frame }) => `${stamp(frame.time)} ${frame.label}`),
      },
    );
    if (sheet) seams.push(sheet);
  }
  return {
    beats: await pages("review-beats", 480, (count) => (count <= 3 ? count : count <= 9 ? 3 : 4)),
    seams,
    phone: await pages("review-phone", REVIEW_PHONE_WIDTH, (count) => Math.min(4, count)),
  };
}