Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

18.7 KB
Raw Blame
import { mkdir, rm, stat, writeFile } from "node:fs/promises";
import { join, relative, resolve, sep } from "node:path";
import puppeteer from "puppeteer-core";
import type { Browser } from "puppeteer-core";
import type { HtmlProgram } from "@hypit/html-program";
import type { MediaFrameRange } from "@hypit/media";
import type { HtmlRasterProgress, resolveExecutionOptions } from "./render.js";
import { assert, mediaExecutablePath, openProcessInput } from "./process.js";
import { verifyOutput } from "./output.js";
import { distributeFrameRange, requestedFrameRanges, sourceFrameAt, sourceWindows, videoSlots } from "./sampling.js";
import type { VideoSlot } from "./sampling.js";
import { CaptureConcurrency, renderWorkerLimit } from "./concurrency.js";
import { createOpaqueFrameCapture } from "./opaque-capture.js";
import { FrameSpanIndex } from "./frame-span-index.js";
import { OrderedFrameSink } from "./ordered-frame-sink.js";
import { decodeSourceFrameWindow, probeSourceVideo } from "./source-frame-decoder.js";
import { SourceFrameStore } from "./source-frame-store.js";
import type { SourceFrameWindow } from "./source-frame-store.js";
import { serveHtmlProgram } from "./server.js";

export type CaptureInput = {
  readonly document: HtmlProgram;
  readonly frames?: readonly number[];
  readonly range: MediaFrameRange;
  readonly config: ReturnType<typeof resolveExecutionOptions> & { readonly chromePath: string };
  readonly directory: string;
  /** Complete browser prelude prepared by the Program owner before entering the child process. */
  readonly frameSelectionPrelude: string;
  /** Program Artifact identities projected onto the already-staged local project. */
  readonly artifactPaths: readonly (readonly [resource: string, path: string])[];
};

/** All browser and codec work belongs to this render's disposable process. */
export async function captureStagedVisual(input: CaptureInput, controller: AbortController,
  onProgress: (event: HtmlRasterProgress) => void): Promise<string> {
  const { document, range, config, directory: work } = input;
  const signal = controller.signal;
  const frameCount = input.frames?.length ?? range.endFrameExclusive - range.startFrame;
  const selected = input.frames === undefined ? undefined : new Set(input.frames);
  const at = (index: number) => input.frames?.[index] ?? range.startFrame + index;
  const fps = { num: document.frameRate.numerator, den: document.frameRate.denominator };
  const limit = renderWorkerLimit(config, frameCount, fps.num / fps.den);
  const concurrency = new CaptureConcurrency(limit, config.workers === "auto");
  // Short contiguous batches retain sequential capture while allowing free browsers
  // to help with expensive passages. This queue exists only inside this render.
  const batchCount = Math.max(limit, Math.ceil(frameCount / Math.max(1, Math.round(fps.num / fps.den))));
  const batches = distributeFrameRange({ startFrame: 0, endFrameExclusive: frameCount }, batchCount);
  let nextBatch = 0;
  const started = performance.now();
  const elapsedMs = () => Math.round(performance.now() - started);
  const [ffmpegPath, ffprobePath] = await Promise.all([
    mediaExecutablePath(config.ffmpegPath), mediaExecutablePath(config.ffprobePath),
  ]);
  const sessions = new Set<Browser>();
  const closing = new Map<Browser, Promise<void>>();
  const close = (session: Browser): Promise<void> => {
    let promise = closing.get(session);
    if (promise === undefined) {
      promise = session.close().finally(() => { sessions.delete(session); });
      closing.set(session, promise);
    }
    return promise;
  };
  const stage = async <T>(subject: string, timeoutMs: number | undefined, run: () => Promise<T>): Promise<T> => {
    signal.throwIfAborted();
    const timeout = timeoutMs === undefined
      ? undefined
      : setTimeout(() => controller.abort(new Error(`HTML renderer ${subject} timed out after ${timeoutMs} ms`)), timeoutMs);
    let onAbort: () => void = () => {};
    const stopped = new Promise<never>((_, reject) => {
      onAbort = () => reject(signal.reason);
      signal.addEventListener("abort", onAbort, { once: true });
    });
    try { return await Promise.race([run(), stopped]); }
    finally {
      if (timeout !== undefined) clearTimeout(timeout);
      signal.removeEventListener("abort", onAbort);
    }
  };
  let server: Awaited<ReturnType<typeof serveHtmlProgram>> | undefined;
  let encoderCompletion: Promise<unknown> | undefined;
  let sourceFrameStore: SourceFrameStore | undefined;
  // Closing pages interrupts an in-flight capture as well as the next loop iteration.
  const abort = () => { for (const session of sessions) void close(session).catch(() => {}); };
  signal.addEventListener("abort", abort, { once: true });
  try {
    signal.throwIfAborted();
    const artifactPaths = new Map(input.artifactPaths);
    const slots = videoSlots(document.frameSources, resource => {
      const path = artifactPaths.get(resource);
      assert(path !== undefined, `HTML renderer Artifact ${resource} has no staged path`);
      return `./${path}`;
    });
    const slotIndex = new FrameSpanIndex(slots);
    const requestedRanges = requestedFrameRanges(range, input.frames);
    const plannedSources = sourceWindows(slots, requestedRanges);
    const sourceTotal = plannedSources.reduce((sum, source) => sum + source.windows.reduce((count, window) =>
      count + window.endFrameExclusive - window.startFrame, 0), 0);
    const sources = new Map<string, { path: string; fps: VideoSlot["sourceFps"];
      metadata: Awaited<ReturnType<typeof probeSourceVideo>> }>();
    // Chrome needs dimensions before initialization. Pixel frames themselves
    // are decoded lazily for the screenshots that lease them.
    for (const source of plannedSources) {
      const path = resolve(work, source.src);
      assert(path.startsWith(`${work}${sep}`), "HTML renderer source is outside the staged project");
      const metadata = await probeSourceVideo({ executable: ffprobePath, path,
        timeoutMs: config.processTimeoutMs, maxOutputBytes: config.maxProcessOutputBytes, signal });
      signal.throwIfAborted();
      sources.set(source.src, { path, fps: source.fps, metadata });
    }
    let extraction = 0;
    sourceFrameStore = new SourceFrameStore({ maxBytes: config.maxDecodedSourceBytes, signal,
      decode: (src, window) => {
        const source = sources.get(src);
        assert(source !== undefined, `HTML renderer source ${src} is outside the requested render`);
        return decodeSourceFrameWindow({ executable: ffmpegPath, path: source.path, outputDir: join(work, "decoded"),
          outputPrefix: `source-${extraction++}`, window, fps: source.fps, metadata: source.metadata,
          timeoutMs: config.processTimeoutMs, maxProcessOutputBytes: config.maxProcessOutputBytes, signal });
      },
      remove: async path => { await rm(path, { force: true }); },
    });
    const sourceDemand = (ranges: readonly MediaFrameRange[]): SourceFrameWindow[] => sourceWindows(slots, ranges, slotIndex)
      .flatMap(source => source.windows.map(window => ({ source: source.src, ...window })));
    const frameRanges = (startIndex: number, endIndex: number): MediaFrameRange[] => input.frames === undefined
      ? [{ startFrame: at(startIndex), endFrameExclusive: at(endIndex - 1) + 1 }]
      : input.frames.slice(startIndex, endIndex).map(frame => ({ startFrame: frame, endFrameExclusive: frame + 1 }));
    const activeFramePayloads = (frame: number) => {
        const payloads = new Map<string, { framePath: string; frameIndex: number }>();
        for (const slot of slotIndex.at(frame)) {
          const frameIndex = sourceFrameAt(slot, frame);
          // Browser initialization may seek outside the requested interval.
          if (selected === undefined ? frame < range.startFrame || frame >= range.endFrameExclusive : !selected.has(frame)) continue;
          const framePath = sourceFrameStore!.path(slot.src, frameIndex);
          assert(framePath !== undefined, `HTML renderer has no decoded frame ${frameIndex} for ${slot.id}`);
          payloads.set(slot.id, { framePath, frameIndex });
        }
        return payloads;
    };
    server = await serveHtmlProgram(work);
    const serverUrl = server.url;
    const outputFrames = join(work, "frames");
    const output = join(work, "visual.mp4");
    const crf = { draft: 28, standard: 23, high: 18 }[config.quality];
    const encoder = input.frames === undefined ? openProcessInput({ executable: ffmpegPath,
      argv: ["-v", "error", "-y", "-f", "image2pipe", "-framerate", `${fps.num}/${fps.den}`, "-vcodec", "png", "-i", "pipe:0",
        "-frames:v", String(frameCount), "-an", "-c:v", "libx264", "-crf", String(crf),
        "-preset", config.quality === "draft" ? "veryfast" : "medium",
        // Chromium composites in sRGB: convert with the BT.709 matrix and tag the stream so players decode it the same way.
        "-vf", "scale=out_color_matrix=bt709:out_range=tv,format=yuv420p,setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv",
        "-movflags", "+faststart", output],
      timeoutMs: config.processTimeoutMs, maxOutputBytes: config.maxProcessOutputBytes, signal,
    }) : undefined;
    const orderedFrames = encoder === undefined ? undefined : new OrderedFrameSink({
      frameCount, maxPendingBytes: config.maxPendingFrameBytes, signal,
      writer: { write: encoder.write, close: async () => { await encoder.close(); } },
    });
    if (encoder === undefined) await mkdir(outputFrames);
    else {
      encoderCompletion = encoder.completed;
      void encoder.completed.catch((error) => {
        orderedFrames!.fail(error);
        controller.abort(error);
      });
    }
    onProgress?.({ phase: "prepared", workers: concurrency.target, sourceFrames: sourceTotal, elapsedMs: elapsedMs() });
    const jobs: Promise<void>[] = [];
    let active = 0, open = 0, initializing = 0, capturedFrames = 0, capturedBytes = 0;
    let sourceAcquireWorkerMs = 0, outputSubmitWorkerMs = 0;
    let startupMs = 0;
    const launch = (): void => {
      const worker = jobs.length;
      jobs.push(runWorker(worker));
      // Failure stops all jobs promptly; all jobs are still awaited below.
      void jobs[worker]!.catch(() => {});
    };
    const runWorker = async (worker: number): Promise<void> => {
      let session: Browser | undefined;
      let ready = false;
      let capturing = false;
      active++;
      open++;
      initializing++;
      const launched = performance.now();
      let setupMs = 0;
      try {
        signal.throwIfAborted();
        session = await puppeteer.launch({ executablePath: config.chromePath, headless: true,
          defaultViewport: { width: document.canvas.width, height: document.canvas.height, deviceScaleFactor: 1 },
          args: ["--no-sandbox", "--disable-dev-shm-usage",
            ...(config.browserGpu === "software" ? ["--disable-gpu"] : []),
          ] });
        sessions.add(session);
        signal.throwIfAborted();
        const page = await session.newPage();
        if (config.initializationTimeoutMs !== undefined) {
          page.setDefaultTimeout(config.initializationTimeoutMs);
          page.setDefaultNavigationTimeout(config.initializationTimeoutMs);
        }
        await page.evaluateOnNewDocument(input.frameSelectionPrelude);
        const browserPid = session.process()?.pid;
        onProgress?.({ phase: "worker-initializing", worker, range, browserPid, elapsedMs: elapsedMs() });
        await stage(`worker ${worker} initialization`, config.initializationTimeoutMs, async () => {
          await page.goto(serverUrl, { waitUntil: "load" });
          await page.evaluate(async () => {
            const program = (window as unknown as { __hypitFrameProgram?: { prepare(): Promise<void> } }).__hypitFrameProgram;
            if (program === undefined) throw new Error("HTML Program page ABI is missing");
            await program.prepare();
          });
        });
        const captureFrame = await createOpaqueFrameCapture(page, document.canvas);
        setupMs = performance.now() - launched;
        startupMs = Math.max(startupMs, setupMs);
        ready = true;
        initializing--;
        if (initializing === 0) concurrency.settled(performance.now());
        onProgress?.({ phase: "worker-start", worker, range, browserPid, elapsedMs: elapsedMs() });
        let completed = 0;
        let lastProgressAt = performance.now();
        const timing = { seekMs: 0, prepareMs: 0, screenshotMs: 0 };
        capturing = true;
        while (true) {
          // Retire only between complete batches, before claiming more work.
          if (active > concurrency.target) break;
          const batch = batches[nextBatch++];
          if (batch === undefined) break;
          for (let index = batch.startFrame; index < batch.endFrameExclusive; index++) {
            const frame = at(index);
            signal.throwIfAborted();
            const required = sourceDemand(frameRanges(index, index + 1));
            // A preference is the unconsumed suffix, not the whole batch: frames
            // behind this Worker must become ordinary eviction candidates.
            const preferred = sourceDemand(frameRanges(index, batch.endFrameExclusive));
            const sourceAcquireStarted = performance.now();
            const sourceLease = await sourceFrameStore!.acquire(required, preferred);
            sourceAcquireWorkerMs += performance.now() - sourceAcquireStarted;
            let captured: Awaited<ReturnType<typeof captureFrame>>;
            try {
              const payloads = activeFramePayloads(frame);
              const sourceUpdates = [...payloads].map(([id, payload]) => ({ id,
                url: new URL(relative(work, payload.framePath).split(sep).map(encodeURIComponent).join("/"), serverUrl).href }));
              captured = await stage(`worker ${worker} frame ${frame}`, config.frameTimeoutMs,
                () => captureFrame(frame, sourceUpdates));
            } finally {
              // Output submission may block behind an earlier frame. Never
              // retain source leases across that independent backpressure.
              sourceLease.release();
            }
            if (input.frames !== undefined) {
              capturedBytes += captured.buffer.byteLength;
              assert(capturedBytes <= config.maxRenderedBytes, "HTML renderer PNG output exceeds its byte limit");
            }
            timing.seekMs += captured.seekMs;
            timing.prepareMs += captured.prepareMs;
            timing.screenshotMs += captured.screenshotMs;
            if (orderedFrames === undefined) {
              await writeFile(join(outputFrames, `${String(index).padStart(9, "0")}.png`), captured.buffer);
            } else {
              const outputSubmitStarted = performance.now();
              await orderedFrames.submit(index, captured.buffer);
              outputSubmitWorkerMs += performance.now() - outputSubmitStarted;
            }
            completed++;
            capturedFrames++;
            if (performance.now() - lastProgressAt >= 1_000) {
              onProgress({ phase: "worker-progress", worker, completed, elapsedMs: elapsedMs() });
              lastProgressAt = performance.now();
            }
          }
          const decision = concurrency.complete({ now: performance.now(), frames: batch.endFrameExclusive - batch.startFrame,
            remainingFrames: frameCount - capturedFrames, startupMs, active: initializing === 0 ? active : 0 });
          if (decision !== undefined) {
            console.log(`Capture auto: ${active} -> ${decision.workers} browsers; ${decision.fps.toFixed(1)} frames/s; ${decision.reason}`);
            while (active < decision.workers && open < limit && nextBatch < batches.length) launch();
          }
        }
        console.log(`Capture worker ${worker}: ${completed} frames; startup ${Math.round(setupMs)} ms; `
          + `seek ${Math.round(timing.seekMs)} ms; prepare ${Math.round(timing.prepareMs)} ms; PNG ${Math.round(timing.screenshotMs)} ms`);
        onProgress({ phase: "worker-complete", worker, completed, browserPid, elapsedMs: elapsedMs() });
      } catch (error) {
        controller.abort(error);
        throw error;
      } finally {
        active--;
        if (!ready) initializing--;
        if (session !== undefined) await close(session);
        open--;
        // A restoring decision can wait for a retiring Chrome to release its
        // slot. Closing processes still count against the reserved ceiling.
        if (!signal.aborted && active < concurrency.target && open < limit && nextBatch < batches.length) launch();
      }
    };
    const initialWorkers = concurrency.target;
    console.log(`Capture: opaque fast PNG; ${config.workers === "auto" ? "auto" : "fixed"} workers ${initialWorkers}, limit ${limit}`);
    for (let i = 0; i < initialWorkers; i++) launch();
    // New workers can join while earlier jobs run; collect every launched job.
    let awaited = 0;
    let failure: unknown;
    while (awaited < jobs.length) {
      const pending = jobs.slice(awaited);
      awaited = jobs.length;
      const results = await Promise.allSettled(pending);
      for (const result of results) if (result.status === "rejected") failure ??= result.reason;
    }
    if (failure !== undefined) throw failure;
    signal.throwIfAborted();
    await Promise.all(closing.values());
    console.log(`Capture source working set: ${sourceTotal} unique frames demanded; decoded ${sourceFrameStore.decodedFrames} frames; `
      + `redecoded ${Math.max(0, sourceFrameStore.decodedFrames - sourceTotal)}; decoder runs ${extraction}; `
      + `decoded bytes ${sourceFrameStore.decodedBytes}; largest frame ${sourceFrameStore.largestDecodedFrameBytes} bytes; `
      + `peak ${sourceFrameStore.peakResidentBytes} bytes; budget ${config.maxDecodedSourceBytes} bytes; `
      + `source acquire ${Math.round(sourceAcquireWorkerMs)} worker-ms; output submit ${Math.round(outputSubmitWorkerMs)} worker-ms`);
    if (input.frames !== undefined) return outputFrames;
    onProgress({ phase: "encoding", elapsedMs: elapsedMs() });
    await orderedFrames!.close();
    const outputStat = await stat(output);
    assert(outputStat.size > 0 && outputStat.size <= config.maxRenderedBytes, "HTML renderer output is empty or exceeds its byte limit");
    await verifyOutput({ path: output, document: { ...document, frameCount }, ffprobePath,
      timeoutMs: config.processTimeoutMs, maxOutputBytes: config.maxProcessOutputBytes, signal });
    signal.throwIfAborted();
    return output;
  } catch (error) {
    controller.abort(error);
    throw error;
  } finally {
    signal.removeEventListener("abort", abort);
    await Promise.allSettled([...sessions].map(close));
    if (sourceFrameStore !== undefined) await sourceFrameStore.close();
    if (encoderCompletion !== undefined) await Promise.allSettled([encoderCompletion]);
    if (server !== undefined) await server.close();
  }
}