Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
import { existsSync } from "node:fs";
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
import { CaptureDirRefusedError } from "./captureErrors.js";
import { join } from "node:path";
import {
downloadAssets,
downloadAndRewriteFonts,
mergeDrops,
noDrops,
totalDrops,
} from "./assetDownloader.js";
import { extractFontMetadata } from "./fontMetadataExtractor.js";
import { normalizeErrorMessage } from "../utils/errorMessage.js";
import { diag } from "../ui/diagnostics.js";
import { serializeTokensForCapture } from "./partialCapture.js";
import type { PartialCaptureState } from "./partialCapture.js";
import type {
CapturePhase,
CapturePhaseProgress,
CaptureResult,
DesignTokens,
ExtractedHtml,
} from "./types.js";
import type { CatalogedAsset } from "./assetCataloger.js";
import type { IconCandidate } from "./faviconRanker.js";
import { createCaptureDownloadBudget } from "./readBoundedResponse.js";
import {
captionImagesWithGemini,
generateAssetDescriptions,
resolveVisionPhaseCompletion,
} from "./contentExtractor.js";
import type { VisionCaptionOutcome } from "./contentExtractor.js";
function hasVisionCredentials(skipVision: boolean): boolean {
return Boolean(
!skipVision &&
(process.env.OPENROUTER_API_KEY ||
process.env.GEMINI_API_KEY ||
process.env.GOOGLE_API_KEY ||
(process.env.HYPERFRAMES_VERTEX_PROJECT_ID &&
process.env.HYPERFRAMES_VERTEX_SERVICE_ACCOUNT)),
);
}
function writeAssetDescriptionsFile(outputDir: string, lines: string[], header: string): void {
writeCaptureFileSync(
join(outputDir, "extracted", "asset-descriptions.md"),
header + lines.map((line) => `- ${line}`).join("\n") + "\n",
"utf-8",
);
}
export interface PostExtractionInput {
state: PartialCaptureState;
outputDir: string;
warnings: string[];
progress: (stage: string, detail?: string) => void;
remainingMs: () => number;
phase: (
name: CapturePhase,
status: CapturePhaseProgress["status"],
reason?: CapturePhaseProgress["reason"],
) => void;
animationCatalog: CaptureResult["animationCatalog"];
catalogedAssets: CatalogedAsset[];
visibleTextContent: string;
faviconLinks: IconCandidate[];
tokens: DesignTokens;
extracted: ExtractedHtml;
skipAssets: boolean;
skipVision: boolean;
downloadByteBudget: ReturnType<typeof createCaptureDownloadBudget>;
assets: CaptureResult["assets"];
dropped: CaptureResult["dropped"];
fontDrops: CaptureResult["dropped"];
canWrite: () => boolean;
}
export interface PostExtractionResult {
assets: CaptureResult["assets"];
dropped: CaptureResult["dropped"];
fontDrops: CaptureResult["dropped"];
extracted: ExtractedHtml;
tokens: DesignTokens;
animationCatalog: CaptureResult["animationCatalog"];
}
export async function runPostExtraction(input: PostExtractionInput): Promise<PostExtractionResult> {
let {
state,
outputDir,
warnings,
progress,
remainingMs,
phase,
animationCatalog,
catalogedAssets,
visibleTextContent,
faviconLinks,
tokens,
extracted,
skipAssets,
skipVision,
downloadByteBudget,
assets,
dropped,
fontDrops,
canWrite,
} = input;
const runFontExtraction = async (): Promise<void> => {
if (!canWrite()) return;
// Download fonts and preserve per-font budget exhaustion in the capture tally.
phase("fonts", "started");
const fontPass = await downloadAndRewriteFonts(extracted.headHtml, outputDir, {
remainingMs,
byteBudget: downloadByteBudget,
});
fontDrops = fontPass.drops;
extracted.headHtml = fontPass.css;
phase(
"fonts",
remainingMs() > 0 ? "completed" : "degraded",
remainingMs() > 0 ? undefined : "budget-exhausted",
);
// Identify each downloaded font by reading its OpenType name table.
// Modern frameworks hash font filenames; this manifest tells the
// downstream pipeline (DESIGN.md authoring, beat sub-agents) which file
// belongs to which family without guessing from filename patterns.
try {
const fontsManifest = extractFontMetadata(
join(outputDir, "assets", "fonts"),
join(outputDir, "extracted", "fonts-manifest.json"),
);
if (fontsManifest.families.length > 0) {
const summary = fontsManifest.families
.map((f) => `${f.family}${f.variable ? " (variable)" : ""} × ${f.fileCount}`)
.join(", ");
// stderr (via diag): `capture --json` writes its envelope to stdout, so
// these progress/advisory lines must not land there.
diag.notice(`Font metadata extracted: ${summary}`);
if (fontsManifest.unidentified.length > 0) {
diag.warn(
` ${fontsManifest.unidentified.length} font file(s) could not be identified — DESIGN.md should flag these explicitly.`,
);
}
}
} catch (err) {
diag.warn("Font metadata extraction failed (non-fatal):", normalizeErrorMessage(err));
}
};
await runFontExtraction();
const runAnimationCatalog = async (): Promise<void> => {
// Save animation catalog — lean version for the agent (not 745 raw CSS declarations)
if (animationCatalog) {
// Extract just what's useful: counts, named animations, a few representative keyframed entries
const uniqueAnimNames = new Set<string>();
for (const d of animationCatalog.cssDeclarations || []) {
if (d.animation?.name) uniqueAnimNames.add(d.animation.name);
}
// Keep up to 10 Web Animations that have actual keyframe data (most useful for recreation)
const representativeAnims = (animationCatalog.webAnimations || [])
.filter((a: { keyframes?: unknown[] }) => a.keyframes && a.keyframes.length > 0)
.slice(0, 10);
const leanCatalog = {
summary: animationCatalog.summary,
namedAnimations: Array.from(uniqueAnimNames),
scrollTriggeredElements: (animationCatalog.scrollTargets || []).length,
representativeAnimations: representativeAnims,
};
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "animations.json"),
JSON.stringify(leanCatalog, null, 2),
"utf-8",
);
}
}
};
await runAnimationCatalog();
const runAssetDownload = async (): Promise<void> => {
// Download assets — single pass using the catalog for best image quality
let assetDrops = noDrops();
const runAssetFetch = async (): Promise<void> => {
if (!skipAssets && canWrite()) {
// Called even with the budget already gone, for the reason the font pass is: the loop that
// skips an asset is the only thing that can say how many it skipped.
phase("assets", "started");
progress("assets", "Downloading assets...");
const assetPass = await downloadAssets(tokens, outputDir, catalogedAssets, faviconLinks, {
remainingMs,
byteBudget: downloadByteBudget,
});
assets = assetPass.assets;
assetDrops = assetPass.drops;
state.assets = assets;
// Which icons the site declared, what each one is, and why one became favicon.<ext>.
// The brand-kit consumer needs the shape to decide which tile an icon belongs in; the
// reason is what stops a substituted headline from being silent again.
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "icons-manifest.json"),
JSON.stringify(assetPass.icons, null, 2),
"utf-8",
);
}
phase(
"assets",
remainingMs() > 0 ? "completed" : "degraded",
remainingMs() > 0 ? undefined : "budget-exhausted",
);
} else {
phase("assets", "degraded", "disabled");
}
};
await runAssetFetch();
const recordAssetDrops = (): void => {
// One capture-wide tally, summed from the two passes that own the drops. The warning is
// DERIVED from it rather than written alongside it, so the prose and the number cannot
// disagree the way two separately-authored budget strings could.
dropped = mergeDrops(fontDrops, assetDrops);
state.dropped = dropped;
const droppedTotal = totalDrops(dropped);
if (droppedTotal > 0) {
const breakdown = Object.entries(dropped)
.filter(([, n]) => typeof n === "number" && n > 0)
.map(([reason, n]) => `${n} ${reason}`)
.join(", ");
warnings.push(
`${droppedTotal} referenced asset(s) are not in this capture (${breakdown}). ` +
"A thin capture with no drops is a thin page; this one was truncated.",
);
}
};
recordAssetDrops();
const rewriteTokenAssets = (): void => {
// Join in-section media URLs → downloaded local paths, then re-write
// tokens.json. Downstream page recreation MUST reference local files:
// remote URLs fail at render time (hotlink/CORS 403, no egress in
// Docker/Lambda, frame-timing blanks for not-yet-loaded images).
const base = (u: string): string => u.split(/[#?]/)[0] ?? u;
const localByUrl = new Map<string, string>();
const collectLocalAssetPaths = (): void => {
for (const a of assets) {
if (!a.url || !a.localPath) continue;
localByUrl.set(a.url, a.localPath);
localByUrl.set(base(a.url), a.localPath);
}
};
collectLocalAssetPaths();
const applyLocalAssetPaths = (): void => {
for (const sec of tokens.sections || []) {
const local: string[] = [];
for (const u of sec.assetUrls || []) {
const hit = localByUrl.get(u) || localByUrl.get(base(u));
if (hit && !local.includes(hit)) local.push(hit);
}
if (local.length) sec.assets = local;
}
};
applyLocalAssetPaths();
if (assets.length && Array.isArray(tokens.sections)) {
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "tokens.json"),
serializeTokensForCapture(tokens),
"utf-8",
);
}
}
};
await rewriteTokenAssets();
};
await runAssetDownload();
// Persist a self-contained page recreation under extracted/, with images already inlined.
try {
const pageHtml = `<!doctype html>\n<html ${extracted.htmlAttrs || ""}>\n<head>\n${extracted.headHtml}\n</head>\n<body>\n${extracted.bodyHtml}\n</body>\n</html>\n`;
state.pageHtml = pageHtml;
if (canWrite())
writeCaptureFileSync(join(outputDir, "extracted", "page.html"), pageHtml, "utf-8");
} catch (err) {
warnings.push(`page.html write failed: ${err}`);
}
// Save visible text content for AI agent to use
if (visibleTextContent) {
if (canWrite()) {
writeCaptureFileSync(
join(outputDir, "extracted", "visible-text.txt"),
visibleTextContent,
"utf-8",
);
}
}
// detected-libraries and assets-catalog removed — 0/8 agents read them in v6 testing
// AI-powered image captioning via Gemini (optional — enriches asset descriptions)
let geminiCaptions: Record<string, string> = {};
const runVision = async (): Promise<void> => {
if (skipVision) {
phase("vision", "degraded", "disabled");
} else if (remainingMs() <= 0 || !canWrite()) {
warnings.push(
"Capture budget exhausted before vision captioning; catalog descriptions were preserved.",
);
phase("vision", "degraded", "budget-exhausted");
} else {
phase("vision", "started");
let visionOutcome: VisionCaptionOutcome = {
timedOutRequests: 0,
failedRequests: 0,
budgetExhausted: false,
};
geminiCaptions = await captionImagesWithGemini(outputDir, progress, warnings, {
remainingMs,
onOutcome: (outcome) => {
visionOutcome = outcome;
},
});
const completion = resolveVisionPhaseCompletion(visionOutcome, remainingMs());
phase(
"vision",
completion.status,
completion.status === "degraded" ? completion.reason : undefined,
);
}
};
await runVision();
// Generate asset descriptions for the AI agent
const runAssetDescriptions = (): void => {
progress("design", "Generating asset descriptions...");
try {
const lines = generateAssetDescriptions(outputDir, tokens, catalogedAssets, geminiCaptions);
if (lines.length === 0 || !canWrite()) return;
const hasVisionKey = hasVisionCredentials(skipVision);
const header = hasVisionKey
? "# Asset Descriptions\n\nOne line per file. Read this instead of opening every image individually.\n\nTo find a specific brand or icon, **grep this file for the brand name in the description text** (e.g. `grep -i 'autodesk' asset-descriptions.md`). The Gemini Vision captions identify what's actually in each file — that's the agent's selector.\n\nThe `logo-<hash>.svg` filename prefix is a cheap structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). It is NOT a content claim — many `logo-*` files are nav icons or decorative shapes. Trust the captions, not the filename prefix.\n\n"
: "# Asset Descriptions\n\n⚠️ No vision credentials — descriptions below are catalog-derived (alt text, headings, section context, filename) instead of Vision-generated. To get richer Vision descriptions on the next capture, set GEMINI_API_KEY (or GOOGLE_API_KEY), or HYPERFRAMES_VERTEX_PROJECT_ID plus HYPERFRAMES_VERTEX_SERVICE_ACCOUNT for Vertex service-account auth, and re-run.\n\nThe `logo-<hash>.svg` filename prefix is a structural hint (DOM said this SVG was inside a `<header>`, home-link `<a>`, or had an aria-label matching the page brand). To pick the actual brand logo without Vision, open the `logo-*` candidates in a previewer or rasterize them with `sharp` before referencing — composing a fake logo ships off-brand in the final video.\n\n";
writeAssetDescriptionsFile(outputDir, lines, header);
progress(
"design",
`${lines.length} asset descriptions written${hasVisionKey ? "" : " (no vision provider — catalog-fallback mode)"}`,
);
} catch {
/* non-critical */
}
};
runAssetDescriptions();
progress("design", "DESIGN.md will be created by your AI agent");
// Generate contact sheets (saves AI agents 50-65% tokens vs reading images individually)
// All functions return string[] — paginated so every image is covered
const runContactSheets = async (): Promise<void> => {
if (remainingMs() > 0 && canWrite()) {
phase("contact-sheets", "started");
try {
const { createScrollContactSheet, createAssetContactSheet, createSvgContactSheet } =
await import("./contactSheet.js");
const contactSheetBudget = { remainingMs };
const runScrollContactSheet = async (): Promise<void> => {
const scrollSheets = await createScrollContactSheet(
join(outputDir, "screenshots"),
join(outputDir, "screenshots", "contact-sheet.jpg"),
contactSheetBudget,
);
if (scrollSheets.length > 0)
progress(
"design",
`Screenshot contact sheet generated (${scrollSheets.length} page${scrollSheets.length > 1 ? "s" : ""})`,
);
};
await runScrollContactSheet();
const runAssetContactSheet = async (): Promise<void> => {
const assetsImgDir = join(outputDir, "assets");
if (existsSync(assetsImgDir)) {
const assetSheets = await createAssetContactSheet(
assetsImgDir,
join(outputDir, "assets", "contact-sheet.jpg"),
contactSheetBudget,
);
if (assetSheets.length > 0)
progress(
"design",
`Asset contact sheet generated (${assetSheets.length} page${assetSheets.length > 1 ? "s" : ""})`,
);
}
};
await runAssetContactSheet();
const runSvgContactSheet = async (): Promise<void> => {
// Scan assets/svgs/ (inline SVGs) AND assets/ root (external SVGs from <img src="*.svg">)
// so sites like huly.io that only use external SVGs still get a grid
const svgsDir = join(outputDir, "assets", "svgs");
const assetsRootDir = join(outputDir, "assets");
const hasSvgsDir = existsSync(svgsDir);
if (hasSvgsDir) ensureCaptureDirSync(outputDir, svgsDir);
const svgOutputPath = hasSvgsDir
? join(outputDir, "assets", "svgs", "contact-sheet.jpg")
: join(outputDir, "assets", "contact-sheet-svgs.jpg");
const svgSheets = await createSvgContactSheet(
svgsDir,
svgOutputPath,
assetsRootDir,
contactSheetBudget,
);
if (svgSheets.length > 0)
progress(
"design",
`SVG contact sheet generated (${svgSheets.length} page${svgSheets.length > 1 ? "s" : ""})`,
);
};
await runSvgContactSheet();
} catch (err) {
if (err instanceof CaptureDirRefusedError)
warnings.push(`SVG contact sheet skipped: ${err.message}`);
/* otherwise non-critical — agent can still read images individually */
}
phase(
"contact-sheets",
remainingMs() > 0 ? "completed" : "degraded",
remainingMs() > 0 ? undefined : "budget-exhausted",
);
} else {
warnings.push(
"Capture budget exhausted before contact sheets; source images were preserved.",
);
phase("contact-sheets", "degraded", "budget-exhausted");
}
};
await runContactSheets();
return { assets, dropped, fontDrops, extracted, tokens, animationCatalog };
}