Skip to content

Write HTML. Render video. Built for agents.

27.8 KB
Raw Blame
/**
 * Download assets (SVGs, images, favicon, video posters) from extracted tokens + asset catalog.
 *
 * Uses the asset catalog (which already deduplicates srcset variants and keeps the highest
 * resolution) as the single source of truth for images. Favicon links are passed separately.
 */

import { isBlockedNetworkHost } from "@hyperframes/engine";
import { publicRequest } from "./publicNetwork.js";
import { ensureCaptureDirSync, writeCaptureFileSync } from "./captureFile.js";
import { join, extname } from "node:path";
import { createHash } from "node:crypto";
import type { DesignTokens, DownloadedAsset } from "./types.js";
import type { CatalogedAsset } from "./assetCataloger.js";
import { CAPTURE_USER_AGENT } from "./userAgent.js";
import { rankIconCandidates, type IconCandidate } from "./faviconRanker.js";
import { classifyIcon, type IconShape } from "./iconClassifier.js";
import {
  readBoundedResponse,
  createCaptureDownloadBudget,
  type DownloadByteBudget,
} from "./readBoundedResponse.js";
import { captureFontExtension, captureFontFilename } from "./captureFontValidation.js";
import { captureImageExtension } from "./captureImageValidation.js";
import { loadSharp } from "../utils/optionalPackages.js";

interface DownloadBudgetOptions {
  byteBudget?: DownloadByteBudget;
  remainingMs?: () => number;
}

/**
 * Why an asset the page referenced is not in the capture.
 *
 * Three of these are DECISIONS this downloader made and one is a FAILURE it hit, which is the
 * split a reader actually needs: a capture that is thin because the page is thin looks exactly
 * like a capture that is thin because a limit truncated it, and neither used to say so.
 *
 * Every member is counted at the single line that performs the drop, so a count can never
 * disagree with the branch it describes.
 */
export type AssetDropReason =
  /** Fetched, then judged too small to be a real asset rather than a spacer or tracking pixel. */
  | "size-floor"
  /** Larger than any one asset may be; skipped whole, never cut. */
  | "size-ceiling"
  /** The post-navigation clock ran out before this one was reached. */
  | "budget-exhausted"
  /** A per-run or per-family limit was already met. */
  | "cap-reached"
  /** The request or the write failed: network error, timeout, refused address, bad status, disk. */
  | "unavailable";

export type AssetDropCounts = Record<AssetDropReason, number>;

/** A tally with every reason at zero — the shape a caller merges into. */
export function noDrops(): AssetDropCounts {
  return {
    "size-floor": 0,
    "size-ceiling": 0,
    "budget-exhausted": 0,
    "cap-reached": 0,
    unavailable: 0,
  };
}

/** Sum two tallies. Used to fold the font pass and the asset pass into one capture-wide count. */
export function mergeDrops(a: AssetDropCounts, b: AssetDropCounts): AssetDropCounts {
  const total = noDrops();
  for (const reason of Object.keys(total) as AssetDropReason[]) {
    total[reason] = a[reason] + b[reason];
  }
  return total;
}

/** How many assets were dropped in total, for a caller deciding whether to say anything at all. */
export function totalDrops(drops: AssetDropCounts): number {
  return Object.values(drops).reduce((sum, n) => sum + n, 0);
}

// SVGs: hash-of-bytes filename so it can't drift from content; label-derived names mis-assigned brands.
function svgContentHashSlug(svgSource: string | Buffer, isLogo: boolean): string {
  const hash = createHash("sha1").update(svgSource).digest("hex").slice(0, 8);
  return isLogo ? `logo-${hash}` : `svg-${hash}`;
}

/**
 * Make a scraped inline `<svg>` usable as a standalone `.svg` file.
 *
 * An inline SVG in an HTML document inherits the SVG namespace from the parser, so the DOM's
 * `outerHTML` does not serialize `xmlns`. That string is fine pasted back into HTML but is NOT
 * a valid standalone document: `<img src="logo-abc123.svg">` renders a broken-image icon, which
 * is how these assets are actually consumed downstream. Declare the namespaces on the way to disk.
 *
 * `xlink:href` is deprecated but still emitted by plenty of sites; an undeclared `xlink:` prefix
 * is a parse error in a standalone document, so declare that too — but only when it is used.
 */
export function toStandaloneSvg(outerHTML: string): string {
  const open = outerHTML.match(/<svg\b[^>]*>/i);
  if (!open) return outerHTML;
  const original = open[0];
  let tag = original;
  const add: string[] = [];
  if (!/\sxmlns\s*=/i.test(tag)) add.push('xmlns="http://www.w3.org/2000/svg"');
  if (/\sxlink:[a-z-]+\s*=/i.test(outerHTML) && !/\sxmlns:xlink\s*=/i.test(tag)) {
    add.push('xmlns:xlink="http://www.w3.org/1999/xlink"');
  }
  if (!add.length) return outerHTML;
  tag = tag.replace(/^<svg\b/i, `<svg ${add.join(" ")}`);
  return outerHTML.replace(original, tag);
}

/** One icon the page declared, as downloaded and inspected. */
export interface IconRecord {
  /** Path inside the capture, e.g. `assets/icon-apple-touch-icon-180x180.png`. */
  file: string;
  url: string;
  rel: string;
  sizes: string | null;
  type: string | null;
  /** Position in the declared-quality ranking; 0 is the best-ranked icon. */
  rank: number;
  shape: IconShape;
  shapeReason: string;
}

/**
 * Every icon a page declared, plus which one became `assets/favicon.<ext>` and why.
 *
 * The `reason` field is the point of this file. The downloader chooses among candidates, and a
 * choice whose losers are invisible is indistinguishable from having had no choice at all — the
 * failure mode that let a silent 403 substitute a worse icon without anything recording it.
 */
export interface IconManifest {
  schema: "hyperframes.capture.icons.v1";
  headline: {
    /** The backwards-compatible stem, e.g. `assets/favicon.png`. */
    file: string;
    /** The `icons[].file` it was copied from, so a consumer can avoid showing it twice. */
    source: string;
    rank: number;
    shape: IconShape;
    reason: string;
  } | null;
  icons: IconRecord[];
}

function emptyIconManifest(): IconManifest {
  return { schema: "hyperframes.capture.icons.v1", headline: null, icons: [] };
}

/** Icons downloaded per capture. Pages declare up to a dozen apple-touch sizes; a few is plenty. */
const MAX_ICONS = 8;

/**
 * Headline preference, deliberately BINARY: a positively identified bare mark first, then the
 * existing declared-quality ranking for everything else.
 *
 * `unknown` is not promoted above `badge`. Ranking it in between looked reasonable and is wrong:
 * a `.ico` cannot be decoded for inspection, so on a site whose icons are all badges the
 * undecodable legacy file would outrank the good SVG purely for being unexaminable. Absence of
 * evidence is not evidence of a bare mark.
 */
const SHAPE_PREFERENCE: Record<IconShape, number> = { "bare-mark": 0, unknown: 1, badge: 1 };

/** `<link rel="apple-touch-icon" sizes="180x180">` -> `icon-apple-touch-icon-180x180`. */
function iconFileStem(icon: IconCandidate): string {
  const slug = `${icon.rel} ${icon.sizes ?? "unsized"}`
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-+|-+$/g, "");
  return `icon-${slug || "unknown"}`;
}

function headlineReason(chosen: IconRecord, all: IconRecord[]): string {
  const badges = all.filter((i) => i.shape === "badge").length;
  if (chosen.shape === "bare-mark") {
    return badges > 0
      ? `bare mark preferred over ${badges} badge(s)`
      : "the only candidate is a bare mark";
  }
  const bare = all.filter((i) => i.shape === "bare-mark").length;
  if (bare > 0) return `best-ranked bare mark was unavailable; used a ${chosen.shape}`;
  return `no bare-mark candidate among ${all.length} icon(s); used the best-ranked ${chosen.shape}`;
}

/**
 * Download every icon the page declared, classify each, and copy the best onto the historical
 * `assets/favicon.<ext>` stem.
 *
 * Downloading all of them rather than stopping at the first success is what makes the choice
 * possible: shape can only be read from bytes, so a ranking that stops early can never know
 * whether the candidate it skipped was the bare mark the brand band actually wants.
 */
/** A stem not yet used in this capture; sites declare the same rel+sizes pair more than once. */
function uniqueStem(icon: IconCandidate, used: Set<string>): string {
  const base = iconFileStem(icon);
  let stem = base;
  for (let n = 2; used.has(stem); n++) stem = `${base}-${n}`;
  used.add(stem);
  return stem;
}

/** Fetch one icon, write it under `stem`, and inspect what it is. Null when it did not arrive. */
async function fetchAndInspectIcon(
  icon: IconCandidate,
  rank: number,
  stem: string,
  outputDir: string,
  timeoutMs: number,
  byteBudget?: DownloadByteBudget,
): Promise<{ record: IconRecord; buffer: Buffer } | null> {
  const buffer = await fetchBuffer(icon.href, timeoutMs, 2 * 1024 * 1024, byteBudget);
  if (!buffer) return null;
  const ext = await captureImageExtension(buffer);
  if (!ext) return null;

  const file = `assets/${stem}${ext}`;
  writeCaptureFileSync(join(outputDir, file), buffer);
  const verdict = await classifyIcon(buffer, ext);
  return {
    buffer,
    record: {
      file,
      url: icon.href,
      rel: icon.rel,
      sizes: icon.sizes ?? null,
      type: icon.type ?? null,
      rank,
      shape: verdict.shape,
      shapeReason: verdict.reason,
    },
  };
}

/** Copy the winning icon onto the historical `assets/favicon.<ext>` stem consumers match on. */
function promoteHeadline(
  manifest: IconManifest,
  bytesByFile: Map<string, Buffer>,
  outputDir: string,
): DownloadedAsset | null {
  const chosen = [...manifest.icons].sort(
    (a, b) => SHAPE_PREFERENCE[a.shape] - SHAPE_PREFERENCE[b.shape] || a.rank - b.rank,
  )[0];
  if (!chosen) return null;

  const file = `assets/favicon${extname(chosen.file)}`;
  writeCaptureFileSync(join(outputDir, file), bytesByFile.get(chosen.file)!);
  manifest.headline = {
    file,
    source: chosen.file,
    rank: chosen.rank,
    shape: chosen.shape,
    reason: headlineReason(chosen, manifest.icons),
  };
  return { url: chosen.url, localPath: file, type: "favicon" };
}

/**
 * Download every icon the page declared, classify each, and promote the best one.
 *
 * Downloading all of them rather than stopping at the first success is what makes the choice
 * possible: shape can only be read from bytes, so a ranking that stops early can never know
 * whether the candidate it skipped was the bare mark the brand band actually wants.
 */
async function downloadDeclaredIcons(
  faviconLinks: IconCandidate[],
  outputDir: string,
  drops: AssetDropCounts,
  options: DownloadBudgetOptions,
): Promise<{ assets: DownloadedAsset[]; manifest: IconManifest }> {
  const ranked = rankIconCandidates(faviconLinks);
  const attempted = ranked.slice(0, MAX_ICONS);
  drops["cap-reached"] += ranked.length - attempted.length;

  const assets: DownloadedAsset[] = [];
  const manifest = emptyIconManifest();
  const bytesByFile = new Map<string, Buffer>();
  const usedStems = new Set<string>();

  for (const [rank, icon] of attempted.entries()) {
    const remainingMs = options.remainingMs?.() ?? 10_000;
    if (remainingMs <= 0) {
      drops["budget-exhausted"] += attempted.length - rank;
      break;
    }
    try {
      const stem = uniqueStem(icon, usedStems);
      const got = await fetchAndInspectIcon(
        icon,
        rank,
        stem,
        outputDir,
        Math.min(10_000, remainingMs),
        options.byteBudget,
      );
      if (!got) {
        drops.unavailable++;
        continue;
      }
      bytesByFile.set(got.record.file, got.buffer);
      manifest.icons.push(got.record);
      assets.push({ url: icon.href, localPath: got.record.file, type: "favicon" });
    } catch {
      drops.unavailable++;
    }
  }

  try {
    const headline = promoteHeadline(manifest, bytesByFile, outputDir);
    if (headline) assets.push(headline);
  } catch {
    drops.unavailable++;
  }

  return { assets, manifest };
}

// fallow-ignore-next-line complexity
export async function downloadAssets(
  tokens: DesignTokens,
  outputDir: string,
  catalogedAssets?: CatalogedAsset[],
  faviconLinks?: IconCandidate[],
  options: DownloadBudgetOptions = {},
): Promise<{ assets: DownloadedAsset[]; drops: AssetDropCounts; icons: IconManifest }> {
  await loadSharp("website capture");
  options = { ...options, byteBudget: options.byteBudget ?? createCaptureDownloadBudget() };
  const assetsDir = join(outputDir, "assets");
  ensureCaptureDirSync(outputDir, assetsDir);

  const assets: DownloadedAsset[] = [];
  const drops = noDrops();
  const downloadedUrls = new Set<string>();
  let icons: IconManifest = emptyIconManifest();

  ensureCaptureDirSync(outputDir, join(outputDir, "assets", "svgs"));
  const usedSvgNames = new Set<string>();
  const MAX_INLINE_SVGS = 30;
  const MAX_INLINE_SVG_BYTES = 2 * 1024 * 1024;
  drops["cap-reached"] += Math.max(0, tokens.svgs.length - MAX_INLINE_SVGS);
  for (let i = 0; i < tokens.svgs.length && i < MAX_INLINE_SVGS; i++) {
    const svg = tokens.svgs[i]!;
    if (!svg.outerHTML || svg.outerHTML.length < 50) {
      drops["size-floor"]++;
      continue;
    }
    // Hash the bytes that actually land on disk, so the filename still can't drift from content.
    const svgFile = toStandaloneSvg(svg.outerHTML);
    if (Buffer.byteLength(svgFile, "utf-8") > MAX_INLINE_SVG_BYTES) {
      drops["size-ceiling"]++;
      continue;
    }
    const slug = svgContentHashSlug(svgFile, !!svg.isLogo);
    let finalSlug = slug;
    let suffix = 2;
    while (usedSvgNames.has(finalSlug)) {
      finalSlug = `${slug}-${suffix}`;
      suffix++;
    }
    usedSvgNames.add(finalSlug);
    const name = `${finalSlug}.svg`;
    const localPath = `assets/svgs/${name}`;
    try {
      writeCaptureFileSync(join(outputDir, localPath), svgFile, "utf-8");
      assets.push({ url: "", localPath, type: "svg" });
    } catch {
      drops.unavailable++;
    }
  }

  // 2. Icons — keep every one the page declares, then choose the headline among them.
  const iconPass = await downloadDeclaredIcons(faviconLinks || [], outputDir, drops, options);
  assets.push(...iconPass.assets);
  icons = iconPass.manifest;

  // 3. Images — use the catalog as the single source of truth (highest resolution, deduplicated)
  // If the catalog is empty, asset download produces zero images — this is surfaced as a warning
  // so the capture doesn't silently produce a half-empty dataset.
  const imageUrls: { url: string; isPoster: boolean }[] = [];

  if (catalogedAssets && catalogedAssets.length > 0) {
    // Use catalog — already deduplicated with highest-res srcset variants
    for (const a of catalogedAssets) {
      if (a.type !== "Image" && a.type !== "Background") continue;
      if (!a.url.startsWith("http")) continue;
      // Skip junk
      if (a.url.includes("pixel") || a.url.includes("beacon") || a.url.includes("analytics"))
        continue;
      if (a.url.includes("/favicon")) continue;
      // Download images from standard img/video contexts + CSS backgrounds (for hero sections, feature illustrations)
      const hasGoodContext = a.contexts.some(
        (c) =>
          c === "img[src]" ||
          c === "img[srcset]" ||
          c === "video[poster]" ||
          c === "source[srcset]" ||
          c === "data-src" ||
          c === "css url()",
      );
      if (!hasGoodContext) continue;
      const isPoster = a.contexts.includes("video[poster]");
      imageUrls.push({ url: a.url, isPoster });
    }
  }

  // Download all images — use catalog context for human-readable filenames.
  // Pre-filter to deduplicate before downloading.
  const toDownload: {
    url: string;
    isPoster: boolean;
    normalized: string;
    catalog?: CatalogedAsset;
  }[] = [];
  for (const { url, isPoster } of imageUrls) {
    const normalized = normalizeUrl(url);
    if (downloadedUrls.has(normalized)) continue;
    downloadedUrls.add(normalized);
    const catalog = catalogedAssets?.find((a) => normalizeUrl(a.url) === normalized);
    toDownload.push({ url, isPoster, normalized, catalog });
  }

  // Download in parallel batches of 5
  const BATCH_SIZE = 5;
  let imgIdx = 0;
  const usedNames = new Set<string>();
  for (let i = 0; i < toDownload.length; i += BATCH_SIZE) {
    const remainingMs = options.remainingMs?.() ?? 10_000;
    if (remainingMs <= 0) {
      drops["budget-exhausted"] += toDownload.length - i;
      break;
    }
    const batch = toDownload.slice(i, i + BATCH_SIZE);
    const results = await Promise.allSettled(
      batch.map(async ({ url, isPoster, catalog }) => {
        const parsedUrl = new URL(url);
        const buffer = await fetchBuffer(
          url,
          Math.min(10_000, remainingMs),
          20 * 1024 * 1024,
          options.byteBudget,
        );
        if (!buffer) {
          drops.unavailable++;
          return null;
        }
        const ext = await captureImageExtension(buffer);
        if (!ext) {
          drops.unavailable++;
          return null;
        }
        const isSvg = ext === ".svg";
        const minSize = isSvg ? 200 : 10000;
        if (buffer.length < minSize) {
          drops["size-floor"]++;
          return null;
        }
        return { url, isPoster, parsedUrl, ext, buffer, catalog };
      }),
    );
    for (const result of results) {
      // A rejection never reached a drop site of its own, so it is counted here. A fulfilled
      // `null` already counted itself above; counting it again here would double it.
      if (result.status === "rejected") {
        drops.unavailable++;
        continue;
      }
      if (!result.value) continue;
      const { url, isPoster, parsedUrl, ext, buffer, catalog } = result.value;
      try {
        let slug: string;
        if (ext === ".svg") {
          const c = catalog;
          const brandRe = /logo|brand|wordmark/i;
          const isLogo = !!(
            c?.inBanner ||
            c?.inHomeLink ||
            c?.matchesTitleBrand ||
            c?.contexts?.some((s) => brandRe.test(s)) ||
            (c?.description && brandRe.test(c.description)) ||
            (c?.nearestHeading && brandRe.test(c.nearestHeading)) ||
            (c?.sectionClasses && brandRe.test(c.sectionClasses))
          );
          slug = svgContentHashSlug(buffer, isLogo);
        } else {
          slug = deriveAssetName(parsedUrl, catalog, isPoster, imgIdx, usedNames);
        }
        const name = `${slug}${ext}`;
        usedNames.add(slug);
        const localPath = `assets/${name}`;
        writeCaptureFileSync(join(outputDir, localPath), buffer);
        assets.push({ url, localPath, type: "image" });
        imgIdx++;
      } catch {
        drops.unavailable++;
      }
    }
  }

  // 4. OG image (if not already downloaded)
  if (tokens.ogImage && !downloadedUrls.has(normalizeUrl(tokens.ogImage))) {
    const remainingMs = options.remainingMs?.() ?? 10_000;
    try {
      if (remainingMs <= 0) {
        drops["budget-exhausted"]++;
      } else {
        const buffer = await fetchBuffer(
          tokens.ogImage,
          Math.min(10_000, remainingMs),
          20 * 1024 * 1024,
          options.byteBudget,
        );
        const ext = buffer && (await captureImageExtension(buffer));
        if (!buffer || !ext) {
          drops.unavailable++;
        } else if (buffer.length <= 5000) {
          drops["size-floor"]++;
        } else {
          const localPath = `assets/og-image${ext}`;
          writeCaptureFileSync(join(outputDir, localPath), buffer);
          assets.push({ url: tokens.ogImage, localPath, type: "image" });
        }
      }
    } catch {
      drops.unavailable++;
    }
  }

  return { assets, drops, icons };
}

/** Normalize URL for deduplication — unwrap Next.js image proxy, strip w/q params */
function normalizeUrl(u: string): string {
  try {
    const parsed = new URL(u);
    if (parsed.pathname.includes("_next/image") && parsed.searchParams.has("url")) {
      return decodeURIComponent(parsed.searchParams.get("url")!);
    }
    parsed.searchParams.delete("w");
    parsed.searchParams.delete("q");
    parsed.searchParams.delete("dpr");
    return parsed.toString();
  } catch {
    return u;
  }
}

/**
 * Download fonts referenced in CSS and rewrite URLs to local paths.
 * Returns the modified CSS string with local font paths.
 */
// fallow-ignore-next-line complexity
export async function downloadAndRewriteFonts(
  css: string,
  outputDir: string,
  options: DownloadBudgetOptions = {},
): Promise<{ css: string; drops: AssetDropCounts }> {
  options = { ...options, byteBudget: options.byteBudget ?? createCaptureDownloadBudget() };
  const assetsDir = join(outputDir, "assets", "fonts");
  ensureCaptureDirSync(outputDir, assetsDir);
  const drops = noDrops();

  const fontUrlRegex = /url\(['"]?(https?:\/\/[^'")\s]+\.(?:woff2?|ttf|otf)[^'")\s]*?)['"]?\)/g;
  const fontUrls = new Set<string>();
  let match;
  while ((match = fontUrlRegex.exec(css)) !== null) {
    if (match[1]) fontUrls.add(match[1]);
  }

  if (fontUrls.size === 0) return { css, drops };

  // Limit font download attempts to bound worst-case egress and latency. Google Fonts serves
  // 20+ unicode-range subsets per weight, so successes alone cannot be the bound: six transient
  // failures can intentionally suppress later URLs in that family. Latin-priority sorting below
  // makes the limited attempts useful while keeping this failure tradeoff explicit.
  const MAX_FONTS_PER_FAMILY = 6;
  const MAX_TOTAL_FONTS = 30;
  const familyCounts = new Map<string, number>();

  // Extract font-family from the @font-face rule containing each URL
  const getFamilyForUrl = (url: string): string => {
    const idx = css.indexOf(url);
    if (idx === -1) return "_unknown";
    const blockStart = css.lastIndexOf("@font-face", idx);
    if (blockStart === -1) return "_unknown";
    const blockSlice = css.slice(blockStart, idx);
    const familyMatch = blockSlice.match(/font-family\s*:\s*['"]?([^'";}\n]+)/i);
    return familyMatch?.[1] ? familyMatch[1].trim().toLowerCase() : "_unknown";
  };

  // Prioritize Latin subsets over CJK/Arabic/etc unicode ranges
  const sortedUrls = Array.from(fontUrls).sort((a, b) => {
    const aLatin = /latin|[A-Za-z0-9]{10,}\.woff/.test(a) ? 0 : 1;
    const bLatin = /latin|[A-Za-z0-9]{10,}\.woff/.test(b) ? 0 : 1;
    return aLatin - bLatin;
  });

  const usedFontNames = new Set<string>();
  let rewritten = css;
  let count = 0;

  for (const [index, fontUrl] of sortedUrls.entries()) {
    const remainingMs = options.remainingMs?.() ?? 10_000;
    if (remainingMs <= 0) {
      drops["budget-exhausted"] += sortedUrls.length - index;
      break;
    }
    if (count >= MAX_TOTAL_FONTS) {
      drops["cap-reached"] += sortedUrls.length - index;
      break;
    }
    const family = getFamilyForUrl(fontUrl);
    const familyCount = familyCounts.get(family) || 0;
    if (familyCount >= MAX_FONTS_PER_FAMILY) {
      drops["cap-reached"]++;
      continue;
    }
    familyCounts.set(family, familyCount + 1);
    count++;

    try {
      const buffer = await fetchBuffer(
        fontUrl,
        Math.min(10_000, remainingMs),
        10 * 1024 * 1024,
        options.byteBudget,
      );
      const extension = buffer && captureFontExtension(buffer);
      if (buffer && extension) {
        const filename = captureFontFilename(fontUrl, extension, usedFontNames);
        const localPath = join(assetsDir, filename);
        const relativePath = `assets/fonts/${filename}`;
        writeCaptureFileSync(localPath, buffer);
        rewritten = rewritten.split(fontUrl).join(relativePath);
      } else {
        drops.unavailable++;
      }
    } catch {
      drops.unavailable++;
    }
  }

  return { css: rewritten, drops };
}

/**
 * Block requests to private/internal hosts to prevent SSRF. WHATWG URL parsing
 * canonicalizes alternate IPv4 encodings (decimal/octal/hex) to dotted-quad
 * before we see them, so only dotted IPv4 and bracketed IPv6 literals reach the
 * shared engine classifier.
 */
export function isPrivateUrl(url: string): boolean {
  try {
    const u = new URL(url);
    if (u.protocol !== "http:" && u.protocol !== "https:") return true; // no file:, etc.
    return isBlockedNetworkHost(u.hostname);
  } catch {
    return true; // reject unparseable URLs
  }
}

/** Max redirect hops safeFetch will follow before giving up. */
const MAX_FETCH_REDIRECTS = 5;

/**
 * fetch() that re-validates the SSRF denylist on EVERY redirect hop. A bare
 * `redirect: "follow"` only checks the initial URL, so a public URL can 30x to
 * an internal/metadata host. We resolve redirects manually and re-run
 * isPrivateUrl on each Location. Returns null when blocked, on too many hops,
 * or on network error.
 */
export async function safeFetch(url: string, init?: RequestInit): Promise<Response | null> {
  let current = url;
  for (let hop = 0; hop <= MAX_FETCH_REDIRECTS; hop++) {
    if (isPrivateUrl(current)) return null;
    const res = await publicRequest(current, { ...init, redirect: "manual" });
    if (res.status >= 300 && res.status < 400) {
      const loc = res.headers.get("location");
      if (!loc) return res;
      await res.body?.cancel().catch(() => {});
      try {
        current = new URL(loc, current).toString();
      } catch {
        return null; // malformed Location header
      }
      continue;
    }
    return res;
  }
  return null; // too many redirects
}

async function fetchBuffer(
  url: string,
  timeoutMs: number,
  maxBytes: number,
  budget: DownloadByteBudget = createCaptureDownloadBudget(),
): Promise<Buffer | null> {
  if (budget.remainingBytes <= 0) return null;
  try {
    const res = await safeFetch(url, {
      signal: AbortSignal.timeout(timeoutMs),
      headers: { "User-Agent": CAPTURE_USER_AGENT },
    });
    if (!res || !res.ok) return null;
    // Reject XML/HTML error pages disguised as 200 OK (common with S3/CloudFront)
    const ct = res.headers.get("content-type") || "";
    if (ct.includes("text/xml") || ct.includes("text/html") || ct.includes("application/xml")) {
      return null;
    }
    return await readBoundedResponse(res, maxBytes, budget);
  } catch {
    return null;
  }
}

function slugify(text: string): string {
  return text
    .toLowerCase()
    .replace(/[^a-z0-9]+/g, "-")
    .replace(/^-|-$/g, "")
    .slice(0, 40);
}

/**
 * Derive a human-readable filename from catalog context.
 * Priority: alt text > nearest heading > meaningful URL path > fallback index.
 */
function deriveAssetName(
  parsedUrl: URL,
  catalog: CatalogedAsset | undefined,
  isPoster: boolean,
  idx: number,
  usedNames: Set<string>,
): string {
  const candidates: string[] = [];

  // 1. Alt text / description from catalog
  if (catalog?.description) {
    const desc = catalog.description.replace(/[^a-zA-Z0-9 -]/g, "").trim();
    if (desc.length > 3 && desc.length < 80) candidates.push(desc);
  }

  // 2. Nearest heading context
  if (catalog?.nearestHeading) {
    const heading = catalog.nearestHeading.replace(/[^a-zA-Z0-9 -]/g, "").trim();
    if (heading.length > 3 && heading.length < 60) candidates.push(heading);
  }

  // 3. Meaningful URL path segment
  const rawName =
    parsedUrl.pathname
      .split("/")
      .pop()
      ?.replace(/\.[^.]+$/, "") || "";
  const isMeaningful =
    rawName.length > 2 &&
    rawName.length < 50 &&
    !/^[a-f0-9]{8,}$/i.test(rawName) &&
    !/^\d+$/.test(rawName) &&
    !rawName.includes("_next") &&
    !rawName.includes("?");
  if (isMeaningful) candidates.push(rawName);

  // 4. Section classes as context
  if (catalog?.sectionClasses) {
    const classes = catalog.sectionClasses
      .split(/\s+/)
      .filter((c) => c.length > 3 && c.length < 30 && !/^(w-|h-|p-|m-|flex|grid|block)/.test(c))
      .slice(0, 2)
      .join("-");
    if (classes.length > 3) candidates.push(classes);
  }

  // Pick the best candidate
  const prefix = isPoster ? "poster" : catalog?.aboveFold ? "hero" : "image";
  let slug = "";

  for (const c of candidates) {
    slug = slugify(c);
    if (slug.length > 3 && !usedNames.has(slug)) break;
  }

  if (!slug || slug.length <= 3 || usedNames.has(slug)) {
    slug = `${prefix}-${idx}`;
  }

  // Deduplicate
  let final = slug;
  let suffix = 2;
  while (usedNames.has(final)) {
    final = `${slug}-${suffix}`;
    suffix++;
  }

  return final;
}