Clone any website into a pixel-accurate Next.js app with AI — Playwright-measured extraction (no eyeballing), spec-gated parallel builder agents, and scored pixel-diff QA at phone/tablet/desktop. Works with Claude Code, Cursor, Copilot + 9 more.
JavaScript aiai-agentsclaudeclaude-codecursornextjspixel-perfectplaywrightscreenshot-to-codetailwindcssweb-cloningwebsite-cloner
#!/usr/bin/env node
// Discover all pages of a site: sitemap.xml + same-origin links on the start page.
// Usage: node scripts/extract/crawl.mjs <url> [--max 25]
// Output: docs/research/<host>/sitemap.json
import { launchPage, gotoAndSettle, hostOf, routeOf, writeJson, parseArgs } from "../lib.mjs";
const args = parseArgs(process.argv.slice(2));
const startUrl = args._[0];
if (!startUrl) {
console.error("Usage: node scripts/extract/crawl.mjs <url> [--max 25]");
process.exit(1);
}
const MAX = Number(args.max ?? 25);
const origin = new URL(startUrl).origin;
// 1. Try sitemap.xml (no browser needed) — runs concurrently with the page load below
// Same-site check that tolerates www./non-www mismatches (very common)
const sameSite = (a, b) => {
try {
return new URL(a).hostname.replace(/^www\./, "") === new URL(b).hostname.replace(/^www\./, "");
} catch {
return false;
}
};
const fetchLocs = async (u) => {
const res = await fetch(u, { signal: AbortSignal.timeout(10000) });
if (!res.ok) return [];
return [...(await res.text()).matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g)].map((m) => m[1].trim());
};
const collectSitemap = async () => {
const fromSitemap = new Set();
try {
const locs = await fetchLocs(`${origin}/sitemap.xml`);
// sitemap index → fetch child sitemaps (first 5) in parallel
const children = locs.filter((u) => u.endsWith(".xml")).slice(0, 5);
const childLocs = await Promise.all(
children.map((child) => fetchLocs(child).catch(() => [] /* skip broken child sitemap */))
);
locs.push(...childLocs.flat());
for (const u of locs) {
if (sameSite(u, origin) && !u.endsWith(".xml")) fromSitemap.add(u);
}
} catch {
// no sitemap — fine, nav links below are the primary source
}
return fromSitemap;
};
// 2. Collect same-origin links from the rendered start page
const collectLinks = async () => {
const { browser, page } = await launchPage();
await gotoAndSettle(page, startUrl);
const links = await page.evaluate(() => {
const inNav = (el) => !!el.closest("header, nav, footer");
const host = location.hostname.replace(/^www\./, "");
const seen = new Map();
for (const a of document.querySelectorAll("a[href]")) {
const href = a.href.split("#")[0].split("?")[0];
let linkHost;
try {
linkHost = new URL(href).hostname.replace(/^www\./, "");
} catch {
continue;
}
if (linkHost !== host) continue;
if (/\.(pdf|zip|jpg|jpeg|png|webp|svg|mp4|xml|ico)$/i.test(href)) continue;
const prev = seen.get(href);
const entry = {
url: href,
text: (a.textContent || "").trim().slice(0, 60),
source: inNav(a) ? "nav" : "body",
};
// nav placement wins over body placement
if (!prev || (prev.source === "body" && entry.source === "nav")) seen.set(href, entry);
}
return [...seen.values()];
});
await browser.close();
return links;
};
// Sitemap fetches and the browser page load are independent — overlap them.
const [fromSitemap, links] = await Promise.all([collectSitemap(), collectLinks()]);
// 3. Merge: start page first, then nav links, then sitemap-only, then body links
const merged = new Map();
const add = (url, source, text = "") => {
const clean = url.replace(/\/+$/, "") || origin;
if (!merged.has(clean)) merged.set(clean, { url: clean, route: routeOf(clean), source, text });
};
add(startUrl, "start");
links.filter((l) => l.source === "nav").forEach((l) => add(l.url, "nav", l.text));
[...fromSitemap].forEach((u) => add(u, "sitemap"));
links.filter((l) => l.source === "body").forEach((l) => add(l.url, "body", l.text));
const all = [...merged.values()];
const pages = all.slice(0, MAX);
const out = {
origin,
startUrl,
generatedAt: new Date().toISOString(),
totalDiscovered: all.length,
capped: all.length > MAX,
pages,
overflow: all.slice(MAX).map((p) => p.url),
};
writeJson(`docs/research/${hostOf(startUrl)}/sitemap.json`, out);
console.log(JSON.stringify({ totalDiscovered: all.length, kept: pages.length, pages: pages.map((p) => `${p.route} (${p.source})`) }, null, 2));