Clone any website into a pixel-accurate Next.js app with AI — Playwright-measured extraction (no eyeballing), spec-gated parallel builder agents, and scored pixel-diff QA at phone/tablet/desktop. Works with Claude Code, Cursor, Copilot + 9 more.
JavaScript aiai-agentsclaudeclaude-codecursornextjspixel-perfectplaywrightscreenshot-to-codetailwindcssweb-cloningwebsite-cloner
#!/usr/bin/env node
// ONE-SHOT page recon: everything Phase 1 + the default-state part of Phase 3a
// need, from THREE page loads total (one per viewport, all concurrent).
//
// From those three live pages this produces:
// docs/research/<host>/tokens.json design tokens (pc)
// docs/research/<host>/css.json real CSS rules/states (pc)
// docs/research/<host>/assets.json asset inventory + downloads to public/
// docs/research/<host>/responsive.json per-section signatures @ all 3 viewports
// docs/research/<host>/sections/<name>.json full computed-style walk per section (pc)
// docs/design-references/<host>/page-{pc,ipad,phone}.png full-page shots
// docs/design-references/<host>/<name>-{pc,ipad,phone}.png per-section shots
//
// Replaces running tokens.mjs + css.mjs + assets.mjs + responsive.mjs +
// screenshot.mjs + a batched section.mjs one after another (~15 page loads).
// Every measurement also comes from the SAME page state, so numbers, walks and
// screenshots can never disagree about what the page looked like.
//
// Usage:
// node scripts/extract/page.mjs <url>
// [--selector "css" --name x ...] explicit sections (skips auto-detect)
// [--depth 5] [--no-download] [--no-shots] [--no-section-shots]
// [--no-site-files] skip tokens.json/css.json (they're
// site-wide — only the first page needs them)
// [--legacy] emit the old full-blob walk format
// (default is compact-v1: style dictionary
// + inherited-prop pruning, ~70% smaller;
// resolve any node with resolve-walk.mjs)
//
// node scripts/extract/page.mjs --rename section-3=hero,section-4=features [--host <h>]
// NO browser: renames the section JSONs (incl. state captures), probe files,
// screenshots, responsive.json and manifest entries in place. Use when
// detection got the sections right but the auto-names wrong — re-running the
// extraction just to rename costs 3 page loads for nothing.
import { mkdirSync, existsSync, renameSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
import {
VIEWPORTS,
openPage,
closeBrowser,
gotoAndSettle,
autoScroll,
freezePage,
shootSectionsFromFullPage,
writeReviewPng,
reviewPathFor,
hostOf,
writeJson,
parseArgs,
toList,
} from "../lib.mjs";
import {
collectTokens,
collectCss,
collectAssets,
downloadAssets,
detectSections,
measureSections,
summarizeResponsive,
walkSections,
nameFromSelector,
detectUtilityCss,
collectSectionHtml,
} from "./collectors.mjs";
import { compactWalk, toLegacy } from "./walk-format.mjs";
import { selfReport, findPage, findSection, advanceStage, updatePageStatus, routeOfUrl } from "../manifest-lib.mjs";
const t0 = Date.now();
const args = parseArgs(process.argv.slice(2));
// ── rename mode: pure file/manifest surgery, exits before any browser work ──
if (args.rename) {
const pairs = String(args.rename).split(",").map((s) => s.trim()).filter(Boolean).map((p) => {
const eq = p.indexOf("=");
if (eq < 1) {
console.error(`Bad --rename "${p}" — use old=new[,old2=new2]`);
process.exit(1);
}
return { from: p.slice(0, eq).trim(), to: p.slice(eq + 1).trim() };
});
const manifestPath = "docs/research/manifest.json";
const manifest = existsSync(manifestPath) ? JSON.parse(readFileSync(manifestPath, "utf8")) : null;
let renameHost = typeof args.host === "string" ? args.host : manifest?.site ? hostOf(manifest.site) : null;
if (!renameHost) {
const dirs = existsSync("docs/research")
? readdirSync("docs/research").filter((d) => existsSync(`docs/research/${d}/sections`))
: [];
if (dirs.length === 1) renameHost = dirs[0];
}
if (!renameHost) {
console.error("--rename needs --host <host> (no manifest or single research dir to derive it from)");
process.exit(1);
}
const sectionsDir = `docs/research/${renameHost}/sections`;
const shotsDir = `docs/design-references/${renameHost}`;
const mv = (from, to) => {
if (!existsSync(from)) return false;
renameSync(from, to);
console.log(` ✓ ${from} → ${to}`);
return true;
};
for (const { from, to } of pairs) {
if (existsSync(`${sectionsDir}/${to}.json`)) {
console.error(` ! target name "${to}" already exists in ${sectionsDir} — skipping "${from}"`);
continue;
}
let any = mv(`${sectionsDir}/${from}.json`, `${sectionsDir}/${to}.json`);
if (existsSync(sectionsDir)) {
for (const f of readdirSync(sectionsDir)) {
// state captures follow their base section (hero-hover.json etc.)
if (f.startsWith(`${from}-`) && f.endsWith(".json")) {
any = mv(`${sectionsDir}/${f}`, `${sectionsDir}/${to}-${f.slice(from.length + 1)}`) || any;
}
}
}
mv(`docs/research/${renameHost}/probe-${from}.json`, `docs/research/${renameHost}/probe-${to}.json`);
if (existsSync(shotsDir)) {
for (const f of readdirSync(shotsDir)) {
if (f.startsWith(`${from}-`) && /^(pc|ipad|phone)(-review)?\.png$/.test(f.slice(from.length + 1))) {
mv(`${shotsDir}/${f}`, `${shotsDir}/${to}-${f.slice(from.length + 1)}`);
}
}
}
const respPath = `docs/research/${renameHost}/responsive.json`;
if (existsSync(respPath)) {
const resp = JSON.parse(readFileSync(respPath, "utf8"));
let hits = 0;
for (const s of resp.sections || []) if (s.name === from) (s.name = to), hits++;
if (hits) {
writeFileSync(respPath, JSON.stringify(resp, null, 2) + "\n");
console.log(` ✓ responsive.json: "${from}" → "${to}"`);
}
}
if (manifest) {
for (const p of manifest.pages || []) {
for (const s of p.sections || []) {
if (s.name !== from) continue;
s.name = to;
if (s.spec?.includes(`/${from}.spec.md`)) {
const ns = s.spec.replace(`/${from}.spec.md`, `/${to}.spec.md`);
mv(s.spec, ns);
s.spec = ns;
}
console.log(` ✓ manifest: ${p.route} "${from}" → "${to}"`);
}
}
}
if (!any) console.error(` ! no files found for section "${from}" under ${sectionsDir}`);
}
if (manifest) {
manifest.updatedAt = new Date().toISOString();
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
}
process.exit(0);
}
const url = args._[0];
if (!url) {
console.error('Usage: node scripts/extract/page.mjs <url> [--selector "css" --name x ...] [--depth 5] [--no-download] [--no-shots] [--no-site-files]');
process.exit(1);
}
const host = hostOf(url);
const depth = Number(args.depth ?? 5);
const explicit = toList(args.selector);
const explicitNames = toList(args.name);
const doDownload = !args["no-download"];
const doShots = !args["no-shots"];
const doSectionShots = doShots && !args["no-section-shots"];
const doSiteFiles = !args["no-site-files"]; // tokens.json + css.json are site-wide — skip on later pages
const shotDir = `docs/design-references/${host}`;
const pageSlug = new URL(url).pathname.replace(/\/+$/, "").replace(/[^a-z0-9]+/gi, "-").replace(/^-|-$/g, "") || "page";
// ── 1. Load all three viewports concurrently (the only page loads) ──────────
const loaded = Object.fromEntries(
await Promise.all(
["pc", "ipad", "phone"].map(async (vp) => {
const { page, close } = await openPage(VIEWPORTS[vp]);
await gotoAndSettle(page, url);
await autoScroll(page, { force: true }); // full-page shots need lazy content everywhere
return [vp, { page, close }];
})
)
);
const pc = loaded.pc.page;
console.error(` ✓ loaded @ pc/ipad/phone in ${((Date.now() - t0) / 1000).toFixed(1)}s`);
// ── 2. Sections: explicit or auto-detected on the desktop layout ────────────
let sections;
if (explicit.length) {
sections = explicit.map((selector, i) => ({ selector, name: explicitNames[i] || nameFromSelector(selector, i) }));
} else {
const detected = await detectSections(pc);
const used = new Set();
sections = detected.map((s, i) => {
let name = nameFromSelector(s.selector, i);
while (used.has(name)) name += "-2";
used.add(name);
return { selector: s.selector, name };
});
}
// ── 3. Measure everything (order matters: measure BEFORE freeze/screenshots) ─
const selectors = sections.map((s) => s.selector);
const [signatures, walks, tokens, css, assetsFound, utilityCss] = await Promise.all([
// signatures at all three viewports — one evaluate per page
(async () => {
const out = {};
await Promise.all(
Object.entries(loaded).map(async ([vp, { page }]) => {
out[vp] = await measureSections(page, selectors);
})
);
return out;
})(),
walkSections(pc, selectors, { depth }),
doSiteFiles ? collectTokens(pc) : null,
doSiteFiles ? collectCss(pc) : null,
collectAssets(pc),
detectUtilityCss(pc),
]);
// Utility-CSS fast path: on Tailwind-style sites the class list IS the spec,
// so each section's cleaned markup rides along in its JSON and spec-scaffold
// quotes it instead of dumping exhaustive computed styles.
const sectionHtml = utilityCss?.detected ? await collectSectionHtml(pc, selectors) : {};
if (utilityCss?.detected) {
console.error(` ✓ utility-CSS detected (${(utilityCss.ratio * 100).toFixed(0)}% of ${utilityCss.classTokens} class tokens) — capturing section markup`);
}
// ── 4. Write research files ─────────────────────────────────────────────────
const meta = { url, generatedAt: new Date().toISOString() };
if (tokens) writeJson(`docs/research/${host}/tokens.json`, { ...meta, utilityCss, ...tokens });
if (css) {
// interactiveCount/blockedCount are stdout-summary fields, not research data
const cssOut = { ...meta, ...css };
delete cssOut.interactiveCount;
delete cssOut.blockedCount;
writeJson(`docs/research/${host}/css.json`, cssOut);
}
writeJson(`docs/research/${host}/responsive.json`, {
...meta,
viewports: { phone: 390, ipad: 768, pc: 1440 },
sections: sections.map((s) => ({
selector: s.selector,
name: s.name,
phone: signatures.phone[s.selector],
ipad: signatures.ipad[s.selector],
pc: signatures.pc[s.selector],
summary: summarizeResponsive(signatures, s.selector),
})),
});
for (const s of sections) {
const walk = walks[s.selector];
const base = { ...meta, viewport: "pc", selector: s.selector, ...(sectionHtml[s.selector] ? { html: sectionHtml[s.selector] } : {}) };
if (args.legacy || walk?.error) {
writeJson(`docs/research/${host}/sections/${s.name}.json`, { ...base, tree: args.legacy ? toLegacy(walk) : walk });
} else {
const { tree, styleTable } = compactWalk(walk);
writeJson(`docs/research/${host}/sections/${s.name}.json`, { ...base, format: "compact-v1", tree, styleTable });
}
}
// Self-report: the walks above ARE the extraction, so the script does its own
// bookkeeping — registering any section the manifest doesn't know yet and
// advancing known ones to `extracted`. No manifest (or untracked route) = no-op.
{
let registered = 0, advanced = 0;
selfReport((m) => {
const p = findPage(m, routeOfUrl(url));
if (!p) return false;
for (const s of sections) {
if (walks[s.selector]?.error) continue;
let sec = findSection(p, s.name);
if (!sec) {
sec = { name: s.name, selector: s.selector, stage: "discovered", spec: null, component: null, scores: {} };
p.sections.push(sec);
registered++;
}
if (advanceStage(sec, "extracted")) advanced++;
}
if (registered || advanced) updatePageStatus(p);
return registered || advanced;
});
if (registered || advanced) {
console.error(` ✓ manifest: ${advanced} section(s) marked extracted${registered ? `, ${registered} newly registered` : ""}`);
}
}
// ── 5. Downloads + screenshots overlap (network vs browser work) ────────────
const jobs = [];
if (doDownload) {
jobs.push(
downloadAssets(assetsFound, { pool: 8 }).then((manifest) => {
writeJson(`docs/research/${host}/assets.json`, { ...meta, ...assetsFound, downloads: manifest });
const ok = manifest.filter((m) => m.ok).length;
const cached = manifest.filter((m) => m.cached).length;
const fail = manifest.filter((m) => !m.ok).length;
console.error(` ✓ assets: ${ok} ok (${cached} already on disk), ${fail} failed`);
})
);
} else {
writeJson(`docs/research/${host}/assets.json`, { ...meta, ...assetsFound, downloads: [] });
}
if (doShots) {
mkdirSync(shotDir, { recursive: true });
jobs.push(
...Object.entries(loaded).map(async ([vp, { page }]) => {
await freezePage(page); // deterministic pixels — safe now, measuring is done
if (doSectionShots) {
// One full-page capture yields the page shot AND every section crop —
// no per-section scroll+shoot chain.
const { failed } = await shootSectionsFromFullPage(page, sections, {
fullPagePath: `${shotDir}/${pageSlug}-${vp}.png`,
pathFor: (s) => `${shotDir}/${s.name}-${vp}.png`,
});
for (const name of failed) console.error(` ! section shot failed: ${name} @ ${vp}`);
} else {
const path = `${shotDir}/${pageSlug}-${vp}.png`;
await page.screenshot({ path, fullPage: true });
writeReviewPng(path, reviewPathFor(path));
}
console.error(` ✓ screenshots @ ${vp}`);
})
);
}
await Promise.all(jobs);
await Promise.all(Object.values(loaded).map((l) => l.close()));
await closeBrowser();
// ── 6. Summary the foreman can read at a glance ─────────────────────────────
console.log(
JSON.stringify(
{
url,
seconds: Number(((Date.now() - t0) / 1000).toFixed(1)),
sections: sections.map((s) => ({
name: s.name,
selector: s.selector,
pcHeight: signatures.pc[s.selector]?.height ?? null,
responsive: summarizeResponsive(signatures, s.selector).changes[0],
})),
utilityCss: utilityCss?.detected ? `detected (${(utilityCss.ratio * 100).toFixed(0)}% utility tokens) — sections carry html` : "not detected",
siteFiles: doSiteFiles ? ["tokens.json", "css.json"] : [],
files: {
research: `docs/research/${host}/`,
screenshots: doShots ? shotDir : null,
},
},
null,
2
)
);