Clone any website into a pixel-accurate Next.js app with AI — Playwright-measured extraction (no eyeballing), spec-gated parallel builder agents, and scored pixel-diff QA at phone/tablet/desktop. Works with Claude Code, Cursor, Copilot + 9 more.
JavaScript aiai-agentsclaudeclaude-codecursornextjspixel-perfectplaywrightscreenshot-to-codetailwindcssweb-cloningwebsite-cloner
// Shared helpers for all extraction/QA scripts.
import { chromium } from "playwright";
import { PNG } from "pngjs";
import { spawnSync } from "node:child_process";
import { mkdirSync, writeFileSync, readFileSync, renameSync } from "node:fs";
import { createHash } from "node:crypto";
import { dirname, join } from "node:path";
// The three viewports every extraction and QA step must cover.
export const VIEWPORTS = {
phone: { width: 390, height: 844 },
ipad: { width: 768, height: 1024 },
pc: { width: 1440, height: 900 },
};
export function hostOf(url) {
return new URL(url).hostname.replace(/^www\./, "");
}
export function slugify(s) {
return (
s
.toLowerCase()
.replace(/^https?:\/\//, "")
.replace(/[^a-z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 80) || "index"
);
}
// URL pathname -> Next.js route ("/about-us" stays "/about-us", "/" stays "/")
export function routeOf(url) {
const p = new URL(url).pathname.replace(/\/+$/, "");
return p === "" ? "/" : p;
}
export function writeJson(path, data) {
mkdirSync(dirname(path), { recursive: true });
writeFileSync(path, JSON.stringify(data, null, 2) + "\n", "utf8");
console.error(` ✓ wrote ${path}`);
}
// ---------------------------------------------------------------------------
// Browser lifecycle
//
// One Chromium per process, shared by every context. Launching costs ~350ms;
// opening a context on a live browser costs ~40ms. Scripts that touch three
// viewports used to pay the launch three times — now they pay it once.
// ---------------------------------------------------------------------------
// Memoise the launch *promise*, not the resolved browser. Concurrent callers
// (forEachViewport opens three contexts at once) would otherwise all observe a
// null browser and each launch their own Chromium — closeBrowser() then closes
// only the last one and the orphans keep the Node event loop alive forever.
let _browserPromise = null;
const CONTEXT_OPTS = {
userAgent:
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
deviceScaleFactor: 1,
// Service workers serve stale shells and bypass request interception — both
// poison extraction determinism and the response cache below.
serviceWorkers: "block",
};
// ---------------------------------------------------------------------------
// Response cache: memory -> disk -> network
//
// Every context has an isolated HTTP cache, so a 3-viewport run downloads every
// image, font and script three times — and a full clone run spawns ~100 separate
// script processes against the same site. The memory layer serves repeats within
// one process; the disk layer (docs/research/.cache/) survives across processes,
// so probe/section/diff runs after page.mjs load subresources from disk instead
// of the network. GET-only, static resource types only (documents and XHR always
// hit the network). `manifest.mjs init --force` wipes the disk layer, and stale
// entries expire after CACHE_TTL (SITE_CLONER_CACHE_TTL_H hours, default 24).
// Set SITE_CLONER_NO_CACHE=1 to bypass caching entirely.
// ---------------------------------------------------------------------------
const CACHEABLE = new Set(["image", "font", "stylesheet", "script"]);
const MAX_ENTRY = 4 * 1024 * 1024; // 4 MB per resource
const MAX_TOTAL = 256 * 1024 * 1024; // 256 MB overall (memory layer)
export const CACHE_DIR = "docs/research/.cache";
const CACHE_TTL_MS = (parseFloat(process.env.SITE_CLONER_CACHE_TTL_H) || 24) * 3600 * 1000;
const _cache = new Map(); // url -> { body: Buffer, contentType: string }
let _cacheBytes = 0;
const _cacheStats = { memHits: 0, diskHits: 0, misses: 0 };
let _cacheDirReady = false;
// The clone under QA serves /_next/static chunks whose CONTENT changes between
// fix iterations under stable names — disk-caching loopback origins would keep
// serving the old build and fake the diff scores. Memory (single process) is safe.
function isLoopback(url) {
const h = new URL(url).hostname;
return h === "localhost" || h === "127.0.0.1" || h === "::1" || h === "[::1]";
}
function diskPathsFor(url) {
const key = createHash("sha256").update(url).digest("hex").slice(0, 24);
return { bin: join(CACHE_DIR, `${key}.bin`), meta: join(CACHE_DIR, `${key}.json`) };
}
function diskRead(url) {
try {
const { bin, meta } = diskPathsFor(url);
const m = JSON.parse(readFileSync(meta, "utf8"));
if (m.url !== url) return null; // hash-prefix collision — treat as miss
if (Date.now() - m.savedAt > CACHE_TTL_MS) return null;
return { body: readFileSync(bin), contentType: m.contentType };
} catch {
return null; // missing/torn/corrupt entry — plain miss
}
}
function diskWrite(url, body, contentType) {
try {
if (!_cacheDirReady) {
mkdirSync(CACHE_DIR, { recursive: true });
_cacheDirReady = true;
}
const { bin, meta } = diskPathsFor(url);
// tmp + rename so a concurrent reader never sees a half-written file. The
// body lands before the sidecar: a sidecar without its .bin reads as a miss.
writeFileSync(`${bin}.${process.pid}.tmp`, body);
renameSync(`${bin}.${process.pid}.tmp`, bin);
writeFileSync(
`${meta}.${process.pid}.tmp`,
JSON.stringify({ url, contentType, savedAt: Date.now(), size: body.length })
);
renameSync(`${meta}.${process.pid}.tmp`, meta);
} catch {
/* disk cache is best-effort — a failed write is just a future miss */
}
}
function memStore(url, body, contentType) {
if (body.length <= MAX_ENTRY && _cacheBytes + body.length <= MAX_TOTAL) {
_cache.set(url, { body, contentType });
_cacheBytes += body.length;
}
}
async function enableResponseCache(context) {
await context.route("**/*", async (route) => {
const req = route.request();
if (req.method() !== "GET" || !CACHEABLE.has(req.resourceType())) {
return route.fallback();
}
const url = req.url();
const hit = _cache.get(url);
if (hit) {
_cacheStats.memHits++;
return route.fulfill({ status: 200, contentType: hit.contentType, body: hit.body });
}
if (!isLoopback(url)) {
const disk = diskRead(url);
if (disk) {
_cacheStats.diskHits++;
memStore(url, disk.body, disk.contentType);
return route.fulfill({ status: 200, contentType: disk.contentType, body: disk.body });
}
}
_cacheStats.misses++;
let res;
try {
res = await route.fetch();
} catch {
return route.fallback(); // fetch aborted (navigation) — let the browser handle it
}
if (res.status() === 200) {
try {
const body = await res.body();
const contentType = res.headers()["content-type"] || "";
memStore(url, body, contentType);
if (body.length <= MAX_ENTRY && !isLoopback(url)) diskWrite(url, body, contentType);
} catch {
/* body unavailable — just pass the response through */
}
}
return route.fulfill({ response: res });
});
}
// Launch Chromium, installing it on the fly if this is a fresh checkout.
// "npx playwright install chromium" as a manual first step is exactly the kind
// of setup friction that makes a template feel broken — the error Playwright
// throws names the fix, so apply it automatically and retry once.
async function launchInstallingIfMissing() {
try {
return await chromium.launch({ headless: true });
} catch (err) {
if (!/playwright install|Executable doesn't exist/i.test(String(err?.message))) throw err;
console.error(" Chromium not installed — running `npx playwright install chromium` (one-time)…");
const r = spawnSync("npx", ["playwright", "install", "chromium"], { stdio: "inherit" });
if (r.status !== 0) throw err;
return chromium.launch({ headless: true });
}
}
export function getBrowser() {
if (!_browserPromise) _browserPromise = launchInstallingIfMissing();
return _browserPromise;
}
// Every script must call this before exiting — the browser process keeps the
// Node event loop alive otherwise.
export async function closeBrowser() {
if (!_browserPromise) return;
const pending = _browserPromise;
_browserPromise = null;
await (await pending).close();
const { memHits, diskHits, misses } = _cacheStats;
if (memHits + diskHits + misses > 0) {
console.error(` cache: ${memHits} mem, ${diskHits} disk, ${misses} network`);
}
}
// Lazy-loading detection: count IntersectionObservers the page constructs, so
// autoScroll() can tell "this page reveals content on scroll" from "this page
// is fully painted already" instead of always paying the scroll cost.
async function instrumentLazySignals(context) {
await context.addInitScript(() => {
window.__ioCount = 0;
const Native = window.IntersectionObserver;
if (!Native) return;
window.IntersectionObserver = class extends Native {
constructor(...a) {
super(...a);
window.__ioCount++;
}
};
});
}
// Open a page on the shared browser. `close()` disposes only this context.
// `cache: false` opts out of response caching (e.g. when probing live-changing pages).
export async function openPage(viewport = VIEWPORTS.pc, { cache = true } = {}) {
const browser = await getBrowser();
const context = await browser.newContext({ viewport, ...CONTEXT_OPTS });
await instrumentLazySignals(context);
if (cache && !process.env.SITE_CLONER_NO_CACHE) await enableResponseCache(context);
const page = await context.newPage();
return { page, context, close: () => context.close() };
}
// Back-compat for single-page scripts written against the older API:
// const { browser, page } = await launchPage(); … await browser.close();
// `browser.close()` tears down the context AND the shared Chromium, matching the
// old semantics exactly. Deferring the teardown to a process exit hook would
// deadlock: Playwright's open socket keeps the event loop alive, so 'beforeExit'
// never fires and the script hangs instead of exiting.
export async function launchPage(viewport = VIEWPORTS.pc) {
const { page, close } = await openPage(viewport);
return {
page,
browser: {
close: async () => {
await close();
await closeBrowser();
},
},
};
}
// Run `fn(page, viewportName)` at several viewports concurrently on one browser.
// Returns results keyed by viewport name. Page loads are the expensive part and
// they overlap completely here.
export async function forEachViewport(names, fn) {
const entries = await Promise.all(
names.map(async (vpName) => {
const { page, close } = await openPage(VIEWPORTS[vpName]);
try {
return [vpName, await fn(page, vpName)];
} finally {
await close();
}
})
);
return Object.fromEntries(entries);
}
// ---------------------------------------------------------------------------
// Waiting — conditions, not sleeps
// ---------------------------------------------------------------------------
// Wait until the document stops growing. Replaces a fixed sleep with the signal
// the sleep was standing in for: late CSS, webfonts and hero images all change
// scrollHeight when they land.
export async function settleLayout(page, { timeout = 3000, quietMs = 150 } = {}) {
await page
.evaluate(
async ({ timeout, quietMs }) => {
const deadline = Date.now() + timeout;
let last = -1;
let stableSince = 0;
while (Date.now() < deadline) {
const h = document.documentElement.scrollHeight;
if (h === last) {
if (!stableSince) stableSince = Date.now();
if (Date.now() - stableSince >= quietMs) return;
} else {
last = h;
stableSince = 0;
}
await new Promise((r) => requestAnimationFrame(r));
}
},
{ timeout, quietMs }
)
.catch(() => {});
}
// Wait until every <img> in the DOM has finished loading (or errored). This is
// the signal networkidle was standing in for on image-heavy pages, and unlike
// networkidle it can't be held hostage by analytics beacons. Polls instead of
// snapshotting once so images inserted AFTER the call starts (late hydration,
// lazy-load) are still waited on — which is what lets it run concurrently with
// networkidle instead of after it.
async function imagesSettled(page, { timeout = 4000 } = {}) {
await page
.evaluate(async (timeout) => {
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
const pending = [...document.images].filter((img) => !img.complete);
if (!pending.length) return;
await Promise.race([
Promise.all(
pending.map(
(img) =>
new Promise((r) => {
img.addEventListener("load", r, { once: true });
img.addEventListener("error", r, { once: true });
})
)
),
new Promise((r) => setTimeout(r, 100)), // tick: re-collect for new <img>s
]);
}
}, timeout)
.catch(() => {});
}
export async function gotoAndSettle(page, url) {
await page.goto(url, { waitUntil: "load", timeout: 60000 });
// Precise signals instead of one long networkidle gamble. The three
// independent waits run concurrently — their caps race instead of summing —
// and settleLayout goes last because stable height depends on fonts and
// images having landed.
await Promise.all([
page.waitForLoadState("networkidle", { timeout: 4000 }).catch(() => {}),
page.evaluate(() => document.fonts?.ready).catch(() => {}),
imagesSettled(page),
]);
await settleLayout(page);
}
// Scroll to bottom in steps (triggers lazy-loaded images/animations), then back
// to top. Skipped entirely when the page shows no sign of lazy content — pass
// `{ force: true }` to scroll regardless.
export async function autoScroll(page, { force = false } = {}) {
const needed = force
? true
: await page
.evaluate(() => {
if (window.__ioCount > 0) return true;
return !!document.querySelector(
'img[loading="lazy"],iframe[loading="lazy"],[data-src],[data-srcset],[data-bg],[data-lazy]'
);
})
.catch(() => true); // if the probe fails, assume the worst and scroll
if (!needed) return;
await page.evaluate(async () => {
// Half-viewport steps: IntersectionObserver thresholds commonly need an
// element substantially in view, and a full-viewport jump can skip them.
const step = window.innerHeight / 2;
for (let y = 0; y <= document.documentElement.scrollHeight; y += step) {
window.scrollTo(0, y);
// Two frames is enough for observer callbacks to fire and request images;
// whether those images *arrive* is what the waits below cover.
await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
}
window.scrollTo(0, 0);
});
// The real lazy-load guarantee is "the images the scroll requested have
// decoded" — imagesSettled states that directly, so networkidle gets the same
// 4s cap it has everywhere else instead of the old 10s gamble.
await Promise.all([
page.waitForLoadState("networkidle", { timeout: 4000 }).catch(() => {}),
imagesSettled(page),
]);
await settleLayout(page);
}
// How long the transitions inside `selector` actually take, in ms — so a state
// capture can wait exactly that long instead of guessing. Covers ::before and
// ::after, where hover overlays usually live.
export async function transitionMs(page, selector, { min = 150, max = 4000 } = {}) {
const ms = await page
.evaluate(
({ selector, max }) => {
const root = document.querySelector(selector);
if (!root) return 0;
const els = [root, ...root.querySelectorAll("*")].slice(0, 500);
const nums = (v) => String(v || "").split(",").map((s) => parseFloat(s) || 0);
let worst = 0;
for (const el of els) {
for (const pe of [null, "::before", "::after"]) {
const cs = getComputedStyle(el, pe);
const dur = nums(cs.transitionDuration);
const delay = nums(cs.transitionDelay);
for (let i = 0; i < dur.length; i++) {
worst = Math.max(worst, (dur[i] + (delay[i] || 0)) * 1000);
}
}
if (worst >= max) return max;
}
return worst;
},
{ selector, max }
)
.catch(() => 0);
return Math.min(max, Math.max(min, Math.round(ms)));
}
// Park animations/transitions/videos so screenshots are deterministic (for QA diffs).
// Animations jump to their END state, not `animation: none`: killing them outright
// reverts `fill-mode: forwards` entry animations to their pre-animation frame, so
// scroll-revealed content screenshots at opacity 0 and QA penalizes a clone that
// is actually correct. A near-zero duration with iteration-count 1 lands every
// animation (including infinite spinners) on its final keyframe deterministically.
export async function freezePage(page) {
await page.addStyleTag({
content:
"*,*::before,*::after{" +
"animation-delay:-0.0001s!important;animation-duration:0.0001s!important;" +
"animation-iteration-count:1!important;" +
"transition-delay:0s!important;transition-duration:0s!important;" +
"caret-color:transparent!important;scroll-behavior:auto!important}",
});
// Two frames so the jumped-to-end animation states actually paint.
await page
.evaluate(() => new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))))
.catch(() => {});
// Park videos on a frame that actually has content. Seeking to 0 lands on the
// first encoded frame, which for a screen recording is often blank — that
// renders as an empty box and costs the section real diff points.
await page.evaluate(async () => {
const vids = [...document.querySelectorAll("video")];
await Promise.all(
vids.map(
(v) =>
new Promise((resolve) => {
v.pause();
const target = Number.isFinite(v.duration) && v.duration > 1 ? 1 : 0;
if (Math.abs(v.currentTime - target) < 0.05) return resolve();
v.addEventListener("seeked", () => resolve(), { once: true });
v.currentTime = target;
setTimeout(resolve, 1000); // don't hang on a video that won't seek
})
)
);
});
}
// ---------------------------------------------------------------------------
// Review-size screenshots
//
// Full-res PNGs (a full clone run produces ~8.5MB of them) are what pixelmatch
// and builder agents need — but the FOREMAN opens screenshots dozens of times
// just to judge "does this look right", and every full-res view costs tokens.
// A 640px-wide -review.png next to each full-res file is visually sufficient
// for judgment work at a fraction of the size.
// ---------------------------------------------------------------------------
export const REVIEW_WIDTH = 640;
export function reviewPathFor(pngPath) {
return pngPath.replace(/\.png$/i, "-review.png");
}
// Box-filter downscale to REVIEW_WIDTH wide. `src` is a decoded image (a PNG
// instance or the plain {width,height,data} object PNG.sync.read returns — it
// is NOT a PNG instance, so duck-type on .data), a PNG buffer, or a file path.
// No-op (returns false) when already narrow enough.
export function writeReviewPng(src, destPath) {
let img;
try {
if (typeof src === "string") img = PNG.sync.read(readFileSync(src));
else if (Buffer.isBuffer(src)) img = PNG.sync.read(src);
else img = src;
} catch {
return false; // unreadable source — review copy is best-effort
}
// Skip marginal downscales (e.g. ipad's 768 -> 640): resampling breaks PNG
// run-length compressibility, so a small shrink often GROWS the file.
if (!img?.data || !img.width || img.width <= REVIEW_WIDTH * 1.25) return false;
const scale = REVIEW_WIDTH / img.width;
const w = REVIEW_WIDTH;
const h = Math.max(1, Math.round(img.height * scale));
const out = new PNG({ width: w, height: h });
const sd = img.data, dd = out.data;
for (let y = 0; y < h; y++) {
const sy0 = Math.floor(y / scale);
const sy1 = Math.min(img.height, Math.max(sy0 + 1, Math.floor((y + 1) / scale)));
for (let x = 0; x < w; x++) {
const sx0 = Math.floor(x / scale);
const sx1 = Math.min(img.width, Math.max(sx0 + 1, Math.floor((x + 1) / scale)));
let r = 0, g = 0, b = 0, a = 0, n = 0;
for (let sy = sy0; sy < sy1; sy++) {
for (let sx = sx0; sx < sx1; sx++) {
const i = (sy * img.width + sx) * 4;
r += sd[i];
g += sd[i + 1];
b += sd[i + 2];
a += sd[i + 3];
n++;
}
}
const j = (y * w + x) * 4;
dd[j] = r / n;
dd[j + 1] = g / n;
dd[j + 2] = b / n;
dd[j + 3] = a / n;
}
}
writeFileSync(destPath, PNG.sync.write(out));
return true;
}
// Screenshot every section from ONE full-page capture: measure each selector's
// page-coordinate box, take a single fullPage shot, and crop the sections out of
// it (deviceScaleFactor is 1, so CSS px == image px). Pixel-identical to a
// per-section locator.screenshot() chain — same bounding boxes, same frozen
// page — minus N sequential scroll+shoot round-trips. Bonus fidelity: sticky
// headers can no longer bleed into mid-page section shots, because nothing
// scrolls between captures. Call AFTER freezePage so boxes match the pixels.
//
// `sections` = [{ name, selector }]; `pathFor(section)` -> output PNG path;
// `fullPagePath` additionally saves the full-page shot itself.
// `review: true` (default) also writes a 640px -review.png beside every PNG —
// pass false for QA captures that only feed pixelmatch.
// Sections the crop can't produce (selector missing, zero box, page taller than
// Chromium's capture limit) fall back to a locator screenshot; returns
// { failed: [names] } for anything neither path could capture.
export async function shootSectionsFromFullPage(page, sections, { fullPagePath = null, pathFor, review = true }) {
const boxes = await page
.evaluate((sels) => {
window.scrollTo(0, 0);
const out = {};
for (const sel of sels) {
const el = document.querySelector(sel);
if (!el) {
out[sel] = null;
continue;
}
const r = el.getBoundingClientRect();
// Round the EDGES, not width/height — for fractional boxes this matches
// how the element screenshot clip lands on whole pixels.
const x = Math.round(r.x + scrollX);
const y = Math.round(r.y + scrollY);
out[sel] = {
x,
y,
width: Math.round(r.x + scrollX + r.width) - x,
height: Math.round(r.y + scrollY + r.height) - y,
};
}
return out;
}, sections.map((s) => s.selector))
.catch(() => ({}));
const buf = await page.screenshot({ fullPage: true, ...(fullPagePath ? { path: fullPagePath } : {}) });
let img = null;
try {
img = PNG.sync.read(buf);
} catch {
/* unreadable capture — every section falls back below */
}
if (review && img && fullPagePath) writeReviewPng(img, reviewPathFor(fullPagePath));
const fallbacks = [];
for (const s of sections) {
const b = img && boxes[s.selector];
const x = b ? Math.max(0, Math.min(b.x, img.width - 1)) : 0;
const y = b ? Math.max(0, Math.min(b.y, img.height - 1)) : 0;
const w = b ? Math.min(b.width, img.width - x) : 0;
const h = b ? Math.min(b.height, img.height - y) : 0;
if (!b || w < 1 || h < 1) {
fallbacks.push(s);
continue;
}
try {
const out = new PNG({ width: w, height: h });
PNG.bitblt(img, out, x, y, w, h, 0, 0);
writeFileSync(pathFor(s), PNG.sync.write(out));
if (review) writeReviewPng(out, reviewPathFor(pathFor(s)));
} catch {
fallbacks.push(s);
}
}
const failed = [];
for (const s of fallbacks) {
try {
const loc = page.locator(s.selector).first();
await loc.scrollIntoViewIfNeeded({ timeout: 3000 });
await loc.screenshot({ path: pathFor(s), timeout: 5000 });
if (review) writeReviewPng(pathFor(s), reviewPathFor(pathFor(s)));
} catch {
failed.push(s.name);
}
}
return { failed };
}
// Tiny CLI arg parser: --key value / --flag -> { _: [positionals], key: value, flag: true }
// A repeated key collects into an array (--selector a --selector b -> ["a","b"]).
export function parseArgs(argv) {
const out = { _: [] };
for (let i = 0; i < argv.length; i++) {
const a = argv[i];
if (a.startsWith("--")) {
const key = a.slice(2);
const next = argv[i + 1];
const val = next !== undefined && !next.startsWith("--") ? (i++, next) : true;
if (key in out) out[key] = [].concat(out[key], val);
else out[key] = val;
} else {
out._.push(a);
}
}
return out;
}
// Normalise a possibly-repeated arg into an array. `--selector x` and
// `--selector x --selector y` both come back as arrays.
export function toList(v) {
if (v === undefined || v === true) return [];
return [].concat(v);
}