Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Audit crawl: conservative OOM retries + window carry-over across chunks (#517)
Ben Senescue5e961bf48ec
diff --git a/src/server/features/audit/AuditScratchpad.ts b/src/server/features/audit/AuditScratchpad.tsindex ca8c3982..59ff6e8d 100644--- a/src/server/features/audit/AuditScratchpad.ts+++ b/src/server/features/audit/AuditScratchpad.ts@@ -26,6 +26,17 @@ export interface ClaimedUrl {inSitemap: boolean;}+interface ClaimedChunk {+ urls: ClaimedUrl[];+ /**+ * True when a prior attempt of this chunk already claimed work — i.e. the+ * caller is a workflow step retry after that attempt died mid-crawl. The+ * crawler uses this to retry far more conservatively (the usual cause of a+ * dead attempt is exceededMemory).+ */+ isRetry: boolean;+}+export interface FrontierStats {/** Pages attempted (crawled or errored) so far. */attempted: number;@@ -147,12 +158,12 @@ export class AuditScratchpad extends DurableObject {* had leased (link-discovered URLs drain before sitemap-only ones, FIFO* within each class — same ordering as the old in-memory queues).*/- async claimChunk(chunkNo: number, limit: number): Promise<ClaimedUrl[]> {+ async claimChunk(chunkNo: number, limit: number): Promise<ClaimedChunk> {const existing = this.selectClaimed(`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'leased' AND chunk_no = ?`,chunkNo,);- if (existing.length > 0) return existing;+ if (existing.length > 0) return { urls: existing, isRetry: true };// A retried step whose earlier attempt already crawled this chunk's// leases must not claim a fresh set under the same chunk number — that// would duplicate work and overshoot the page budget.@@ -164,8 +175,8 @@ export class AuditScratchpad extends DurableObject {chunkNo,).one();- if (done.n > 0) return [];- if (limit <= 0) return [];+ if (done.n > 0) return { urls: [], isRetry: true };+ if (limit <= 0) return { urls: [], isRetry: false };const fresh = this.selectClaimed(`SELECT url, depth, in_sitemap FROM frontier WHERE state = 'pending'@@ -179,7 +190,7 @@ export class AuditScratchpad extends DurableObject {row.url,);}- return fresh;+ return { urls: fresh, isRetry: false };}/** Persist one crawled sub-batch: completions, mirror rows, links, frontier. */diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.tsindex 5c3bde74..3cdcf9dc 100644--- a/src/server/lib/audit/crawl-window.test.ts+++ b/src/server/lib/audit/crawl-window.test.ts@@ -1,5 +1,8 @@import { describe, expect, it } from "vitest";-import { adjustCrawlWindow } from "@/server/lib/audit/crawl-window";+import {+ adjustCrawlWindow,+ RETRY_CRAWL_WINDOW,+} from "@/server/lib/audit/crawl-window";import type {CrawledPageResult,PageFetchClass,@@ -79,11 +82,30 @@ describe("adjustCrawlWindow", () => {});it("caps the window so heavy pages stay inside the byte budget", () => {- // 1 MiB average pages: 16 MiB budget / 1 MiB = window of 16.+ // 1 MiB average pages: 8 MiB budget / 1 MiB = window of 8.const recent = Array.from({ length: 25 }, () =>page("ok", 300, 1024 * 1024),);- expect(adjustCrawlWindow(20, recent)).toBe(16);+ expect(adjustCrawlWindow(20, recent)).toBe(8);+ });++ it("does not grow on a small sample, but still applies the byte bound", () => {+ const fastSmall = Array.from({ length: 5 }, () => page("ok", 400));+ expect(adjustCrawlWindow(10, fastSmall)).toBe(10);+ const fastHeavy = Array.from({ length: 5 }, () =>+ page("ok", 400, 1024 * 1024),+ );+ expect(adjustCrawlWindow(10, fastHeavy)).toBe(8);+ });++ it("retry limits keep the window small even on a clean, fast site", () => {+ const recent = Array.from({ length: 25 }, () => page("ok", 400));+ expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(5);+ });++ it("retry limits shrink below the normal minimum on trouble", () => {+ const recent = Array.from({ length: 10 }, () => page("error", 15_000));+ expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(2);});it("keeps the byte bound at the minimum window even for huge pages", () => {diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.tsindex 145eb1f6..7e2793bf 100644--- a/src/server/lib/audit/crawl-window.ts+++ b/src/server/lib/audit/crawl-window.ts@@ -7,32 +7,69 @@ import type { CrawledPageResult } from "@/server/lib/audit/types";* slowly (politeness toward struggling or defensive sites) and grows when* the site answers fast.*/-export const INITIAL_CRAWL_WINDOW = 10;-const MIN_WINDOW = 5;-const MAX_WINDOW = 20;-const SLOW_RESPONSE_MS = 10_000;-const FAST_RESPONSE_MS = 1_500;+interface CrawlWindowLimits {+ /** Window size a chunk starts with, before any observations. */+ initial: number;+ min: number;+ max: number;+ /**+ * Total HTML the in-flight window may buffer at once. Each in-flight page+ * holds its body (decoded to a UTF-16 string, roughly doubling it), and the+ * workflow shares its isolate's 128 MB memory limit with the rest of the+ * worker — production audits died with exceededMemory when a fast site let+ * the window grow unchecked.+ */+ budgetBytes: number;+}++export const CRAWL_WINDOW: CrawlWindowLimits = {+ initial: 10,+ min: 5,+ max: 20,+ budgetBytes: 8 * 1024 * 1024,+};/**- * Total HTML the in-flight window may buffer at once. Each in-flight page- * holds its body (decoded to a UTF-16 string, roughly doubling it), and the- * workflow shares its isolate's 128 MB memory limit with the rest of the- * worker — production audits died with exceededMemory when a fast site let- * the window grow unchecked.+ * Limits for a chunk whose earlier attempt died mid-crawl — in production+ * almost always exceededMemory on a heavy-page site. The retry is the+ * chunk's last attempt (retry limit 1), so it must not re-run the exact+ * memory profile that just killed the isolate.*/-const IN_FLIGHT_HTML_BUDGET_BYTES = 16 * 1024 * 1024;+export const RETRY_CRAWL_WINDOW: CrawlWindowLimits = {+ initial: 3,+ min: 2,+ max: 5,+ budgetBytes: 4 * 1024 * 1024,+};++const SLOW_RESPONSE_MS = 10_000;+const FAST_RESPONSE_MS = 1_500;/** Floor for the observed page size so tiny-page sites can't void the bound. */const MIN_ASSUMED_PAGE_BYTES = 64 * 1024;+/**+ * Growth requires a full-size sample. The first persist sub-batch is small+ * (so the byte bound reacts to heavy pages early), and a handful of fast+ * pages proves too little to widen the window.+ */+const GROWTH_MIN_SAMPLE = 25;++export function clampCrawlWindow(+ size: number,+ limits: CrawlWindowLimits,+): number {+ return Math.min(Math.max(size, limits.min), limits.max);+}/*** Adapt the window to the last persisted sub-batch. Shrinks on trouble- * (errors, blocks, very slow responses), grows only on a clean and mostly- * fast batch, and is always capped so the batch's average page size times- * the window stays inside the in-flight byte budget.+ * (errors, blocks, very slow responses), grows only on a clean, mostly fast,+ * full-size batch, and is always capped so the batch's average page size+ * times the window stays inside the in-flight byte budget.*/export function adjustCrawlWindow(windowSize: number,recent: CrawledPageResult[],+ limits: CrawlWindowLimits = CRAWL_WINDOW,): number {if (recent.length === 0) return windowSize;const troubled = recent.filter(@@ -42,15 +79,19 @@ export function adjustCrawlWindow().length;let next = windowSize;if (troubled * 3 >= recent.length) {- next = Math.max(MIN_WINDOW, Math.floor(windowSize / 2));+ next = Math.max(limits.min, Math.floor(windowSize / 2));} else {const fast = recent.filter((page) =>page.fetchClass === "ok" &&(page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,).length;- if (troubled === 0 && fast * 2 >= recent.length) {- next = Math.min(MAX_WINDOW, windowSize + 5);+ if (+ troubled === 0 &&+ fast * 2 >= recent.length &&+ recent.length >= GROWTH_MIN_SAMPLE+ ) {+ next = Math.min(limits.max, windowSize + 5);}}@@ -59,8 +100,8 @@ export function adjustCrawlWindow(MIN_ASSUMED_PAGE_BYTES,);const byteBound = Math.max(- MIN_WINDOW,- Math.floor(IN_FLIGHT_HTML_BUDGET_BYTES / avgPageBytes),+ limits.min,+ Math.floor(limits.budgetBytes / avgPageBytes),);return Math.min(next, byteBound);}diff --git a/src/server/lib/audit/page-analyzer.test.ts b/src/server/lib/audit/page-analyzer.test.tsindex ccab9838..557a5360 100644--- a/src/server/lib/audit/page-analyzer.test.ts+++ b/src/server/lib/audit/page-analyzer.test.ts@@ -206,3 +206,24 @@ describe("analyzeHtml parity with the DOM reference", () => {</body>`);});});++describe("analyzeHtml extraction caps", () => {+ it("caps links and images per page", () => {+ const links = Array.from(+ { length: 1_100 },+ (_, i) => `<a href="/p/${i}">link ${i}</a>`,+ ).join("");+ const images = Array.from(+ { length: 1_100 },+ (_, i) => `<img src="/i/${i}.png">`,+ ).join("");+ const analysis = analyzeHtml(+ `<body>${links}${images}</body>`,+ "https://example.com/",+ 200,+ 100,+ );+ expect(analysis.links).toHaveLength(1_000);+ expect(analysis.images).toHaveLength(1_000);+ });+});diff --git a/src/server/lib/audit/page-analyzer.ts b/src/server/lib/audit/page-analyzer.tsindex 5abc9d30..47db8ce5 100644--- a/src/server/lib/audit/page-analyzer.ts+++ b/src/server/lib/audit/page-analyzer.ts@@ -26,6 +26,15 @@ const HEADING_LEVELS: Record<string, number> = {h6: 6,};const MAX_ANCHOR_CHARS = 200;+/**+ * Per-page caps on the extracted collections. Crawler-trap and mega-menu+ * pages can carry thousands of links/images per page, and crawled pages sit+ * in memory in 25-page persist batches — uncapped collections were part of+ * the audit engine's exceededMemory profile. Counts derived from these+ * arrays saturate at the cap on such pathological pages.+ */+const MAX_EXTRACTED_LINKS = 1_000;+const MAX_EXTRACTED_IMAGES = 1_000;interface OpenAnchor {href: string;@@ -103,6 +112,7 @@ export function analyzeHtml(if (!openAnchor) return;const { href, rel, text } = openAnchor;openAnchor = null;+ if (linksByTarget.size >= MAX_EXTRACTED_LINKS) return;const resolved = normalizeUrl(href, pageUrl);if (!resolved || linksByTarget.has(resolved)) return;const anchor = text@@ -148,10 +158,12 @@ export function analyzeHtml(handleLinkTag(attribs);break;case "img":- images.push({- src: attribs["src"] ?? null,- alt: "alt" in attribs ? attribs["alt"] : null,- });+ if (images.length < MAX_EXTRACTED_IMAGES) {+ images.push({+ src: attribs["src"] ?? null,+ alt: "alt" in attribs ? attribs["alt"] : null,+ });+ }break;case "script":if (attribs["type"] === "application/ld+json") {diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.tsindex 5080ce05..b090955f 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts@@ -15,7 +15,9 @@ import {import { AuditProgressKV } from "@/server/lib/audit/progress-kv";import {adjustCrawlWindow,- INITIAL_CRAWL_WINDOW,+ clampCrawlWindow,+ CRAWL_WINDOW,+ RETRY_CRAWL_WINDOW,} from "@/server/lib/audit/crawl-window";import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";import { pgStep } from "@/server/workflows/pgStep";@@ -34,6 +36,14 @@ const CHUNK_TARGET_PAGES = 200;const CHUNK_SOFT_DEADLINE_MS = 90_000;/** Crawled pages are persisted in sub-batches of this size. */const PERSIST_BATCH_SIZE = 25;+/**+ * The first sub-batch of a chunk is deliberately small: the crawl window+ * only adapts when a sub-batch persists, and on a heavy-page site a full+ * 25-page batch crawled at the starting window was already enough to+ * exceed the isolate's memory. Five pages tell the byte bound what the+ * site's pages weigh before the window commits to more.+ */+const FIRST_PERSIST_BATCH_SIZE = 5;/*** Stop launching new fetches while more than this many persist sub-batches* are waiting: persistence is sequential, so when the DB falls behind a fast@@ -91,6 +101,11 @@ export async function runCrawlPhase(let attemptedTotal = 0;let pending = params.seededCount;let zeroProgressChunks = 0;+ // The adapted window carries across chunks via durable step results:+ // without this every chunk restarted at the initial window and re-learned+ // the site's page weight the hard way — on heavy-page sites that meant an+ // exceededMemory death every ~200 pages.+ let windowHint = CRAWL_WINDOW.initial;while (pending > 0 && attemptedTotal < params.maxPages) {chunkNo += 1;@@ -103,6 +118,7 @@ export async function runCrawlPhase(...params,chunkNo,attemptedBefore: attemptedTotal,+ startWindow: windowHint,}),);// Apply the chunk's counters even when it did no new work (a retried@@ -110,6 +126,9 @@ export async function runCrawlPhase(// with up-to-date scratchpad totals) — finalize must not see stale ones.attemptedTotal = result.attempted;pending = result.pending;+ // `?? initial`: an instance in flight across a deploy replays cached+ // step results from before endWindow existed.+ windowHint = result.endWindow ?? CRAWL_WINDOW.initial;// One zero-attempt chunk is normal (retry of a completed chunk number);// two in a row means the frontier is unservable — stop with what we// have instead of spinning forever.@@ -122,8 +141,17 @@ export async function runCrawlPhase(}async function runCrawlChunk(- input: CrawlPhaseParams & { chunkNo: number; attemptedBefore: number },-): Promise<{ attemptedInChunk: number; attempted: number; pending: number }> {+ input: CrawlPhaseParams & {+ chunkNo: number;+ attemptedBefore: number;+ startWindow: number;+ },+): Promise<{+ attemptedInChunk: number;+ attempted: number;+ pending: number;+ endWindow: number;+}> {const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =input;const scratchpad = getAuditScratchpad(auditId);@@ -132,23 +160,34 @@ async function runCrawlChunk(CHUNK_TARGET_PAGES,maxPages - input.attemptedBefore,);- const claimed = await scratchpad.claimChunk(chunkNo, claimLimit);+ const { urls: claimed, isRetry } = await scratchpad.claimChunk(+ chunkNo,+ claimLimit,+ );if (claimed.length === 0) {const stats = await scratchpad.getStats();return {attemptedInChunk: 0,attempted: stats.attempted,pending: stats.pending,+ endWindow: input.startWindow,};}const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;- let windowSize = INITIAL_CRAWL_WINDOW;+ // A retry means the previous attempt died mid-crawl (in production almost+ // always exceededMemory), and it is the chunk's last attempt — so it runs+ // under drastically reduced limits instead of the profile that just failed.+ const limits = isRetry ? RETRY_CRAWL_WINDOW : CRAWL_WINDOW;+ let windowSize = isRetry+ ? limits.initial+ : clampCrawlWindow(input.startWindow, limits);let nextIndex = 0;let attemptedInChunk = 0;const inFlight = new Set<Promise<void>>();+ let persistThreshold = FIRST_PERSIST_BATCH_SIZE;let batch: CrawledPageResult[] = [];// Persistence runs concurrently with fetching (pipelined) but sequentially// with itself, so DB write pressure stays bounded at one batch at a time.@@ -159,7 +198,8 @@ async function runCrawlChunk(if (batch.length === 0) return;const pages = batch;batch = [];- windowSize = adjustCrawlWindow(windowSize, pages);+ persistThreshold = PERSIST_BATCH_SIZE;+ windowSize = adjustCrawlWindow(windowSize, pages, limits);queuedPersists += 1;persistChain = persistChain.then(() =>@@ -184,7 +224,7 @@ async function runCrawlChunk(.then((page) => {attemptedInChunk += 1;batch.push(page);- if (batch.length >= PERSIST_BATCH_SIZE) flush();+ if (batch.length >= persistThreshold) flush();}).finally(() => {inFlight.delete(promise);@@ -239,6 +279,7 @@ async function runCrawlChunk(attemptedInChunk,attempted: stats.attempted,pending: stats.pending,+ endWindow: windowSize,};}