Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
import type { CrawledPageResult } from "@/server/lib/audit/types";
/**
* Rolling fetch-concurrency window for the crawl. Unlike fixed batches, a
* slow page only occupies one slot instead of stalling a whole batch. The
* window adapts to the site: it shrinks when fetches error/block/crawl
* slowly (politeness toward struggling or defensive sites) and grows when
* the site answers fast.
*/
interface CrawlWindowLimits {
/** Window size a chunk starts with, before any observations. */
initial: number;
min: number;
max: number;
/**
* Total HTML the in-flight window may buffer at once. Each in-flight page
* holds its body (decoded to a UTF-16 string, roughly doubling it), and the
* workflow shares its isolate's 128 MB memory limit with the rest of the
* worker — production audits died with exceededMemory when a fast site let
* the window grow unchecked.
*/
budgetBytes: number;
}
export const CRAWL_WINDOW: CrawlWindowLimits = {
initial: 2,
min: 1,
max: 2,
budgetBytes: 8 * 1024 * 1024,
};
/**
* Limits for a chunk whose earlier attempt died mid-crawl — in production
* almost always exceededMemory on a heavy-page site. The retry is the
* chunk's last attempt (retry limit 1), so it must not re-run the exact
* memory profile that just killed the isolate.
*/
export const RETRY_CRAWL_WINDOW: CrawlWindowLimits = {
initial: 1,
min: 1,
max: 1,
budgetBytes: 4 * 1024 * 1024,
};
const SLOW_RESPONSE_MS = 10_000;
const FAST_RESPONSE_MS = 1_500;
/** Floor for the observed page size so tiny-page sites can't void the bound. */
const MIN_ASSUMED_PAGE_BYTES = 64 * 1024;
/**
* Growth requires a full-size sample. The first persist sub-batch is small
* (so the byte bound reacts to heavy pages early), and a handful of fast
* pages proves too little to widen the window.
*/
const GROWTH_MIN_SAMPLE = 25;
export function clampCrawlWindow(
size: number,
limits: CrawlWindowLimits,
): number {
return Math.min(Math.max(size, limits.min), limits.max);
}
/**
* Adapt the window to the last persisted sub-batch. Shrinks on trouble
* (errors, blocks, very slow responses), grows only on a clean, mostly fast,
* full-size batch, and is always capped so the batch's average page size
* times the window stays inside the in-flight byte budget.
*/
export function adjustCrawlWindow(
windowSize: number,
recent: CrawledPageResult[],
limits: CrawlWindowLimits = CRAWL_WINDOW,
): number {
if (recent.length === 0) return windowSize;
const troubled = recent.filter(
(page) =>
page.fetchClass !== "ok" ||
// A 429 the retries recovered from still says we are crawling faster
// than the site allows.
page.rateLimited ||
(page.responseTimeMs ?? 0) >= SLOW_RESPONSE_MS,
).length;
let next = windowSize;
if (troubled * 3 >= recent.length) {
next = Math.max(limits.min, Math.floor(windowSize / 2));
} else {
const fast = recent.filter(
(page) =>
page.fetchClass === "ok" &&
(page.responseTimeMs ?? Infinity) <= FAST_RESPONSE_MS,
).length;
if (
troubled === 0 &&
fast * 2 >= recent.length &&
recent.length >= GROWTH_MIN_SAMPLE
) {
next = Math.min(limits.max, windowSize + 5);
}
}
const avgPageBytes = Math.max(
recent.reduce((sum, page) => sum + page.htmlBytes, 0) / recent.length,
MIN_ASSUMED_PAGE_BYTES,
);
const byteBound = Math.max(
limits.min,
Math.floor(limits.budgetBytes / avgPageBytes),
);
return Math.min(next, byteBound);
}