Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Pace site audits and resume after longer rate-limit cooldowns (#595)
Ben Senescu6edf052c32e2
diff --git a/src/server/features/audit/AuditScratchpad.ts b/src/server/features/audit/AuditScratchpad.tsindex d473f21c..b6b1ebce 100644--- a/src/server/features/audit/AuditScratchpad.ts+++ b/src/server/features/audit/AuditScratchpad.ts@@ -14,11 +14,12 @@* write racing in after destroy() — is wiped after 7 days (failed audits'* state doubles as the resume/debug artifact until then).*- * All methods are synchronous inside (SQLite in DOs is sync), so each RPC is- * effectively atomic. Writes are idempotent: the workflow retries steps, so+ * SQL mutations are synchronous; key-value checkpoints await durable writes.+ * Writes are idempotent: the workflow retries steps, so* every insert is OR IGNORE / OR REPLACE on a stable key.*/import { DurableObject, env } from "cloudflare:workers";+import type { CrawlThrottleState } from "@/server/lib/audit/crawl-throttle";import {BROKEN_LINKS_SQL,ORPHAN_PAGES_SQL,@@ -119,6 +120,14 @@ export class AuditScratchpad extends DurableObject {);}+ async getCrawlThrottle(): Promise<CrawlThrottleState | undefined> {+ return this.ctx.storage.get<CrawlThrottleState>("crawl-throttle");+ }++ async saveCrawlThrottle(state: CrawlThrottleState): Promise<void> {+ await this.ctx.storage.put("crawl-throttle", state);+ }+async seedSitemapUrls(urls: string[]): Promise<void> {for (const url of urls) {// Upsert so a URL that already exists (e.g. the start URL) still getsdiff --git a/src/server/lib/audit/crawl-throttle.test.ts b/src/server/lib/audit/crawl-throttle.test.tsindex 3c9cce49..d1945f4d 100644--- a/src/server/lib/audit/crawl-throttle.test.ts+++ b/src/server/lib/audit/crawl-throttle.test.ts@@ -1,83 +1,177 @@-import { afterEach, describe, expect, it, vi } from "vitest";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";-afterEach(() => {- vi.useRealTimers();+beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(0);});+afterEach(() => vi.useRealTimers());describe("createCrawlThrottle", () => {- it("backs off exponentially and still pauses after a URL exhausts its retries", async () => {- vi.useFakeTimers();- const throttle = createCrawlThrottle(Date.now() + 90_000);- for (const [attempt, delay] of [- [1, 1_000],- [2, 2_000],- [3, 4_000],- [4, 8_000],- ]) {- expect(throttle.backoff(attempt, null)).toBe(attempt <= 3);- let ready = false;- const waiting = throttle.ready().then((result) => {- ready = result;- });- await vi.advanceTimersByTimeAsync(delay - 1);- expect(ready).toBe(false);- await vi.advanceTimersByTimeAsync(1);- await waiting;- expect(ready).toBe(true);- }+ it("keeps oversized Retry-After values finite even with overlapping refusals", async () => {+ const throttle = createCrawlThrottle(90_000);+ await Promise.all([+ throttle.backoff(1, "9".repeat(310)),+ throttle.backoff(1, "60"),+ ]);+ expect(throttle.stopped).toBe(true);+ expect(Object.values(throttle.state).every(Number.isFinite)).toBe(true);+ expect(await throttle.ready()).toBe(false);});- it.each(["60", "Thu, 01 Jan 2026 00:01:00 GMT"])(- "honors a full sixty-second Retry-After: %s",- async (retryAfter) => {- vi.useFakeTimers();- vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));- const throttle = createCrawlThrottle(Date.now() + 90_000);- expect(throttle.backoff(1, retryAfter)).toBe(true);- let ready = false;- const waiting = throttle.ready().then((result) => {- ready = result;+ it("waits for cooldown persistence before allowing another request", async () => {+ let save: () => void = vi.fn();+ const throttle = createCrawlThrottle(+ 90_000,+ undefined,+ () =>+ new Promise<void>((resolve) => {+ save = resolve;+ }),+ );+ const backoff = throttle.backoff(1, "1");+ await vi.advanceTimersByTimeAsync(0);+ let started = false;+ const request = throttle.ready().then(() => {+ started = true;+ });+ await vi.advanceTimersByTimeAsync(10_000);+ expect(started).toBe(false);+ save();+ await backoff;+ await request;+ expect(started).toBe(true);+ });++ it("spaces simultaneous request starts instead of releasing a burst", async () => {+ const throttle = createCrawlThrottle(90_000);+ const starts: number[] = [];+ const requests = Promise.all(+ Array.from({ length: 5 }, async () => {+ if (await throttle.ready()) starts.push(Date.now());+ }),+ );+ await vi.runAllTimersAsync();+ await requests;+ expect(starts).toEqual([0, 1_000, 2_000, 3_000, 4_000]);+ });++ it("pauses for thirty seconds on the first 429 and spaces retries", async () => {+ const throttle = createCrawlThrottle(90_000);+ expect(await throttle.ready()).toBe(true);+ expect(await throttle.backoff(1, null)).toBe(true);+ const starts: number[] = [];+ const requests = Promise.all(+ Array.from({ length: 3 }, async () => {+ if (await throttle.ready()) starts.push(Date.now());+ }),+ );+ await vi.advanceTimersByTimeAsync(29_999);+ expect(starts).toEqual([]);+ await vi.runAllTimersAsync();+ await requests;+ expect(starts).toEqual([30_000, 32_000, 34_000]);+ });++ it("rechecks a cooldown extended while requests are waiting", async () => {+ const throttle = createCrawlThrottle(90_000);+ await throttle.backoff(1, "5");+ const starts: number[] = [];+ const request = throttle.ready().then(() => starts.push(Date.now()));+ await vi.advanceTimersByTimeAsync(4_000);+ await throttle.backoff(1, "10");+ await vi.advanceTimersByTimeAsync(9_999);+ expect(starts).toEqual([]);+ await vi.advanceTimersByTimeAsync(1);+ await request;+ expect(starts).toEqual([14_000]);+ });++ it.each(["60", "Thu, 01 Jan 1970 00:01:00 GMT"])(+ "honors Retry-After: %s",+ async (header) => {+ const throttle = createCrawlThrottle(90_000);+ await throttle.backoff(1, header);+ let started = false;+ const request = throttle.ready().then(() => {+ started = true;});await vi.advanceTimersByTimeAsync(59_999);- expect(ready).toBe(false);+ expect(started).toBe(false);await vi.advanceTimersByTimeAsync(1);- await waiting;- expect(ready).toBe(true);+ await request;+ expect(started).toBe(true);},);- it("does not label an ordinary chunk deadline as a rate-limit stop", async () => {- vi.useFakeTimers();- const throttle = createCrawlThrottle(Date.now() + 90_000);- await vi.advanceTimersByTimeAsync(90_000);+ it("backs off across chunks and stops after four consecutive refusals", async () => {+ let throttle = createCrawlThrottle(90_000);+ for (const [index, delay] of [30_000, 60_000, 120_000, 240_000].entries()) {+ expect(await throttle.backoff(1, null)).toBe(index < 3);+ expect(throttle.state.pausedUntil - Date.now()).toBe(delay);+ if (index < 3) {+ await vi.advanceTimersByTimeAsync(delay);+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);+ }+ }+ expect(throttle.stopped).toBe(true);expect(await throttle.ready()).toBe(false);+ });++ it("retains slower pacing after recovery and caps further slowdowns", async () => {+ let throttle = createCrawlThrottle(90_000);+ for (let i = 0; i < 8; i++) {+ await throttle.backoff(1, "1");+ await vi.advanceTimersByTimeAsync(30_000);+ await throttle.recovered();+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);+ }+ expect(throttle.state.intervalMs).toBe(30_000);expect(throttle.stopped).toBe(false);});- it("stops without scheduling a timer when the cooldown exceeds the crawl budget", async () => {- vi.useFakeTimers();- const throttle = createCrawlThrottle(Date.now() + 90_000);- expect(throttle.backoff(1, "600")).toBe(false);- expect(throttle.stopped).toBe(true);+ it("defers a ten-minute wait without marking the audit stopped", async () => {+ const throttle = createCrawlThrottle(90_000);+ expect(await throttle.backoff(1, "600")).toBe(true);expect(await throttle.ready()).toBe(false);+ expect(throttle.stopped).toBe(false);+ expect(throttle.state.pausedUntil).toBe(600_000);expect(vi.getTimerCount()).toBe(0);+ await vi.advanceTimersByTimeAsync(600_000);+ const resumed = createCrawlThrottle(690_000, throttle.state);+ expect(await resumed.ready()).toBe(true);});- it("holds a waiter through a pause that another 429 extends", async () => {- vi.useFakeTimers();- const throttle = createCrawlThrottle(Date.now() + 90_000);- throttle.backoff(1, "1");- let released = false;- const waiting = throttle.ready().then((result) => {- released = result;+ it("preserves request spacing across a chunk boundary", async () => {+ const first = createCrawlThrottle(90_000);+ await first.ready();+ const second = createCrawlThrottle(90_000, first.state);+ let startedAt = -1;+ const request = second.ready().then(() => {+ startedAt = Date.now();});- await vi.advanceTimersByTimeAsync(900);- throttle.backoff(1, "1");- await vi.advanceTimersByTimeAsync(200);- expect(released).toBe(false);- await vi.advanceTimersByTimeAsync(800);- await waiting;- expect(released).toBe(true);+ await vi.runAllTimersAsync();+ await request;+ expect(startedAt).toBe(1_000);+ });++ it("does not label an ordinary chunk deadline as a rate-limit stop", async () => {+ const throttle = createCrawlThrottle(90_000);+ await vi.advanceTimersByTimeAsync(90_000);+ expect(await throttle.ready()).toBe(false);+ expect(throttle.stopped).toBe(false);+ });++ it("stops when cumulative cooldowns exceed thirty minutes, without retrying early", async () => {+ let throttle = createCrawlThrottle(90_000);+ for (let i = 0; i < 3; i++) {+ expect(await throttle.backoff(1, "600")).toBe(true);+ await vi.advanceTimersByTimeAsync(600_000);+ await throttle.recovered();+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);+ }+ expect(await throttle.backoff(1, "600")).toBe(false);+ expect(await throttle.ready()).toBe(false);+ expect(vi.getTimerCount()).toBe(0);});});diff --git a/src/server/lib/audit/crawl-throttle.ts b/src/server/lib/audit/crawl-throttle.tsindex 70ef9b15..ee055ad1 100644--- a/src/server/lib/audit/crawl-throttle.ts+++ b/src/server/lib/audit/crawl-throttle.ts@@ -1,12 +1,25 @@/** Retries per URL after its first 429. Every 429 still pauses new URLs. */const MAX_RETRIES = 3;-const FIRST_DELAY_MS = 1_000;+const FIRST_DELAY_MS = 30_000;+const MAX_INTERVAL_MS = 30_000;+const MAX_COOLDOWN_MS = 30 * 60_000;++/** Checkpointed between chunks so a new step does not forget site limits. */+export type CrawlThrottleState = {+ intervalMs: number;+ nextRequestAt: number;+ pausedUntil: number;+ consecutiveRateLimits: number;+ cooldownMs: number;+};/** `Retry-After` is either delay-seconds or an HTTP-date. */function parseRetryAfterMs(header: string | null): number | null {if (!header) return null;const value = header.trim();- if (/^\d+$/.test(value)) return Number(value) * 1_000;+ // Keep over-budget values finite for checkpoint serialization/arithmetic.+ if (/^\d+$/.test(value))+ return Math.min(Number(value) * 1_000, MAX_COOLDOWN_MS + 1);const at = Date.parse(value);return Number.isNaN(at) ? null : Math.max(0, at - Date.now());}@@ -15,38 +28,89 @@ export interface CrawlThrottle {/** False when this chunk can no longer start a request. */ready(): Promise<boolean>;/** Pause the origin on every 429; return whether this URL may retry. */- backoff(attempt: number, retryAfter: string | null): boolean;+ backoff(attempt: number, retryAfter: string | null): Promise<boolean>;+ /** A non-429 response breaks a run of consecutive refusals. */+ recovered(): Promise<void>;+ readonly checkpointFailed: boolean;readonly stopped: boolean;+ readonly state: CrawlThrottleState;}-/** One origin cooldown, bounded by the scheduler's existing chunk deadline. */-export function createCrawlThrottle(deadlineAt: number): CrawlThrottle {- let pausedUntil = 0;- let stopped = false;+/** One audit's origin pacing. Long waits are resumed by the workflow. */+export function createCrawlThrottle(+ deadlineAt: number,+ previous?: CrawlThrottleState,+ persist?: (state: CrawlThrottleState) => Promise<void>,+): CrawlThrottle {+ const state: CrawlThrottleState = previous+ ? { ...previous }+ : {+ intervalMs: 1_000,+ nextRequestAt: 0,+ pausedUntil: 0,+ consecutiveRateLimits: 0,+ cooldownMs: 0,+ };+ let checkpointFailed = false;+ let checkpoint = Promise.resolve();+ const save = () => {+ const snapshot = { ...state };+ checkpoint = checkpoint+ .then(() => persist?.(snapshot))+ .catch((error) => {+ checkpointFailed = true;+ throw error;+ });+ return checkpoint;+ };+ const stopped = () =>+ checkpointFailed ||+ state.consecutiveRateLimits > MAX_RETRIES ||+ state.cooldownMs > MAX_COOLDOWN_MS;return {async ready() {- // Another request can extend the shared pause while this one waits.- while (pausedUntil > Date.now()) {- if (stopped) return false;- await new Promise((resolve) =>- setTimeout(resolve, pausedUntil - Date.now()),- );+ while (!stopped()) {+ await checkpoint;+ const now = Date.now();+ const readyAt = Math.max(state.pausedUntil, state.nextRequestAt);+ if (now >= deadlineAt || readyAt >= deadlineAt) return false;+ if (readyAt <= now) {+ // Reserve synchronously: waiters waking together must take turns.+ state.nextRequestAt = now + state.intervalMs;+ return true;+ }+ await new Promise((resolve) => setTimeout(resolve, readyAt - now));}- return !stopped && Date.now() < deadlineAt;+ return false;},- backoff(attempt, retryAfter) {+ async backoff(attempt, retryAfter) {+ state.consecutiveRateLimits += 1;+ state.intervalMs = Math.min(MAX_INTERVAL_MS, state.intervalMs * 2);const delayMs = Math.max(- FIRST_DELAY_MS,- parseRetryAfterMs(retryAfter) ?? FIRST_DELAY_MS * 2 ** (attempt - 1),+ state.intervalMs,+ parseRetryAfterMs(retryAfter) ??+ FIRST_DELAY_MS * 2 ** (state.consecutiveRateLimits - 1),);- pausedUntil = Math.max(pausedUntil, Date.now() + delayMs);- // Never shorten the site's requested wait to fit our budget. Stop the- // crawl instead; the scheduler preserves unvisited URLs as incomplete.- if (pausedUntil >= deadlineAt) stopped = true;- return !stopped && attempt <= MAX_RETRIES;+ const now = Date.now();+ const pausedUntil = Math.max(state.pausedUntil, now + delayMs);+ state.cooldownMs += pausedUntil - Math.max(now, state.pausedUntil);+ state.pausedUntil = pausedUntil;+ await save();+ return !stopped() && attempt <= MAX_RETRIES;+ },+ async recovered() {+ if (state.consecutiveRateLimits === 0) return;+ state.consecutiveRateLimits = 0;+ await save();+ },+ get checkpointFailed() {+ return checkpointFailed;},get stopped() {- return stopped;+ return stopped();+ },+ get state() {+ return { ...state };},};}diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.tsindex 5cac4a09..587096ff 100644--- a/src/server/lib/audit/crawl-window.test.ts+++ b/src/server/lib/audit/crawl-window.test.ts@@ -53,82 +53,44 @@ function page(describe("adjustCrawlWindow", () => {it("keeps the window on an empty batch", () => {- expect(adjustCrawlWindow(10, [])).toBe(10);+ expect(adjustCrawlWindow(2, [])).toBe(2);});- it("halves the window when a third of the batch is troubled", () => {- const recent = [- ...Array.from({ length: 9 }, () => page("error", 15_000)),- ...Array.from({ length: 16 }, () => page("ok", 500)),- ];- expect(adjustCrawlWindow(25, recent)).toBe(12);- });-- it("treats blocked fetches as trouble", () => {- const recent = Array.from({ length: 10 }, () => page("blocked", 300));- expect(adjustCrawlWindow(20, recent)).toBe(10);- });+ it.each(["error", "blocked", "rate_limited"] as const)(+ "reduces concurrency on %s",+ (fetchClass) => {+ const recent = Array.from({ length: 10 }, () => page(fetchClass, 300));+ expect(adjustCrawlWindow(2, recent)).toBe(1);+ },+ );- it("treats a 429 the retries recovered from as trouble", () => {+ it("treats recovered 429s as trouble", () => {const recent = Array.from({ length: 10 }, () => ({...page("ok", 300),rateLimited: true,}));- expect(adjustCrawlWindow(20, recent)).toBe(10);+ expect(adjustCrawlWindow(2, recent)).toBe(1);});- it("never shrinks below the minimum", () => {- const recent = Array.from({ length: 10 }, () => page("error", 15_000));- expect(adjustCrawlWindow(6, recent)).toBe(5);+ it("never shrinks below one request", () => {+ expect(adjustCrawlWindow(1, [page("error", 15_000)])).toBe(1);});- it("grows on a clean, fast batch up to the cap", () => {+ it("never grows beyond two concurrent requests, even on a fast site", () => {const recent = Array.from({ length: 25 }, () => page("ok", 400));- expect(adjustCrawlWindow(10, recent)).toBe(15);- expect(adjustCrawlWindow(20, recent)).toBe(20);- });-- it("caps the window so heavy pages stay inside the byte budget", () => {- // 1 MiB average pages: 8 MiB budget / 1 MiB = window of 8.- const recent = Array.from({ length: 25 }, () =>- page("ok", 300, 1024 * 1024),- );- expect(adjustCrawlWindow(20, recent)).toBe(8);- });-- it("does not grow on a small sample, but still applies the byte bound", () => {- const fastSmall = Array.from({ length: 5 }, () => page("ok", 400));- expect(adjustCrawlWindow(10, fastSmall)).toBe(10);- const fastHeavy = Array.from({ length: 5 }, () =>- page("ok", 400, 1024 * 1024),- );- expect(adjustCrawlWindow(10, fastHeavy)).toBe(8);+ expect(adjustCrawlWindow(2, recent)).toBe(2);+ expect(adjustCrawlWindow(1, recent)).toBe(2);});- it("retry limits keep the window small even on a clean, fast site", () => {+ it("keeps retry chunks at one concurrent request", () => {const recent = Array.from({ length: 25 }, () => page("ok", 400));- expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(5);- });-- it("retry limits shrink below the normal minimum on trouble", () => {- const recent = Array.from({ length: 10 }, () => page("error", 15_000));- expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(2);+ expect(adjustCrawlWindow(1, recent, RETRY_CRAWL_WINDOW)).toBe(1);});- it("keeps the byte bound at the minimum window even for huge pages", () => {+ it("preserves the byte budget if page sizes increase", () => {const recent = Array.from({ length: 25 }, () =>- page("ok", 300, 4 * 1024 * 1024),+ page("ok", 300, 5 * 1024 * 1024),);- expect(adjustCrawlWindow(20, recent)).toBe(5);- });-- it("lets small pages use the full window cap", () => {- const recent = Array.from({ length: 25 }, () => page("ok", 400, 10_000));- expect(adjustCrawlWindow(20, recent)).toBe(20);- });-- it("holds steady on a clean but slow batch", () => {- const recent = Array.from({ length: 25 }, () => page("ok", 5_000));- expect(adjustCrawlWindow(15, recent)).toBe(15);+ expect(adjustCrawlWindow(2, recent)).toBe(1);});});diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.tsindex c30b8bc4..515b9391 100644--- a/src/server/lib/audit/crawl-window.ts+++ b/src/server/lib/audit/crawl-window.ts@@ -23,9 +23,9 @@ interface CrawlWindowLimits {}export const CRAWL_WINDOW: CrawlWindowLimits = {- initial: 10,- min: 5,- max: 20,+ initial: 2,+ min: 1,+ max: 2,budgetBytes: 8 * 1024 * 1024,};@@ -36,9 +36,9 @@ export const CRAWL_WINDOW: CrawlWindowLimits = {* memory profile that just killed the isolate.*/export const RETRY_CRAWL_WINDOW: CrawlWindowLimits = {- initial: 3,- min: 2,- max: 5,+ initial: 1,+ min: 1,+ max: 1,budgetBytes: 4 * 1024 * 1024,};diff --git a/src/server/workflows/site-audit-workflow-helpers.test.ts b/src/server/workflows/site-audit-workflow-helpers.test.tsindex 9c19df79..a3c33ebb 100644--- a/src/server/workflows/site-audit-workflow-helpers.test.ts+++ b/src/server/workflows/site-audit-workflow-helpers.test.ts@@ -83,7 +83,7 @@ describe("crawlPage", () => {await vi.advanceTimersByTimeAsync(4_000);expect(fetched).toEqual([`${PAGE_URL}/first`]);- await vi.advanceTimersByTimeAsync(2_000);+ await vi.advanceTimersByTimeAsync(3_000);const pages = await Promise.all([first, second]);expect(fetched).toHaveLength(3);expect(pages.map((page) => page?.fetchClass)).toEqual(["ok", "ok"]);@@ -91,7 +91,7 @@ describe("crawlPage", () => {it("records a page the site keeps rate limiting, without calling it blocked", async () => {vi.useFakeTimers();- const fetchMock = stubFetch({ status: 429 });+ const fetchMock = stubFetch({ status: 429, retryAfter: "1" });const crawled = crawl();await vi.advanceTimersByTimeAsync(30_000);@@ -101,39 +101,17 @@ describe("crawlPage", () => {expect(page?.fetchClass).toBe("rate_limited");});- it("keeps the origin paused after a URL exhausts its retries", async () => {- vi.useFakeTimers();- vi.setSystemTime(0);- const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(- async () =>- new Response(PAGE_HTML, {- status: Date.now() < 15_000 ? 429 : 200,- headers: { "content-type": "text/html" },- }),- );- const throttle = createCrawlThrottle(90_000);- const first = crawlPage(`${PAGE_URL}/first`, 0, false, throttle);- await vi.advanceTimersByTimeAsync(7_000);- expect((await first)?.fetchClass).toBe("rate_limited");- expect(fetchMock).toHaveBeenCalledTimes(4);-- const second = crawlPage(`${PAGE_URL}/second`, 0, false, throttle);- await vi.advanceTimersByTimeAsync(7_999);- expect(fetchMock).toHaveBeenCalledTimes(4);- await vi.advanceTimersByTimeAsync(1);- expect((await second)?.fetchClass).toBe("ok");- });-- it("recovers concurrent pages when the origin asks for sixty seconds", async () => {+ it("recovers concurrent pages without a burst after sixty seconds", async () => {vi.useFakeTimers();vi.setSystemTime(0);- const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(- async () =>- new Response(PAGE_HTML, {- status: Date.now() < 60_000 ? 429 : 200,- headers: { "content-type": "text/html", "retry-after": "60" },- }),- );+ const starts: number[] = [];+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {+ starts.push(Date.now());+ return new Response(PAGE_HTML, {+ status: Date.now() < 60_000 ? 429 : 200,+ headers: { "content-type": "text/html", "retry-after": "60" },+ });+ });const throttle = createCrawlThrottle(90_000);const pages = Promise.all(Array.from({ length: 5 }, (_, i) =>@@ -141,24 +119,34 @@ describe("crawlPage", () => {),);await vi.advanceTimersByTimeAsync(59_999);- expect(fetchMock).toHaveBeenCalledTimes(5);- await vi.advanceTimersByTimeAsync(1);+ expect(starts).toEqual([0]);+ await vi.runAllTimersAsync();expect((await pages).map((page) => page?.fetchClass)).toEqual(Array(5).fill("ok"),);- expect(fetchMock).toHaveBeenCalledTimes(10);+ expect(starts).toEqual([0, 60_000, 62_000, 64_000, 66_000, 68_000]);});- it("records the observed 429 but does not fetch another URL when the wait cannot fit", async () => {+ it("defers the refused URL when its cooldown exceeds the chunk deadline", async () => {vi.useFakeTimers();const fetchMock = stubFetch({ status: 429, retryAfter: "600" });const throttle = createCrawlThrottle(Date.now() + 90_000);- expect((await crawlPage(PAGE_URL, 0, false, throttle))?.fetchClass).toBe(- "rate_limited",- );+ expect(await crawlPage(PAGE_URL, 0, false, throttle)).toBeNull();expect(await crawlPage(`${PAGE_URL}/unvisited`, 0, false, throttle),).toBeNull();+ expect(throttle.stopped).toBe(false);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ it("records a final 429 when the requested wait exceeds the audit's cooldown budget", async () => {+ vi.useFakeTimers();+ const fetchMock = stubFetch({ status: 429, retryAfter: "3600" });+ const throttle = createCrawlThrottle(Date.now() + 90_000);+ expect((await crawlPage(PAGE_URL, 0, false, throttle))?.fetchClass).toBe(+ "rate_limited",+ );+ expect(throttle.stopped).toBe(true);expect(fetchMock).toHaveBeenCalledTimes(1);});@@ -302,7 +290,8 @@ describe("crawlPage", () => {const before = await collect();const pages = [];for (let i = 0; i < 50; i++) {- pages.push(await crawlPage("https://example.com/" + i, 0, true, throttle));+ pages.push(await crawlPage("https://example.com/" + i, 0, true,+ createCrawlThrottle(Date.now() + 90_000)));}const retained = (await collect()) - before;assert.equal(pages.length, 50);diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.tsindex e838c826..7d207aea 100644--- a/src/server/workflows/site-audit-workflow-helpers.ts+++ b/src/server/workflows/site-audit-workflow-helpers.ts@@ -86,9 +86,12 @@ async function fetchPage(url: string, throttle: CrawlThrottle) {// the last retry is already classified rate_limited and needs no flag.rateLimited: attempt > 1,};- if (response.status !== 429) return result;+ if (response.status !== 429) {+ await throttle.recovered();+ return result;+ }- const retry = throttle.backoff(+ const retry = await throttle.backoff(attempt,response.headers.get("retry-after"),);@@ -227,6 +230,9 @@ export async function crawlPage(inSitemap,});} catch (error) {+ // Losing a durable cooldown must fail the workflow, not become a page+ // error that lets the scheduler continue making requests.+ if (throttle.checkpointFailed) throw error;const responseTimeMs = Date.now() - startTime;console.warn(`Failed to crawl ${url}:`, error);return emptyPageResult({diff --git a/src/server/workflows/siteAuditWorkflowCrawl.test.ts b/src/server/workflows/siteAuditWorkflowCrawl.test.tsindex bac76b3c..4bbc0c2e 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.test.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.test.ts@@ -11,6 +11,12 @@ const mocks = vi.hoisted(() => ({releaseUrls: vi.fn<(urls: string[]) => Promise<void>>(),insertCrawledBatch: vi.fn(),pgStep: vi.fn(),+ sleepUntil: vi.fn(),+ getCrawlThrottle: vi.fn(),+ saveCrawlThrottle: vi.fn(),+}));+vi.mock("cloudflare:workflows", () => ({+ NonRetryableError: class extends Error {},}));vi.mock("@/server/features/audit/AuditScratchpad", () => ({getAuditScratchpad: () => mocks,@@ -36,6 +42,7 @@ import { parseRobotsTxt } from "@/server/lib/audit/discovery";const ORIGIN = "https://example.com";const HTML = "<html><title>A page</title><body><h1>A page</h1></body></html>";let saved: CrawledPageResult[];+let urls: string[];beforeEach(async () => {// Load the lazy HTML parser before advancing the fake network clock.@@ -43,23 +50,38 @@ beforeEach(async () => {vi.useFakeTimers();vi.setSystemTime(0);saved = [];+ let checkpoint: unknown;+ mocks.getCrawlThrottle.mockImplementation(async () =>+ structuredClone(checkpoint),+ );+ mocks.saveCrawlThrottle.mockImplementation(async (state) => {+ checkpoint = structuredClone(state);+ });mocks.pgStep.mockImplementation((_step, _name, _config, fn: () => unknown) =>fn(),);+ mocks.sleepUntil.mockImplementation(async (_name, at: number) => {+ await new Promise((resolve) =>+ setTimeout(resolve, Math.max(0, at - Date.now())),+ );+ });mocks.claimChunk.mockImplementation(async (_chunk: number, limit: number) => ({- urls: Array.from({ length: limit }, (_, i) => ({- url: `${ORIGIN}/${i}`,- depth: 0,- inSitemap: true,- })),+ urls: urls+ .filter((url) => !saved.some((page) => page.url === url))+ .slice(0, limit)+ .map((url) => ({+ url,+ depth: 0,+ inSitemap: true,+ })),isRetry: false,}),);mocks.getStats.mockImplementation(async () => ({attempted: saved.length,- pending: 100 - saved.length,- seen: 100,+ pending: urls.length - saved.length,+ seen: urls.length,}));mocks.recordBatch.mockImplementation(() => mocks.getStats());mocks.insertCrawledBatch.mockImplementation(@@ -74,15 +96,21 @@ afterEach(() => {});function crawl(maxPages = 100) {+ urls = Array.from({ length: maxPages }, (_, i) => `${ORIGIN}/${i}`);return runCrawlPhase(- { do: vi.fn(), sleep: vi.fn(), sleepUntil: vi.fn(), waitForEvent: vi.fn() },+ {+ do: vi.fn(),+ sleep: vi.fn(),+ sleepUntil: mocks.sleepUntil,+ waitForEvent: vi.fn(),+ },{auditId: "audit",workflowInstanceId: "workflow",origin: ORIGIN,maxPages,seededCount: maxPages,- robots: parseRobotsTxt("", ORIGIN),+ robots: parseRobotsTxt(ORIGIN, ""),},);}@@ -101,85 +129,162 @@ function serve(status: (now: number) => number, retryAfter?: string) {});}-describe("crawl rate-limit budget", () => {- it("honors the final URL's cooldown before starting the next chunk", async () => {- const total = 210;- mocks.claimChunk.mockImplementation(- async (chunk: number, limit: number) => ({- urls: Array.from({ length: limit }, (_, i) => ({- url: `${ORIGIN}/${(chunk - 1) * 200 + i}`,- depth: 0,- inSitemap: true,- })),- isRetry: false,- }),+describe("crawl pacing and cooldowns", () => {+ it("spaces a fast site's requests across chunks without duplicates", async () => {+ const starts: number[] = [];+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {+ starts.push(Date.now());+ return new Response(HTML, { headers: { "content-type": "text/html" } });+ });+ const result = crawl(210);+ await vi.runAllTimersAsync();+ expect(await result).toEqual({ pagesCrawled: 210, completed: true });+ expect(new Set(saved.map((page) => page.url)).size).toBe(210);+ expect(starts).toHaveLength(210);+ expect(starts.slice(1).every((at, i) => at - starts[i] >= 1_000)).toBe(+ true,);- mocks.getStats.mockImplementation(async () => ({- attempted: saved.length,- pending: total - saved.length,- seen: total,- }));- let lastRefusalAt = 0;- let nextChunkStartedAt = 0;- vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {- const url = new Request(input).url;- if (url === `${ORIGIN}/200`) nextChunkStartedAt = Date.now();- await new Promise((resolve) => setTimeout(resolve, 1));- const refused = url === `${ORIGIN}/199`;- if (refused) lastRefusalAt = Date.now();+ expect(mocks.claimChunk.mock.calls.length).toBeGreaterThan(1);+ });++ it("never has more than two fetches in flight on a slow site", async () => {+ let inFlight = 0;+ let peak = 0;+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {+ inFlight += 1;+ peak = Math.max(peak, inFlight);+ await new Promise((resolve) => setTimeout(resolve, 3_000));+ inFlight -= 1;+ return new Response(HTML, { headers: { "content-type": "text/html" } });+ });+ const result = crawl(10);+ await vi.runAllTimersAsync();+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });+ expect(peak).toBe(2);+ });++ it("retains slower pacing across chunks after recovering from the first 429", async () => {+ const starts: number[] = [];+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {+ starts.push(Date.now());return new Response(HTML, {- status: refused ? 429 : 200,- headers: { "content-type": "text/html", "retry-after": "5" },+ status: starts.length === 1 ? 429 : 200,+ headers: { "content-type": "text/html" },});});-- const result = crawl(total);- await vi.advanceTimersByTimeAsync(30_000);- expect(await result).toEqual({ pagesCrawled: total, completed: true });- expect(mocks.claimChunk).toHaveBeenCalledTimes(2);- expect(lastRefusalAt).toBeGreaterThan(0);- expect(nextChunkStartedAt - lastRefusalAt).toBeGreaterThanOrEqual(5_000);+ const result = crawl(100);+ await vi.runAllTimersAsync();+ expect(await result).toEqual({ pagesCrawled: 100, completed: true });+ expect(starts[1]).toBe(30_000);+ expect(starts.slice(2).every((at, i) => at - starts[i + 1] >= 2_000)).toBe(+ true,+ );+ expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);+ expect(mocks.claimChunk.mock.calls.length).toBeGreaterThan(1);});- it("finishes all pages after a sixty-second origin cooldown", async () => {- const fetchMock = serve((now) => (now < 60_000 ? 429 : 200), "60");- const result = crawl();- await vi.advanceTimersByTimeAsync(59_999);- expect(fetchMock).toHaveBeenCalledTimes(10);- await vi.advanceTimersByTimeAsync(1);- await vi.advanceTimersByTimeAsync(10_000);- expect(await result).toEqual({ pagesCrawled: 100, completed: true });+ it("sleeps durably for ten minutes then retries the deferred URL", async () => {+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");+ const result = crawl(10);+ await vi.advanceTimersByTimeAsync(599_999);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(saved).toHaveLength(0);+ expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(10);+ expect(mocks.sleepUntil).toHaveBeenCalledWith("crawl-cooldown-1", 600_001);+ await vi.runAllTimersAsync();+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });+ expect(fetchMock).toHaveBeenCalledTimes(11);+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);});- it("stops a permanently limited origin without restarting in another chunk", async () => {- const fetchMock = serve(() => 429);- const result = crawl();- await vi.advanceTimersByTimeAsync(100_000);- const outcome = await result;- expect(outcome).toMatchObject({ completed: false, rateLimited: true });- expect(outcome.pagesCrawled).toBeLessThan(100);- expect(fetchMock.mock.calls.length).toBeLessThan(400);- expect(saved.every((page) => page.fetchClass === "rate_limited")).toBe(- true,+ it("replays the same durable cooldown after a workflow restart", async () => {+ const cache = new Map<string, unknown>();+ mocks.pgStep.mockImplementation(+ async (_step, name: string, _config, fn: () => Promise<unknown>) => {+ if (!cache.has(name)) cache.set(name, await fn());+ return cache.get(name);+ },+ );+ mocks.sleepUntil.mockRejectedValueOnce(new Error("isolate restarted"));+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");+ const first = expect(crawl(10)).rejects.toThrow("isolate restarted");+ await vi.advanceTimersByTimeAsync(1_000);+ await first;+ expect(fetchMock).toHaveBeenCalledTimes(1);+ // Resume after the timestamp has passed: the same sleep must still replay.+ vi.setSystemTime(700_000);+ const resumed = crawl(10);+ await vi.runAllTimersAsync();+ expect(await resumed).toEqual({ pagesCrawled: 10, completed: true });+ expect(mocks.sleepUntil.mock.calls).toEqual([+ ["crawl-cooldown-1", 600_001],+ ["crawl-cooldown-1", 600_001],+ ]);+ expect(fetchMock).toHaveBeenCalledTimes(11);+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);+ });++ it("honors the saved cooldown when the chunk fails before returning", async () => {+ mocks.releaseUrls.mockRejectedValueOnce(+ new Error("scratchpad unavailable"),+ );+ mocks.pgStep.mockImplementation(+ async (_step, _name, _config, fn: () => Promise<unknown>) => {+ try {+ return await fn();+ } catch {+ await new Promise((resolve) => setTimeout(resolve, 10_000));+ return fn();+ }+ },);- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);- expect(mocks.releaseUrls).toHaveBeenCalledTimes(1);- expect(mocks.releaseUrls.mock.calls[0][0].length).toBe(100 - saved.length);+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");+ const result = crawl(10);+ await vi.advanceTimersByTimeAsync(599_999);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ await vi.runAllTimersAsync();+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });+ expect(fetchMock).toHaveBeenCalledTimes(11);+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);});- it("leaves URLs unvisited when Retry-After is longer than the entire budget", async () => {+ it("fails without more requests if a cooldown cannot be checkpointed", async () => {+ mocks.saveCrawlThrottle.mockRejectedValueOnce(+ new Error("storage unavailable"),+ );const fetchMock = serve(() => 429, "600");+ const result = expect(crawl(10)).rejects.toThrow(+ "Unable to save the site's crawl cooldown.",+ );+ await vi.runAllTimersAsync();+ await result;+ expect(fetchMock).toHaveBeenCalledTimes(1);+ });++ it("stops a permanently limited site after four refusals across chunks", async () => {+ const fetchMock = serve(() => 429);+ const result = crawl();+ await vi.runAllTimersAsync();+ expect(await result).toMatchObject({ completed: false, rateLimited: true });+ expect(fetchMock).toHaveBeenCalledTimes(4);+ expect(saved).toHaveLength(1);+ expect(saved[0].fetchClass).toBe("rate_limited");+ expect(mocks.sleepUntil.mock.calls.length).toBeGreaterThan(0);+ expect(mocks.releaseUrls).toHaveBeenCalled();+ });++ it("stops without sleeping or retrying early when Retry-After exceeds the cooldown budget", async () => {+ const fetchMock = serve(() => 429, "3600");const result = crawl();- await vi.advanceTimersByTimeAsync(100);+ await vi.runAllTimersAsync();expect(await result).toEqual({- pagesCrawled: 10,+ pagesCrawled: 1,completed: false,rateLimited: true,});- expect(fetchMock).toHaveBeenCalledTimes(10);- expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(90);- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(mocks.sleepUntil).not.toHaveBeenCalled();+ expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(99);});});diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.tsindex 448840dc..f5fa8426 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts@@ -1,4 +1,5 @@import type { WorkflowStep } from "cloudflare:workers";+import { NonRetryableError } from "cloudflare:workflows";import type { RobotsResult } from "@/server/lib/audit/discovery";import type { CrawledPageResult } from "@/server/lib/audit/types";import { isSameOrigin } from "@/server/lib/audit/url-utils";@@ -19,7 +20,10 @@ import {CRAWL_WINDOW,RETRY_CRAWL_WINDOW,} from "@/server/lib/audit/crawl-window";-import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";+import {+ createCrawlThrottle,+ type CrawlThrottleState,+} from "@/server/lib/audit/crawl-throttle";import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";import { pgStep } from "@/server/workflows/pgStep";import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";@@ -109,6 +113,7 @@ export async function runCrawlPhase(// the site's page weight the hard way — on heavy-page sites that meant an// exceededMemory death every ~200 pages.let windowHint = CRAWL_WINDOW.initial;+ let throttleState: CrawlThrottleState | undefined;while (pending > 0 && attemptedTotal < params.maxPages) {chunkNo += 1;@@ -122,6 +127,7 @@ export async function runCrawlPhase(chunkNo,attemptedBefore: attemptedTotal,startWindow: windowHint,+ throttleState,}),);// Apply the chunk's counters even when it did no new work (a retried@@ -139,6 +145,14 @@ export async function runCrawlPhase(// `?? initial`: an instance in flight across a deploy replays cached// step results from before endWindow existed.windowHint = result.endWindow ?? CRAWL_WINDOW.initial;+ throttleState = result.throttleState;+ if (pending > 0 && attemptedTotal < params.maxPages && result.resumeAt) {+ // The timestamp comes from the persisted chunk result. Always replay+ // the same sleep step, even when that timestamp is now in the past.+ await step.sleepUntil(`crawl-cooldown-${chunkNo}`, result.resumeAt);+ zeroProgressChunks = 0;+ continue;+ }// One zero-attempt chunk is normal (retry of a completed chunk number);// two in a row means the frontier is unservable — stop with what we// have instead of spinning forever.@@ -155,6 +169,7 @@ async function runCrawlChunk(chunkNo: number;attemptedBefore: number;startWindow: number;+ throttleState?: CrawlThrottleState;},): Promise<{attemptedInChunk: number;@@ -162,10 +177,14 @@ async function runCrawlChunk(pending: number;endWindow: number;rateLimited?: boolean;+ throttleState?: CrawlThrottleState;+ resumeAt?: number;}> {const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =input;const scratchpad = getAuditScratchpad(auditId);+ let previousThrottle =+ (await scratchpad.getCrawlThrottle()) ?? input.throttleState;const claimLimit = Math.min(CHUNK_TARGET_PAGES,@@ -175,6 +194,31 @@ async function runCrawlChunk(chunkNo,claimLimit,);+ const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;+ if (isRetry && previousThrottle) {+ // Requests made since the last checkpoint may have consumed a slot.+ previousThrottle = {+ ...previousThrottle,+ nextRequestAt: Math.max(+ previousThrottle.nextRequestAt,+ Date.now() + previousThrottle.intervalMs,+ ),+ };+ }+ const throttle = createCrawlThrottle(+ deadlineAt,+ previousThrottle,+ async (state) => {+ try {+ await scratchpad.saveCrawlThrottle(state);+ } catch {+ // A retry cannot safely honor a cooldown that failed to checkpoint.+ throw new NonRetryableError(+ "Unable to save the site's crawl cooldown.",+ );+ }+ },+ );if (claimed.length === 0) {const stats = await scratchpad.getStats();return {@@ -182,11 +226,16 @@ async function runCrawlChunk(attempted: stats.attempted,pending: stats.pending,endWindow: input.startWindow,+ throttleState: throttle.state,+ rateLimited: throttle.stopped,+ resumeAt:+ throttle.state.pausedUntil > Date.now()+ ? throttle.state.pausedUntil+ : undefined,};}const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));- const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;// A retry means the previous attempt died mid-crawl (in production almost// always exceededMemory), and it is the chunk's last attempt — so it runs@@ -198,8 +247,6 @@ async function runCrawlChunk(let nextIndex = 0;let attemptedInChunk = 0;const inFlight = new Set<Promise<void>>();- // Shared by every fetch in the chunk: one page's 429 pauses them all.- const throttle = createCrawlThrottle(deadlineAt);const deferred: string[] = [];let persistThreshold = FIRST_PERSIST_BATCH_SIZE;let batch: CrawledPageResult[] = [];@@ -284,6 +331,7 @@ async function runCrawlChunk(}flush();await persistChain;+ await scratchpad.saveCrawlThrottle(throttle.state);// Preserve URLs without a page result, including slots stopped by a cooldown.const unattempted = [@@ -298,17 +346,18 @@ async function runCrawlChunk(// persistCrawledPages), so a chunk that dies mid-way underreports by at// most one sub-batch, not a whole chunk.const stats = await scratchpad.getStats();- // The next chunk creates a fresh throttle. Honor the last response's- // cooldown before allowing it to fetch more URLs.- if (stats.pending > 0 && stats.attempted < maxPages) {- await throttle.ready();- }+ const throttleState = throttle.state;return {attemptedInChunk,attempted: stats.attempted,pending: stats.pending,endWindow: windowSize,rateLimited: throttle.stopped,+ throttleState,+ resumeAt:+ throttleState.pausedUntil > Date.now()+ ? throttleState.pausedUntil+ : undefined,};}