Skip to content

Open source alternative to Semrush and Ahrefs

Pace site audits and resume after longer rate-limit cooldowns (#595)

Ben Senescu6edf052c32e2
diff --git a/src/server/features/audit/AuditScratchpad.ts b/src/server/features/audit/AuditScratchpad.ts
index d473f21c..b6b1ebce 100644
--- a/src/server/features/audit/AuditScratchpad.ts
+++ b/src/server/features/audit/AuditScratchpad.ts
@@ -14,11 +14,12 @@
* write racing in after destroy() — is wiped after 7 days (failed audits'
* state doubles as the resume/debug artifact until then).
*
- * All methods are synchronous inside (SQLite in DOs is sync), so each RPC is
- * effectively atomic. Writes are idempotent: the workflow retries steps, so
+ * SQL mutations are synchronous; key-value checkpoints await durable writes.
+ * Writes are idempotent: the workflow retries steps, so
* every insert is OR IGNORE / OR REPLACE on a stable key.
*/
import { DurableObject, env } from "cloudflare:workers";
+import type { CrawlThrottleState } from "@/server/lib/audit/crawl-throttle";
import {
BROKEN_LINKS_SQL,
ORPHAN_PAGES_SQL,
@@ -119,6 +120,14 @@ export class AuditScratchpad extends DurableObject {
);
}
+ async getCrawlThrottle(): Promise<CrawlThrottleState | undefined> {
+ return this.ctx.storage.get<CrawlThrottleState>("crawl-throttle");
+ }
+
+ async saveCrawlThrottle(state: CrawlThrottleState): Promise<void> {
+ await this.ctx.storage.put("crawl-throttle", state);
+ }
+
async seedSitemapUrls(urls: string[]): Promise<void> {
for (const url of urls) {
// Upsert so a URL that already exists (e.g. the start URL) still gets
diff --git a/src/server/lib/audit/crawl-throttle.test.ts b/src/server/lib/audit/crawl-throttle.test.ts
index 3c9cce49..d1945f4d 100644
--- a/src/server/lib/audit/crawl-throttle.test.ts
+++ b/src/server/lib/audit/crawl-throttle.test.ts
@@ -1,83 +1,177 @@
-import { afterEach, describe, expect, it, vi } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";
-afterEach(() => {
- vi.useRealTimers();
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(0);
});
+afterEach(() => vi.useRealTimers());
describe("createCrawlThrottle", () => {
- it("backs off exponentially and still pauses after a URL exhausts its retries", async () => {
- vi.useFakeTimers();
- const throttle = createCrawlThrottle(Date.now() + 90_000);
- for (const [attempt, delay] of [
- [1, 1_000],
- [2, 2_000],
- [3, 4_000],
- [4, 8_000],
- ]) {
- expect(throttle.backoff(attempt, null)).toBe(attempt <= 3);
- let ready = false;
- const waiting = throttle.ready().then((result) => {
- ready = result;
- });
- await vi.advanceTimersByTimeAsync(delay - 1);
- expect(ready).toBe(false);
- await vi.advanceTimersByTimeAsync(1);
- await waiting;
- expect(ready).toBe(true);
- }
+ it("keeps oversized Retry-After values finite even with overlapping refusals", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ await Promise.all([
+ throttle.backoff(1, "9".repeat(310)),
+ throttle.backoff(1, "60"),
+ ]);
+ expect(throttle.stopped).toBe(true);
+ expect(Object.values(throttle.state).every(Number.isFinite)).toBe(true);
+ expect(await throttle.ready()).toBe(false);
});
- it.each(["60", "Thu, 01 Jan 2026 00:01:00 GMT"])(
- "honors a full sixty-second Retry-After: %s",
- async (retryAfter) => {
- vi.useFakeTimers();
- vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));
- const throttle = createCrawlThrottle(Date.now() + 90_000);
- expect(throttle.backoff(1, retryAfter)).toBe(true);
- let ready = false;
- const waiting = throttle.ready().then((result) => {
- ready = result;
+ it("waits for cooldown persistence before allowing another request", async () => {
+ let save: () => void = vi.fn();
+ const throttle = createCrawlThrottle(
+ 90_000,
+ undefined,
+ () =>
+ new Promise<void>((resolve) => {
+ save = resolve;
+ }),
+ );
+ const backoff = throttle.backoff(1, "1");
+ await vi.advanceTimersByTimeAsync(0);
+ let started = false;
+ const request = throttle.ready().then(() => {
+ started = true;
+ });
+ await vi.advanceTimersByTimeAsync(10_000);
+ expect(started).toBe(false);
+ save();
+ await backoff;
+ await request;
+ expect(started).toBe(true);
+ });
+
+ it("spaces simultaneous request starts instead of releasing a burst", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ const starts: number[] = [];
+ const requests = Promise.all(
+ Array.from({ length: 5 }, async () => {
+ if (await throttle.ready()) starts.push(Date.now());
+ }),
+ );
+ await vi.runAllTimersAsync();
+ await requests;
+ expect(starts).toEqual([0, 1_000, 2_000, 3_000, 4_000]);
+ });
+
+ it("pauses for thirty seconds on the first 429 and spaces retries", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ expect(await throttle.ready()).toBe(true);
+ expect(await throttle.backoff(1, null)).toBe(true);
+ const starts: number[] = [];
+ const requests = Promise.all(
+ Array.from({ length: 3 }, async () => {
+ if (await throttle.ready()) starts.push(Date.now());
+ }),
+ );
+ await vi.advanceTimersByTimeAsync(29_999);
+ expect(starts).toEqual([]);
+ await vi.runAllTimersAsync();
+ await requests;
+ expect(starts).toEqual([30_000, 32_000, 34_000]);
+ });
+
+ it("rechecks a cooldown extended while requests are waiting", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ await throttle.backoff(1, "5");
+ const starts: number[] = [];
+ const request = throttle.ready().then(() => starts.push(Date.now()));
+ await vi.advanceTimersByTimeAsync(4_000);
+ await throttle.backoff(1, "10");
+ await vi.advanceTimersByTimeAsync(9_999);
+ expect(starts).toEqual([]);
+ await vi.advanceTimersByTimeAsync(1);
+ await request;
+ expect(starts).toEqual([14_000]);
+ });
+
+ it.each(["60", "Thu, 01 Jan 1970 00:01:00 GMT"])(
+ "honors Retry-After: %s",
+ async (header) => {
+ const throttle = createCrawlThrottle(90_000);
+ await throttle.backoff(1, header);
+ let started = false;
+ const request = throttle.ready().then(() => {
+ started = true;
});
await vi.advanceTimersByTimeAsync(59_999);
- expect(ready).toBe(false);
+ expect(started).toBe(false);
await vi.advanceTimersByTimeAsync(1);
- await waiting;
- expect(ready).toBe(true);
+ await request;
+ expect(started).toBe(true);
},
);
- it("does not label an ordinary chunk deadline as a rate-limit stop", async () => {
- vi.useFakeTimers();
- const throttle = createCrawlThrottle(Date.now() + 90_000);
- await vi.advanceTimersByTimeAsync(90_000);
+ it("backs off across chunks and stops after four consecutive refusals", async () => {
+ let throttle = createCrawlThrottle(90_000);
+ for (const [index, delay] of [30_000, 60_000, 120_000, 240_000].entries()) {
+ expect(await throttle.backoff(1, null)).toBe(index < 3);
+ expect(throttle.state.pausedUntil - Date.now()).toBe(delay);
+ if (index < 3) {
+ await vi.advanceTimersByTimeAsync(delay);
+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);
+ }
+ }
+ expect(throttle.stopped).toBe(true);
expect(await throttle.ready()).toBe(false);
+ });
+
+ it("retains slower pacing after recovery and caps further slowdowns", async () => {
+ let throttle = createCrawlThrottle(90_000);
+ for (let i = 0; i < 8; i++) {
+ await throttle.backoff(1, "1");
+ await vi.advanceTimersByTimeAsync(30_000);
+ await throttle.recovered();
+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);
+ }
+ expect(throttle.state.intervalMs).toBe(30_000);
expect(throttle.stopped).toBe(false);
});
- it("stops without scheduling a timer when the cooldown exceeds the crawl budget", async () => {
- vi.useFakeTimers();
- const throttle = createCrawlThrottle(Date.now() + 90_000);
- expect(throttle.backoff(1, "600")).toBe(false);
- expect(throttle.stopped).toBe(true);
+ it("defers a ten-minute wait without marking the audit stopped", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ expect(await throttle.backoff(1, "600")).toBe(true);
expect(await throttle.ready()).toBe(false);
+ expect(throttle.stopped).toBe(false);
+ expect(throttle.state.pausedUntil).toBe(600_000);
expect(vi.getTimerCount()).toBe(0);
+ await vi.advanceTimersByTimeAsync(600_000);
+ const resumed = createCrawlThrottle(690_000, throttle.state);
+ expect(await resumed.ready()).toBe(true);
});
- it("holds a waiter through a pause that another 429 extends", async () => {
- vi.useFakeTimers();
- const throttle = createCrawlThrottle(Date.now() + 90_000);
- throttle.backoff(1, "1");
- let released = false;
- const waiting = throttle.ready().then((result) => {
- released = result;
+ it("preserves request spacing across a chunk boundary", async () => {
+ const first = createCrawlThrottle(90_000);
+ await first.ready();
+ const second = createCrawlThrottle(90_000, first.state);
+ let startedAt = -1;
+ const request = second.ready().then(() => {
+ startedAt = Date.now();
});
- await vi.advanceTimersByTimeAsync(900);
- throttle.backoff(1, "1");
- await vi.advanceTimersByTimeAsync(200);
- expect(released).toBe(false);
- await vi.advanceTimersByTimeAsync(800);
- await waiting;
- expect(released).toBe(true);
+ await vi.runAllTimersAsync();
+ await request;
+ expect(startedAt).toBe(1_000);
+ });
+
+ it("does not label an ordinary chunk deadline as a rate-limit stop", async () => {
+ const throttle = createCrawlThrottle(90_000);
+ await vi.advanceTimersByTimeAsync(90_000);
+ expect(await throttle.ready()).toBe(false);
+ expect(throttle.stopped).toBe(false);
+ });
+
+ it("stops when cumulative cooldowns exceed thirty minutes, without retrying early", async () => {
+ let throttle = createCrawlThrottle(90_000);
+ for (let i = 0; i < 3; i++) {
+ expect(await throttle.backoff(1, "600")).toBe(true);
+ await vi.advanceTimersByTimeAsync(600_000);
+ await throttle.recovered();
+ throttle = createCrawlThrottle(Date.now() + 90_000, throttle.state);
+ }
+ expect(await throttle.backoff(1, "600")).toBe(false);
+ expect(await throttle.ready()).toBe(false);
+ expect(vi.getTimerCount()).toBe(0);
});
});
diff --git a/src/server/lib/audit/crawl-throttle.ts b/src/server/lib/audit/crawl-throttle.ts
index 70ef9b15..ee055ad1 100644
--- a/src/server/lib/audit/crawl-throttle.ts
+++ b/src/server/lib/audit/crawl-throttle.ts
@@ -1,12 +1,25 @@
/** Retries per URL after its first 429. Every 429 still pauses new URLs. */
const MAX_RETRIES = 3;
-const FIRST_DELAY_MS = 1_000;
+const FIRST_DELAY_MS = 30_000;
+const MAX_INTERVAL_MS = 30_000;
+const MAX_COOLDOWN_MS = 30 * 60_000;
+
+/** Checkpointed between chunks so a new step does not forget site limits. */
+export type CrawlThrottleState = {
+ intervalMs: number;
+ nextRequestAt: number;
+ pausedUntil: number;
+ consecutiveRateLimits: number;
+ cooldownMs: number;
+};
/** `Retry-After` is either delay-seconds or an HTTP-date. */
function parseRetryAfterMs(header: string | null): number | null {
if (!header) return null;
const value = header.trim();
- if (/^\d+$/.test(value)) return Number(value) * 1_000;
+ // Keep over-budget values finite for checkpoint serialization/arithmetic.
+ if (/^\d+$/.test(value))
+ return Math.min(Number(value) * 1_000, MAX_COOLDOWN_MS + 1);
const at = Date.parse(value);
return Number.isNaN(at) ? null : Math.max(0, at - Date.now());
}
@@ -15,38 +28,89 @@ export interface CrawlThrottle {
/** False when this chunk can no longer start a request. */
ready(): Promise<boolean>;
/** Pause the origin on every 429; return whether this URL may retry. */
- backoff(attempt: number, retryAfter: string | null): boolean;
+ backoff(attempt: number, retryAfter: string | null): Promise<boolean>;
+ /** A non-429 response breaks a run of consecutive refusals. */
+ recovered(): Promise<void>;
+ readonly checkpointFailed: boolean;
readonly stopped: boolean;
+ readonly state: CrawlThrottleState;
}
-/** One origin cooldown, bounded by the scheduler's existing chunk deadline. */
-export function createCrawlThrottle(deadlineAt: number): CrawlThrottle {
- let pausedUntil = 0;
- let stopped = false;
+/** One audit's origin pacing. Long waits are resumed by the workflow. */
+export function createCrawlThrottle(
+ deadlineAt: number,
+ previous?: CrawlThrottleState,
+ persist?: (state: CrawlThrottleState) => Promise<void>,
+): CrawlThrottle {
+ const state: CrawlThrottleState = previous
+ ? { ...previous }
+ : {
+ intervalMs: 1_000,
+ nextRequestAt: 0,
+ pausedUntil: 0,
+ consecutiveRateLimits: 0,
+ cooldownMs: 0,
+ };
+ let checkpointFailed = false;
+ let checkpoint = Promise.resolve();
+ const save = () => {
+ const snapshot = { ...state };
+ checkpoint = checkpoint
+ .then(() => persist?.(snapshot))
+ .catch((error) => {
+ checkpointFailed = true;
+ throw error;
+ });
+ return checkpoint;
+ };
+ const stopped = () =>
+ checkpointFailed ||
+ state.consecutiveRateLimits > MAX_RETRIES ||
+ state.cooldownMs > MAX_COOLDOWN_MS;
return {
async ready() {
- // Another request can extend the shared pause while this one waits.
- while (pausedUntil > Date.now()) {
- if (stopped) return false;
- await new Promise((resolve) =>
- setTimeout(resolve, pausedUntil - Date.now()),
- );
+ while (!stopped()) {
+ await checkpoint;
+ const now = Date.now();
+ const readyAt = Math.max(state.pausedUntil, state.nextRequestAt);
+ if (now >= deadlineAt || readyAt >= deadlineAt) return false;
+ if (readyAt <= now) {
+ // Reserve synchronously: waiters waking together must take turns.
+ state.nextRequestAt = now + state.intervalMs;
+ return true;
+ }
+ await new Promise((resolve) => setTimeout(resolve, readyAt - now));
}
- return !stopped && Date.now() < deadlineAt;
+ return false;
},
- backoff(attempt, retryAfter) {
+ async backoff(attempt, retryAfter) {
+ state.consecutiveRateLimits += 1;
+ state.intervalMs = Math.min(MAX_INTERVAL_MS, state.intervalMs * 2);
const delayMs = Math.max(
- FIRST_DELAY_MS,
- parseRetryAfterMs(retryAfter) ?? FIRST_DELAY_MS * 2 ** (attempt - 1),
+ state.intervalMs,
+ parseRetryAfterMs(retryAfter) ??
+ FIRST_DELAY_MS * 2 ** (state.consecutiveRateLimits - 1),
);
- pausedUntil = Math.max(pausedUntil, Date.now() + delayMs);
- // Never shorten the site's requested wait to fit our budget. Stop the
- // crawl instead; the scheduler preserves unvisited URLs as incomplete.
- if (pausedUntil >= deadlineAt) stopped = true;
- return !stopped && attempt <= MAX_RETRIES;
+ const now = Date.now();
+ const pausedUntil = Math.max(state.pausedUntil, now + delayMs);
+ state.cooldownMs += pausedUntil - Math.max(now, state.pausedUntil);
+ state.pausedUntil = pausedUntil;
+ await save();
+ return !stopped() && attempt <= MAX_RETRIES;
+ },
+ async recovered() {
+ if (state.consecutiveRateLimits === 0) return;
+ state.consecutiveRateLimits = 0;
+ await save();
+ },
+ get checkpointFailed() {
+ return checkpointFailed;
},
get stopped() {
- return stopped;
+ return stopped();
+ },
+ get state() {
+ return { ...state };
},
};
}
diff --git a/src/server/lib/audit/crawl-window.test.ts b/src/server/lib/audit/crawl-window.test.ts
index 5cac4a09..587096ff 100644
--- a/src/server/lib/audit/crawl-window.test.ts
+++ b/src/server/lib/audit/crawl-window.test.ts
@@ -53,82 +53,44 @@ function page(
describe("adjustCrawlWindow", () => {
it("keeps the window on an empty batch", () => {
- expect(adjustCrawlWindow(10, [])).toBe(10);
+ expect(adjustCrawlWindow(2, [])).toBe(2);
});
- it("halves the window when a third of the batch is troubled", () => {
- const recent = [
- ...Array.from({ length: 9 }, () => page("error", 15_000)),
- ...Array.from({ length: 16 }, () => page("ok", 500)),
- ];
- expect(adjustCrawlWindow(25, recent)).toBe(12);
- });
-
- it("treats blocked fetches as trouble", () => {
- const recent = Array.from({ length: 10 }, () => page("blocked", 300));
- expect(adjustCrawlWindow(20, recent)).toBe(10);
- });
+ it.each(["error", "blocked", "rate_limited"] as const)(
+ "reduces concurrency on %s",
+ (fetchClass) => {
+ const recent = Array.from({ length: 10 }, () => page(fetchClass, 300));
+ expect(adjustCrawlWindow(2, recent)).toBe(1);
+ },
+ );
- it("treats a 429 the retries recovered from as trouble", () => {
+ it("treats recovered 429s as trouble", () => {
const recent = Array.from({ length: 10 }, () => ({
...page("ok", 300),
rateLimited: true,
}));
- expect(adjustCrawlWindow(20, recent)).toBe(10);
+ expect(adjustCrawlWindow(2, recent)).toBe(1);
});
- it("never shrinks below the minimum", () => {
- const recent = Array.from({ length: 10 }, () => page("error", 15_000));
- expect(adjustCrawlWindow(6, recent)).toBe(5);
+ it("never shrinks below one request", () => {
+ expect(adjustCrawlWindow(1, [page("error", 15_000)])).toBe(1);
});
- it("grows on a clean, fast batch up to the cap", () => {
+ it("never grows beyond two concurrent requests, even on a fast site", () => {
const recent = Array.from({ length: 25 }, () => page("ok", 400));
- expect(adjustCrawlWindow(10, recent)).toBe(15);
- expect(adjustCrawlWindow(20, recent)).toBe(20);
- });
-
- it("caps the window so heavy pages stay inside the byte budget", () => {
- // 1 MiB average pages: 8 MiB budget / 1 MiB = window of 8.
- const recent = Array.from({ length: 25 }, () =>
- page("ok", 300, 1024 * 1024),
- );
- expect(adjustCrawlWindow(20, recent)).toBe(8);
- });
-
- it("does not grow on a small sample, but still applies the byte bound", () => {
- const fastSmall = Array.from({ length: 5 }, () => page("ok", 400));
- expect(adjustCrawlWindow(10, fastSmall)).toBe(10);
- const fastHeavy = Array.from({ length: 5 }, () =>
- page("ok", 400, 1024 * 1024),
- );
- expect(adjustCrawlWindow(10, fastHeavy)).toBe(8);
+ expect(adjustCrawlWindow(2, recent)).toBe(2);
+ expect(adjustCrawlWindow(1, recent)).toBe(2);
});
- it("retry limits keep the window small even on a clean, fast site", () => {
+ it("keeps retry chunks at one concurrent request", () => {
const recent = Array.from({ length: 25 }, () => page("ok", 400));
- expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(5);
- });
-
- it("retry limits shrink below the normal minimum on trouble", () => {
- const recent = Array.from({ length: 10 }, () => page("error", 15_000));
- expect(adjustCrawlWindow(3, recent, RETRY_CRAWL_WINDOW)).toBe(2);
+ expect(adjustCrawlWindow(1, recent, RETRY_CRAWL_WINDOW)).toBe(1);
});
- it("keeps the byte bound at the minimum window even for huge pages", () => {
+ it("preserves the byte budget if page sizes increase", () => {
const recent = Array.from({ length: 25 }, () =>
- page("ok", 300, 4 * 1024 * 1024),
+ page("ok", 300, 5 * 1024 * 1024),
);
- expect(adjustCrawlWindow(20, recent)).toBe(5);
- });
-
- it("lets small pages use the full window cap", () => {
- const recent = Array.from({ length: 25 }, () => page("ok", 400, 10_000));
- expect(adjustCrawlWindow(20, recent)).toBe(20);
- });
-
- it("holds steady on a clean but slow batch", () => {
- const recent = Array.from({ length: 25 }, () => page("ok", 5_000));
- expect(adjustCrawlWindow(15, recent)).toBe(15);
+ expect(adjustCrawlWindow(2, recent)).toBe(1);
});
});
diff --git a/src/server/lib/audit/crawl-window.ts b/src/server/lib/audit/crawl-window.ts
index c30b8bc4..515b9391 100644
--- a/src/server/lib/audit/crawl-window.ts
+++ b/src/server/lib/audit/crawl-window.ts
@@ -23,9 +23,9 @@ interface CrawlWindowLimits {
}
export const CRAWL_WINDOW: CrawlWindowLimits = {
- initial: 10,
- min: 5,
- max: 20,
+ initial: 2,
+ min: 1,
+ max: 2,
budgetBytes: 8 * 1024 * 1024,
};
@@ -36,9 +36,9 @@ export const CRAWL_WINDOW: CrawlWindowLimits = {
* memory profile that just killed the isolate.
*/
export const RETRY_CRAWL_WINDOW: CrawlWindowLimits = {
- initial: 3,
- min: 2,
- max: 5,
+ initial: 1,
+ min: 1,
+ max: 1,
budgetBytes: 4 * 1024 * 1024,
};
diff --git a/src/server/workflows/site-audit-workflow-helpers.test.ts b/src/server/workflows/site-audit-workflow-helpers.test.ts
index 9c19df79..a3c33ebb 100644
--- a/src/server/workflows/site-audit-workflow-helpers.test.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.test.ts
@@ -83,7 +83,7 @@ describe("crawlPage", () => {
await vi.advanceTimersByTimeAsync(4_000);
expect(fetched).toEqual([`${PAGE_URL}/first`]);
- await vi.advanceTimersByTimeAsync(2_000);
+ await vi.advanceTimersByTimeAsync(3_000);
const pages = await Promise.all([first, second]);
expect(fetched).toHaveLength(3);
expect(pages.map((page) => page?.fetchClass)).toEqual(["ok", "ok"]);
@@ -91,7 +91,7 @@ describe("crawlPage", () => {
it("records a page the site keeps rate limiting, without calling it blocked", async () => {
vi.useFakeTimers();
- const fetchMock = stubFetch({ status: 429 });
+ const fetchMock = stubFetch({ status: 429, retryAfter: "1" });
const crawled = crawl();
await vi.advanceTimersByTimeAsync(30_000);
@@ -101,39 +101,17 @@ describe("crawlPage", () => {
expect(page?.fetchClass).toBe("rate_limited");
});
- it("keeps the origin paused after a URL exhausts its retries", async () => {
- vi.useFakeTimers();
- vi.setSystemTime(0);
- const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(
- async () =>
- new Response(PAGE_HTML, {
- status: Date.now() < 15_000 ? 429 : 200,
- headers: { "content-type": "text/html" },
- }),
- );
- const throttle = createCrawlThrottle(90_000);
- const first = crawlPage(`${PAGE_URL}/first`, 0, false, throttle);
- await vi.advanceTimersByTimeAsync(7_000);
- expect((await first)?.fetchClass).toBe("rate_limited");
- expect(fetchMock).toHaveBeenCalledTimes(4);
-
- const second = crawlPage(`${PAGE_URL}/second`, 0, false, throttle);
- await vi.advanceTimersByTimeAsync(7_999);
- expect(fetchMock).toHaveBeenCalledTimes(4);
- await vi.advanceTimersByTimeAsync(1);
- expect((await second)?.fetchClass).toBe("ok");
- });
-
- it("recovers concurrent pages when the origin asks for sixty seconds", async () => {
+ it("recovers concurrent pages without a burst after sixty seconds", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
- const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(
- async () =>
- new Response(PAGE_HTML, {
- status: Date.now() < 60_000 ? 429 : 200,
- headers: { "content-type": "text/html", "retry-after": "60" },
- }),
- );
+ const starts: number[] = [];
+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+ starts.push(Date.now());
+ return new Response(PAGE_HTML, {
+ status: Date.now() < 60_000 ? 429 : 200,
+ headers: { "content-type": "text/html", "retry-after": "60" },
+ });
+ });
const throttle = createCrawlThrottle(90_000);
const pages = Promise.all(
Array.from({ length: 5 }, (_, i) =>
@@ -141,24 +119,34 @@ describe("crawlPage", () => {
),
);
await vi.advanceTimersByTimeAsync(59_999);
- expect(fetchMock).toHaveBeenCalledTimes(5);
- await vi.advanceTimersByTimeAsync(1);
+ expect(starts).toEqual([0]);
+ await vi.runAllTimersAsync();
expect((await pages).map((page) => page?.fetchClass)).toEqual(
Array(5).fill("ok"),
);
- expect(fetchMock).toHaveBeenCalledTimes(10);
+ expect(starts).toEqual([0, 60_000, 62_000, 64_000, 66_000, 68_000]);
});
- it("records the observed 429 but does not fetch another URL when the wait cannot fit", async () => {
+ it("defers the refused URL when its cooldown exceeds the chunk deadline", async () => {
vi.useFakeTimers();
const fetchMock = stubFetch({ status: 429, retryAfter: "600" });
const throttle = createCrawlThrottle(Date.now() + 90_000);
- expect((await crawlPage(PAGE_URL, 0, false, throttle))?.fetchClass).toBe(
- "rate_limited",
- );
+ expect(await crawlPage(PAGE_URL, 0, false, throttle)).toBeNull();
expect(
await crawlPage(`${PAGE_URL}/unvisited`, 0, false, throttle),
).toBeNull();
+ expect(throttle.stopped).toBe(false);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("records a final 429 when the requested wait exceeds the audit's cooldown budget", async () => {
+ vi.useFakeTimers();
+ const fetchMock = stubFetch({ status: 429, retryAfter: "3600" });
+ const throttle = createCrawlThrottle(Date.now() + 90_000);
+ expect((await crawlPage(PAGE_URL, 0, false, throttle))?.fetchClass).toBe(
+ "rate_limited",
+ );
+ expect(throttle.stopped).toBe(true);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
@@ -302,7 +290,8 @@ describe("crawlPage", () => {
const before = await collect();
const pages = [];
for (let i = 0; i < 50; i++) {
- pages.push(await crawlPage("https://example.com/" + i, 0, true, throttle));
+ pages.push(await crawlPage("https://example.com/" + i, 0, true,
+ createCrawlThrottle(Date.now() + 90_000)));
}
const retained = (await collect()) - before;
assert.equal(pages.length, 50);
diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.ts
index e838c826..7d207aea 100644
--- a/src/server/workflows/site-audit-workflow-helpers.ts
+++ b/src/server/workflows/site-audit-workflow-helpers.ts
@@ -86,9 +86,12 @@ async function fetchPage(url: string, throttle: CrawlThrottle) {
// the last retry is already classified rate_limited and needs no flag.
rateLimited: attempt > 1,
};
- if (response.status !== 429) return result;
+ if (response.status !== 429) {
+ await throttle.recovered();
+ return result;
+ }
- const retry = throttle.backoff(
+ const retry = await throttle.backoff(
attempt,
response.headers.get("retry-after"),
);
@@ -227,6 +230,9 @@ export async function crawlPage(
inSitemap,
});
} catch (error) {
+ // Losing a durable cooldown must fail the workflow, not become a page
+ // error that lets the scheduler continue making requests.
+ if (throttle.checkpointFailed) throw error;
const responseTimeMs = Date.now() - startTime;
console.warn(`Failed to crawl ${url}:`, error);
return emptyPageResult({
diff --git a/src/server/workflows/siteAuditWorkflowCrawl.test.ts b/src/server/workflows/siteAuditWorkflowCrawl.test.ts
index bac76b3c..4bbc0c2e 100644
--- a/src/server/workflows/siteAuditWorkflowCrawl.test.ts
+++ b/src/server/workflows/siteAuditWorkflowCrawl.test.ts
@@ -11,6 +11,12 @@ const mocks = vi.hoisted(() => ({
releaseUrls: vi.fn<(urls: string[]) => Promise<void>>(),
insertCrawledBatch: vi.fn(),
pgStep: vi.fn(),
+ sleepUntil: vi.fn(),
+ getCrawlThrottle: vi.fn(),
+ saveCrawlThrottle: vi.fn(),
+}));
+vi.mock("cloudflare:workflows", () => ({
+ NonRetryableError: class extends Error {},
}));
vi.mock("@/server/features/audit/AuditScratchpad", () => ({
getAuditScratchpad: () => mocks,
@@ -36,6 +42,7 @@ import { parseRobotsTxt } from "@/server/lib/audit/discovery";
const ORIGIN = "https://example.com";
const HTML = "<html><title>A page</title><body><h1>A page</h1></body></html>";
let saved: CrawledPageResult[];
+let urls: string[];
beforeEach(async () => {
// Load the lazy HTML parser before advancing the fake network clock.
@@ -43,23 +50,38 @@ beforeEach(async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
saved = [];
+ let checkpoint: unknown;
+ mocks.getCrawlThrottle.mockImplementation(async () =>
+ structuredClone(checkpoint),
+ );
+ mocks.saveCrawlThrottle.mockImplementation(async (state) => {
+ checkpoint = structuredClone(state);
+ });
mocks.pgStep.mockImplementation((_step, _name, _config, fn: () => unknown) =>
fn(),
);
+ mocks.sleepUntil.mockImplementation(async (_name, at: number) => {
+ await new Promise((resolve) =>
+ setTimeout(resolve, Math.max(0, at - Date.now())),
+ );
+ });
mocks.claimChunk.mockImplementation(
async (_chunk: number, limit: number) => ({
- urls: Array.from({ length: limit }, (_, i) => ({
- url: `${ORIGIN}/${i}`,
- depth: 0,
- inSitemap: true,
- })),
+ urls: urls
+ .filter((url) => !saved.some((page) => page.url === url))
+ .slice(0, limit)
+ .map((url) => ({
+ url,
+ depth: 0,
+ inSitemap: true,
+ })),
isRetry: false,
}),
);
mocks.getStats.mockImplementation(async () => ({
attempted: saved.length,
- pending: 100 - saved.length,
- seen: 100,
+ pending: urls.length - saved.length,
+ seen: urls.length,
}));
mocks.recordBatch.mockImplementation(() => mocks.getStats());
mocks.insertCrawledBatch.mockImplementation(
@@ -74,15 +96,21 @@ afterEach(() => {
});
function crawl(maxPages = 100) {
+ urls = Array.from({ length: maxPages }, (_, i) => `${ORIGIN}/${i}`);
return runCrawlPhase(
- { do: vi.fn(), sleep: vi.fn(), sleepUntil: vi.fn(), waitForEvent: vi.fn() },
+ {
+ do: vi.fn(),
+ sleep: vi.fn(),
+ sleepUntil: mocks.sleepUntil,
+ waitForEvent: vi.fn(),
+ },
{
auditId: "audit",
workflowInstanceId: "workflow",
origin: ORIGIN,
maxPages,
seededCount: maxPages,
- robots: parseRobotsTxt("", ORIGIN),
+ robots: parseRobotsTxt(ORIGIN, ""),
},
);
}
@@ -101,85 +129,162 @@ function serve(status: (now: number) => number, retryAfter?: string) {
});
}
-describe("crawl rate-limit budget", () => {
- it("honors the final URL's cooldown before starting the next chunk", async () => {
- const total = 210;
- mocks.claimChunk.mockImplementation(
- async (chunk: number, limit: number) => ({
- urls: Array.from({ length: limit }, (_, i) => ({
- url: `${ORIGIN}/${(chunk - 1) * 200 + i}`,
- depth: 0,
- inSitemap: true,
- })),
- isRetry: false,
- }),
+describe("crawl pacing and cooldowns", () => {
+ it("spaces a fast site's requests across chunks without duplicates", async () => {
+ const starts: number[] = [];
+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+ starts.push(Date.now());
+ return new Response(HTML, { headers: { "content-type": "text/html" } });
+ });
+ const result = crawl(210);
+ await vi.runAllTimersAsync();
+ expect(await result).toEqual({ pagesCrawled: 210, completed: true });
+ expect(new Set(saved.map((page) => page.url)).size).toBe(210);
+ expect(starts).toHaveLength(210);
+ expect(starts.slice(1).every((at, i) => at - starts[i] >= 1_000)).toBe(
+ true,
);
- mocks.getStats.mockImplementation(async () => ({
- attempted: saved.length,
- pending: total - saved.length,
- seen: total,
- }));
- let lastRefusalAt = 0;
- let nextChunkStartedAt = 0;
- vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
- const url = new Request(input).url;
- if (url === `${ORIGIN}/200`) nextChunkStartedAt = Date.now();
- await new Promise((resolve) => setTimeout(resolve, 1));
- const refused = url === `${ORIGIN}/199`;
- if (refused) lastRefusalAt = Date.now();
+ expect(mocks.claimChunk.mock.calls.length).toBeGreaterThan(1);
+ });
+
+ it("never has more than two fetches in flight on a slow site", async () => {
+ let inFlight = 0;
+ let peak = 0;
+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+ inFlight += 1;
+ peak = Math.max(peak, inFlight);
+ await new Promise((resolve) => setTimeout(resolve, 3_000));
+ inFlight -= 1;
+ return new Response(HTML, { headers: { "content-type": "text/html" } });
+ });
+ const result = crawl(10);
+ await vi.runAllTimersAsync();
+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });
+ expect(peak).toBe(2);
+ });
+
+ it("retains slower pacing across chunks after recovering from the first 429", async () => {
+ const starts: number[] = [];
+ vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
+ starts.push(Date.now());
return new Response(HTML, {
- status: refused ? 429 : 200,
- headers: { "content-type": "text/html", "retry-after": "5" },
+ status: starts.length === 1 ? 429 : 200,
+ headers: { "content-type": "text/html" },
});
});
-
- const result = crawl(total);
- await vi.advanceTimersByTimeAsync(30_000);
- expect(await result).toEqual({ pagesCrawled: total, completed: true });
- expect(mocks.claimChunk).toHaveBeenCalledTimes(2);
- expect(lastRefusalAt).toBeGreaterThan(0);
- expect(nextChunkStartedAt - lastRefusalAt).toBeGreaterThanOrEqual(5_000);
+ const result = crawl(100);
+ await vi.runAllTimersAsync();
+ expect(await result).toEqual({ pagesCrawled: 100, completed: true });
+ expect(starts[1]).toBe(30_000);
+ expect(starts.slice(2).every((at, i) => at - starts[i + 1] >= 2_000)).toBe(
+ true,
+ );
+ expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);
+ expect(mocks.claimChunk.mock.calls.length).toBeGreaterThan(1);
});
- it("finishes all pages after a sixty-second origin cooldown", async () => {
- const fetchMock = serve((now) => (now < 60_000 ? 429 : 200), "60");
- const result = crawl();
- await vi.advanceTimersByTimeAsync(59_999);
- expect(fetchMock).toHaveBeenCalledTimes(10);
- await vi.advanceTimersByTimeAsync(1);
- await vi.advanceTimersByTimeAsync(10_000);
- expect(await result).toEqual({ pagesCrawled: 100, completed: true });
+ it("sleeps durably for ten minutes then retries the deferred URL", async () => {
+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");
+ const result = crawl(10);
+ await vi.advanceTimersByTimeAsync(599_999);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(saved).toHaveLength(0);
+ expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(10);
+ expect(mocks.sleepUntil).toHaveBeenCalledWith("crawl-cooldown-1", 600_001);
+ await vi.runAllTimersAsync();
+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });
+ expect(fetchMock).toHaveBeenCalledTimes(11);
+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);
expect(saved.every((page) => page.fetchClass === "ok")).toBe(true);
- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
});
- it("stops a permanently limited origin without restarting in another chunk", async () => {
- const fetchMock = serve(() => 429);
- const result = crawl();
- await vi.advanceTimersByTimeAsync(100_000);
- const outcome = await result;
- expect(outcome).toMatchObject({ completed: false, rateLimited: true });
- expect(outcome.pagesCrawled).toBeLessThan(100);
- expect(fetchMock.mock.calls.length).toBeLessThan(400);
- expect(saved.every((page) => page.fetchClass === "rate_limited")).toBe(
- true,
+ it("replays the same durable cooldown after a workflow restart", async () => {
+ const cache = new Map<string, unknown>();
+ mocks.pgStep.mockImplementation(
+ async (_step, name: string, _config, fn: () => Promise<unknown>) => {
+ if (!cache.has(name)) cache.set(name, await fn());
+ return cache.get(name);
+ },
+ );
+ mocks.sleepUntil.mockRejectedValueOnce(new Error("isolate restarted"));
+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");
+ const first = expect(crawl(10)).rejects.toThrow("isolate restarted");
+ await vi.advanceTimersByTimeAsync(1_000);
+ await first;
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ // Resume after the timestamp has passed: the same sleep must still replay.
+ vi.setSystemTime(700_000);
+ const resumed = crawl(10);
+ await vi.runAllTimersAsync();
+ expect(await resumed).toEqual({ pagesCrawled: 10, completed: true });
+ expect(mocks.sleepUntil.mock.calls).toEqual([
+ ["crawl-cooldown-1", 600_001],
+ ["crawl-cooldown-1", 600_001],
+ ]);
+ expect(fetchMock).toHaveBeenCalledTimes(11);
+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);
+ });
+
+ it("honors the saved cooldown when the chunk fails before returning", async () => {
+ mocks.releaseUrls.mockRejectedValueOnce(
+ new Error("scratchpad unavailable"),
+ );
+ mocks.pgStep.mockImplementation(
+ async (_step, _name, _config, fn: () => Promise<unknown>) => {
+ try {
+ return await fn();
+ } catch {
+ await new Promise((resolve) => setTimeout(resolve, 10_000));
+ return fn();
+ }
+ },
);
- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
- expect(mocks.releaseUrls).toHaveBeenCalledTimes(1);
- expect(mocks.releaseUrls.mock.calls[0][0].length).toBe(100 - saved.length);
+ const fetchMock = serve((now) => (now < 600_000 ? 429 : 200), "600");
+ const result = crawl(10);
+ await vi.advanceTimersByTimeAsync(599_999);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ await vi.runAllTimersAsync();
+ expect(await result).toEqual({ pagesCrawled: 10, completed: true });
+ expect(fetchMock).toHaveBeenCalledTimes(11);
+ expect(new Set(saved.map((page) => page.url)).size).toBe(10);
});
- it("leaves URLs unvisited when Retry-After is longer than the entire budget", async () => {
+ it("fails without more requests if a cooldown cannot be checkpointed", async () => {
+ mocks.saveCrawlThrottle.mockRejectedValueOnce(
+ new Error("storage unavailable"),
+ );
const fetchMock = serve(() => 429, "600");
+ const result = expect(crawl(10)).rejects.toThrow(
+ "Unable to save the site's crawl cooldown.",
+ );
+ await vi.runAllTimersAsync();
+ await result;
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ });
+
+ it("stops a permanently limited site after four refusals across chunks", async () => {
+ const fetchMock = serve(() => 429);
+ const result = crawl();
+ await vi.runAllTimersAsync();
+ expect(await result).toMatchObject({ completed: false, rateLimited: true });
+ expect(fetchMock).toHaveBeenCalledTimes(4);
+ expect(saved).toHaveLength(1);
+ expect(saved[0].fetchClass).toBe("rate_limited");
+ expect(mocks.sleepUntil.mock.calls.length).toBeGreaterThan(0);
+ expect(mocks.releaseUrls).toHaveBeenCalled();
+ });
+
+ it("stops without sleeping or retrying early when Retry-After exceeds the cooldown budget", async () => {
+ const fetchMock = serve(() => 429, "3600");
const result = crawl();
- await vi.advanceTimersByTimeAsync(100);
+ await vi.runAllTimersAsync();
expect(await result).toEqual({
- pagesCrawled: 10,
+ pagesCrawled: 1,
completed: false,
rateLimited: true,
});
- expect(fetchMock).toHaveBeenCalledTimes(10);
- expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(90);
- expect(mocks.claimChunk).toHaveBeenCalledTimes(1);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(mocks.sleepUntil).not.toHaveBeenCalled();
+ expect(mocks.releaseUrls.mock.calls[0][0]).toHaveLength(99);
});
});
diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.ts
index 448840dc..f5fa8426 100644
--- a/src/server/workflows/siteAuditWorkflowCrawl.ts
+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts
@@ -1,4 +1,5 @@
import type { WorkflowStep } from "cloudflare:workers";
+import { NonRetryableError } from "cloudflare:workflows";
import type { RobotsResult } from "@/server/lib/audit/discovery";
import type { CrawledPageResult } from "@/server/lib/audit/types";
import { isSameOrigin } from "@/server/lib/audit/url-utils";
@@ -19,7 +20,10 @@ import {
CRAWL_WINDOW,
RETRY_CRAWL_WINDOW,
} from "@/server/lib/audit/crawl-window";
-import { createCrawlThrottle } from "@/server/lib/audit/crawl-throttle";
+import {
+ createCrawlThrottle,
+ type CrawlThrottleState,
+} from "@/server/lib/audit/crawl-throttle";
import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";
import { pgStep } from "@/server/workflows/pgStep";
import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";
@@ -109,6 +113,7 @@ export async function runCrawlPhase(
// the site's page weight the hard way — on heavy-page sites that meant an
// exceededMemory death every ~200 pages.
let windowHint = CRAWL_WINDOW.initial;
+ let throttleState: CrawlThrottleState | undefined;
while (pending > 0 && attemptedTotal < params.maxPages) {
chunkNo += 1;
@@ -122,6 +127,7 @@ export async function runCrawlPhase(
chunkNo,
attemptedBefore: attemptedTotal,
startWindow: windowHint,
+ throttleState,
}),
);
// Apply the chunk's counters even when it did no new work (a retried
@@ -139,6 +145,14 @@ export async function runCrawlPhase(
// `?? initial`: an instance in flight across a deploy replays cached
// step results from before endWindow existed.
windowHint = result.endWindow ?? CRAWL_WINDOW.initial;
+ throttleState = result.throttleState;
+ if (pending > 0 && attemptedTotal < params.maxPages && result.resumeAt) {
+ // The timestamp comes from the persisted chunk result. Always replay
+ // the same sleep step, even when that timestamp is now in the past.
+ await step.sleepUntil(`crawl-cooldown-${chunkNo}`, result.resumeAt);
+ zeroProgressChunks = 0;
+ continue;
+ }
// One zero-attempt chunk is normal (retry of a completed chunk number);
// two in a row means the frontier is unservable — stop with what we
// have instead of spinning forever.
@@ -155,6 +169,7 @@ async function runCrawlChunk(
chunkNo: number;
attemptedBefore: number;
startWindow: number;
+ throttleState?: CrawlThrottleState;
},
): Promise<{
attemptedInChunk: number;
@@ -162,10 +177,14 @@ async function runCrawlChunk(
pending: number;
endWindow: number;
rateLimited?: boolean;
+ throttleState?: CrawlThrottleState;
+ resumeAt?: number;
}> {
const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =
input;
const scratchpad = getAuditScratchpad(auditId);
+ let previousThrottle =
+ (await scratchpad.getCrawlThrottle()) ?? input.throttleState;
const claimLimit = Math.min(
CHUNK_TARGET_PAGES,
@@ -175,6 +194,31 @@ async function runCrawlChunk(
chunkNo,
claimLimit,
);
+ const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;
+ if (isRetry && previousThrottle) {
+ // Requests made since the last checkpoint may have consumed a slot.
+ previousThrottle = {
+ ...previousThrottle,
+ nextRequestAt: Math.max(
+ previousThrottle.nextRequestAt,
+ Date.now() + previousThrottle.intervalMs,
+ ),
+ };
+ }
+ const throttle = createCrawlThrottle(
+ deadlineAt,
+ previousThrottle,
+ async (state) => {
+ try {
+ await scratchpad.saveCrawlThrottle(state);
+ } catch {
+ // A retry cannot safely honor a cooldown that failed to checkpoint.
+ throw new NonRetryableError(
+ "Unable to save the site's crawl cooldown.",
+ );
+ }
+ },
+ );
if (claimed.length === 0) {
const stats = await scratchpad.getStats();
return {
@@ -182,11 +226,16 @@ async function runCrawlChunk(
attempted: stats.attempted,
pending: stats.pending,
endWindow: input.startWindow,
+ throttleState: throttle.state,
+ rateLimited: throttle.stopped,
+ resumeAt:
+ throttle.state.pausedUntil > Date.now()
+ ? throttle.state.pausedUntil
+ : undefined,
};
}
const depthByUrl = new Map(claimed.map((entry) => [entry.url, entry.depth]));
- const deadlineAt = Date.now() + CHUNK_SOFT_DEADLINE_MS;
// A retry means the previous attempt died mid-crawl (in production almost
// always exceededMemory), and it is the chunk's last attempt — so it runs
@@ -198,8 +247,6 @@ async function runCrawlChunk(
let nextIndex = 0;
let attemptedInChunk = 0;
const inFlight = new Set<Promise<void>>();
- // Shared by every fetch in the chunk: one page's 429 pauses them all.
- const throttle = createCrawlThrottle(deadlineAt);
const deferred: string[] = [];
let persistThreshold = FIRST_PERSIST_BATCH_SIZE;
let batch: CrawledPageResult[] = [];
@@ -284,6 +331,7 @@ async function runCrawlChunk(
}
flush();
await persistChain;
+ await scratchpad.saveCrawlThrottle(throttle.state);
// Preserve URLs without a page result, including slots stopped by a cooldown.
const unattempted = [
@@ -298,17 +346,18 @@ async function runCrawlChunk(
// persistCrawledPages), so a chunk that dies mid-way underreports by at
// most one sub-batch, not a whole chunk.
const stats = await scratchpad.getStats();
- // The next chunk creates a fresh throttle. Honor the last response's
- // cooldown before allowing it to fetch more URLs.
- if (stats.pending > 0 && stats.attempted < maxPages) {
- await throttle.ready();
- }
+ const throttleState = throttle.state;
return {
attemptedInChunk,
attempted: stats.attempted,
pending: stats.pending,
endWindow: windowSize,
rateLimited: throttle.stopped,
+ throttleState,
+ resumeAt:
+ throttleState.pausedUntil > Date.now()
+ ? throttleState.pausedUntil
+ : undefined,
};
}