Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
Revert "fix(audit): back off and retry on 429 instead of recording the page a…" (#564)
Ben Senescuac9ee482d2b4
diff --git a/src/server/lib/audit/crawl-throttle.test.ts b/src/server/lib/audit/crawl-throttle.test.tsdeleted file mode 100644index 120ea394..00000000--- a/src/server/lib/audit/crawl-throttle.test.ts+++ /dev/null@@ -1,42 +0,0 @@-import { describe, expect, it } from "vitest";-import {- backOff,- IDLE_THROTTLE,- parseRetryAfterMs,-} from "@/server/lib/audit/crawl-throttle";--describe("parseRetryAfterMs", () => {- it("reads delay-seconds and HTTP-dates, ignores garbage", () => {- expect(parseRetryAfterMs("5")).toBe(5_000);- const inTenSeconds = new Date(Date.now() + 10_000).toUTCString();- const ms = parseRetryAfterMs(inTenSeconds);- expect(ms).toBeGreaterThan(8_000);- expect(ms).toBeLessThanOrEqual(10_000);- expect(parseRetryAfterMs("soon")).toBeUndefined();- expect(parseRetryAfterMs(null)).toBeUndefined();- });-});--describe("backOff", () => {- it("pauses for Retry-After within bounds and starts spacing launches", () => {- expect(backOff(IDLE_THROTTLE, 5_000, 1_000)).toEqual({- pausedUntil: 6_000,- launchGapMs: 250,- });- expect(backOff(IDLE_THROTTLE, undefined, 1_000).pausedUntil).toBe(3_000);- expect(backOff(IDLE_THROTTLE, 0, 1_000).pausedUntil).toBe(2_000);- expect(backOff(IDLE_THROTTLE, 3_600_000, 1_000).pausedUntil).toBe(21_000);- });-- it("treats a burst during one pause as one signal, then doubles per episode", () => {- const first = backOff(IDLE_THROTTLE, 2_000, 1_000);- const burst = backOff(first, 2_000, 1_500);- expect(burst).toEqual({ pausedUntil: 3_500, launchGapMs: 250 });-- let throttle = burst;- for (const now of [10_000, 20_000, 30_000, 40_000, 50_000]) {- throttle = backOff(throttle, 2_000, now);- }- expect(throttle.launchGapMs).toBe(4_000);- });-});diff --git a/src/server/lib/audit/crawl-throttle.ts b/src/server/lib/audit/crawl-throttle.tsdeleted file mode 100644index 056cf75b..00000000--- a/src/server/lib/audit/crawl-throttle.ts+++ /dev/null@@ -1,58 +0,0 @@-/**- * Site-wide backoff for rate limiting. A 429 means "too fast", not "go- * away": the whole crawl pauses for the server's Retry-After (or a default),- * every launch after that is spaced out, and the URL is retried a bounded- * number of times before it is recorded as blocked.- */-export interface CrawlThrottle {- /** No fetch launches before this time (ms epoch). */- pausedUntil: number;- /** Minimum spacing between launches; 0 until the site rate-limits us. */- launchGapMs: number;-}--/** Retries per URL within one chunk before a 429 is recorded as blocked. */-export const MAX_RATE_LIMIT_RETRIES = 3;-const DEFAULT_PAUSE_MS = 2_000;-const MIN_PAUSE_MS = 1_000;-/** Longer Retry-After values are cut short: the chunk has a 90s deadline. */-const MAX_PAUSE_MS = 20_000;-const MIN_LAUNCH_GAP_MS = 250;-const MAX_LAUNCH_GAP_MS = 4_000;--export const IDLE_THROTTLE: CrawlThrottle = { pausedUntil: 0, launchGapMs: 0 };--/** `Retry-After` is either delay-seconds or an HTTP-date. */-export function parseRetryAfterMs(header: string | null): number | undefined {- if (!header) return undefined;- const value = header.trim();- if (/^\d+$/.test(value)) return Number(value) * 1000;- const at = Date.parse(value);- return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now());-}--/**- * Register a 429. The concurrent fetches of one window tend to fail- * together, so a burst arriving while already paused counts as one signal:- * it can extend the pause but does not compound the launch gap.- */-export function backOff(- throttle: CrawlThrottle,- retryAfterMs: number | undefined,- now = Date.now(),-): CrawlThrottle {- const pause = Math.min(- Math.max(retryAfterMs ?? DEFAULT_PAUSE_MS, MIN_PAUSE_MS),- MAX_PAUSE_MS,- );- const alreadyPaused = now < throttle.pausedUntil;- return {- pausedUntil: Math.max(throttle.pausedUntil, now + pause),- launchGapMs: alreadyPaused- ? throttle.launchGapMs- : Math.min(- Math.max(throttle.launchGapMs * 2, MIN_LAUNCH_GAP_MS),- MAX_LAUNCH_GAP_MS,- ),- };-}diff --git a/src/server/lib/audit/types.ts b/src/server/lib/audit/types.tsindex 253617a7..2fbe4991 100644--- a/src/server/lib/audit/types.ts+++ b/src/server/lib/audit/types.ts@@ -146,11 +146,6 @@ export interface CrawledPageResult {* response time is measured at headers and says nothing about body size.*/htmlBytes: number;- /**- * Server-requested wait before retrying, from a 429's Retry-After header.- * Only set on 429 responses. Transient — drives the crawl throttle.- */- retryAfterMs?: number;imagesTotal: number;imagesMissingAlt: number;images: Array<{ src: string | null; alt: string | null }>;diff --git a/src/server/workflows/site-audit-workflow-helpers.ts b/src/server/workflows/site-audit-workflow-helpers.tsindex d18bcb94..a03463f3 100644--- a/src/server/workflows/site-audit-workflow-helpers.ts+++ b/src/server/workflows/site-audit-workflow-helpers.ts@@ -4,7 +4,6 @@ import type {} from "@/server/lib/audit/types";import { sha256Hex } from "@/server/lib/audit/ids";import { normalizeUrl } from "@/server/lib/audit/url-utils";-import { parseRetryAfterMs } from "@/server/lib/audit/crawl-throttle";const CRAWL_USER_AGENT = "OpenSEO-Audit/1.0";const MAX_HTML_BYTES = 1024 * 1024;@@ -128,10 +127,6 @@ export async function crawlPage(// The body was still fetched and buffered; report its size so the// crawl window's byte budget sees blocked/error pages too.htmlBytes: body.length,- retryAfterMs:- statusCode === 429- ? parseRetryAfterMs(response.headers.get("retry-after"))- : undefined,});}@@ -253,7 +248,6 @@ function emptyPageResult(input: {crawlDepth: number | null;inSitemap: boolean;htmlBytes?: number;- retryAfterMs?: number;}): CrawledPageResult {return {id: crypto.randomUUID(),@@ -281,7 +275,6 @@ function emptyPageResult(input: {contentHash: null,isHtml: false,htmlBytes: input.htmlBytes ?? 0,- retryAfterMs: input.retryAfterMs,imagesTotal: 0,imagesMissingAlt: 0,images: [],diff --git a/src/server/workflows/siteAuditWorkflowCrawl.ts b/src/server/workflows/siteAuditWorkflowCrawl.tsindex ac16017c..b090955f 100644--- a/src/server/workflows/siteAuditWorkflowCrawl.ts+++ b/src/server/workflows/siteAuditWorkflowCrawl.ts@@ -19,12 +19,6 @@ import {CRAWL_WINDOW,RETRY_CRAWL_WINDOW,} from "@/server/lib/audit/crawl-window";-import {- backOff,- IDLE_THROTTLE,- MAX_RATE_LIMIT_RETRIES,- type CrawlThrottle,-} from "@/server/lib/audit/crawl-throttle";import { crawlPage } from "@/server/workflows/site-audit-workflow-helpers";import { pgStep } from "@/server/workflows/pgStep";import { CRAWL_CHUNK_STEP } from "@/server/workflows/auditStepConfigs";@@ -112,9 +106,6 @@ export async function runCrawlPhase(// the site's page weight the hard way — on heavy-page sites that meant an// exceededMemory death every ~200 pages.let windowHint = CRAWL_WINDOW.initial;- // Same for the launch gap a rate-limiting site taught us: relearning it- // per chunk would cost a fresh round of 429s every 200 pages.- let launchGapHint = IDLE_THROTTLE.launchGapMs;while (pending > 0 && attemptedTotal < params.maxPages) {chunkNo += 1;@@ -128,7 +119,6 @@ export async function runCrawlPhase(chunkNo,attemptedBefore: attemptedTotal,startWindow: windowHint,- startLaunchGapMs: launchGapHint,}),);// Apply the chunk's counters even when it did no new work (a retried@@ -139,7 +129,6 @@ export async function runCrawlPhase(// `?? initial`: an instance in flight across a deploy replays cached// step results from before endWindow existed.windowHint = result.endWindow ?? CRAWL_WINDOW.initial;- launchGapHint = result.endLaunchGapMs ?? IDLE_THROTTLE.launchGapMs;// One zero-attempt chunk is normal (retry of a completed chunk number);// two in a row means the frontier is unservable — stop with what we// have instead of spinning forever.@@ -156,14 +145,12 @@ async function runCrawlChunk(chunkNo: number;attemptedBefore: number;startWindow: number;- startLaunchGapMs: number;},): Promise<{attemptedInChunk: number;attempted: number;pending: number;endWindow: number;- endLaunchGapMs: number;}> {const { auditId, workflowInstanceId, origin, maxPages, robots, chunkNo } =input;@@ -184,7 +171,6 @@ async function runCrawlChunk(attempted: stats.attempted,pending: stats.pending,endWindow: input.startWindow,- endLaunchGapMs: input.startLaunchGapMs,};}@@ -201,15 +187,6 @@ async function runCrawlChunk(let nextIndex = 0;let attemptedInChunk = 0;const inFlight = new Set<Promise<void>>();- // 429s are the site saying "slower", not "blocked": the URL goes back on- // the queue, the whole window pauses, and launches are spaced out.- let throttle: CrawlThrottle = {- ...IDLE_THROTTLE,- launchGapMs: input.startLaunchGapMs,- };- let nextLaunchAt = 0;- const retryQueue: ClaimedUrl[] = [];- const retriesByUrl = new Map<string, number>();let persistThreshold = FIRST_PERSIST_BATCH_SIZE;let batch: CrawledPageResult[] = [];// Persistence runs concurrently with fetching (pipelined) but sequentially@@ -245,15 +222,6 @@ async function runCrawlChunk(const launch = (entry: ClaimedUrl) => {const promise = crawlPage(entry.url, entry.depth, entry.inSitemap).then((page) => {- if (page.statusCode === 429) {- throttle = backOff(throttle, page.retryAfterMs);- const retries = retriesByUrl.get(entry.url) ?? 0;- if (retries < MAX_RATE_LIMIT_RETRIES) {- retriesByUrl.set(entry.url, retries + 1);- retryQueue.push(entry);- return;- }- }attemptedInChunk += 1;batch.push(page);if (batch.length >= persistThreshold) flush();@@ -264,26 +232,18 @@ async function runCrawlChunk(inFlight.add(promise);};- const hasNext = () => retryQueue.length > 0 || nextIndex < claimed.length;-while (true) {- const now = Date.now();- const canLaunch =- hasNext() &&- now < deadlineAt &&- queuedPersists <= MAX_QUEUED_PERSIST_BATCHES;- if (canLaunch && inFlight.size < windowSize) {- const waitMs = Math.min(- Math.max(throttle.pausedUntil, nextLaunchAt) - now,- deadlineAt - now,- );- if (waitMs > 0) {- await sleep(waitMs);- continue;- }- launch(retryQueue.shift() ?? claimed[nextIndex++]);- nextLaunchAt = Date.now() + throttle.launchGapMs;- continue;+ while (+ inFlight.size < windowSize &&+ nextIndex < claimed.length &&+ Date.now() < deadlineAt+ ) {+ // queuedPersists changes when persistChain settles. Keep it out of the+ // loop condition because the type-aware linter cannot see that async+ // mutation and flags the otherwise valid backpressure check.+ if (queuedPersists > MAX_QUEUED_PERSIST_BATCHES) break;+ launch(claimed[nextIndex]);+ nextIndex += 1;}if (inFlight.size > 0) {await Promise.race(inFlight);@@ -294,8 +254,8 @@ async function runCrawlChunk(// chunk is done (leases exhausted or soft deadline hit).if (queuedPersists > MAX_QUEUED_PERSIST_BATCHES &&- hasNext() &&- now < deadlineAt+ nextIndex < claimed.length &&+ Date.now() < deadlineAt) {await persistChain;continue;@@ -305,12 +265,8 @@ async function runCrawlChunk(flush();await persistChain;- // Leases we never launched or finished retrying (soft deadline) go back- // to the queue; the next chunk retries them with a fresh budget.- const unattempted = [- ...retryQueue.map((entry) => entry.url),- ...claimed.slice(nextIndex).map((entry) => entry.url),- ];+ // Leases we never launched (soft deadline) go back to the queue.+ const unattempted = claimed.slice(nextIndex).map((entry) => entry.url);if (unattempted.length > 0) {await scratchpad.releaseUrls(unattempted);}@@ -324,14 +280,9 @@ async function runCrawlChunk(attempted: stats.attempted,pending: stats.pending,endWindow: windowSize,- endLaunchGapMs: throttle.launchGapMs,};}-function sleep(ms: number) {- return new Promise<void>((resolve) => setTimeout(resolve, ms));-}-async function persistCrawledPages(input: {auditId: string;workflowInstanceId: string;diff --git a/src/shared/audit-issues.ts b/src/shared/audit-issues.tsindex 8b306a62..baddfa7f 100644--- a/src/shared/audit-issues.ts+++ b/src/shared/audit-issues.ts@@ -20,7 +20,7 @@ export const AUDIT_ISSUE_TYPES = {severity: "critical",title: "Crawler was blocked",explanation:- "The site returned a bot challenge or access denial (e.g. a Cloudflare challenge, 403, or a 429 that persisted after we slowed down and retried) instead of the page. We report this honestly rather than pretending the page is broken — but it means this page could not be audited, and other crawlers like search engines may face similar friction.",+ "The site returned a bot challenge or access denial (e.g. a Cloudflare challenge, 403, or 429) instead of the page. We report this honestly rather than pretending the page is broken — but it means this page could not be audited, and other crawlers like search engines may face similar friction.",howToFix:'If you own this site, allowlist the "OpenSEO-Audit" user agent in your WAF/bot-protection settings (on Cloudflare: a WAF custom rule that skips bot protection when the user agent contains "OpenSEO-Audit"; on some free tiers you may need to relax bot protection). Then re-run the audit.',},