Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): add retry limits to prevent infinite scrape loops (#2660)
misza-onec547eec9576c
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 5a009f7b4..ab6c1e81f 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -96,6 +96,13 @@ const configSchema = z.object({SCRAPEURL_AB_EXTEND_MAXAGE: z.stringbool().optional(),SCRAPEURL_ENGINE_WATERFALL_DELAY_MS: z.coerce.number().default(0),+ // Scrape Retry Limits+ SCRAPE_MAX_ATTEMPTS: z.coerce.number().int().positive().default(6),+ SCRAPE_MAX_FEATURE_TOGGLES: z.coerce.number().int().positive().default(3),+ SCRAPE_MAX_FEATURE_REMOVALS: z.coerce.number().int().positive().default(3),+ SCRAPE_MAX_PDF_PREFETCHES: z.coerce.number().int().positive().default(2),+ SCRAPE_MAX_DOCUMENT_PREFETCHES: z.coerce.number().int().positive().default(2),+// Search ServicesSEARXNG_ENDPOINT: z.string().optional(),SEARXNG_ENGINES: z.string().optional(),diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 9dc7ba709..2300a6d43 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -24,6 +24,8 @@ import {SSLError,ProxySelectionError,NoCachedDataError,+ ScrapeJobCancelledError,+ ScrapeRetryLimitError,} from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -37,6 +39,8 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_PROXY_SELECTION_ERROR: ProxySelectionError,SCRAPE_PDF_PREFETCH_FAILED: PDFPrefetchFailed,SCRAPE_DOCUMENT_PREFETCH_FAILED: DocumentPrefetchFailed,+ SCRAPE_JOB_CANCELLED: ScrapeJobCancelledError,+ SCRAPE_RETRY_LIMIT: ScrapeRetryLimitError,SCRAPE_ZDR_VIOLATION_ERROR: ZDRViolationError,SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex f45769a69..dc0157753 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -8,6 +8,8 @@ export type ErrorCodes =| "SCRAPE_PROXY_SELECTION_ERROR"| "SCRAPE_PDF_PREFETCH_FAILED"| "SCRAPE_DOCUMENT_PREFETCH_FAILED"+ | "SCRAPE_JOB_CANCELLED"+ | "SCRAPE_RETRY_LIMIT"| "SCRAPE_ZDR_VIOLATION_ERROR"| "SCRAPE_DNS_RESOLUTION_ERROR"| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 4939d44ab..24e5662e6 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -453,3 +453,69 @@ export class WaterfallNextEngineSignal extends Error {super("Waterfall next engine");}}++export class ScrapeJobCancelledError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_JOB_CANCELLED",+ "Scrape job was cancelled before completion.",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ScrapeJobCancelledError();+ x.stack = data.stack;+ return x;+ }+}++export type ScrapeRetryLimitReason =+ | "global"+ | "feature_toggle"+ | "feature_removal"+ | "pdf_antibot"+ | "document_antibot";++export type ScrapeRetryStats = {+ totalAttempts: number;+ addFeatureAttempts: number;+ removeFeatureAttempts: number;+ pdfAntibotAttempts: number;+ documentAntibotAttempts: number;+};++export class ScrapeRetryLimitError extends TransportableError {+ constructor(+ public reason: ScrapeRetryLimitReason,+ public stats: ScrapeRetryStats,+ ) {+ super(+ "SCRAPE_RETRY_LIMIT",+ `Scrape aborted after exceeding retry limit (${reason}).`,+ );+ }++ serialize() {+ return {+ ...super.serialize(),+ reason: this.reason,+ stats: this.stats,+ };+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ScrapeRetryLimitError(data.reason, data.stats);+ x.stack = data.stack;+ return x;+ }+}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 172ad5eaa..e5ddd7842 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -47,7 +47,9 @@ import {WaterfallNextEngineSignal,EngineUnsuccessfulError,ProxySelectionError,+ ScrapeRetryLimitError,} from "./error";+import { ScrapeRetryTracker } from "./retryTracker";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";import { urlSpecificParams } from "./lib/urlSpecificParams";@@ -971,11 +973,19 @@ export async function scrapeURL(});}- try {- // temporary fix until new scrapeURL system- let documentReattempted = false;- let pdfReattempted = false;+ // Initialize retry tracker with configured limits+ const retryTracker = new ScrapeRetryTracker(+ {+ maxAttempts: config.SCRAPE_MAX_ATTEMPTS,+ maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,+ maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,+ maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,+ maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,+ },+ meta.logger,+ );+ try {let result: ScrapeUrlResponse;while (true) {try {@@ -987,7 +997,7 @@ export async function scrapeURL((meta.internalOptions.forceEngine === undefined ||Array.isArray(meta.internalOptions.forceEngine))) {- // note: we might want to reattempt check here too+ retryTracker.record("feature_toggle", error);meta.logger.debug("More feature flags requested by scraper: adding " +error.featureFlags.join(", "),@@ -1007,6 +1017,7 @@ export async function scrapeURL((meta.internalOptions.forceEngine === undefined ||Array.isArray(meta.internalOptions.forceEngine))) {+ retryTracker.record("feature_removal", error);meta.logger.debug("Incorrect feature flags reported by scraper: removing " +error.featureFlags.join(","),@@ -1027,21 +1038,13 @@ export async function scrapeURL();throw error;} else {- if (!pdfReattempted) {- meta.logger.debug(- "PDF was blocked by anti-bot, prefetching with chrome-cdp",- );-- pdfReattempted = true;- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "pdf"),- );- } else {- meta.logger.debug(- "PDF was blocked by anti-bot, skipping as it was already attempted",- );- throw error;- }+ retryTracker.record("pdf_antibot", error);+ meta.logger.debug(+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "pdf"),+ );}} else if (error instanceof DocumentAntibotError &&@@ -1053,21 +1056,13 @@ export async function scrapeURL();throw error;} else {- if (!documentReattempted) {- meta.logger.debug(- "Document was blocked by anti-bot, prefetching with chrome-cdp",- );-- documentReattempted = true;- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "document"),- );- } else {- meta.logger.debug(- "Document was blocked by anti-bot, skipping as it was already attempted",- );- throw error;- }+ retryTracker.record("document_antibot", error);+ meta.logger.debug(+ "Document was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "document"),+ );}} else {throw error;@@ -1220,6 +1215,12 @@ export async function scrapeURL(} else if (error instanceof DNSResolutionError) {errorType = "DNSResolutionError";meta.logger.warn("scrapeURL: DNS resolution error", { error });+ } else if (error instanceof ScrapeRetryLimitError) {+ errorType = "ScrapeRetryLimitError";+ meta.logger.warn("scrapeURL: Retry limit reached", {+ error,+ retryStats: error.stats,+ });} else if (error instanceof AbortManagerThrownError) {errorType = "AbortManagerThrownError";throw error.inner;diff --git a/apps/api/src/scraper/scrapeURL/retryTracker.ts b/apps/api/src/scraper/scrapeURL/retryTracker.tsnew file mode 100644index 000000000..d017cbac6--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/retryTracker.ts@@ -0,0 +1,89 @@+import type { Logger } from "winston";+import {+ ScrapeRetryLimitError,+ ScrapeRetryLimitReason,+ ScrapeRetryStats,+} from "./error";++type RetryReason = Exclude<ScrapeRetryLimitReason, "global">;++interface ScrapeRetryTrackerConfig {+ maxAttempts: number;+ maxFeatureToggles: number;+ maxFeatureRemovals: number;+ maxPdfPrefetches: number;+ maxDocumentPrefetches: number;+}++export class ScrapeRetryTracker {+ private stats: ScrapeRetryStats = {+ totalAttempts: 0,+ addFeatureAttempts: 0,+ removeFeatureAttempts: 0,+ pdfAntibotAttempts: 0,+ documentAntibotAttempts: 0,+ };++ constructor(+ private readonly config: ScrapeRetryTrackerConfig,+ private readonly logger: Logger,+ ) {}++ getSnapshot(): ScrapeRetryStats {+ return { ...this.stats };+ }++ record(reason: RetryReason, lastError: unknown) {+ this.stats.totalAttempts += 1;+ if (this.stats.totalAttempts > this.config.maxAttempts) {+ this.throwLimit("global", lastError);+ }++ switch (reason) {+ case "feature_toggle":+ this.stats.addFeatureAttempts += 1;+ if (this.stats.addFeatureAttempts > this.config.maxFeatureToggles) {+ this.throwLimit(reason, lastError);+ }+ break;+ case "feature_removal":+ this.stats.removeFeatureAttempts += 1;+ if (this.stats.removeFeatureAttempts > this.config.maxFeatureRemovals) {+ this.throwLimit(reason, lastError);+ }+ break;+ case "pdf_antibot":+ this.stats.pdfAntibotAttempts += 1;+ if (this.stats.pdfAntibotAttempts > this.config.maxPdfPrefetches) {+ this.throwLimit(reason, lastError);+ }+ break;+ case "document_antibot":+ this.stats.documentAntibotAttempts += 1;+ if (+ this.stats.documentAntibotAttempts > this.config.maxDocumentPrefetches+ ) {+ this.throwLimit(reason, lastError);+ }+ break;+ }++ this.logger.warn("scrapeURL retrying after handled error", {+ reason,+ retryStats: this.getSnapshot(),+ lastError:+ lastError instanceof Error ? lastError.message : (lastError ?? null),+ });+ }++ private throwLimit(reason: ScrapeRetryLimitReason, lastError: unknown) {+ const snapshot = this.getSnapshot();+ this.logger.error("scrapeURL retry limit reached", {+ reason,+ retryStats: snapshot,+ lastError:+ lastError instanceof Error ? lastError.message : (lastError ?? null),+ });+ throw new ScrapeRetryLimitError(reason, snapshot);+ }+}