Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): add retry limits to prevent infinite scrape loops (#2660)

misza-onec547eec9576c
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 5a009f7b4..ab6c1e81f 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -96,6 +96,13 @@ const configSchema = z.object({
SCRAPEURL_AB_EXTEND_MAXAGE: z.stringbool().optional(),
SCRAPEURL_ENGINE_WATERFALL_DELAY_MS: z.coerce.number().default(0),
+ // Scrape Retry Limits
+ SCRAPE_MAX_ATTEMPTS: z.coerce.number().int().positive().default(6),
+ SCRAPE_MAX_FEATURE_TOGGLES: z.coerce.number().int().positive().default(3),
+ SCRAPE_MAX_FEATURE_REMOVALS: z.coerce.number().int().positive().default(3),
+ SCRAPE_MAX_PDF_PREFETCHES: z.coerce.number().int().positive().default(2),
+ SCRAPE_MAX_DOCUMENT_PREFETCHES: z.coerce.number().int().positive().default(2),
+
// Search Services
SEARXNG_ENDPOINT: z.string().optional(),
SEARXNG_ENGINES: z.string().optional(),
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 9dc7ba709..2300a6d43 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -24,6 +24,8 @@ import {
SSLError,
ProxySelectionError,
NoCachedDataError,
+ ScrapeJobCancelledError,
+ ScrapeRetryLimitError,
} from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -37,6 +39,8 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_PROXY_SELECTION_ERROR: ProxySelectionError,
SCRAPE_PDF_PREFETCH_FAILED: PDFPrefetchFailed,
SCRAPE_DOCUMENT_PREFETCH_FAILED: DocumentPrefetchFailed,
+ SCRAPE_JOB_CANCELLED: ScrapeJobCancelledError,
+ SCRAPE_RETRY_LIMIT: ScrapeRetryLimitError,
SCRAPE_ZDR_VIOLATION_ERROR: ZDRViolationError,
SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,
SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index f45769a69..dc0157753 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -8,6 +8,8 @@ export type ErrorCodes =
| "SCRAPE_PROXY_SELECTION_ERROR"
| "SCRAPE_PDF_PREFETCH_FAILED"
| "SCRAPE_DOCUMENT_PREFETCH_FAILED"
+ | "SCRAPE_JOB_CANCELLED"
+ | "SCRAPE_RETRY_LIMIT"
| "SCRAPE_ZDR_VIOLATION_ERROR"
| "SCRAPE_DNS_RESOLUTION_ERROR"
| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 4939d44ab..24e5662e6 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -453,3 +453,69 @@ export class WaterfallNextEngineSignal extends Error {
super("Waterfall next engine");
}
}
+
+export class ScrapeJobCancelledError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_JOB_CANCELLED",
+ "Scrape job was cancelled before completion.",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ScrapeJobCancelledError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export type ScrapeRetryLimitReason =
+ | "global"
+ | "feature_toggle"
+ | "feature_removal"
+ | "pdf_antibot"
+ | "document_antibot";
+
+export type ScrapeRetryStats = {
+ totalAttempts: number;
+ addFeatureAttempts: number;
+ removeFeatureAttempts: number;
+ pdfAntibotAttempts: number;
+ documentAntibotAttempts: number;
+};
+
+export class ScrapeRetryLimitError extends TransportableError {
+ constructor(
+ public reason: ScrapeRetryLimitReason,
+ public stats: ScrapeRetryStats,
+ ) {
+ super(
+ "SCRAPE_RETRY_LIMIT",
+ `Scrape aborted after exceeding retry limit (${reason}).`,
+ );
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ reason: this.reason,
+ stats: this.stats,
+ };
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ScrapeRetryLimitError(data.reason, data.stats);
+ x.stack = data.stack;
+ return x;
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 172ad5eaa..e5ddd7842 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -47,7 +47,9 @@ import {
WaterfallNextEngineSignal,
EngineUnsuccessfulError,
ProxySelectionError,
+ ScrapeRetryLimitError,
} from "./error";
+import { ScrapeRetryTracker } from "./retryTracker";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
import { urlSpecificParams } from "./lib/urlSpecificParams";
@@ -971,11 +973,19 @@ export async function scrapeURL(
});
}
- try {
- // temporary fix until new scrapeURL system
- let documentReattempted = false;
- let pdfReattempted = false;
+ // Initialize retry tracker with configured limits
+ const retryTracker = new ScrapeRetryTracker(
+ {
+ maxAttempts: config.SCRAPE_MAX_ATTEMPTS,
+ maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,
+ maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,
+ maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,
+ maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,
+ },
+ meta.logger,
+ );
+ try {
let result: ScrapeUrlResponse;
while (true) {
try {
@@ -987,7 +997,7 @@ export async function scrapeURL(
(meta.internalOptions.forceEngine === undefined ||
Array.isArray(meta.internalOptions.forceEngine))
) {
- // note: we might want to reattempt check here too
+ retryTracker.record("feature_toggle", error);
meta.logger.debug(
"More feature flags requested by scraper: adding " +
error.featureFlags.join(", "),
@@ -1007,6 +1017,7 @@ export async function scrapeURL(
(meta.internalOptions.forceEngine === undefined ||
Array.isArray(meta.internalOptions.forceEngine))
) {
+ retryTracker.record("feature_removal", error);
meta.logger.debug(
"Incorrect feature flags reported by scraper: removing " +
error.featureFlags.join(","),
@@ -1027,21 +1038,13 @@ export async function scrapeURL(
);
throw error;
} else {
- if (!pdfReattempted) {
- meta.logger.debug(
- "PDF was blocked by anti-bot, prefetching with chrome-cdp",
- );
-
- pdfReattempted = true;
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "pdf"),
- );
- } else {
- meta.logger.debug(
- "PDF was blocked by anti-bot, skipping as it was already attempted",
- );
- throw error;
- }
+ retryTracker.record("pdf_antibot", error);
+ meta.logger.debug(
+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "pdf"),
+ );
}
} else if (
error instanceof DocumentAntibotError &&
@@ -1053,21 +1056,13 @@ export async function scrapeURL(
);
throw error;
} else {
- if (!documentReattempted) {
- meta.logger.debug(
- "Document was blocked by anti-bot, prefetching with chrome-cdp",
- );
-
- documentReattempted = true;
- meta.featureFlags = new Set(
- [...meta.featureFlags].filter(x => x !== "document"),
- );
- } else {
- meta.logger.debug(
- "Document was blocked by anti-bot, skipping as it was already attempted",
- );
- throw error;
- }
+ retryTracker.record("document_antibot", error);
+ meta.logger.debug(
+ "Document was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "document"),
+ );
}
} else {
throw error;
@@ -1220,6 +1215,12 @@ export async function scrapeURL(
} else if (error instanceof DNSResolutionError) {
errorType = "DNSResolutionError";
meta.logger.warn("scrapeURL: DNS resolution error", { error });
+ } else if (error instanceof ScrapeRetryLimitError) {
+ errorType = "ScrapeRetryLimitError";
+ meta.logger.warn("scrapeURL: Retry limit reached", {
+ error,
+ retryStats: error.stats,
+ });
} else if (error instanceof AbortManagerThrownError) {
errorType = "AbortManagerThrownError";
throw error.inner;
diff --git a/apps/api/src/scraper/scrapeURL/retryTracker.ts b/apps/api/src/scraper/scrapeURL/retryTracker.ts
new file mode 100644
index 000000000..d017cbac6
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/retryTracker.ts
@@ -0,0 +1,89 @@
+import type { Logger } from "winston";
+import {
+ ScrapeRetryLimitError,
+ ScrapeRetryLimitReason,
+ ScrapeRetryStats,
+} from "./error";
+
+type RetryReason = Exclude<ScrapeRetryLimitReason, "global">;
+
+interface ScrapeRetryTrackerConfig {
+ maxAttempts: number;
+ maxFeatureToggles: number;
+ maxFeatureRemovals: number;
+ maxPdfPrefetches: number;
+ maxDocumentPrefetches: number;
+}
+
+export class ScrapeRetryTracker {
+ private stats: ScrapeRetryStats = {
+ totalAttempts: 0,
+ addFeatureAttempts: 0,
+ removeFeatureAttempts: 0,
+ pdfAntibotAttempts: 0,
+ documentAntibotAttempts: 0,
+ };
+
+ constructor(
+ private readonly config: ScrapeRetryTrackerConfig,
+ private readonly logger: Logger,
+ ) {}
+
+ getSnapshot(): ScrapeRetryStats {
+ return { ...this.stats };
+ }
+
+ record(reason: RetryReason, lastError: unknown) {
+ this.stats.totalAttempts += 1;
+ if (this.stats.totalAttempts > this.config.maxAttempts) {
+ this.throwLimit("global", lastError);
+ }
+
+ switch (reason) {
+ case "feature_toggle":
+ this.stats.addFeatureAttempts += 1;
+ if (this.stats.addFeatureAttempts > this.config.maxFeatureToggles) {
+ this.throwLimit(reason, lastError);
+ }
+ break;
+ case "feature_removal":
+ this.stats.removeFeatureAttempts += 1;
+ if (this.stats.removeFeatureAttempts > this.config.maxFeatureRemovals) {
+ this.throwLimit(reason, lastError);
+ }
+ break;
+ case "pdf_antibot":
+ this.stats.pdfAntibotAttempts += 1;
+ if (this.stats.pdfAntibotAttempts > this.config.maxPdfPrefetches) {
+ this.throwLimit(reason, lastError);
+ }
+ break;
+ case "document_antibot":
+ this.stats.documentAntibotAttempts += 1;
+ if (
+ this.stats.documentAntibotAttempts > this.config.maxDocumentPrefetches
+ ) {
+ this.throwLimit(reason, lastError);
+ }
+ break;
+ }
+
+ this.logger.warn("scrapeURL retrying after handled error", {
+ reason,
+ retryStats: this.getSnapshot(),
+ lastError:
+ lastError instanceof Error ? lastError.message : (lastError ?? null),
+ });
+ }
+
+ private throwLimit(reason: ScrapeRetryLimitReason, lastError: unknown) {
+ const snapshot = this.getSnapshot();
+ this.logger.error("scrapeURL retry limit reached", {
+ reason,
+ retryStats: snapshot,
+ lastError:
+ lastError instanceof Error ? lastError.message : (lastError ?? null),
+ });
+ throw new ScrapeRetryLimitError(reason, snapshot);
+ }
+}