Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/pdf): fall back to chrome-cdp prefetch when the pdf engine's direct download fails at the proxy (#4477)
GergΕ MΓ³ricz1f193af5750d
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex ef7be19f6..4f973e356 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -17,6 +17,8 @@ import {UnsupportedFileError,PDFAntibotError,DocumentAntibotError,+ PDFFetchProxyError,+ DocumentFetchProxyError,PDFInsufficientTimeError,PDFOCRRequiredError,NoEnginesLeftError,@@ -60,8 +62,10 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,+ SCRAPE_PDF_FETCH_PROXY_ERROR: PDFFetchProxyError,SCRAPE_PDF_OCR_REQUIRED: PDFOCRRequiredError,SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,+ SCRAPE_DOCUMENT_FETCH_PROXY_ERROR: DocumentFetchProxyError,SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,SCRAPE_NO_CACHED_DATA: NoCachedDataError,SCRAPE_LOCKDOWN_CACHE_MISS: LockdownMissError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 000776781..63814624a 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -15,8 +15,10 @@ export type ErrorCodes =| "SCRAPE_DNS_RESOLUTION_ERROR"| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"| "SCRAPE_PDF_ANTIBOT_ERROR"+ | "SCRAPE_PDF_FETCH_PROXY_ERROR"| "SCRAPE_PDF_OCR_REQUIRED"| "SCRAPE_DOCUMENT_ANTIBOT_ERROR"+ | "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR"| "SCRAPE_UNSUPPORTED_FILE_ERROR"| "SCRAPE_ACTION_ERROR"| "SCRAPE_RACED_REDIRECT_ERROR"diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex 6bde2f0bf..c6666af19 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -1,10 +1,17 @@import { Meta } from "../..";import { EngineScrapeResult } from "..";-import { fetchFileToBuffer } from "../utils/downloadFile";+import {+ fetchFileGuardingProxyFailure,+ fetchFileToBuffer,+} from "../utils/downloadFile";import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";import { safeMarkdownToHtml } from "../pdf/markdownToHtml";import type { Response } from "undici";-import { DocumentAntibotError, EngineUnsuccessfulError } from "../../error";+import {+ DocumentAntibotError,+ DocumentFetchProxyError,+ EngineUnsuccessfulError,+} from "../../error";import {documentContentTypeFromExtension,documentExtensionFromContentType,@@ -39,13 +46,15 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {proxyUsed = meta.documentPrefetch.proxyUsed;} else {// Fetch the document normally- const result = await fetchFileToBuffer(- meta.rewrittenUrl ?? meta.url,- meta.options.skipTlsVerification,- {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- },+ const result = await fetchDocumentFileGuardingProxyFailure(meta, () =>+ fetchFileToBuffer(+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ ),);response = result.response;buffer = result.buffer;@@ -113,3 +122,32 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {export function documentMaxReasonableTime(meta: Meta): number {return 15000;}++/**+ * Guards the document engine's direct undici download: a proxy tunneling+ * failure converts into DocumentFetchProxyError, which the scrapeURL retry+ * loop handles exactly like DocumentAntibotError (clear the "document" flag,+ * re-run the waterfall, browser engine fetches the file). See+ * fetchFileGuardingProxyFailure for the conversion eligibility rules.+ */+function fetchDocumentFileGuardingProxyFailure<T>(+ meta: Meta,+ fetch: () => Promise<T>,+): Promise<T> {+ return fetchFileGuardingProxyFailure(+ {+ prefetch: meta.documentPrefetch,+ // Same conversion-eligibility rules as the pdf engine: only when+ // forceEngine is unset (retry loop recovers via browser fallback) or+ // scalar-pinned to this engine (clean error surfaces); arrays keep the+ // raw error so the waterfall continues through remaining forced+ // engines.+ flagMandated:+ (meta.internalOptions.forceEngine === undefined &&+ meta.featureFlags.has("document")) ||+ meta.internalOptions.forceEngine === "document",+ makeError: () => new DocumentFetchProxyError(),+ },+ fetch,+ );+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 798888f1a..bda10f147 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -1,10 +1,15 @@import { Meta } from "../..";import { config } from "../../../../config";import { EngineScrapeResult } from "..";-import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";+import {+ downloadFile,+ fetchFileGuardingProxyFailure,+ fetchFileToBuffer,+} from "../utils/downloadFile";import { safeMarkdownToHtml } from "./markdownToHtml";import {PDFAntibotError,+ PDFFetchProxyError,PDFInsufficientTimeError,PDFOCRRequiredError,PDFPrefetchFailed,@@ -69,6 +74,38 @@ function getIneligibleReason(return null;}+/**+ * Guards the pdf engine's direct undici downloads: a proxy tunneling+ * failure converts into PDFFetchProxyError, which the scrapeURL retry loop+ * handles exactly like PDFAntibotError (clear the "pdf" flag, re-run the+ * waterfall, browser engine fetches the file). See+ * fetchFileGuardingProxyFailure for the conversion eligibility rules.+ */+function fetchPdfFileGuardingProxyFailure<T>(+ meta: Meta,+ fetch: () => Promise<T>,+): Promise<T> {+ return fetchFileGuardingProxyFailure(+ {+ prefetch: meta.pdfPrefetch,+ // Convert only where the outcome is actionable: with forceEngine+ // unset, the retry loop recovers PDFFetchProxyError via the browser+ // fallback; with a scalar forceEngine=pdf, this engine is pinned+ // with no fallback in the list, so the clean error surfaces instead+ // of the raw TypeError. An ARRAY forceEngine must NOT convert β the+ // retry loop bypasses recovery for any forceEngine, and the raw+ // error is what lets the waterfall continue through the remaining+ // forced engines.+ flagMandated:+ (meta.internalOptions.forceEngine === undefined &&+ meta.featureFlags.has("pdf")) ||+ meta.internalOptions.forceEngine === "pdf",+ makeError: () => new PDFFetchProxyError(),+ },+ fetch,+ );+}+export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const shouldParse = shouldParsePDF(meta.options.parsers);const maxPages = getPDFMaxPages(meta.options.parsers);@@ -118,19 +155,24 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {proxyUsed: meta.pdfPrefetch.proxyUsed,};} else {- const file = await fetchFileToBuffer(- meta.rewrittenUrl ?? meta.url,- meta.options.skipTlsVerification,- {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- },- PDF_DOWNLOAD_MAX_FILE_SIZE,+ const file = await fetchPdfFileGuardingProxyFailure(meta, () =>+ fetchFileToBuffer(+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ PDF_DOWNLOAD_MAX_FILE_SIZE,+ ),);if (!isPdfBuffer(file.buffer)) {// downloaded content isn't a valid PDF- if (meta.pdfPrefetch === undefined) {+ // (null prefetch = browser round trip ran but delivered no file β+ // still PDFAntibotError so the retry loop can give the browser+ // another shot, exactly like the no-prefetch case)+ if (meta.pdfPrefetch == null) {// for non-PDF URLs, this is expected, not anti-botif (!meta.featureFlags.has("pdf")) {throw new EngineUnsuccessfulError("pdf");@@ -183,20 +225,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {const { response, tempFilePath } =meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }- : await downloadFile(- meta.id,- meta.rewrittenUrl ?? meta.url,- meta.options.skipTlsVerification,- {- headers: meta.options.headers,- signal: meta.abort.asSignal(),- },- // Parse path streams to disk and can hand large files to FirePDF- // by GCS reference, so it admits more than the raw fetch path β- // up to the requesting team's large-PDF limit.- byReferenceReachable- ? largePdfLimitBytes(meta)- : PDF_DOWNLOAD_MAX_FILE_SIZE,+ : await fetchPdfFileGuardingProxyFailure(meta, () =>+ downloadFile(+ meta.id,+ meta.rewrittenUrl ?? meta.url,+ meta.options.skipTlsVerification,+ {+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ },+ // Parse path streams to disk and can hand large files to FirePDF+ // by GCS reference, so it admits more than the raw fetch path β+ // up to the requesting team's large-PDF limit.+ byReferenceReachable+ ? largePdfLimitBytes(meta)+ : PDF_DOWNLOAD_MAX_FILE_SIZE,+ ),);try {@@ -216,7 +260,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {}if (!isPdfBuffer(header.subarray(0, headerBytesRead))) {- if (meta.pdfPrefetch === undefined) {+ // (null prefetch = browser round trip ran but delivered no file β+ // still PDFAntibotError so the retry loop can give the browser+ // another shot, exactly like the no-prefetch case)+ if (meta.pdfPrefetch == null) {if (!meta.featureFlags.has("pdf")) {throw new EngineUnsuccessfulError("pdf");} else {diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex 380da34d3..c119de58b 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -16,6 +16,90 @@ import * as undici from "undici";import { getSecureDispatcher } from "./safeFetch";import { logger } from "../../../../lib/logger";+/**+ * Matches undici ProxyAgent tunnel failures: the proxy's CONNECT response was+ * not 200 (502, 407, ...). undici emits `RequestAbortedError` (code+ * UND_ERR_ABORTED) with the message "Proxy response (NNN) !== 200 when HTTP+ * Tunneling" β see undici's lib/dispatcher/proxy-agent.js.+ */+const PROXY_TUNNEL_FAILURE_MESSAGE =+ /Proxy response \(\d+\) !== 200 when HTTP Tunneling/;++/**+ * Detects a proxy tunneling failure anywhere in an error's cause chain.+ *+ * undici wraps these several levels deep β+ * TypeError("fetch failed") β DOMException("Request was cancelled.") β+ * RequestAbortedError("Proxy response (502) !== 200 when HTTP Tunneling") β+ * so, unlike mapUndiciError, we walk the whole chain. Callers use this to+ * retry the download through a different transport (e.g. the browser engine's+ * own proxy infra) instead of failing the scrape.+ */+function isProxyFetchFailure(error: unknown): boolean {+ let current: unknown = error;+ // Bounded walk with a visited guard: cause chains are short in practice,+ // but a malformed/cyclic chain must never spin or crash the scrape.+ const visited = new Set<unknown>();+ while (+ typeof current === "object" &&+ current !== null &&+ !visited.has(current)+ ) {+ visited.add(current);+ const message = (current as { message?: unknown }).message;+ if (+ typeof message === "string" &&+ PROXY_TUNNEL_FAILURE_MESSAGE.test(message)+ ) {+ return true;+ }+ current = (current as { cause?: unknown }).cause;+ }+ return false;+}++/**+ * Wraps a file engine's direct undici download so a proxy tunneling failure+ * converts into an engine-level error that the scrapeURL retry loop treats+ * exactly like the antibot case: clear the file flag and re-run the waterfall,+ * letting the browser engine fetch the file through fire-engine's own proxy+ * infrastructure instead of ours (see PDFFetchProxyError /+ * DocumentFetchProxyError).+ *+ * Converts when the engine is the flag-mandated handler for the file type+ * and no usable prefetch exists. The retry loop distinguishes the two empty+ * prefetch states: `undefined` ("browser never attempted") triggers the+ * browser fallback; `null` ("browser attempted, delivered no file") fails+ * fast with this error rather than burning another prefetch round trip.+ * A real prefetch object never reaches the direct download at all.+ */+export async function fetchFileGuardingProxyFailure<T>(+ opts: {+ /** meta's prefetch state for this file type; undefined = not attempted,+ * null = attempted and came back empty, object = use the file. */+ prefetch: unknown;+ /** Whether the engine is the flag-mandated handler for this file type+ * (feature flag, or a scalar forceEngine pinning this engine). */+ flagMandated: boolean;+ /** Error that triggers the browser-engine fallback in the retry loop. */+ makeError: () => Error;+ },+ fetch: () => Promise<T>,+): Promise<T> {+ try {+ return await fetch();+ } catch (error) {+ if (+ opts.prefetch == null &&+ opts.flagMandated &&+ isProxyFetchFailure(error)+ ) {+ throw opts.makeError();+ }+ throw error;+ }+}+const mapUndiciError = (url: string, skipTlsVerification: boolean, e: any) => {const code = e?.code ?? e?.cause?.code ?? e?.errno ?? e?.name;if (e?.name === "AbortError") {diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 81d67487a..670761925 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -269,6 +269,29 @@ export class PDFAntibotError extends TransportableError {}}+export class PDFFetchProxyError extends TransportableError {+ constructor() {+ const message = isSelfHosted()+ ? "Failed to download the PDF file because the configured proxy could not establish a connection to the website (the proxy rejected the tunneling request). Check your proxy configuration (PROXY_SERVER) and whether the target site is reachable from it, then retry."+ : "Failed to download the PDF file β Firecrawl could not establish a connection to the website while downloading it. Please retry your request, and if the issue persists, contact [email protected] with your request ID for investigation.";++ super("SCRAPE_PDF_FETCH_PROXY_ERROR", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new PDFFetchProxyError();+ x.stack = data.stack;+ return x;+ }+}+export class PDFInsufficientTimeError extends TransportableError {constructor(public pageCount: number,@@ -468,6 +491,29 @@ export class DocumentAntibotError extends TransportableError {}}+export class DocumentFetchProxyError extends TransportableError {+ constructor() {+ const message = isSelfHosted()+ ? "Failed to download the document file because the configured proxy could not establish a connection to the website (the proxy rejected the tunneling request). Check your proxy configuration (PROXY_SERVER) and whether the target site is reachable from it, then retry."+ : "Failed to download the document file β Firecrawl could not establish a connection to the website while downloading it. Please retry your request, and if the issue persists, contact [email protected] with your request ID for investigation.";++ super("SCRAPE_DOCUMENT_FETCH_PROXY_ERROR", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new DocumentFetchProxyError();+ x.stack = data.stack;+ return x;+ }+}+export class DocumentPrefetchFailed extends TransportableError {constructor() {const message = isSelfHosted()@@ -727,7 +773,9 @@ export type ScrapeRetryLimitReason =| "feature_toggle"| "feature_removal"| "pdf_antibot"- | "document_antibot";+ | "document_antibot"+ | "pdf_fetch_proxy"+ | "document_fetch_proxy";export type ScrapeRetryStats = {totalAttempts: number;@@ -735,6 +783,8 @@ export type ScrapeRetryStats = {removeFeatureAttempts: number;pdfAntibotAttempts: number;documentAntibotAttempts: number;+ pdfFetchProxyAttempts: number;+ documentFetchProxyAttempts: number;};export class ScrapeRetryLimitError extends TransportableError {@@ -760,7 +810,14 @@ export class ScrapeRetryLimitError extends TransportableError {_: ErrorCodes,data: ReturnType<typeof this.prototype.serialize>,) {- const x = new ScrapeRetryLimitError(data.reason, data.stats);+ // Counters added after this error first shipped default to 0: during a+ // rolling deploy, workers on the older build serialize stats payloads+ // without them, and the ScrapeRetryStats contract types them as numbers.+ const x = new ScrapeRetryLimitError(data.reason, {+ ...data.stats,+ pdfFetchProxyAttempts: data.stats.pdfFetchProxyAttempts ?? 0,+ documentFetchProxyAttempts: data.stats.documentFetchProxyAttempts ?? 0,+ });x.stack = data.stack;return x;}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 4c0b0802b..d94d5e224 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -32,7 +32,9 @@ import {EngineError,NoEnginesLeftError,PDFAntibotError,+ PDFFetchProxyError,DocumentAntibotError,+ DocumentFetchProxyError,RemoveFeatureError,SiteError,UnsupportedFileError,@@ -168,6 +170,9 @@ export type Meta = {}| null| undefined; // undefined: no prefetch yet, null: prefetch came back empty+ // (null is preserved through the retry loop's AddFeatureError handler so+ // antibot/proxy-failure handling can tell "never attempted" apart from+ // "attempted, browser delivered no file")/** Live state of a by-reference FirePDF job (large PDFs) this scrape* submitted or adopted. Such jobs outlive an abandoned scrape BY* DESIGN (see fire-pdf/async.ts's cancel policy), so a SCRAPE_TIMEOUT@@ -204,6 +209,7 @@ export type Meta = {}| null| undefined; // undefined: no prefetch yet, null: prefetch came back empty+ // (null preserved through the retry loop, same as pdfPrefetch)fetchPrefetch:| {url?: string;@@ -718,6 +724,12 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {// Skip when the content was already prefetched (a browser engine already// ran the actions and downloaded the file); the re-run only needs the// document/pdf engine to parse it, which does not support actions.+ // Skip when a browser engine already ran the actions β i.e. any+ // prefetch state exists, including the null sentinel (browser ran,+ // delivered no file): the re-run only needs the pdf/document engine+ // to parse the file, and the actions check must not preempt the+ // antibot/proxy recovery paths below with a misleading+ // ActionsNotSupportedError after the actions already executed.if (meta.featureFlags.has("actions") &&meta.pdfPrefetch === undefined &&@@ -881,8 +893,10 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ActionError ||error.error instanceof UnsupportedFileError ||error.error instanceof PDFAntibotError ||+ error.error instanceof PDFFetchProxyError ||error.error instanceof PDFOCRRequiredError ||error.error instanceof DocumentAntibotError ||+ error.error instanceof DocumentFetchProxyError ||error.error instanceof PDFInsufficientTimeError ||error.error instanceof ProxySelectionError ||error.error instanceof NoCachedDataError ||@@ -1294,9 +1308,17 @@ export async function scrapeURL();if (error.pdfPrefetch) {meta.pdfPrefetch = error.pdfPrefetch;+ } else if (error.pdfPrefetch === null) {+ // Browser round trip ran but delivered no file. Preserve the+ // null sentinel: the antibot branches below still retry (the+ // empty handoff may be transient), but the proxy-failure+ // branches fail fast instead of re-running the browser.+ meta.pdfPrefetch = null;}if (error.documentPrefetch) {meta.documentPrefetch = error.documentPrefetch;+ } else if (error.documentPrefetch === null) {+ meta.documentPrefetch = null;}} else if (error instanceof RemoveFeatureError &&@@ -1318,7 +1340,10 @@ export async function scrapeURL(error instanceof PDFAntibotError &&meta.internalOptions.forceEngine === undefined) {- if (meta.pdfPrefetch !== undefined) {+ // null = browser ran but delivered no file (possibly transient) β+ // still worth one more browser round trip, so only a real+ // prefetch object fails here.+ if (meta.pdfPrefetch != null) {meta.logger.error("PDF was prefetched and still blocked by antibot, failing",);@@ -1332,11 +1357,37 @@ export async function scrapeURL([...meta.featureFlags].filter(x => x !== "pdf"),);}+ } else if (+ error instanceof PDFFetchProxyError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // meta.pdfPrefetch distinguishes "browser never attempted"+ // (undefined β clear the pdf flag so the browser engine fetches+ // the file through fire-engine's proxies) from "browser attempted,+ // came back empty" (null β fail fast: another round trip would+ // only burn the shared antibot+proxy prefetch budget).+ if (meta.pdfPrefetch !== undefined) {+ meta.logger.error(+ "PDF was prefetched and the direct fetch still failed at the proxy, failing",+ );+ throw error;+ } else {+ retryTracker.record("pdf_fetch_proxy", error);+ meta.logger.debug(+ "PDF direct download failed at the proxy, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "pdf"),+ );+ }} else if (error instanceof DocumentAntibotError &&meta.internalOptions.forceEngine === undefined) {- if (meta.documentPrefetch !== undefined) {+ // null = browser ran but delivered no file (possibly transient) β+ // still worth one more browser round trip, so only a real+ // prefetch object fails here.+ if (meta.documentPrefetch != null) {meta.logger.error("Document was prefetched and still blocked by antibot, failing",);@@ -1350,6 +1401,25 @@ export async function scrapeURL([...meta.featureFlags].filter(x => x !== "document"),);}+ } else if (+ error instanceof DocumentFetchProxyError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // Same undefined-vs-null distinction as the PDF branch above.+ if (meta.documentPrefetch !== undefined) {+ meta.logger.error(+ "Document was prefetched and the direct fetch still failed at the proxy, failing",+ );+ throw error;+ } else {+ retryTracker.record("document_fetch_proxy", error);+ meta.logger.debug(+ "Document direct download failed at the proxy, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "document"),+ );+ }} else {throw error;}@@ -1560,6 +1630,18 @@ export async function scrapeURL("scrapeURL: Failed to prefetch document that is protected by anti-bot",{ error },);+ } else if (error instanceof PDFFetchProxyError) {+ errorType = "PDFFetchProxyError";+ meta.logger.warn(+ "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",+ { error },+ );+ } else if (error instanceof DocumentFetchProxyError) {+ errorType = "DocumentFetchProxyError";+ meta.logger.warn(+ "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",+ { error },+ );} else if (error instanceof BrandingNotSupportedError) {errorType = "BrandingNotSupportedError";meta.logger.warn("scrapeURL: Branding not supported for this content", {diff --git a/apps/api/src/scraper/scrapeURL/retryTracker.ts b/apps/api/src/scraper/scrapeURL/retryTracker.tsindex d017cbac6..b0b231d71 100644--- a/apps/api/src/scraper/scrapeURL/retryTracker.ts+++ b/apps/api/src/scraper/scrapeURL/retryTracker.ts@@ -22,6 +22,8 @@ export class ScrapeRetryTracker {removeFeatureAttempts: 0,pdfAntibotAttempts: 0,documentAntibotAttempts: 0,+ pdfFetchProxyAttempts: 0,+ documentFetchProxyAttempts: 0,};constructor(@@ -53,19 +55,40 @@ export class ScrapeRetryTracker {}break;case "pdf_antibot":- this.stats.pdfAntibotAttempts += 1;- if (this.stats.pdfAntibotAttempts > this.config.maxPdfPrefetches) {+ case "pdf_fetch_proxy": {+ if (reason === "pdf_antibot") {+ this.stats.pdfAntibotAttempts += 1;+ } else {+ this.stats.pdfFetchProxyAttempts += 1;+ }+ // Antibot and proxy-failure retries consume the same resource β one+ // browser-engine prefetch round trip β so they share the budget: a+ // scrape must not be granted maxPdfPrefetches of EACH.+ if (+ this.stats.pdfAntibotAttempts + this.stats.pdfFetchProxyAttempts >+ this.config.maxPdfPrefetches+ ) {this.throwLimit(reason, lastError);}break;+ }case "document_antibot":- this.stats.documentAntibotAttempts += 1;+ case "document_fetch_proxy": {+ if (reason === "document_antibot") {+ this.stats.documentAntibotAttempts += 1;+ } else {+ this.stats.documentFetchProxyAttempts += 1;+ }+ // Same shared-budget rationale as the PDF counters above.if (- this.stats.documentAntibotAttempts > this.config.maxDocumentPrefetches+ this.stats.documentAntibotAttempts ++ this.stats.documentFetchProxyAttempts >+ this.config.maxDocumentPrefetches) {this.throwLimit(reason, lastError);}break;+ }}this.logger.warn("scrapeURL retrying after handled error", {diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsindex 7e38f119f..f02ae61b9 100644--- a/apps/api/src/services/sentry.ts+++ b/apps/api/src/services/sentry.ts@@ -35,10 +35,12 @@ const transportableErrorCodes = ["SCRAPE_ZDR_VIOLATION_ERROR","SCRAPE_UNSUPPORTED_FILE_ERROR","SCRAPE_PDF_ANTIBOT_ERROR",+ "SCRAPE_PDF_FETCH_PROXY_ERROR","SCRAPE_ACTION_ERROR","SCRAPE_PDF_INSUFFICIENT_TIME_ERROR","SCRAPE_PDF_PREFETCH_FAILED","SCRAPE_DOCUMENT_ANTIBOT_ERROR",+ "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR","SCRAPE_DOCUMENT_PREFETCH_FAILED","SCRAPE_ACTIONS_NOT_SUPPORTED","SCRAPE_TIMEOUT",