Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api/pdf): fall back to chrome-cdp prefetch when the pdf engine's direct download fails at the proxy (#4477)

GergΕ‘ MΓ³ricz1f193af5750d
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index ef7be19f6..4f973e356 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -17,6 +17,8 @@ import {
UnsupportedFileError,
PDFAntibotError,
DocumentAntibotError,
+ PDFFetchProxyError,
+ DocumentFetchProxyError,
PDFInsufficientTimeError,
PDFOCRRequiredError,
NoEnginesLeftError,
@@ -60,8 +62,10 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,
SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,
SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,
+ SCRAPE_PDF_FETCH_PROXY_ERROR: PDFFetchProxyError,
SCRAPE_PDF_OCR_REQUIRED: PDFOCRRequiredError,
SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,
+ SCRAPE_DOCUMENT_FETCH_PROXY_ERROR: DocumentFetchProxyError,
SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,
SCRAPE_NO_CACHED_DATA: NoCachedDataError,
SCRAPE_LOCKDOWN_CACHE_MISS: LockdownMissError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 000776781..63814624a 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -15,8 +15,10 @@ export type ErrorCodes =
| "SCRAPE_DNS_RESOLUTION_ERROR"
| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"
| "SCRAPE_PDF_ANTIBOT_ERROR"
+ | "SCRAPE_PDF_FETCH_PROXY_ERROR"
| "SCRAPE_PDF_OCR_REQUIRED"
| "SCRAPE_DOCUMENT_ANTIBOT_ERROR"
+ | "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR"
| "SCRAPE_UNSUPPORTED_FILE_ERROR"
| "SCRAPE_ACTION_ERROR"
| "SCRAPE_RACED_REDIRECT_ERROR"
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index 6bde2f0bf..c6666af19 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -1,10 +1,17 @@
import { Meta } from "../..";
import { EngineScrapeResult } from "..";
-import { fetchFileToBuffer } from "../utils/downloadFile";
+import {
+ fetchFileGuardingProxyFailure,
+ fetchFileToBuffer,
+} from "../utils/downloadFile";
import { convertDocumentToMarkdown } from "@mendable/firecrawl-rs";
import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
import type { Response } from "undici";
-import { DocumentAntibotError, EngineUnsuccessfulError } from "../../error";
+import {
+ DocumentAntibotError,
+ DocumentFetchProxyError,
+ EngineUnsuccessfulError,
+} from "../../error";
import {
documentContentTypeFromExtension,
documentExtensionFromContentType,
@@ -39,13 +46,15 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
proxyUsed = meta.documentPrefetch.proxyUsed;
} else {
// Fetch the document normally
- const result = await fetchFileToBuffer(
- meta.rewrittenUrl ?? meta.url,
- meta.options.skipTlsVerification,
- {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- },
+ const result = await fetchDocumentFileGuardingProxyFailure(meta, () =>
+ fetchFileToBuffer(
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ ),
);
response = result.response;
buffer = result.buffer;
@@ -113,3 +122,32 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
export function documentMaxReasonableTime(meta: Meta): number {
return 15000;
}
+
+/**
+ * Guards the document engine's direct undici download: a proxy tunneling
+ * failure converts into DocumentFetchProxyError, which the scrapeURL retry
+ * loop handles exactly like DocumentAntibotError (clear the "document" flag,
+ * re-run the waterfall, browser engine fetches the file). See
+ * fetchFileGuardingProxyFailure for the conversion eligibility rules.
+ */
+function fetchDocumentFileGuardingProxyFailure<T>(
+ meta: Meta,
+ fetch: () => Promise<T>,
+): Promise<T> {
+ return fetchFileGuardingProxyFailure(
+ {
+ prefetch: meta.documentPrefetch,
+ // Same conversion-eligibility rules as the pdf engine: only when
+ // forceEngine is unset (retry loop recovers via browser fallback) or
+ // scalar-pinned to this engine (clean error surfaces); arrays keep the
+ // raw error so the waterfall continues through remaining forced
+ // engines.
+ flagMandated:
+ (meta.internalOptions.forceEngine === undefined &&
+ meta.featureFlags.has("document")) ||
+ meta.internalOptions.forceEngine === "document",
+ makeError: () => new DocumentFetchProxyError(),
+ },
+ fetch,
+ );
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 798888f1a..bda10f147 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -1,10 +1,15 @@
import { Meta } from "../..";
import { config } from "../../../../config";
import { EngineScrapeResult } from "..";
-import { downloadFile, fetchFileToBuffer } from "../utils/downloadFile";
+import {
+ downloadFile,
+ fetchFileGuardingProxyFailure,
+ fetchFileToBuffer,
+} from "../utils/downloadFile";
import { safeMarkdownToHtml } from "./markdownToHtml";
import {
PDFAntibotError,
+ PDFFetchProxyError,
PDFInsufficientTimeError,
PDFOCRRequiredError,
PDFPrefetchFailed,
@@ -69,6 +74,38 @@ function getIneligibleReason(
return null;
}
+/**
+ * Guards the pdf engine's direct undici downloads: a proxy tunneling
+ * failure converts into PDFFetchProxyError, which the scrapeURL retry loop
+ * handles exactly like PDFAntibotError (clear the "pdf" flag, re-run the
+ * waterfall, browser engine fetches the file). See
+ * fetchFileGuardingProxyFailure for the conversion eligibility rules.
+ */
+function fetchPdfFileGuardingProxyFailure<T>(
+ meta: Meta,
+ fetch: () => Promise<T>,
+): Promise<T> {
+ return fetchFileGuardingProxyFailure(
+ {
+ prefetch: meta.pdfPrefetch,
+ // Convert only where the outcome is actionable: with forceEngine
+ // unset, the retry loop recovers PDFFetchProxyError via the browser
+ // fallback; with a scalar forceEngine=pdf, this engine is pinned
+ // with no fallback in the list, so the clean error surfaces instead
+ // of the raw TypeError. An ARRAY forceEngine must NOT convert β€” the
+ // retry loop bypasses recovery for any forceEngine, and the raw
+ // error is what lets the waterfall continue through the remaining
+ // forced engines.
+ flagMandated:
+ (meta.internalOptions.forceEngine === undefined &&
+ meta.featureFlags.has("pdf")) ||
+ meta.internalOptions.forceEngine === "pdf",
+ makeError: () => new PDFFetchProxyError(),
+ },
+ fetch,
+ );
+}
+
export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const shouldParse = shouldParsePDF(meta.options.parsers);
const maxPages = getPDFMaxPages(meta.options.parsers);
@@ -118,19 +155,24 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
proxyUsed: meta.pdfPrefetch.proxyUsed,
};
} else {
- const file = await fetchFileToBuffer(
- meta.rewrittenUrl ?? meta.url,
- meta.options.skipTlsVerification,
- {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- },
- PDF_DOWNLOAD_MAX_FILE_SIZE,
+ const file = await fetchPdfFileGuardingProxyFailure(meta, () =>
+ fetchFileToBuffer(
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ PDF_DOWNLOAD_MAX_FILE_SIZE,
+ ),
);
if (!isPdfBuffer(file.buffer)) {
// downloaded content isn't a valid PDF
- if (meta.pdfPrefetch === undefined) {
+ // (null prefetch = browser round trip ran but delivered no file β€”
+ // still PDFAntibotError so the retry loop can give the browser
+ // another shot, exactly like the no-prefetch case)
+ if (meta.pdfPrefetch == null) {
// for non-PDF URLs, this is expected, not anti-bot
if (!meta.featureFlags.has("pdf")) {
throw new EngineUnsuccessfulError("pdf");
@@ -183,20 +225,22 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
const { response, tempFilePath } =
meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null
? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }
- : await downloadFile(
- meta.id,
- meta.rewrittenUrl ?? meta.url,
- meta.options.skipTlsVerification,
- {
- headers: meta.options.headers,
- signal: meta.abort.asSignal(),
- },
- // Parse path streams to disk and can hand large files to FirePDF
- // by GCS reference, so it admits more than the raw fetch path β€”
- // up to the requesting team's large-PDF limit.
- byReferenceReachable
- ? largePdfLimitBytes(meta)
- : PDF_DOWNLOAD_MAX_FILE_SIZE,
+ : await fetchPdfFileGuardingProxyFailure(meta, () =>
+ downloadFile(
+ meta.id,
+ meta.rewrittenUrl ?? meta.url,
+ meta.options.skipTlsVerification,
+ {
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ },
+ // Parse path streams to disk and can hand large files to FirePDF
+ // by GCS reference, so it admits more than the raw fetch path β€”
+ // up to the requesting team's large-PDF limit.
+ byReferenceReachable
+ ? largePdfLimitBytes(meta)
+ : PDF_DOWNLOAD_MAX_FILE_SIZE,
+ ),
);
try {
@@ -216,7 +260,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
}
if (!isPdfBuffer(header.subarray(0, headerBytesRead))) {
- if (meta.pdfPrefetch === undefined) {
+ // (null prefetch = browser round trip ran but delivered no file β€”
+ // still PDFAntibotError so the retry loop can give the browser
+ // another shot, exactly like the no-prefetch case)
+ if (meta.pdfPrefetch == null) {
if (!meta.featureFlags.has("pdf")) {
throw new EngineUnsuccessfulError("pdf");
} else {
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
index 380da34d3..c119de58b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -16,6 +16,90 @@ import * as undici from "undici";
import { getSecureDispatcher } from "./safeFetch";
import { logger } from "../../../../lib/logger";
+/**
+ * Matches undici ProxyAgent tunnel failures: the proxy's CONNECT response was
+ * not 200 (502, 407, ...). undici emits `RequestAbortedError` (code
+ * UND_ERR_ABORTED) with the message "Proxy response (NNN) !== 200 when HTTP
+ * Tunneling" β€” see undici's lib/dispatcher/proxy-agent.js.
+ */
+const PROXY_TUNNEL_FAILURE_MESSAGE =
+ /Proxy response \(\d+\) !== 200 when HTTP Tunneling/;
+
+/**
+ * Detects a proxy tunneling failure anywhere in an error's cause chain.
+ *
+ * undici wraps these several levels deep β€”
+ * TypeError("fetch failed") β†’ DOMException("Request was cancelled.") β†’
+ * RequestAbortedError("Proxy response (502) !== 200 when HTTP Tunneling") β€”
+ * so, unlike mapUndiciError, we walk the whole chain. Callers use this to
+ * retry the download through a different transport (e.g. the browser engine's
+ * own proxy infra) instead of failing the scrape.
+ */
+function isProxyFetchFailure(error: unknown): boolean {
+ let current: unknown = error;
+ // Bounded walk with a visited guard: cause chains are short in practice,
+ // but a malformed/cyclic chain must never spin or crash the scrape.
+ const visited = new Set<unknown>();
+ while (
+ typeof current === "object" &&
+ current !== null &&
+ !visited.has(current)
+ ) {
+ visited.add(current);
+ const message = (current as { message?: unknown }).message;
+ if (
+ typeof message === "string" &&
+ PROXY_TUNNEL_FAILURE_MESSAGE.test(message)
+ ) {
+ return true;
+ }
+ current = (current as { cause?: unknown }).cause;
+ }
+ return false;
+}
+
+/**
+ * Wraps a file engine's direct undici download so a proxy tunneling failure
+ * converts into an engine-level error that the scrapeURL retry loop treats
+ * exactly like the antibot case: clear the file flag and re-run the waterfall,
+ * letting the browser engine fetch the file through fire-engine's own proxy
+ * infrastructure instead of ours (see PDFFetchProxyError /
+ * DocumentFetchProxyError).
+ *
+ * Converts when the engine is the flag-mandated handler for the file type
+ * and no usable prefetch exists. The retry loop distinguishes the two empty
+ * prefetch states: `undefined` ("browser never attempted") triggers the
+ * browser fallback; `null` ("browser attempted, delivered no file") fails
+ * fast with this error rather than burning another prefetch round trip.
+ * A real prefetch object never reaches the direct download at all.
+ */
+export async function fetchFileGuardingProxyFailure<T>(
+ opts: {
+ /** meta's prefetch state for this file type; undefined = not attempted,
+ * null = attempted and came back empty, object = use the file. */
+ prefetch: unknown;
+ /** Whether the engine is the flag-mandated handler for this file type
+ * (feature flag, or a scalar forceEngine pinning this engine). */
+ flagMandated: boolean;
+ /** Error that triggers the browser-engine fallback in the retry loop. */
+ makeError: () => Error;
+ },
+ fetch: () => Promise<T>,
+): Promise<T> {
+ try {
+ return await fetch();
+ } catch (error) {
+ if (
+ opts.prefetch == null &&
+ opts.flagMandated &&
+ isProxyFetchFailure(error)
+ ) {
+ throw opts.makeError();
+ }
+ throw error;
+ }
+}
+
const mapUndiciError = (url: string, skipTlsVerification: boolean, e: any) => {
const code = e?.code ?? e?.cause?.code ?? e?.errno ?? e?.name;
if (e?.name === "AbortError") {
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 81d67487a..670761925 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -269,6 +269,29 @@ export class PDFAntibotError extends TransportableError {
}
}
+export class PDFFetchProxyError extends TransportableError {
+ constructor() {
+ const message = isSelfHosted()
+ ? "Failed to download the PDF file because the configured proxy could not establish a connection to the website (the proxy rejected the tunneling request). Check your proxy configuration (PROXY_SERVER) and whether the target site is reachable from it, then retry."
+ : "Failed to download the PDF file β€” Firecrawl could not establish a connection to the website while downloading it. Please retry your request, and if the issue persists, contact [email protected] with your request ID for investigation.";
+
+ super("SCRAPE_PDF_FETCH_PROXY_ERROR", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new PDFFetchProxyError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class PDFInsufficientTimeError extends TransportableError {
constructor(
public pageCount: number,
@@ -468,6 +491,29 @@ export class DocumentAntibotError extends TransportableError {
}
}
+export class DocumentFetchProxyError extends TransportableError {
+ constructor() {
+ const message = isSelfHosted()
+ ? "Failed to download the document file because the configured proxy could not establish a connection to the website (the proxy rejected the tunneling request). Check your proxy configuration (PROXY_SERVER) and whether the target site is reachable from it, then retry."
+ : "Failed to download the document file β€” Firecrawl could not establish a connection to the website while downloading it. Please retry your request, and if the issue persists, contact [email protected] with your request ID for investigation.";
+
+ super("SCRAPE_DOCUMENT_FETCH_PROXY_ERROR", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new DocumentFetchProxyError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class DocumentPrefetchFailed extends TransportableError {
constructor() {
const message = isSelfHosted()
@@ -727,7 +773,9 @@ export type ScrapeRetryLimitReason =
| "feature_toggle"
| "feature_removal"
| "pdf_antibot"
- | "document_antibot";
+ | "document_antibot"
+ | "pdf_fetch_proxy"
+ | "document_fetch_proxy";
export type ScrapeRetryStats = {
totalAttempts: number;
@@ -735,6 +783,8 @@ export type ScrapeRetryStats = {
removeFeatureAttempts: number;
pdfAntibotAttempts: number;
documentAntibotAttempts: number;
+ pdfFetchProxyAttempts: number;
+ documentFetchProxyAttempts: number;
};
export class ScrapeRetryLimitError extends TransportableError {
@@ -760,7 +810,14 @@ export class ScrapeRetryLimitError extends TransportableError {
_: ErrorCodes,
data: ReturnType<typeof this.prototype.serialize>,
) {
- const x = new ScrapeRetryLimitError(data.reason, data.stats);
+ // Counters added after this error first shipped default to 0: during a
+ // rolling deploy, workers on the older build serialize stats payloads
+ // without them, and the ScrapeRetryStats contract types them as numbers.
+ const x = new ScrapeRetryLimitError(data.reason, {
+ ...data.stats,
+ pdfFetchProxyAttempts: data.stats.pdfFetchProxyAttempts ?? 0,
+ documentFetchProxyAttempts: data.stats.documentFetchProxyAttempts ?? 0,
+ });
x.stack = data.stack;
return x;
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 4c0b0802b..d94d5e224 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -32,7 +32,9 @@ import {
EngineError,
NoEnginesLeftError,
PDFAntibotError,
+ PDFFetchProxyError,
DocumentAntibotError,
+ DocumentFetchProxyError,
RemoveFeatureError,
SiteError,
UnsupportedFileError,
@@ -168,6 +170,9 @@ export type Meta = {
}
| null
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
+ // (null is preserved through the retry loop's AddFeatureError handler so
+ // antibot/proxy-failure handling can tell "never attempted" apart from
+ // "attempted, browser delivered no file")
/** Live state of a by-reference FirePDF job (large PDFs) this scrape
* submitted or adopted. Such jobs outlive an abandoned scrape BY
* DESIGN (see fire-pdf/async.ts's cancel policy), so a SCRAPE_TIMEOUT
@@ -204,6 +209,7 @@ export type Meta = {
}
| null
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
+ // (null preserved through the retry loop, same as pdfPrefetch)
fetchPrefetch:
| {
url?: string;
@@ -718,6 +724,12 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
// Skip when the content was already prefetched (a browser engine already
// ran the actions and downloaded the file); the re-run only needs the
// document/pdf engine to parse it, which does not support actions.
+ // Skip when a browser engine already ran the actions β€” i.e. any
+ // prefetch state exists, including the null sentinel (browser ran,
+ // delivered no file): the re-run only needs the pdf/document engine
+ // to parse the file, and the actions check must not preempt the
+ // antibot/proxy recovery paths below with a misleading
+ // ActionsNotSupportedError after the actions already executed.
if (
meta.featureFlags.has("actions") &&
meta.pdfPrefetch === undefined &&
@@ -881,8 +893,10 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ActionError ||
error.error instanceof UnsupportedFileError ||
error.error instanceof PDFAntibotError ||
+ error.error instanceof PDFFetchProxyError ||
error.error instanceof PDFOCRRequiredError ||
error.error instanceof DocumentAntibotError ||
+ error.error instanceof DocumentFetchProxyError ||
error.error instanceof PDFInsufficientTimeError ||
error.error instanceof ProxySelectionError ||
error.error instanceof NoCachedDataError ||
@@ -1294,9 +1308,17 @@ export async function scrapeURL(
);
if (error.pdfPrefetch) {
meta.pdfPrefetch = error.pdfPrefetch;
+ } else if (error.pdfPrefetch === null) {
+ // Browser round trip ran but delivered no file. Preserve the
+ // null sentinel: the antibot branches below still retry (the
+ // empty handoff may be transient), but the proxy-failure
+ // branches fail fast instead of re-running the browser.
+ meta.pdfPrefetch = null;
}
if (error.documentPrefetch) {
meta.documentPrefetch = error.documentPrefetch;
+ } else if (error.documentPrefetch === null) {
+ meta.documentPrefetch = null;
}
} else if (
error instanceof RemoveFeatureError &&
@@ -1318,7 +1340,10 @@ export async function scrapeURL(
error instanceof PDFAntibotError &&
meta.internalOptions.forceEngine === undefined
) {
- if (meta.pdfPrefetch !== undefined) {
+ // null = browser ran but delivered no file (possibly transient) β€”
+ // still worth one more browser round trip, so only a real
+ // prefetch object fails here.
+ if (meta.pdfPrefetch != null) {
meta.logger.error(
"PDF was prefetched and still blocked by antibot, failing",
);
@@ -1332,11 +1357,37 @@ export async function scrapeURL(
[...meta.featureFlags].filter(x => x !== "pdf"),
);
}
+ } else if (
+ error instanceof PDFFetchProxyError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // meta.pdfPrefetch distinguishes "browser never attempted"
+ // (undefined β€” clear the pdf flag so the browser engine fetches
+ // the file through fire-engine's proxies) from "browser attempted,
+ // came back empty" (null β€” fail fast: another round trip would
+ // only burn the shared antibot+proxy prefetch budget).
+ if (meta.pdfPrefetch !== undefined) {
+ meta.logger.error(
+ "PDF was prefetched and the direct fetch still failed at the proxy, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("pdf_fetch_proxy", error);
+ meta.logger.debug(
+ "PDF direct download failed at the proxy, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "pdf"),
+ );
+ }
} else if (
error instanceof DocumentAntibotError &&
meta.internalOptions.forceEngine === undefined
) {
- if (meta.documentPrefetch !== undefined) {
+ // null = browser ran but delivered no file (possibly transient) β€”
+ // still worth one more browser round trip, so only a real
+ // prefetch object fails here.
+ if (meta.documentPrefetch != null) {
meta.logger.error(
"Document was prefetched and still blocked by antibot, failing",
);
@@ -1350,6 +1401,25 @@ export async function scrapeURL(
[...meta.featureFlags].filter(x => x !== "document"),
);
}
+ } else if (
+ error instanceof DocumentFetchProxyError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ // Same undefined-vs-null distinction as the PDF branch above.
+ if (meta.documentPrefetch !== undefined) {
+ meta.logger.error(
+ "Document was prefetched and the direct fetch still failed at the proxy, failing",
+ );
+ throw error;
+ } else {
+ retryTracker.record("document_fetch_proxy", error);
+ meta.logger.debug(
+ "Document direct download failed at the proxy, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "document"),
+ );
+ }
} else {
throw error;
}
@@ -1560,6 +1630,18 @@ export async function scrapeURL(
"scrapeURL: Failed to prefetch document that is protected by anti-bot",
{ error },
);
+ } else if (error instanceof PDFFetchProxyError) {
+ errorType = "PDFFetchProxyError";
+ meta.logger.warn(
+ "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",
+ { error },
+ );
+ } else if (error instanceof DocumentFetchProxyError) {
+ errorType = "DocumentFetchProxyError";
+ meta.logger.warn(
+ "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",
+ { error },
+ );
} else if (error instanceof BrandingNotSupportedError) {
errorType = "BrandingNotSupportedError";
meta.logger.warn("scrapeURL: Branding not supported for this content", {
diff --git a/apps/api/src/scraper/scrapeURL/retryTracker.ts b/apps/api/src/scraper/scrapeURL/retryTracker.ts
index d017cbac6..b0b231d71 100644
--- a/apps/api/src/scraper/scrapeURL/retryTracker.ts
+++ b/apps/api/src/scraper/scrapeURL/retryTracker.ts
@@ -22,6 +22,8 @@ export class ScrapeRetryTracker {
removeFeatureAttempts: 0,
pdfAntibotAttempts: 0,
documentAntibotAttempts: 0,
+ pdfFetchProxyAttempts: 0,
+ documentFetchProxyAttempts: 0,
};
constructor(
@@ -53,19 +55,40 @@ export class ScrapeRetryTracker {
}
break;
case "pdf_antibot":
- this.stats.pdfAntibotAttempts += 1;
- if (this.stats.pdfAntibotAttempts > this.config.maxPdfPrefetches) {
+ case "pdf_fetch_proxy": {
+ if (reason === "pdf_antibot") {
+ this.stats.pdfAntibotAttempts += 1;
+ } else {
+ this.stats.pdfFetchProxyAttempts += 1;
+ }
+ // Antibot and proxy-failure retries consume the same resource β€” one
+ // browser-engine prefetch round trip β€” so they share the budget: a
+ // scrape must not be granted maxPdfPrefetches of EACH.
+ if (
+ this.stats.pdfAntibotAttempts + this.stats.pdfFetchProxyAttempts >
+ this.config.maxPdfPrefetches
+ ) {
this.throwLimit(reason, lastError);
}
break;
+ }
case "document_antibot":
- this.stats.documentAntibotAttempts += 1;
+ case "document_fetch_proxy": {
+ if (reason === "document_antibot") {
+ this.stats.documentAntibotAttempts += 1;
+ } else {
+ this.stats.documentFetchProxyAttempts += 1;
+ }
+ // Same shared-budget rationale as the PDF counters above.
if (
- this.stats.documentAntibotAttempts > this.config.maxDocumentPrefetches
+ this.stats.documentAntibotAttempts +
+ this.stats.documentFetchProxyAttempts >
+ this.config.maxDocumentPrefetches
) {
this.throwLimit(reason, lastError);
}
break;
+ }
}
this.logger.warn("scrapeURL retrying after handled error", {
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
index 7e38f119f..f02ae61b9 100644
--- a/apps/api/src/services/sentry.ts
+++ b/apps/api/src/services/sentry.ts
@@ -35,10 +35,12 @@ const transportableErrorCodes = [
"SCRAPE_ZDR_VIOLATION_ERROR",
"SCRAPE_UNSUPPORTED_FILE_ERROR",
"SCRAPE_PDF_ANTIBOT_ERROR",
+ "SCRAPE_PDF_FETCH_PROXY_ERROR",
"SCRAPE_ACTION_ERROR",
"SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",
"SCRAPE_PDF_PREFETCH_FAILED",
"SCRAPE_DOCUMENT_ANTIBOT_ERROR",
+ "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR",
"SCRAPE_DOCUMENT_PREFETCH_FAILED",
"SCRAPE_ACTIONS_NOT_SUPPORTED",
"SCRAPE_TIMEOUT",