Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): make proxy selection error fatal

Gergő Móricz9723b5137fec
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 45f79d604..7fdd0847c 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -5,6 +5,7 @@ export type ErrorCodes =
| "SCRAPE_ALL_ENGINES_FAILED"
| "SCRAPE_SSL_ERROR"
| "SCRAPE_SITE_ERROR"
+ | "SCRAPE_PROXY_SELECTION_ERROR"
| "SCRAPE_PDF_PREFETCH_FAILED"
| "SCRAPE_ZDR_VIOLATION_ERROR"
| "SCRAPE_DNS_RESOLUTION_ERROR"
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 7854614b8..d0391d77e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -11,6 +11,7 @@ import {
UnsupportedFileError,
DNSResolutionError,
FEPageLoadFailed,
+ ProxySelectionError,
} from "../../error";
import { MockState } from "../../lib/mock";
import { fireEngineStagingURL, fireEngineURL } from "./scrape";
@@ -182,6 +183,11 @@ export async function fireEngineCheckStatus(
} else {
throw new SiteError(code);
}
+ } else if (
+ typeof status.error === "string" &&
+ status.error.includes("proxies available for")
+ ) {
+ throw new ProxySelectionError();
} else if (
typeof status.error === "string" &&
status.error.includes("Dns resolution error for hostname: ")
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 17d527d4d..899ee2c6a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -21,6 +21,7 @@ import {
SSLError,
UnsupportedFileError,
FEPageLoadFailed,
+ ProxySelectionError,
} from "../../error";
import * as Sentry from "@sentry/node";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
@@ -100,7 +101,8 @@ async function performFireEngineScrape<
error instanceof DNSResolutionError ||
error instanceof ActionError ||
error instanceof UnsupportedFileError ||
- error instanceof FEPageLoadFailed
+ error instanceof FEPageLoadFailed ||
+ error instanceof ProxySelectionError
) {
fireEngineDelete(
logger.child({
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 565688ad7..7a3ea0b40 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -11,6 +11,7 @@ import {
DNSResolutionError,
EngineError,
FEPageLoadFailed,
+ ProxySelectionError,
SSLError,
SiteError,
UnsupportedFileError,
@@ -263,6 +264,11 @@ export async function fireEngineScrape<
status.error.includes("Javascript execution failed"))
) {
throw new ActionError(status.error.split("Error: ")[1]);
+ } else if (
+ typeof status.error === "string" &&
+ status.error.includes("proxies available for")
+ ) {
+ throw new ProxySelectionError();
} else {
throw new EngineError("Scrape job failed", {
cause: {
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 1f07e0ae9..faae68f1d 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -113,6 +113,30 @@ export class SiteError extends TransportableError {
}
}
+export class ProxySelectionError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_PROXY_SELECTION_ERROR",
+ "Specified proxy cannot be selected. Change `location` or `proxy` in your scrape request.",
+ );
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ };
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new ProxySelectionError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class ActionError extends TransportableError {
constructor(public errorCode: string) {
super(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 86076e0f6..688708a06 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -38,6 +38,7 @@ import {
EngineSnipedError,
WaterfallNextEngineSignal,
EngineUnsuccessfulError,
+ ProxySelectionError,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@@ -528,7 +529,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ActionError ||
error.error instanceof UnsupportedFileError ||
error.error instanceof PDFAntibotError ||
- error.error instanceof PDFInsufficientTimeError
+ error.error instanceof PDFInsufficientTimeError ||
+ error.error instanceof ProxySelectionError
) {
throw error.error;
} else if (error.error instanceof LLMRefusalError) {
@@ -864,6 +866,8 @@ export async function scrapeURL(
"scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
{ error },
);
+ } else if (error instanceof ProxySelectionError) {
+ meta.logger.warn("scrapeURL: Proxy selection error", { error });
} else if (error instanceof AbortManagerThrownError) {
throw error.inner;
} else {