Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): make proxy selection error fatal
Gergő Móricz9723b5137fec
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 45f79d604..7fdd0847c 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -5,6 +5,7 @@ export type ErrorCodes =| "SCRAPE_ALL_ENGINES_FAILED"| "SCRAPE_SSL_ERROR"| "SCRAPE_SITE_ERROR"+ | "SCRAPE_PROXY_SELECTION_ERROR"| "SCRAPE_PDF_PREFETCH_FAILED"| "SCRAPE_ZDR_VIOLATION_ERROR"| "SCRAPE_DNS_RESOLUTION_ERROR"diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 7854614b8..d0391d77e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -11,6 +11,7 @@ import {UnsupportedFileError,DNSResolutionError,FEPageLoadFailed,+ ProxySelectionError,} from "../../error";import { MockState } from "../../lib/mock";import { fireEngineStagingURL, fireEngineURL } from "./scrape";@@ -182,6 +183,11 @@ export async function fireEngineCheckStatus(} else {throw new SiteError(code);}+ } else if (+ typeof status.error === "string" &&+ status.error.includes("proxies available for")+ ) {+ throw new ProxySelectionError();} else if (typeof status.error === "string" &&status.error.includes("Dns resolution error for hostname: ")diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 17d527d4d..899ee2c6a 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -21,6 +21,7 @@ import {SSLError,UnsupportedFileError,FEPageLoadFailed,+ ProxySelectionError,} from "../../error";import * as Sentry from "@sentry/node";import { specialtyScrapeCheck } from "../utils/specialtyHandler";@@ -100,7 +101,8 @@ async function performFireEngineScrape<error instanceof DNSResolutionError ||error instanceof ActionError ||error instanceof UnsupportedFileError ||- error instanceof FEPageLoadFailed+ error instanceof FEPageLoadFailed ||+ error instanceof ProxySelectionError) {fireEngineDelete(logger.child({diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 565688ad7..7a3ea0b40 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -11,6 +11,7 @@ import {DNSResolutionError,EngineError,FEPageLoadFailed,+ ProxySelectionError,SSLError,SiteError,UnsupportedFileError,@@ -263,6 +264,11 @@ export async function fireEngineScrape<status.error.includes("Javascript execution failed"))) {throw new ActionError(status.error.split("Error: ")[1]);+ } else if (+ typeof status.error === "string" &&+ status.error.includes("proxies available for")+ ) {+ throw new ProxySelectionError();} else {throw new EngineError("Scrape job failed", {cause: {diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 1f07e0ae9..faae68f1d 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -113,6 +113,30 @@ export class SiteError extends TransportableError {}}+export class ProxySelectionError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_PROXY_SELECTION_ERROR",+ "Specified proxy cannot be selected. Change `location` or `proxy` in your scrape request.",+ );+ }++ serialize() {+ return {+ ...super.serialize(),+ };+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new ProxySelectionError();+ x.stack = data.stack;+ return x;+ }+}+export class ActionError extends TransportableError {constructor(public errorCode: string) {super(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 86076e0f6..688708a06 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -38,6 +38,7 @@ import {EngineSnipedError,WaterfallNextEngineSignal,EngineUnsuccessfulError,+ ProxySelectionError,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -528,7 +529,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ActionError ||error.error instanceof UnsupportedFileError ||error.error instanceof PDFAntibotError ||- error.error instanceof PDFInsufficientTimeError+ error.error instanceof PDFInsufficientTimeError ||+ error.error instanceof ProxySelectionError) {throw error.error;} else if (error.error instanceof LLMRefusalError) {@@ -864,6 +866,8 @@ export async function scrapeURL("scrapeURL: Failed to prefetch PDF that is protected by anti-bot",{ error },);+ } else if (error instanceof ProxySelectionError) {+ meta.logger.warn("scrapeURL: Proxy selection error", { error });} else if (error instanceof AbortManagerThrownError) {throw error.inner;} else {