Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): hide "Raced redirect error" from users (#2030)
Gergő Móricz61e38dc893a5
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 45084c741..baf531832 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -61,6 +61,9 @@ export async function crawlErrorsController(res.status(200).json({errors: (await getJobs(failedJobIDs)).map((x) => {const error = deserializeTransportableError(x.failedReason) as TransportableError | null;+ if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {+ return null;+ }return {id: x.id,timestamp:@@ -75,7 +78,7 @@ export async function crawlErrorsController(error: x.failedReason,}),};- }),+ }).filter((x) => x !== null),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 45084c741..baf531832 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -61,6 +61,9 @@ export async function crawlErrorsController(res.status(200).json({errors: (await getJobs(failedJobIDs)).map((x) => {const error = deserializeTransportableError(x.failedReason) as TransportableError | null;+ if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {+ return null;+ }return {id: x.id,timestamp:@@ -75,7 +78,7 @@ export async function crawlErrorsController(error: x.failedReason,}),};- }),+ }).filter((x) => x !== null),robotsBlocked: await redisEvictConnection.smembers("crawl:" + req.params.jobId + ":robots_blocked",),diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 13468e84a..ddb40fe18 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -1,4 +1,4 @@-import { ErrorCodes, MapTimeoutError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";+import { ErrorCodes, MapTimeoutError, RacedRedirectError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";import { ActionError, DNSResolutionError, UnsupportedFileError, PDFAntibotError, PDFInsufficientTimeError, NoEnginesLeftError, ZDRViolationError, PDFPrefetchFailed, SiteError, SSLError } from "../scraper/scrapeURL/error";// TODO: figure out correct typing for this@@ -16,6 +16,7 @@ const errorMap: Record<ErrorCodes, any> = {"SCRAPE_PDF_ANTIBOT_ERROR": PDFAntibotError,"SCRAPE_UNSUPPORTED_FILE_ERROR": UnsupportedFileError,"SCRAPE_ACTION_ERROR": ActionError,+ "SCRAPE_RACED_REDIRECT_ERROR": RacedRedirectError,// Zod errors"BAD_REQUEST": null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 068ed2011..bb94cead6 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -12,6 +12,7 @@ export type ErrorCodes ="SCRAPE_PDF_ANTIBOT_ERROR" |"SCRAPE_UNSUPPORTED_FILE_ERROR" |"SCRAPE_ACTION_ERROR" |+ "SCRAPE_RACED_REDIRECT_ERROR" |"BAD_REQUEST_INVALID_JSON" |"BAD_REQUEST";@@ -89,4 +90,20 @@ export class MapTimeoutError extends TransportableError {x.stack = data.stack;return x;}-}\ No newline at end of file+}++export class RacedRedirectError extends TransportableError {+ constructor() {+ super("SCRAPE_RACED_REDIRECT_ERROR", "Raced redirect error");+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new RacedRedirectError();+ x.stack = data.stack;+ return x;+ }+}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex ae6299a82..7b0e3bac6 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -49,19 +49,13 @@ import { finishCrawlIfNeeded } from "./crawl-logic";import { LangfuseExporter } from "langfuse-vercel";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";import { NodeSDK } from "@opentelemetry/sdk-node";-import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";+import { RacedRedirectError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";import { serializeTransportableError } from "../../lib/error-serde";import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { resourceFromAttributes } from "@opentelemetry/resources";import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";-class RacedRedirectError extends Error {- constructor() {- super("Raced redirect error");- }-}-const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));configDotenv();