Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): hide "Raced redirect error" from users (#2030)

Gergő Móricz61e38dc893a5
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 45084c741..baf531832 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -61,6 +61,9 @@ export async function crawlErrorsController(
res.status(200).json({
errors: (await getJobs(failedJobIDs)).map((x) => {
const error = deserializeTransportableError(x.failedReason) as TransportableError | null;
+ if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {
+ return null;
+ }
return {
id: x.id,
timestamp:
@@ -75,7 +78,7 @@ export async function crawlErrorsController(
error: x.failedReason,
}),
};
- }),
+ }).filter((x) => x !== null),
robotsBlocked: await redisEvictConnection.smembers(
"crawl:" + req.params.jobId + ":robots_blocked",
),
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 45084c741..baf531832 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -61,6 +61,9 @@ export async function crawlErrorsController(
res.status(200).json({
errors: (await getJobs(failedJobIDs)).map((x) => {
const error = deserializeTransportableError(x.failedReason) as TransportableError | null;
+ if (error?.code === "SCRAPE_RACED_REDIRECT_ERROR") {
+ return null;
+ }
return {
id: x.id,
timestamp:
@@ -75,7 +78,7 @@ export async function crawlErrorsController(
error: x.failedReason,
}),
};
- }),
+ }).filter((x) => x !== null),
robotsBlocked: await redisEvictConnection.smembers(
"crawl:" + req.params.jobId + ":robots_blocked",
),
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 13468e84a..ddb40fe18 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -1,4 +1,4 @@
-import { ErrorCodes, MapTimeoutError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";
+import { ErrorCodes, MapTimeoutError, RacedRedirectError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";
import { ActionError, DNSResolutionError, UnsupportedFileError, PDFAntibotError, PDFInsufficientTimeError, NoEnginesLeftError, ZDRViolationError, PDFPrefetchFailed, SiteError, SSLError } from "../scraper/scrapeURL/error";
// TODO: figure out correct typing for this
@@ -16,6 +16,7 @@ const errorMap: Record<ErrorCodes, any> = {
"SCRAPE_PDF_ANTIBOT_ERROR": PDFAntibotError,
"SCRAPE_UNSUPPORTED_FILE_ERROR": UnsupportedFileError,
"SCRAPE_ACTION_ERROR": ActionError,
+ "SCRAPE_RACED_REDIRECT_ERROR": RacedRedirectError,
// Zod errors
"BAD_REQUEST": null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 068ed2011..bb94cead6 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -12,6 +12,7 @@ export type ErrorCodes =
"SCRAPE_PDF_ANTIBOT_ERROR" |
"SCRAPE_UNSUPPORTED_FILE_ERROR" |
"SCRAPE_ACTION_ERROR" |
+ "SCRAPE_RACED_REDIRECT_ERROR" |
"BAD_REQUEST_INVALID_JSON" |
"BAD_REQUEST";
@@ -89,4 +90,20 @@ export class MapTimeoutError extends TransportableError {
x.stack = data.stack;
return x;
}
-}
\ No newline at end of file
+}
+
+export class RacedRedirectError extends TransportableError {
+ constructor() {
+ super("SCRAPE_RACED_REDIRECT_ERROR", "Raced redirect error");
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new RacedRedirectError();
+ x.stack = data.stack;
+ return x;
+ }
+}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index ae6299a82..7b0e3bac6 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -49,19 +49,13 @@ import { finishCrawlIfNeeded } from "./crawl-logic";
import { LangfuseExporter } from "langfuse-vercel";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { NodeSDK } from "@opentelemetry/sdk-node";
-import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";
+import { RacedRedirectError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";
import { serializeTransportableError } from "../../lib/error-serde";
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-class RacedRedirectError extends Error {
- constructor() {
- super("Raced redirect error");
- }
-}
-
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
configDotenv();