Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2): error handling (#1957)

Gergő Móriczb32fa6fad5b3
diff --git a/apps/api/src/__tests__/e2e_map/v2_map.test.ts b/apps/api/src/__tests__/e2e_map/v2_map.test.ts
index 5bc842710..aed1103f9 100644
--- a/apps/api/src/__tests__/e2e_map/v2_map.test.ts
+++ b/apps/api/src/__tests__/e2e_map/v2_map.test.ts
@@ -118,7 +118,7 @@ describe("E2E Tests for Map V2 API Routes", () => {
expect(response.statusCode).toBe(408);
expect(response.body).toHaveProperty("success", false);
- expect(response.body).toHaveProperty("error", "Request timed out");
+ expect(response.body).toHaveProperty("error", "Map timed out");
},
10000,
);
diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
index 60a752bfd..efc3c731c 100644
--- a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
+++ b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts
@@ -278,7 +278,7 @@ describe("E2E Tests for v1 API Routes", () => {
if (!("error" in response.body)) {
throw new Error("Expected response body to have 'error' property");
}
- expect(response.body.error).toBe("Request timed out");
+ expect(response.body.error).toContain("Scrape timed out");
expect(response.body.success).toBe(false);
},
30000,
diff --git a/apps/api/src/__tests__/snips/v1/map.test.ts b/apps/api/src/__tests__/snips/v1/map.test.ts
index 304c9dd39..2cea25595 100644
--- a/apps/api/src/__tests__/snips/v1/map.test.ts
+++ b/apps/api/src/__tests__/snips/v1/map.test.ts
@@ -27,7 +27,7 @@ describe("Map tests", () => {
expect(response.statusCode).toBe(408);
expect(response.body.success).toBe(false);
- expect(response.body.error).toBe("Request timed out");
+ expect(response.body.error).toBe("Map timed out");
}, 10000);
it.concurrent("handles query parameters correctly", async () => {
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index 23c031df7..562717cc8 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -27,7 +27,7 @@ describe("Map tests", () => {
expect(response.statusCode).toBe(408);
expect(response.body.success).toBe(false);
- expect(response.body.error).toBe("Request timed out");
+ expect(response.body.error).toBe("Map timed out");
}, 10000);
it.concurrent("handles query parameters correctly", async () => {
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index d3425dff3..122b3ca34 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -33,6 +33,7 @@ import { Document as V0Document } from "./../../lib/entities";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { fromV0Combo } from "../v2/types";
+import { ScrapeJobTimeoutError } from "../../lib/error";
export async function scrapeHelper(
jobId: string,
@@ -97,13 +98,10 @@ export async function scrapeHelper(
try {
doc = await waitForJob(jobId, timeout);
} catch (e) {
- if (
- e instanceof Error &&
- (e.message.startsWith("Job wait") || e.message === "timeout")
- ) {
+ if (e instanceof ScrapeJobTimeoutError) {
return {
success: false,
- error: "Request timed out",
+ error: e.message,
returnCode: 408,
};
} else if (
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index a96220883..04200ba9e 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -26,6 +26,7 @@ import {
} from "../v1/types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { fromV0Combo } from "../v2/types";
+import { ScrapeJobTimeoutError } from "../../lib/error";
export async function searchHelper(
jobId: string,
@@ -234,11 +235,8 @@ export async function searchController(req: Request, res: Response) {
});
return res.status(result.returnCode).json(result);
} catch (error) {
- if (
- error instanceof Error &&
- (error.message.startsWith("Job wait") || error.message === "timeout")
- ) {
- return res.status(408).json({ error: "Request timed out" });
+ if (error instanceof ScrapeJobTimeoutError) {
+ return res.status(408).json({ error: error.message });
}
Sentry.captureException(error);
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 0405a2571..ee48fed68 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -5,7 +5,6 @@ import {
mapRequestSchema,
RequestWithAuth,
TeamFlags,
- TimeoutSignal,
} from "./types";
import { scrapeOptions } from "../v2/types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
@@ -24,6 +23,7 @@ import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
import Redis from "ioredis";
import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel } from "../../services/index";
+import { MapTimeoutError } from "../../lib/error";
configDotenv();
const redis = new Redis(process.env.REDIS_URL!);
@@ -353,16 +353,17 @@ export async function mapController(
}),
...(req.body.timeout !== undefined ? [
new Promise((resolve, reject) => setTimeout(() => {
- abort.abort(new TimeoutSignal());
- reject(new TimeoutSignal());
+ abort.abort(new MapTimeoutError());
+ reject(new MapTimeoutError());
}, req.body.timeout))
] : []),
]) as any;
} catch (error) {
- if (error instanceof TimeoutSignal || error === "timeout") {
+ if (error instanceof MapTimeoutError) {
return res.status(408).json({
success: false,
- error: "Request timed out",
+ code: error.code,
+ error: error.message,
});
} else {
throw error;
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index d1c92fc86..b5b82ca43 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -12,6 +12,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import { getScrapeQueue } from "../../services/queue-service";
import { fromV1ScrapeOptions } from "../v2/types";
+import { TransportableError } from "../../lib/error";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -107,17 +108,16 @@ export async function scrapeController(
await getScrapeQueue().remove(jobId);
}
- if (
- e instanceof Error &&
- (e.message.startsWith("Job wait") || e.message === "timeout")
- ) {
- return res.status(408).json({
+ if (e instanceof TransportableError) {
+ return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({
success: false,
- error: "Request timed out",
+ code: e.code,
+ error: e.message,
});
} else {
return res.status(500).json({
success: false,
+ code: "UNKNOWN_ERROR",
error: `(Internal server error) - ${e && e.message ? e.message : e}`,
});
}
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index c6c714288..e0a2340e1 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -26,6 +26,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
+import { ScrapeJobTimeoutError } from "../../lib/error";
interface DocumentWithCostTracking {
document: Document;
@@ -397,13 +398,11 @@ export async function searchController(
return res.status(200).json(responseData);
} catch (error) {
- if (
- error instanceof Error &&
- (error.message.startsWith("Job wait") || error.message === "timeout")
- ) {
+ if (error instanceof ScrapeJobTimeoutError) {
return res.status(408).json({
success: false,
- error: "Request timed out",
+ code: error.code,
+ error: error.message,
});
}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 59fc89a7a..05c01156d 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -10,6 +10,8 @@ import {
} from "../../lib/entities";
import { InternalOptions } from "../../scraper/scrapeURL";
import { getURLDepth } from "../../scraper/WebScraper/utils/maxDepthUtils";
+import Ajv from "ajv";
+import { ErrorCodes } from "../../lib/error";
export enum IntegrationEnum {
DIFY = "dify",
@@ -535,9 +537,6 @@ export type ScrapeOptions = BaseScrapeOptions & {
},
};
-import Ajv from "ajv";
-import type { CostTracking } from "../../lib/extract/extraction-service";
-
const ajv = new Ajv();
export const extractV1Options = z
@@ -921,6 +920,7 @@ export type Document = {
export type ErrorResponse = {
success: false;
+ code?: ErrorCodes;
error: string;
details?: any;
};
@@ -1441,10 +1441,3 @@ export const generateLLMsTextRequestSchema = z.object({
export type GenerateLLMsTextRequest = z.infer<
typeof generateLLMsTextRequestSchema
>;
-
-export class TimeoutSignal extends Error {
- constructor() {
- super("Operation timed out");
- this.name = "TimeoutSignal";
- }
-}
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index dec7485e6..ff0a66659 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -23,6 +23,7 @@ import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
+import { ScrapeJobTimeoutError } from "../../lib/error";
interface DocumentWithCostTracking {
document: Document;
@@ -349,13 +350,11 @@ export async function x402SearchController(
return res.status(200).json(responseData);
} catch (error) {
- if (
- error instanceof Error &&
- (error.message.startsWith("Job wait") || error.message === "timeout")
- ) {
+ if (error instanceof ScrapeJobTimeoutError) {
return res.status(408).json({
success: false,
- error: "Request timed out",
+ code: error.code,
+ error: error.message,
});
}
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 4199f910e..9e5295109 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -1,6 +1,7 @@
import { Response } from "express";
import { z } from "zod";
import {
+ ErrorResponse,
RequestWithAuth,
crawlRequestSchema,
} from "./types";
@@ -18,7 +19,7 @@ const crawlParamsPreviewRequestSchema = z.object({
type CrawlParamsPreviewRequest = z.infer<typeof crawlParamsPreviewRequestSchema>;
type CrawlParamsPreviewResponse = {
- success: boolean;
+ success: true;
data?: {
url: string;
includePaths?: string[];
@@ -34,8 +35,7 @@ type CrawlParamsPreviewResponse = {
delay?: number;
limit?: number;
};
- error?: string;
-};
+} | ErrorResponse;
export async function crawlParamsPreviewController(
req: RequestWithAuth<{}, CrawlParamsPreviewResponse, CrawlParamsPreviewRequest>,
@@ -94,6 +94,7 @@ export async function crawlParamsPreviewController(
if (error instanceof z.ZodError) {
return res.status(400).json({
success: false,
+ code: "BAD_REQUEST",
error: "Invalid request parameters: " + error.errors.map(e => e.message).join(", "),
});
}
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 87d2577b0..7581854b4 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -5,7 +5,6 @@ import {
RequestWithAuth,
scrapeOptions,
TeamFlags,
- TimeoutSignal,
MapRequest,
MapDocument,
MapResponse,
@@ -24,6 +23,7 @@ import { logger } from "../../lib/logger";
import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";
import { redisEvictConnection } from "../../services/redis";
import { performCosineSimilarityV2 } from "../../lib/map-cosine";
+import { MapTimeoutError } from "../../lib/error";
configDotenv();
@@ -340,16 +340,17 @@ export async function mapController(
}),
...(req.body.timeout !== undefined ? [
new Promise((resolve, reject) => setTimeout(() => {
- abort.abort(new TimeoutSignal());
- reject(new TimeoutSignal());
+ abort.abort(new MapTimeoutError());
+ reject(new MapTimeoutError());
}, req.body.timeout))
] : []),
]) as any;
} catch (error) {
- if (error instanceof TimeoutSignal || error === "timeout") {
+ if (error instanceof MapTimeoutError) {
return res.status(408).json({
success: false,
- error: "Request timed out",
+ code: error.code,
+ error: error.message,
});
} else {
throw error;
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index df7cf2220..b34fe5a31 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -12,6 +12,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import { getScrapeQueue } from "../../services/queue-service";
import { hasFormatOfType } from "../../lib/format-utils";
+import { TransportableError } from "../../lib/error";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -101,19 +102,18 @@ export async function scrapeController(
} catch (e) {
logger.error(`Error in scrapeController`, {
startTime,
+ error: e,
});
if (zeroDataRetention) {
await getScrapeQueue().remove(jobId);
}
- if (
- e instanceof Error &&
- (e.message.startsWith("Job wait") || e.message === "timeout")
- ) {
- return res.status(408).json({
+ if (e instanceof TransportableError) {
+ return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({
success: false,
- error: "Request timed out",
+ code: e.code,
+ error: e.message,
});
} else {
return res.status(500).json({
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 90ddf4407..158ff149d 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -25,6 +25,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
import { SearchResult, SearchV2Response } from "../../lib/entities";
+import { ScrapeJobTimeoutError } from "../../lib/error";
interface DocumentWithCostTracking {
document: Document;
@@ -425,13 +426,11 @@ export async function searchController(
data: searchResponse,
});
} catch (error) {
- if (
- error instanceof Error &&
- (error.message.startsWith("Job wait") || error.message === "timeout")
- ) {
+ if (error instanceof ScrapeJobTimeoutError) {
return res.status(408).json({
success: false,
- error: "Request timed out",
+ code: error.code,
+ error: error.message,
});
}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 3f833ccd0..9068eda7a 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -11,6 +11,8 @@ import {
} from "../../lib/entities";
import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";
import { InternalOptions } from "../../scraper/scrapeURL";
+import { ErrorCodes } from "../../lib/error";
+import Ajv from "ajv";
export enum IntegrationEnum {
DIFY = "dify",
@@ -386,9 +388,6 @@ export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;
export type ScrapeOptions = BaseScrapeOptions;
-import Ajv from "ajv";
-import type { CostTracking } from "../../lib/extract/extraction-service";
-
const ajv = new Ajv();
export const extractOptions = z
@@ -727,6 +726,7 @@ export type Document = {
export type ErrorResponse = {
success: false;
+ code?: ErrorCodes;
error: string;
details?: any;
};
@@ -1438,9 +1438,3 @@ export const generateLLMsTextRequestSchema = z.object({
export type GenerateLLMsTextRequest = z.infer<
typeof generateLLMsTextRequestSchema
>;
-
-export class TimeoutSignal extends Error {
- constructor() {
- super("Operation timed out");
- }
-}
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 9ebf1a04f..bee6925b5 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -230,7 +230,7 @@ app.use(
res
.status(400)
- .json({ success: false, error: "Bad Request", details: err.errors });
+ .json({ success: false, code: "BAD_REQUEST", error: "Bad Request", details: err.errors });
} else {
next(err);
}
@@ -254,7 +254,7 @@ app.use(
) {
return res
.status(400)
- .json({ success: false, error: "Bad request, malformed JSON" });
+ .json({ success: false, code: "BAD_REQUEST_INVALID_JSON", error: "Bad request, malformed JSON" });
}
const id = res.sentry ?? uuidv4();
@@ -268,6 +268,7 @@ app.use(
{ error: err, errorId: id, path: req.path, teamId: req.acuc?.team_id, team_id: req.acuc?.team_id });
res.status(500).json({
success: false,
+ code: "UNKNOWN_ERROR",
error:
"An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +
id,
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
new file mode 100644
index 000000000..13468e84a
--- /dev/null
+++ b/apps/api/src/lib/error-serde.ts
@@ -0,0 +1,36 @@
+import { ErrorCodes, MapTimeoutError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";
+import { ActionError, DNSResolutionError, UnsupportedFileError, PDFAntibotError, PDFInsufficientTimeError, NoEnginesLeftError, ZDRViolationError, PDFPrefetchFailed, SiteError, SSLError } from "../scraper/scrapeURL/error";
+
+// TODO: figure out correct typing for this
+const errorMap: Record<ErrorCodes, any> = {
+ "SCRAPE_TIMEOUT": ScrapeJobTimeoutError,
+ "MAP_TIMEOUT": MapTimeoutError,
+ "UNKNOWN_ERROR": UnknownError,
+ "SCRAPE_ALL_ENGINES_FAILED": NoEnginesLeftError,
+ "SCRAPE_SSL_ERROR": SSLError,
+ "SCRAPE_SITE_ERROR": SiteError,
+ "SCRAPE_PDF_PREFETCH_FAILED": PDFPrefetchFailed,
+ "SCRAPE_ZDR_VIOLATION_ERROR": ZDRViolationError,
+ "SCRAPE_DNS_RESOLUTION_ERROR": DNSResolutionError,
+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR": PDFInsufficientTimeError,
+ "SCRAPE_PDF_ANTIBOT_ERROR": PDFAntibotError,
+ "SCRAPE_UNSUPPORTED_FILE_ERROR": UnsupportedFileError,
+ "SCRAPE_ACTION_ERROR": ActionError,
+
+ // Zod errors
+ "BAD_REQUEST": null,
+ "BAD_REQUEST_INVALID_JSON": null,
+};
+
+export function serializeTransportableError(error: TransportableError) {
+ return `${error.code}|${JSON.stringify(error.serialize())}`;
+}
+
+export function deserializeTransportableError(data: string): InstanceType<typeof errorMap[keyof typeof errorMap]> | null {
+ const [code, ...serialized] = data.split("|");
+ const x = errorMap[code];
+ if (!x) {
+ return null;
+ }
+ return x.deserialize(code, JSON.parse(serialized.join("|")));
+}
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
new file mode 100644
index 000000000..068ed2011
--- /dev/null
+++ b/apps/api/src/lib/error.ts
@@ -0,0 +1,92 @@
+export type ErrorCodes =
+ "SCRAPE_TIMEOUT" |
+ "MAP_TIMEOUT" |
+ "UNKNOWN_ERROR" |
+ "SCRAPE_ALL_ENGINES_FAILED" |
+ "SCRAPE_SSL_ERROR" |
+ "SCRAPE_SITE_ERROR" |
+ "SCRAPE_PDF_PREFETCH_FAILED" |
+ "SCRAPE_ZDR_VIOLATION_ERROR" |
+ "SCRAPE_DNS_RESOLUTION_ERROR" |
+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR" |
+ "SCRAPE_PDF_ANTIBOT_ERROR" |
+ "SCRAPE_UNSUPPORTED_FILE_ERROR" |
+ "SCRAPE_ACTION_ERROR" |
+ "BAD_REQUEST_INVALID_JSON" |
+ "BAD_REQUEST";
+
+export class TransportableError extends Error {
+ public readonly code: ErrorCodes;
+
+ constructor(code: ErrorCodes, message?: string, options?: ErrorOptions) {
+ super(message, options);
+ this.code = code;
+ }
+
+ serialize() {
+ return {
+ cause: this.cause,
+ stack: this.stack,
+ message: this.message,
+ }
+ }
+
+ static deserialize(code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new TransportableError(code, data.message, { cause: data.cause });
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export class ScrapeJobTimeoutError extends TransportableError {
+ constructor(message: string) {
+ super("SCRAPE_TIMEOUT", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new ScrapeJobTimeoutError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export class UnknownError extends TransportableError {
+ constructor(inner: unknown) {
+ super("UNKNOWN_ERROR", `(Internal server error) - ${(inner && inner instanceof Error) ? inner.message : inner}`);
+
+ if (inner instanceof Error) {
+ this.stack = inner.stack;
+ }
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new UnknownError("dummy");
+ x.message = data.message;
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export class MapTimeoutError extends TransportableError {
+ constructor() {
+ super("MAP_TIMEOUT", "Map timed out");
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new MapTimeoutError();
+ x.stack = data.stack;
+ return x;
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 3672097d1..8bf82e8fe 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -10,9 +10,9 @@ import { logger as _logger } from "../../lib/logger";
import https from "https";
import { redisEvictConnection } from "../../services/redis";
import { extractLinks } from "../../lib/html-transformer";
-import { TimeoutSignal } from "../../controllers/v1/types";
import { filterLinks } from "../../lib/crawler";
import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";
+import { ScrapeJobTimeoutError } from "../../lib/error";
export interface FilterResult {
allowed: boolean;
@@ -755,7 +755,7 @@ export class WebCrawler {
mock,
);
} catch (error) {
- if (error instanceof TimeoutSignal) {
+ if (error instanceof ScrapeJobTimeoutError) {
throw error;
} else {
this.logger.debug(`Failed to fetch sitemap from ${sitemapUrl}`, {
@@ -805,7 +805,7 @@ export class WebCrawler {
mock,
);
} catch (error) {
- if (error instanceof TimeoutSignal) {
+ if (error instanceof ScrapeJobTimeoutError) {
throw error;
} else {
this.logger.debug(
@@ -816,7 +816,7 @@ export class WebCrawler {
}
}
} catch (error) {
- if (error instanceof TimeoutSignal) {
+ if (error instanceof ScrapeJobTimeoutError) {
throw error;
} else {
this.logger.debug(`Error processing main domain sitemap`, {
@@ -840,7 +840,7 @@ export class WebCrawler {
mock,
);
} catch (error) {
- if (error instanceof TimeoutSignal) {
+ if (error instanceof ScrapeJobTimeoutError) {
throw error;
} else {
this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index c4429431e..4971a6f13 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -1,10 +1,11 @@
import { parseStringPromise } from "xml2js";
import { WebCrawler } from "./crawler";
import { scrapeURL } from "../scrapeURL";
-import { scrapeOptions, TimeoutSignal } from "../../controllers/v2/types";
+import { scrapeOptions } from "../../controllers/v2/types";
import type { Logger } from "winston";
import { CostTracking } from "../../lib/extract/extraction-service";
import { parseSitemapXml, processSitemap } from "../../lib/crawler";
+import { ScrapeJobTimeoutError } from "../../lib/error";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
process.env.FIRE_ENGINE_BETA_URL !== undefined;
@@ -87,7 +88,7 @@ export async function getLinksFromSitemap(
return 0;
}
} catch (error) {
- if (error instanceof TimeoutSignal) {
+ if (error instanceof ScrapeJobTimeoutError) {
throw error;
} else {
logger.error(`Request failed for sitemap fetch`, {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index f63ddaaeb..818d9ab4d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -260,19 +260,6 @@ export async function scrapePDF(
error instanceof AbortManagerThrownError
) {
throw error;
- // } else if (
- // (error instanceof Error && error.name === "TimeoutError") ||
- // (error instanceof Error &&
- // error.message === "Request failed" &&
- // error.cause &&
- // error.cause instanceof Error &&
- // error.cause.name === "TimeoutError") ||
- // (error instanceof Error && error.name === "TimeoutSignal")
- // ) {
- // meta.logger.warn("RunPod MU timed out", { timeout: remainingTime });
- // throw new TimeoutError(
- // "PDF parsing timed out, please increase the timeout parameter in your scrape request",
- // );
}
meta.logger.warn(
"RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 295b208b0..10461f5ba 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -1,3 +1,4 @@
+import { ErrorCodes, TransportableError } from "../../lib/error";
import { Meta } from ".";
import { Engine, FeatureFlag } from "./engines";
@@ -7,15 +8,29 @@ export class EngineError extends Error {
}
}
-export class NoEnginesLeftError extends Error {
+export class NoEnginesLeftError extends TransportableError {
public fallbackList: Engine[];
constructor(fallbackList: Engine[]) {
super(
+ "SCRAPE_ALL_ENGINES_FAILED",
"All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].",
);
this.fallbackList = fallbackList;
}
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ fallbackList: this.fallbackList,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new NoEnginesLeftError(data.fallbackList);
+ x.stack = data.stack;
+ return x;
+ }
}
export class AddFeatureError extends Error {
@@ -41,58 +56,143 @@ export class RemoveFeatureError extends Error {
}
}
-export class SSLError extends Error {
- constructor(skipTlsVerification: boolean) {
+export class SSLError extends TransportableError {
+ constructor(public skipTlsVerification: boolean) {
super(
+ "SCRAPE_SSL_ERROR",
"An SSL error occurred while scraping the URL. "
+ (skipTlsVerification
? "Since you have `skipTlsVerification` enabled, this means that the TLS configuration of the target site is completely broken. Try scraping the plain HTTP version of the page."
: "If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`.")
);
}
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ skipTlsVerification: this.skipTlsVerification,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new SSLError(data.skipTlsVerification);
+ x.stack = data.stack;
+ return x;
+ }
}
-export class SiteError extends Error {
- public code: string;
- constructor(code: string) {
+export class SiteError extends TransportableError {
+ constructor(public errorCode: string) {
super(
- "Specified URL is failing to load in the browser. Error code: " + code,
+ "SCRAPE_SITE_ERROR",
+ "Specified URL is failing to load in the browser. Error code: " + errorCode,
);
- this.code = code;
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ errorCode: this.errorCode,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new SiteError(data.errorCode);
+ x.stack = data.stack;
+ return x;
}
}
-export class ActionError extends Error {
- public code: string;
- constructor(code: string) {
- super("Action(s) failed to complete. Error code: " + code);
- this.code = code;
+export class ActionError extends TransportableError {
+ constructor(public errorCode: string) {
+ super("SCRAPE_ACTION_ERROR", "Action(s) failed to complete. Error code: " + errorCode);
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ errorCode: this.errorCode,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new ActionError(data.errorCode);
+ x.stack = data.stack;
+ return x;
}
}
-export class UnsupportedFileError extends Error {
- public reason: string;
- constructor(reason: string) {
- super("Scrape resulted in unsupported file: " + reason);
- this.reason = reason;
+export class UnsupportedFileError extends TransportableError {
+ constructor(public reason: string) {
+ super("SCRAPE_UNSUPPORTED_FILE_ERROR", "Scrape resulted in unsupported file: " + reason);
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ reason: this.reason,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new UnsupportedFileError(data.reason);
+ x.stack = data.stack;
+ return x;
}
}
-export class PDFAntibotError extends Error {
+export class PDFAntibotError extends TransportableError {
constructor() {
- super("PDF scrape was prevented by anti-bot")
+ super("SCRAPE_PDF_ANTIBOT_ERROR", "PDF scrape was prevented by anti-bot")
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new PDFAntibotError();
+ x.stack = data.stack;
+ return x;
}
}
-export class PDFInsufficientTimeError extends Error {
- constructor(pageCount: number, minTimeout: number) {
- super(`Insufficient time to process PDF of ${pageCount} pages. Please increase the timeout parameter in your scrape request to at least ${minTimeout}ms.`);
+export class PDFInsufficientTimeError extends TransportableError {
+ constructor(public pageCount: number, public minTimeout: number) {
+ super("SCRAPE_PDF_INSUFFICIENT_TIME_ERROR", `Insufficient time to process PDF of ${pageCount} pages. Please increase the timeout parameter in your scrape request to at least ${minTimeout}ms.`);
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ pageCount: this.pageCount,
+ minTimeout: this.minTimeout,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new PDFInsufficientTimeError(data.pageCount, data.minTimeout);
+ x.stack = data.stack;
+ return x;
}
}
-export class DNSResolutionError extends Error {
- constructor(hostname: string) {
- super(`DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);
+export class DNSResolutionError extends TransportableError {
+ constructor(public hostname: string) {
+ super("SCRAPE_DNS_RESOLUTION_ERROR", `DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ hostname: this.hostname,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new DNSResolutionError(data.hostname);
+ x.stack = data.stack;
+ return x;
}
}
@@ -102,39 +202,48 @@ export class IndexMissError extends Error {
}
}
-export class ZDRViolationError extends Error {
- constructor(feature: string) {
- super(`${feature} is not supported when using zeroDataRetention. Please contact [email protected] to unblock this feature.`);
+export class ZDRViolationError extends TransportableError {
+ constructor(public feature: string) {
+ super("SCRAPE_ZDR_VIOLATION_ERROR", `${feature} is not supported when using zeroDataRetention. Please contact [email protected] to unblock this feature.`);
}
-}
-export class PDFPrefetchFailed extends Error {
- constructor() {
- super("Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");
+ serialize() {
+ return {
+ ...super.serialize(),
+ feature: this.feature,
+ }
+ }
+
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new ZDRViolationError(data.feature);
+ x.stack = data.stack;
+ return x;
}
}
-export class FEPageLoadFailed extends Error {
+export class PDFPrefetchFailed extends TransportableError {
constructor() {
- super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");
+ super("SCRAPE_PDF_PREFETCH_FAILED", "Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");
}
-}
-export class ScrapeURLScopedTimeoutError extends Error {
- constructor(message: string, options?: ErrorOptions) {
- super(message, options);
+ serialize() {
+ return super.serialize();
}
-}
-export class ScrapeTimeoutError extends ScrapeURLScopedTimeoutError {
- name = "ScrapeTimeoutError";
+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {
+ const x = new PDFPrefetchFailed();
+ x.stack = data.stack;
+ return x;
+ }
+}
+export class FEPageLoadFailed extends Error {
constructor() {
- super("Scrape timed out");
+ super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");
}
}
-export class EngineSnipedError extends ScrapeURLScopedTimeoutError {
+export class EngineSnipedError extends Error {
name = "EngineSnipedError";
constructor() {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index bb22a2875..30901a637 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,7 +1,7 @@
import { Logger } from "winston";
import * as Sentry from "@sentry/node";
-import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v2/types";
+import { Document, ScrapeOptions, TeamFlags } from "../../controllers/v2/types";
import { ScrapeOptions as ScrapeOptionsV1 } from "../../controllers/v1/types";
import { logger as _logger } from "../../lib/logger";
import {
@@ -29,7 +29,6 @@ import {
ZDRViolationError,
PDFPrefetchFailed,
FEPageLoadFailed,
- ScrapeTimeoutError,
EngineSnipedError,
WaterfallNextEngineSignal,
EngineUnsuccessfulError,
@@ -43,6 +42,7 @@ import { robustFetch } from "./lib/fetch";
import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
import { checkRobotsTxt } from "../../lib/robots-txt";
import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";
+import { ScrapeJobTimeoutError } from "../../lib/error";
export type ScrapeUrlResponse = (
| {
@@ -203,7 +203,7 @@ async function buildMetaObject(
tier: "scrape",
timesOutAt: new Date(Date.now() + options.timeout),
throwable() {
- return new ScrapeTimeoutError();
+ return new ScrapeJobTimeoutError("Scrape timed out");
},
} : undefined,
),
@@ -413,7 +413,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
setTimeout(() => {
try {
meta.abort.throwIfAborted();
- meta.logger.warn("throwIfAborted did not throw");
+
+ // Fallback error if above doesn't throw
+ throw new ScrapeJobTimeoutError("Scrape timed out due to maximum length of 5 minutes");
} catch (error) {
reject(error);
}
@@ -496,18 +498,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
snipeAbortController.abort();
if (result === null) {
- // TODO: rectify TODOv2
- // if (meta.results["pdf"]?.state === "timeout") {
- // throw meta.results["pdf"].error ?? new TimeoutSignal();
- // }
- // if (Object.values(meta.results).every(x => x.state === "timeout")) {
- // throw new TimeoutSignal();
- // } else {
- throw new NoEnginesLeftError(
- fallbackList.map((x) => x.engine),
- // meta.results,
- );
- // }
+ throw new NoEnginesLeftError(
+ fallbackList.map((x) => x.engine),
+ );
}
let document: Document = {
@@ -769,8 +762,6 @@ export async function scrapeURL(
meta.logger.warn("scrapeURL: Insufficient time to process PDF", { error });
} else if (error instanceof PDFPrefetchFailed) {
meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });
- } else if (error instanceof TimeoutSignal) {
- throw error;
} else if (error instanceof AbortManagerThrownError) {
throw error.inner;
} else {
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index 84e18db7e..9c5798b5d 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -2,12 +2,11 @@ import { Logger } from "winston";
import { z, ZodError } from "zod";
import * as Sentry from "@sentry/node";
import { MockState, saveMock } from "./mock";
-import { TimeoutSignal } from "../../../controllers/v1/types";
import { fireEngineURL } from "../engines/fire-engine/scrape";
-import { fetch, RequestInit, Response, FormData, Agent } from "undici";
+import { fetch, Response, FormData, Agent } from "undici";
import { cacheableLookup } from "./cacheableLookup";
-import { log } from "console";
import dns from "dns";
+import { AbortManagerThrownError } from "./abortManager";
export type RobustFetchParams<Schema extends z.Schema<any>> = {
url: string;
@@ -125,8 +124,8 @@ export async function robustFetch<
: {}),
});
} catch (error) {
- if (error instanceof TimeoutSignal || (error instanceof Error && error.name === "TimeoutError") || (error instanceof Error && error.message === "Operation timed out")) {
- throw new TimeoutSignal();
+ if (error instanceof AbortManagerThrownError) {
+ throw error;
} else if (!ignoreFailure) {
Sentry.captureException(error);
if (tryCount > 1) {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 47fd4e0a5..a4a5ab922 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -20,6 +20,8 @@ import { Document } from "../controllers/v1/types";
import { getCrawl } from "../lib/crawl-redis";
import { Logger } from "winston";
import { Job } from "bullmq";
+import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";
+import { deserializeTransportableError } from "../lib/error-serde";
/**
* Checks if a job is a crawl or batch scrape based on its options
@@ -327,17 +329,33 @@ export async function waitForJob(
await new Promise(resolve => setTimeout(resolve, 500));
job = await queue.getJob(_job as string);
if (Date.now() - start > (timeout ?? 180000)) {
- throw new Error("Job wait (concurrency limited)");
+ throw new ScrapeJobTimeoutError("Scrape timed out while waiting in the concurrency limit queue");
+ }
+ }
+ let doc: Document;
+ try {
+ doc = await Promise.race([
+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
+ timeout !== null ? new Promise((resolve, reject) => {
+ setTimeout(() => {
+ reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));
+ }, Math.max(0, timeout - (Date.now() - start)));
+ }) : null,
+ ].filter(x => x !== null));
+ } catch (e) {
+ if (e instanceof TransportableError) {
+ throw e;
+ } else if (e instanceof Error) {
+ const x = deserializeTransportableError(e.message);
+ if (x) {
+ throw x;
+ } else {
+ throw e;
+ }
+ } else {
+ throw e;
}
}
- let doc: Document = await Promise.race([
- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
- timeout !== null ? new Promise((resolve, reject) => {
- setTimeout(() => {
- reject(new Error("Job wait " + (typeof _job == "string" ? "(was concurrency limited)" : "")));
- }, Math.max(0, timeout - (Date.now() - start)));
- }) : null,
- ].filter(x => x !== null));
logger.debug("Got job");
if (!doc) {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e25d85632..a761ba6b2 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -49,6 +49,8 @@ import { finishCrawlIfNeeded } from "./crawl-logic";
import { LangfuseExporter } from "langfuse-vercel";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { NodeSDK } from "@opentelemetry/sdk-node";
+import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";
+import { serializeTransportableError } from "../../lib/error-serde";
class RacedRedirectError extends Error {
constructor() {
@@ -145,7 +147,7 @@ async function processJob(job: Job & { id: string }) {
});
if (remainingTime !== undefined && remainingTime < 0) {
- throw new Error("timeout");
+ throw new ScrapeJobTimeoutError("Scrape timed out");
}
const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;
@@ -166,7 +168,7 @@ async function processJob(job: Job & { id: string }) {
? [
(async () => {
await sleep(remainingTime);
- throw new Error("timeout");
+ throw new ScrapeJobTimeoutError("Scrape timed out");
})(),
]
: []),
@@ -175,7 +177,7 @@ async function processJob(job: Job & { id: string }) {
try {
signal?.throwIfAborted();
} catch (e) {
- throw new Error("timeout");
+ throw new ScrapeJobTimeoutError("Scrape timed out");
}
if (!pipeline.success) {
@@ -370,7 +372,7 @@ async function processJob(job: Job & { id: string }) {
try {
signal?.throwIfAborted();
} catch (e) {
- throw new Error("timeout");
+ throw new ScrapeJobTimeoutError("Scrape timed out");
}
const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
@@ -426,7 +428,7 @@ async function processJob(job: Job & { id: string }) {
try {
signal?.throwIfAborted();
} catch (e) {
- throw new Error("timeout");
+ throw new ScrapeJobTimeoutError("Scrape timed out");
}
const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);
@@ -472,7 +474,7 @@ async function processJob(job: Job & { id: string }) {
}
const isEarlyTimeout =
- error instanceof Error && error.message === "timeout";
+ error instanceof ScrapeJobTimeoutError;
const isCancelled =
error instanceof Error &&
error.message === "Parent crawl/batch scrape was cancelled";
@@ -846,7 +848,11 @@ export const processJobInternal = async (job: Job & { id: string }) => {
} catch (error) {
logger.debug("Job failed", { error });
Sentry.captureException(error);
- throw error;
+ if (error instanceof TransportableError) {
+ throw new Error(serializeTransportableError(error));
+ } else {
+ throw new Error(serializeTransportableError(new UnknownError(error)));
+ }
}
};