Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2): error handling (#1957)
Gergő Móriczb32fa6fad5b3
diff --git a/apps/api/src/__tests__/e2e_map/v2_map.test.ts b/apps/api/src/__tests__/e2e_map/v2_map.test.tsindex 5bc842710..aed1103f9 100644--- a/apps/api/src/__tests__/e2e_map/v2_map.test.ts+++ b/apps/api/src/__tests__/e2e_map/v2_map.test.ts@@ -118,7 +118,7 @@ describe("E2E Tests for Map V2 API Routes", () => {expect(response.statusCode).toBe(408);expect(response.body).toHaveProperty("success", false);- expect(response.body).toHaveProperty("error", "Request timed out");+ expect(response.body).toHaveProperty("error", "Map timed out");},10000,);diff --git a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.tsindex 60a752bfd..efc3c731c 100644--- a/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts+++ b/apps/api/src/__tests__/e2e_v1_withAuth_all_params/index.test.ts@@ -278,7 +278,7 @@ describe("E2E Tests for v1 API Routes", () => {if (!("error" in response.body)) {throw new Error("Expected response body to have 'error' property");}- expect(response.body.error).toBe("Request timed out");+ expect(response.body.error).toContain("Scrape timed out");expect(response.body.success).toBe(false);},30000,diff --git a/apps/api/src/__tests__/snips/v1/map.test.ts b/apps/api/src/__tests__/snips/v1/map.test.tsindex 304c9dd39..2cea25595 100644--- a/apps/api/src/__tests__/snips/v1/map.test.ts+++ b/apps/api/src/__tests__/snips/v1/map.test.ts@@ -27,7 +27,7 @@ describe("Map tests", () => {expect(response.statusCode).toBe(408);expect(response.body.success).toBe(false);- expect(response.body.error).toBe("Request timed out");+ expect(response.body.error).toBe("Map timed out");}, 10000);it.concurrent("handles query parameters correctly", async () => {diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex 23c031df7..562717cc8 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -27,7 +27,7 @@ describe("Map tests", () => {expect(response.statusCode).toBe(408);expect(response.body.success).toBe(false);- expect(response.body.error).toBe("Request timed out");+ expect(response.body.error).toBe("Map timed out");}, 10000);it.concurrent("handles query parameters correctly", async () => {diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex d3425dff3..122b3ca34 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -33,6 +33,7 @@ import { Document as V0Document } from "./../../lib/entities";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { getJobFromGCS } from "../../lib/gcs-jobs";import { fromV0Combo } from "../v2/types";+import { ScrapeJobTimeoutError } from "../../lib/error";export async function scrapeHelper(jobId: string,@@ -97,13 +98,10 @@ export async function scrapeHelper(try {doc = await waitForJob(jobId, timeout);} catch (e) {- if (- e instanceof Error &&- (e.message.startsWith("Job wait") || e.message === "timeout")- ) {+ if (e instanceof ScrapeJobTimeoutError) {return {success: false,- error: "Request timed out",+ error: e.message,returnCode: 408,};} else if (diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex a96220883..04200ba9e 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -26,6 +26,7 @@ import {} from "../v1/types";import { getJobFromGCS } from "../../lib/gcs-jobs";import { fromV0Combo } from "../v2/types";+import { ScrapeJobTimeoutError } from "../../lib/error";export async function searchHelper(jobId: string,@@ -234,11 +235,8 @@ export async function searchController(req: Request, res: Response) {});return res.status(result.returnCode).json(result);} catch (error) {- if (- error instanceof Error &&- (error.message.startsWith("Job wait") || error.message === "timeout")- ) {- return res.status(408).json({ error: "Request timed out" });+ if (error instanceof ScrapeJobTimeoutError) {+ return res.status(408).json({ error: error.message });}Sentry.captureException(error);diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 0405a2571..ee48fed68 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -5,7 +5,6 @@ import {mapRequestSchema,RequestWithAuth,TeamFlags,- TimeoutSignal,} from "./types";import { scrapeOptions } from "../v2/types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";@@ -24,6 +23,7 @@ import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";import Redis from "ioredis";import { generateURLSplits, queryIndexAtDomainSplitLevel, queryIndexAtSplitLevel } from "../../services/index";+import { MapTimeoutError } from "../../lib/error";configDotenv();const redis = new Redis(process.env.REDIS_URL!);@@ -353,16 +353,17 @@ export async function mapController(}),...(req.body.timeout !== undefined ? [new Promise((resolve, reject) => setTimeout(() => {- abort.abort(new TimeoutSignal());- reject(new TimeoutSignal());+ abort.abort(new MapTimeoutError());+ reject(new MapTimeoutError());}, req.body.timeout))] : []),]) as any;} catch (error) {- if (error instanceof TimeoutSignal || error === "timeout") {+ if (error instanceof MapTimeoutError) {return res.status(408).json({success: false,- error: "Request timed out",+ code: error.code,+ error: error.message,});} else {throw error;diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex d1c92fc86..b5b82ca43 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -12,6 +12,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";import { fromV1ScrapeOptions } from "../v2/types";+import { TransportableError } from "../../lib/error";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -107,17 +108,16 @@ export async function scrapeController(await getScrapeQueue().remove(jobId);}- if (- e instanceof Error &&- (e.message.startsWith("Job wait") || e.message === "timeout")- ) {- return res.status(408).json({+ if (e instanceof TransportableError) {+ return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({success: false,- error: "Request timed out",+ code: e.code,+ error: e.message,});} else {return res.status(500).json({success: false,+ code: "UNKNOWN_ERROR",error: `(Internal server error) - ${e && e.message ? e.message : e}`,});}diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex c6c714288..e0a2340e1 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -26,6 +26,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";+import { ScrapeJobTimeoutError } from "../../lib/error";interface DocumentWithCostTracking {document: Document;@@ -397,13 +398,11 @@ export async function searchController(return res.status(200).json(responseData);} catch (error) {- if (- error instanceof Error &&- (error.message.startsWith("Job wait") || error.message === "timeout")- ) {+ if (error instanceof ScrapeJobTimeoutError) {return res.status(408).json({success: false,- error: "Request timed out",+ code: error.code,+ error: error.message,});}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 59fc89a7a..05c01156d 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -10,6 +10,8 @@ import {} from "../../lib/entities";import { InternalOptions } from "../../scraper/scrapeURL";import { getURLDepth } from "../../scraper/WebScraper/utils/maxDepthUtils";+import Ajv from "ajv";+import { ErrorCodes } from "../../lib/error";export enum IntegrationEnum {DIFY = "dify",@@ -535,9 +537,6 @@ export type ScrapeOptions = BaseScrapeOptions & {},};-import Ajv from "ajv";-import type { CostTracking } from "../../lib/extract/extraction-service";-const ajv = new Ajv();export const extractV1Options = z@@ -921,6 +920,7 @@ export type Document = {export type ErrorResponse = {success: false;+ code?: ErrorCodes;error: string;details?: any;};@@ -1441,10 +1441,3 @@ export const generateLLMsTextRequestSchema = z.object({export type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;--export class TimeoutSignal extends Error {- constructor() {- super("Operation timed out");- this.name = "TimeoutSignal";- }-}diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex dec7485e6..ff0a66659 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -23,6 +23,7 @@ import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";+import { ScrapeJobTimeoutError } from "../../lib/error";interface DocumentWithCostTracking {document: Document;@@ -349,13 +350,11 @@ export async function x402SearchController(return res.status(200).json(responseData);} catch (error) {- if (- error instanceof Error &&- (error.message.startsWith("Job wait") || error.message === "timeout")- ) {+ if (error instanceof ScrapeJobTimeoutError) {return res.status(408).json({success: false,- error: "Request timed out",+ code: error.code,+ error: error.message,});}diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 4199f910e..9e5295109 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -1,6 +1,7 @@import { Response } from "express";import { z } from "zod";import {+ ErrorResponse,RequestWithAuth,crawlRequestSchema,} from "./types";@@ -18,7 +19,7 @@ const crawlParamsPreviewRequestSchema = z.object({type CrawlParamsPreviewRequest = z.infer<typeof crawlParamsPreviewRequestSchema>;type CrawlParamsPreviewResponse = {- success: boolean;+ success: true;data?: {url: string;includePaths?: string[];@@ -34,8 +35,7 @@ type CrawlParamsPreviewResponse = {delay?: number;limit?: number;};- error?: string;-};+} | ErrorResponse;export async function crawlParamsPreviewController(req: RequestWithAuth<{}, CrawlParamsPreviewResponse, CrawlParamsPreviewRequest>,@@ -94,6 +94,7 @@ export async function crawlParamsPreviewController(if (error instanceof z.ZodError) {return res.status(400).json({success: false,+ code: "BAD_REQUEST",error: "Invalid request parameters: " + error.errors.map(e => e.message).join(", "),});}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 87d2577b0..7581854b4 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -5,7 +5,6 @@ import {RequestWithAuth,scrapeOptions,TeamFlags,- TimeoutSignal,MapRequest,MapDocument,MapResponse,@@ -24,6 +23,7 @@ import { logger } from "../../lib/logger";import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";import { redisEvictConnection } from "../../services/redis";import { performCosineSimilarityV2 } from "../../lib/map-cosine";+import { MapTimeoutError } from "../../lib/error";configDotenv();@@ -340,16 +340,17 @@ export async function mapController(}),...(req.body.timeout !== undefined ? [new Promise((resolve, reject) => setTimeout(() => {- abort.abort(new TimeoutSignal());- reject(new TimeoutSignal());+ abort.abort(new MapTimeoutError());+ reject(new MapTimeoutError());}, req.body.timeout))] : []),]) as any;} catch (error) {- if (error instanceof TimeoutSignal || error === "timeout") {+ if (error instanceof MapTimeoutError) {return res.status(408).json({success: false,- error: "Request timed out",+ code: error.code,+ error: error.message,});} else {throw error;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex df7cf2220..b34fe5a31 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -12,6 +12,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { getScrapeQueue } from "../../services/queue-service";import { hasFormatOfType } from "../../lib/format-utils";+import { TransportableError } from "../../lib/error";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -101,19 +102,18 @@ export async function scrapeController(} catch (e) {logger.error(`Error in scrapeController`, {startTime,+ error: e,});if (zeroDataRetention) {await getScrapeQueue().remove(jobId);}- if (- e instanceof Error &&- (e.message.startsWith("Job wait") || e.message === "timeout")- ) {- return res.status(408).json({+ if (e instanceof TransportableError) {+ return res.status(e.code === "SCRAPE_TIMEOUT" ? 408 : 500).json({success: false,- error: "Request timed out",+ code: e.code,+ error: e.message,});} else {return res.status(500).json({diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 90ddf4407..158ff149d 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -25,6 +25,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";import { SearchResult, SearchV2Response } from "../../lib/entities";+import { ScrapeJobTimeoutError } from "../../lib/error";interface DocumentWithCostTracking {document: Document;@@ -425,13 +426,11 @@ export async function searchController(data: searchResponse,});} catch (error) {- if (- error instanceof Error &&- (error.message.startsWith("Job wait") || error.message === "timeout")- ) {+ if (error instanceof ScrapeJobTimeoutError) {return res.status(408).json({success: false,- error: "Request timed out",+ code: error.code,+ error: error.message,});}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 3f833ccd0..9068eda7a 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -11,6 +11,8 @@ import {} from "../../lib/entities";import { agentOptionsExtract, ScrapeOptions as V1ScrapeOptions } from "../v1/types";import { InternalOptions } from "../../scraper/scrapeURL";+import { ErrorCodes } from "../../lib/error";+import Ajv from "ajv";export enum IntegrationEnum {DIFY = "dify",@@ -386,9 +388,6 @@ export type BaseScrapeOptions = z.infer<typeof baseScrapeOptions>;export type ScrapeOptions = BaseScrapeOptions;-import Ajv from "ajv";-import type { CostTracking } from "../../lib/extract/extraction-service";-const ajv = new Ajv();export const extractOptions = z@@ -727,6 +726,7 @@ export type Document = {export type ErrorResponse = {success: false;+ code?: ErrorCodes;error: string;details?: any;};@@ -1438,9 +1438,3 @@ export const generateLLMsTextRequestSchema = z.object({export type GenerateLLMsTextRequest = z.infer<typeof generateLLMsTextRequestSchema>;--export class TimeoutSignal extends Error {- constructor() {- super("Operation timed out");- }-}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 9ebf1a04f..bee6925b5 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -230,7 +230,7 @@ app.use(res.status(400)- .json({ success: false, error: "Bad Request", details: err.errors });+ .json({ success: false, code: "BAD_REQUEST", error: "Bad Request", details: err.errors });} else {next(err);}@@ -254,7 +254,7 @@ app.use() {return res.status(400)- .json({ success: false, error: "Bad request, malformed JSON" });+ .json({ success: false, code: "BAD_REQUEST_INVALID_JSON", error: "Bad request, malformed JSON" });}const id = res.sentry ?? uuidv4();@@ -268,6 +268,7 @@ app.use({ error: err, errorId: id, path: req.path, teamId: req.acuc?.team_id, team_id: req.acuc?.team_id });res.status(500).json({success: false,+ code: "UNKNOWN_ERROR",error:"An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +id,diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsnew file mode 100644index 000000000..13468e84a--- /dev/null+++ b/apps/api/src/lib/error-serde.ts@@ -0,0 +1,36 @@+import { ErrorCodes, MapTimeoutError, ScrapeJobTimeoutError, TransportableError, UnknownError } from "./error";+import { ActionError, DNSResolutionError, UnsupportedFileError, PDFAntibotError, PDFInsufficientTimeError, NoEnginesLeftError, ZDRViolationError, PDFPrefetchFailed, SiteError, SSLError } from "../scraper/scrapeURL/error";++// TODO: figure out correct typing for this+const errorMap: Record<ErrorCodes, any> = {+ "SCRAPE_TIMEOUT": ScrapeJobTimeoutError,+ "MAP_TIMEOUT": MapTimeoutError,+ "UNKNOWN_ERROR": UnknownError,+ "SCRAPE_ALL_ENGINES_FAILED": NoEnginesLeftError,+ "SCRAPE_SSL_ERROR": SSLError,+ "SCRAPE_SITE_ERROR": SiteError,+ "SCRAPE_PDF_PREFETCH_FAILED": PDFPrefetchFailed,+ "SCRAPE_ZDR_VIOLATION_ERROR": ZDRViolationError,+ "SCRAPE_DNS_RESOLUTION_ERROR": DNSResolutionError,+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR": PDFInsufficientTimeError,+ "SCRAPE_PDF_ANTIBOT_ERROR": PDFAntibotError,+ "SCRAPE_UNSUPPORTED_FILE_ERROR": UnsupportedFileError,+ "SCRAPE_ACTION_ERROR": ActionError,++ // Zod errors+ "BAD_REQUEST": null,+ "BAD_REQUEST_INVALID_JSON": null,+};++export function serializeTransportableError(error: TransportableError) {+ return `${error.code}|${JSON.stringify(error.serialize())}`;+}++export function deserializeTransportableError(data: string): InstanceType<typeof errorMap[keyof typeof errorMap]> | null {+ const [code, ...serialized] = data.split("|");+ const x = errorMap[code];+ if (!x) {+ return null;+ }+ return x.deserialize(code, JSON.parse(serialized.join("|")));+}diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsnew file mode 100644index 000000000..068ed2011--- /dev/null+++ b/apps/api/src/lib/error.ts@@ -0,0 +1,92 @@+export type ErrorCodes =+ "SCRAPE_TIMEOUT" |+ "MAP_TIMEOUT" |+ "UNKNOWN_ERROR" |+ "SCRAPE_ALL_ENGINES_FAILED" |+ "SCRAPE_SSL_ERROR" |+ "SCRAPE_SITE_ERROR" |+ "SCRAPE_PDF_PREFETCH_FAILED" |+ "SCRAPE_ZDR_VIOLATION_ERROR" |+ "SCRAPE_DNS_RESOLUTION_ERROR" |+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR" |+ "SCRAPE_PDF_ANTIBOT_ERROR" |+ "SCRAPE_UNSUPPORTED_FILE_ERROR" |+ "SCRAPE_ACTION_ERROR" |+ "BAD_REQUEST_INVALID_JSON" |+ "BAD_REQUEST";++export class TransportableError extends Error {+ public readonly code: ErrorCodes;++ constructor(code: ErrorCodes, message?: string, options?: ErrorOptions) {+ super(message, options);+ this.code = code;+ }++ serialize() {+ return {+ cause: this.cause,+ stack: this.stack,+ message: this.message,+ }+ }++ static deserialize(code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new TransportableError(code, data.message, { cause: data.cause });+ x.stack = data.stack;+ return x;+ }+}++export class ScrapeJobTimeoutError extends TransportableError {+ constructor(message: string) {+ super("SCRAPE_TIMEOUT", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new ScrapeJobTimeoutError(data.message);+ x.stack = data.stack;+ return x;+ }+}++export class UnknownError extends TransportableError {+ constructor(inner: unknown) {+ super("UNKNOWN_ERROR", `(Internal server error) - ${(inner && inner instanceof Error) ? inner.message : inner}`);++ if (inner instanceof Error) {+ this.stack = inner.stack;+ }+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new UnknownError("dummy");+ x.message = data.message;+ x.stack = data.stack;+ return x;+ }+}++export class MapTimeoutError extends TransportableError {+ constructor() {+ super("MAP_TIMEOUT", "Map timed out");+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(_code: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new MapTimeoutError();+ x.stack = data.stack;+ return x;+ }+}\ No newline at end of filediff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 3672097d1..8bf82e8fe 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -10,9 +10,9 @@ import { logger as _logger } from "../../lib/logger";import https from "https";import { redisEvictConnection } from "../../services/redis";import { extractLinks } from "../../lib/html-transformer";-import { TimeoutSignal } from "../../controllers/v1/types";import { filterLinks } from "../../lib/crawler";import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";+import { ScrapeJobTimeoutError } from "../../lib/error";export interface FilterResult {allowed: boolean;@@ -755,7 +755,7 @@ export class WebCrawler {mock,);} catch (error) {- if (error instanceof TimeoutSignal) {+ if (error instanceof ScrapeJobTimeoutError) {throw error;} else {this.logger.debug(`Failed to fetch sitemap from ${sitemapUrl}`, {@@ -805,7 +805,7 @@ export class WebCrawler {mock,);} catch (error) {- if (error instanceof TimeoutSignal) {+ if (error instanceof ScrapeJobTimeoutError) {throw error;} else {this.logger.debug(@@ -816,7 +816,7 @@ export class WebCrawler {}}} catch (error) {- if (error instanceof TimeoutSignal) {+ if (error instanceof ScrapeJobTimeoutError) {throw error;} else {this.logger.debug(`Error processing main domain sitemap`, {@@ -840,7 +840,7 @@ export class WebCrawler {mock,);} catch (error) {- if (error instanceof TimeoutSignal) {+ if (error instanceof ScrapeJobTimeoutError) {throw error;} else {this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex c4429431e..4971a6f13 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -1,10 +1,11 @@import { parseStringPromise } from "xml2js";import { WebCrawler } from "./crawler";import { scrapeURL } from "../scrapeURL";-import { scrapeOptions, TimeoutSignal } from "../../controllers/v2/types";+import { scrapeOptions } from "../../controllers/v2/types";import type { Logger } from "winston";import { CostTracking } from "../../lib/extract/extraction-service";import { parseSitemapXml, processSitemap } from "../../lib/crawler";+import { ScrapeJobTimeoutError } from "../../lib/error";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&process.env.FIRE_ENGINE_BETA_URL !== undefined;@@ -87,7 +88,7 @@ export async function getLinksFromSitemap(return 0;}} catch (error) {- if (error instanceof TimeoutSignal) {+ if (error instanceof ScrapeJobTimeoutError) {throw error;} else {logger.error(`Request failed for sitemap fetch`, {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex f63ddaaeb..818d9ab4d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -260,19 +260,6 @@ export async function scrapePDF(error instanceof AbortManagerThrownError) {throw error;- // } else if (- // (error instanceof Error && error.name === "TimeoutError") ||- // (error instanceof Error &&- // error.message === "Request failed" &&- // error.cause &&- // error.cause instanceof Error &&- // error.cause.name === "TimeoutError") ||- // (error instanceof Error && error.name === "TimeoutSignal")- // ) {- // meta.logger.warn("RunPod MU timed out", { timeout: remainingTime });- // throw new TimeoutError(- // "PDF parsing timed out, please increase the timeout parameter in your scrape request",- // );}meta.logger.warn("RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 295b208b0..10461f5ba 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -1,3 +1,4 @@+import { ErrorCodes, TransportableError } from "../../lib/error";import { Meta } from ".";import { Engine, FeatureFlag } from "./engines";@@ -7,15 +8,29 @@ export class EngineError extends Error {}}-export class NoEnginesLeftError extends Error {+export class NoEnginesLeftError extends TransportableError {public fallbackList: Engine[];constructor(fallbackList: Engine[]) {super(+ "SCRAPE_ALL_ENGINES_FAILED","All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].",);this.fallbackList = fallbackList;}++ serialize() {+ return {+ ...super.serialize(),+ fallbackList: this.fallbackList,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new NoEnginesLeftError(data.fallbackList);+ x.stack = data.stack;+ return x;+ }}export class AddFeatureError extends Error {@@ -41,58 +56,143 @@ export class RemoveFeatureError extends Error {}}-export class SSLError extends Error {- constructor(skipTlsVerification: boolean) {+export class SSLError extends TransportableError {+ constructor(public skipTlsVerification: boolean) {super(+ "SCRAPE_SSL_ERROR","An SSL error occurred while scraping the URL. "+ (skipTlsVerification? "Since you have `skipTlsVerification` enabled, this means that the TLS configuration of the target site is completely broken. Try scraping the plain HTTP version of the page.": "If you're not inputting any sensitive data, try scraping with `skipTlsVerification: true`."));}++ serialize() {+ return {+ ...super.serialize(),+ skipTlsVerification: this.skipTlsVerification,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new SSLError(data.skipTlsVerification);+ x.stack = data.stack;+ return x;+ }}-export class SiteError extends Error {- public code: string;- constructor(code: string) {+export class SiteError extends TransportableError {+ constructor(public errorCode: string) {super(- "Specified URL is failing to load in the browser. Error code: " + code,+ "SCRAPE_SITE_ERROR",+ "Specified URL is failing to load in the browser. Error code: " + errorCode,);- this.code = code;+ }++ serialize() {+ return {+ ...super.serialize(),+ errorCode: this.errorCode,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new SiteError(data.errorCode);+ x.stack = data.stack;+ return x;}}-export class ActionError extends Error {- public code: string;- constructor(code: string) {- super("Action(s) failed to complete. Error code: " + code);- this.code = code;+export class ActionError extends TransportableError {+ constructor(public errorCode: string) {+ super("SCRAPE_ACTION_ERROR", "Action(s) failed to complete. Error code: " + errorCode);+ }++ serialize() {+ return {+ ...super.serialize(),+ errorCode: this.errorCode,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new ActionError(data.errorCode);+ x.stack = data.stack;+ return x;}}-export class UnsupportedFileError extends Error {- public reason: string;- constructor(reason: string) {- super("Scrape resulted in unsupported file: " + reason);- this.reason = reason;+export class UnsupportedFileError extends TransportableError {+ constructor(public reason: string) {+ super("SCRAPE_UNSUPPORTED_FILE_ERROR", "Scrape resulted in unsupported file: " + reason);+ }++ serialize() {+ return {+ ...super.serialize(),+ reason: this.reason,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new UnsupportedFileError(data.reason);+ x.stack = data.stack;+ return x;}}-export class PDFAntibotError extends Error {+export class PDFAntibotError extends TransportableError {constructor() {- super("PDF scrape was prevented by anti-bot")+ super("SCRAPE_PDF_ANTIBOT_ERROR", "PDF scrape was prevented by anti-bot")+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new PDFAntibotError();+ x.stack = data.stack;+ return x;}}-export class PDFInsufficientTimeError extends Error {- constructor(pageCount: number, minTimeout: number) {- super(`Insufficient time to process PDF of ${pageCount} pages. Please increase the timeout parameter in your scrape request to at least ${minTimeout}ms.`);+export class PDFInsufficientTimeError extends TransportableError {+ constructor(public pageCount: number, public minTimeout: number) {+ super("SCRAPE_PDF_INSUFFICIENT_TIME_ERROR", `Insufficient time to process PDF of ${pageCount} pages. Please increase the timeout parameter in your scrape request to at least ${minTimeout}ms.`);+ }++ serialize() {+ return {+ ...super.serialize(),+ pageCount: this.pageCount,+ minTimeout: this.minTimeout,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new PDFInsufficientTimeError(data.pageCount, data.minTimeout);+ x.stack = data.stack;+ return x;}}-export class DNSResolutionError extends Error {- constructor(hostname: string) {- super(`DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);+export class DNSResolutionError extends TransportableError {+ constructor(public hostname: string) {+ super("SCRAPE_DNS_RESOLUTION_ERROR", `DNS resolution failed for hostname: ${hostname}. Please check if the domain is valid and accessible.`);+ }++ serialize() {+ return {+ ...super.serialize(),+ hostname: this.hostname,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new DNSResolutionError(data.hostname);+ x.stack = data.stack;+ return x;}}@@ -102,39 +202,48 @@ export class IndexMissError extends Error {}}-export class ZDRViolationError extends Error {- constructor(feature: string) {- super(`${feature} is not supported when using zeroDataRetention. Please contact [email protected] to unblock this feature.`);+export class ZDRViolationError extends TransportableError {+ constructor(public feature: string) {+ super("SCRAPE_ZDR_VIOLATION_ERROR", `${feature} is not supported when using zeroDataRetention. Please contact [email protected] to unblock this feature.`);}-}-export class PDFPrefetchFailed extends Error {- constructor() {- super("Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");+ serialize() {+ return {+ ...super.serialize(),+ feature: this.feature,+ }+ }++ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new ZDRViolationError(data.feature);+ x.stack = data.stack;+ return x;}}-export class FEPageLoadFailed extends Error {+export class PDFPrefetchFailed extends TransportableError {constructor() {- super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");+ super("SCRAPE_PDF_PREFETCH_FAILED", "Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]");}-}-export class ScrapeURLScopedTimeoutError extends Error {- constructor(message: string, options?: ErrorOptions) {- super(message, options);+ serialize() {+ return super.serialize();}-}-export class ScrapeTimeoutError extends ScrapeURLScopedTimeoutError {- name = "ScrapeTimeoutError";+ static deserialize(_: ErrorCodes, data: ReturnType<typeof this.prototype.serialize>) {+ const x = new PDFPrefetchFailed();+ x.stack = data.stack;+ return x;+ }+}+export class FEPageLoadFailed extends Error {constructor() {- super("Scrape timed out");+ super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");}}-export class EngineSnipedError extends ScrapeURLScopedTimeoutError {+export class EngineSnipedError extends Error {name = "EngineSnipedError";constructor() {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex bb22a2875..30901a637 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,7 +1,7 @@import { Logger } from "winston";import * as Sentry from "@sentry/node";-import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v2/types";+import { Document, ScrapeOptions, TeamFlags } from "../../controllers/v2/types";import { ScrapeOptions as ScrapeOptionsV1 } from "../../controllers/v1/types";import { logger as _logger } from "../../lib/logger";import {@@ -29,7 +29,6 @@ import {ZDRViolationError,PDFPrefetchFailed,FEPageLoadFailed,- ScrapeTimeoutError,EngineSnipedError,WaterfallNextEngineSignal,EngineUnsuccessfulError,@@ -43,6 +42,7 @@ import { robustFetch } from "./lib/fetch";import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";import { checkRobotsTxt } from "../../lib/robots-txt";import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";+import { ScrapeJobTimeoutError } from "../../lib/error";export type ScrapeUrlResponse = (| {@@ -203,7 +203,7 @@ async function buildMetaObject(tier: "scrape",timesOutAt: new Date(Date.now() + options.timeout),throwable() {- return new ScrapeTimeoutError();+ return new ScrapeJobTimeoutError("Scrape timed out");},} : undefined,),@@ -413,7 +413,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {setTimeout(() => {try {meta.abort.throwIfAborted();- meta.logger.warn("throwIfAborted did not throw");++ // Fallback error if above doesn't throw+ throw new ScrapeJobTimeoutError("Scrape timed out due to maximum length of 5 minutes");} catch (error) {reject(error);}@@ -496,18 +498,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {snipeAbortController.abort();if (result === null) {- // TODO: rectify TODOv2- // if (meta.results["pdf"]?.state === "timeout") {- // throw meta.results["pdf"].error ?? new TimeoutSignal();- // }- // if (Object.values(meta.results).every(x => x.state === "timeout")) {- // throw new TimeoutSignal();- // } else {- throw new NoEnginesLeftError(- fallbackList.map((x) => x.engine),- // meta.results,- );- // }+ throw new NoEnginesLeftError(+ fallbackList.map((x) => x.engine),+ );}let document: Document = {@@ -769,8 +762,6 @@ export async function scrapeURL(meta.logger.warn("scrapeURL: Insufficient time to process PDF", { error });} else if (error instanceof PDFPrefetchFailed) {meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });- } else if (error instanceof TimeoutSignal) {- throw error;} else if (error instanceof AbortManagerThrownError) {throw error.inner;} else {diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 84e18db7e..9c5798b5d 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -2,12 +2,11 @@ import { Logger } from "winston";import { z, ZodError } from "zod";import * as Sentry from "@sentry/node";import { MockState, saveMock } from "./mock";-import { TimeoutSignal } from "../../../controllers/v1/types";import { fireEngineURL } from "../engines/fire-engine/scrape";-import { fetch, RequestInit, Response, FormData, Agent } from "undici";+import { fetch, Response, FormData, Agent } from "undici";import { cacheableLookup } from "./cacheableLookup";-import { log } from "console";import dns from "dns";+import { AbortManagerThrownError } from "./abortManager";export type RobustFetchParams<Schema extends z.Schema<any>> = {url: string;@@ -125,8 +124,8 @@ export async function robustFetch<: {}),});} catch (error) {- if (error instanceof TimeoutSignal || (error instanceof Error && error.name === "TimeoutError") || (error instanceof Error && error.message === "Operation timed out")) {- throw new TimeoutSignal();+ if (error instanceof AbortManagerThrownError) {+ throw error;} else if (!ignoreFailure) {Sentry.captureException(error);if (tryCount > 1) {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 47fd4e0a5..a4a5ab922 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -20,6 +20,8 @@ import { Document } from "../controllers/v1/types";import { getCrawl } from "../lib/crawl-redis";import { Logger } from "winston";import { Job } from "bullmq";+import { ScrapeJobTimeoutError, TransportableError } from "../lib/error";+import { deserializeTransportableError } from "../lib/error-serde";/*** Checks if a job is a crawl or batch scrape based on its options@@ -327,17 +329,33 @@ export async function waitForJob(await new Promise(resolve => setTimeout(resolve, 500));job = await queue.getJob(_job as string);if (Date.now() - start > (timeout ?? 180000)) {- throw new Error("Job wait (concurrency limited)");+ throw new ScrapeJobTimeoutError("Scrape timed out while waiting in the concurrency limit queue");+ }+ }+ let doc: Document;+ try {+ doc = await Promise.race([+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),+ timeout !== null ? new Promise((resolve, reject) => {+ setTimeout(() => {+ reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));+ }, Math.max(0, timeout - (Date.now() - start)));+ }) : null,+ ].filter(x => x !== null));+ } catch (e) {+ if (e instanceof TransportableError) {+ throw e;+ } else if (e instanceof Error) {+ const x = deserializeTransportableError(e.message);+ if (x) {+ throw x;+ } else {+ throw e;+ }+ } else {+ throw e;}}- let doc: Document = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),- timeout !== null ? new Promise((resolve, reject) => {- setTimeout(() => {- reject(new Error("Job wait " + (typeof _job == "string" ? "(was concurrency limited)" : "")));- }, Math.max(0, timeout - (Date.now() - start)));- }) : null,- ].filter(x => x !== null));logger.debug("Got job");if (!doc) {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e25d85632..a761ba6b2 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -49,6 +49,8 @@ import { finishCrawlIfNeeded } from "./crawl-logic";import { LangfuseExporter } from "langfuse-vercel";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";import { NodeSDK } from "@opentelemetry/sdk-node";+import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";+import { serializeTransportableError } from "../../lib/error-serde";class RacedRedirectError extends Error {constructor() {@@ -145,7 +147,7 @@ async function processJob(job: Job & { id: string }) {});if (remainingTime !== undefined && remainingTime < 0) {- throw new Error("timeout");+ throw new ScrapeJobTimeoutError("Scrape timed out");}const signal = remainingTime ? AbortSignal.timeout(remainingTime) : undefined;@@ -166,7 +168,7 @@ async function processJob(job: Job & { id: string }) {? [(async () => {await sleep(remainingTime);- throw new Error("timeout");+ throw new ScrapeJobTimeoutError("Scrape timed out");})(),]: []),@@ -175,7 +177,7 @@ async function processJob(job: Job & { id: string }) {try {signal?.throwIfAborted();} catch (e) {- throw new Error("timeout");+ throw new ScrapeJobTimeoutError("Scrape timed out");}if (!pipeline.success) {@@ -370,7 +372,7 @@ async function processJob(job: Job & { id: string }) {try {signal?.throwIfAborted();} catch (e) {- throw new Error("timeout");+ throw new ScrapeJobTimeoutError("Scrape timed out");}const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);@@ -426,7 +428,7 @@ async function processJob(job: Job & { id: string }) {try {signal?.throwIfAborted();} catch (e) {- throw new Error("timeout");+ throw new ScrapeJobTimeoutError("Scrape timed out");}const credits_billed = await billScrapeJob(job, doc, logger, costTracking, (await getACUCTeam(job.data.team_id))?.flags ?? null);@@ -472,7 +474,7 @@ async function processJob(job: Job & { id: string }) {}const isEarlyTimeout =- error instanceof Error && error.message === "timeout";+ error instanceof ScrapeJobTimeoutError;const isCancelled =error instanceof Error &&error.message === "Parent crawl/batch scrape was cancelled";@@ -846,7 +848,11 @@ export const processJobInternal = async (job: Job & { id: string }) => {} catch (error) {logger.debug("Job failed", { error });Sentry.captureException(error);- throw error;+ if (error instanceof TransportableError) {+ throw new Error(serializeTransportableError(error));+ } else {+ throw new Error(serializeTransportableError(new UnknownError(error)));+ }}};