Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/avgrab): pass error
mogerydeb8d407115f
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 5983db766..cb0803115 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -9,7 +9,7 @@ import { configDotenv } from "dotenv";import { billTeam } from "../../services/billing/credit_billing";import { logMap, logRequest } from "../../services/logging/log_job";import { logger as _logger } from "../../lib/logger";-import { MapTimeoutError } from "../../lib/error";+import { MapTimeoutError, MapFailedError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";import { getMapResults, MapResult } from "../../lib/map-utils";import { v7 as uuidv7 } from "uuid";@@ -121,6 +121,12 @@ export async function mapController(});}} catch (error) {+ if (error instanceof MapFailedError) {+ return res.status(500).json({+ success: false,+ error: error.message,+ });+ }logger.warn("avgrab resolve failed, falling back to standard map", {error,});diff --git a/apps/api/src/lib/avgrab-resolve.ts b/apps/api/src/lib/avgrab-resolve.tsindex b41a2d873..e48b852d2 100644--- a/apps/api/src/lib/avgrab-resolve.ts+++ b/apps/api/src/lib/avgrab-resolve.ts@@ -1,6 +1,7 @@import { Agent, fetch } from "undici";import { config } from "../config";import { MapDocument } from "../controllers/v2/types";+import { MapFailedError } from "./error";import * as winston from "winston";const avgrabAgent = new Agent({@@ -27,7 +28,10 @@ async function getResolveRegex(): Promise<RegExp | null> {);}- const data = (await res.json().catch(() => null)) as Record<string, unknown> | null;+ const data = (await res.json().catch(() => null)) as Record<+ string,+ unknown+ > | null;if (!data || typeof data.resolve_regex !== "string") {throw new Error("avgrab service returned invalid resolve URL pattern");}@@ -84,15 +88,17 @@ export async function resolveViaAvgrab(});if (!response.ok) {- const error = await response+ const body = (await response.json()- .catch(() => ({ detail: "Unknown error" }));+ .catch(() => ({ detail: "Unknown error" }))) as Record<string, unknown>;+ const detail =+ typeof body.detail === "string" ? body.detail : "Unknown error";logger.error("avgrab resolve failed", {url,status: response.status,- error,+ detail,});- return null;+ throw new MapFailedError(detail);}const data = (await response.json()) as AvgrabResolveResponse;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex e280ab801..def6dff34 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -2,6 +2,7 @@ import {ActionsNotSupportedError,CrawlDenialError,ErrorCodes,+ MapFailedError,MapTimeoutError,RacedRedirectError,ScrapeJobTimeoutError,@@ -61,6 +62,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,+ MAP_FAILED: MapFailedError,// Zod errorsBAD_REQUEST: null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 640e846da..a3097355e 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -26,6 +26,7 @@ export type ErrorCodes =| "AGENT_INDEX_ONLY"| "SCRAPE_AUDIO_UNSUPPORTED_URL"| "CRAWL_DENIAL"+ | "MAP_FAILED"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST";@@ -127,6 +128,25 @@ export class MapTimeoutError extends TransportableError {}}+export class MapFailedError extends TransportableError {+ constructor(message: string) {+ super("MAP_FAILED", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _code: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new MapFailedError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class RacedRedirectError extends TransportableError {constructor() {super(