Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/avgrab): pass error

mogerydeb8d407115f
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 5983db766..cb0803115 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -9,7 +9,7 @@ import { configDotenv } from "dotenv";
import { billTeam } from "../../services/billing/credit_billing";
import { logMap, logRequest } from "../../services/logging/log_job";
import { logger as _logger } from "../../lib/logger";
-import { MapTimeoutError } from "../../lib/error";
+import { MapTimeoutError, MapFailedError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import { getMapResults, MapResult } from "../../lib/map-utils";
import { v7 as uuidv7 } from "uuid";
@@ -121,6 +121,12 @@ export async function mapController(
});
}
} catch (error) {
+ if (error instanceof MapFailedError) {
+ return res.status(500).json({
+ success: false,
+ error: error.message,
+ });
+ }
logger.warn("avgrab resolve failed, falling back to standard map", {
error,
});
diff --git a/apps/api/src/lib/avgrab-resolve.ts b/apps/api/src/lib/avgrab-resolve.ts
index b41a2d873..e48b852d2 100644
--- a/apps/api/src/lib/avgrab-resolve.ts
+++ b/apps/api/src/lib/avgrab-resolve.ts
@@ -1,6 +1,7 @@
import { Agent, fetch } from "undici";
import { config } from "../config";
import { MapDocument } from "../controllers/v2/types";
+import { MapFailedError } from "./error";
import * as winston from "winston";
const avgrabAgent = new Agent({
@@ -27,7 +28,10 @@ async function getResolveRegex(): Promise<RegExp | null> {
);
}
- const data = (await res.json().catch(() => null)) as Record<string, unknown> | null;
+ const data = (await res.json().catch(() => null)) as Record<
+ string,
+ unknown
+ > | null;
if (!data || typeof data.resolve_regex !== "string") {
throw new Error("avgrab service returned invalid resolve URL pattern");
}
@@ -84,15 +88,17 @@ export async function resolveViaAvgrab(
});
if (!response.ok) {
- const error = await response
+ const body = (await response
.json()
- .catch(() => ({ detail: "Unknown error" }));
+ .catch(() => ({ detail: "Unknown error" }))) as Record<string, unknown>;
+ const detail =
+ typeof body.detail === "string" ? body.detail : "Unknown error";
logger.error("avgrab resolve failed", {
url,
status: response.status,
- error,
+ detail,
});
- return null;
+ throw new MapFailedError(detail);
}
const data = (await response.json()) as AvgrabResolveResponse;
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index e280ab801..def6dff34 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -2,6 +2,7 @@ import {
ActionsNotSupportedError,
CrawlDenialError,
ErrorCodes,
+ MapFailedError,
MapTimeoutError,
RacedRedirectError,
ScrapeJobTimeoutError,
@@ -61,6 +62,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_SITEMAP_ERROR: SitemapError,
CRAWL_DENIAL: CrawlDenialError,
SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,
+ MAP_FAILED: MapFailedError,
// Zod errors
BAD_REQUEST: null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 640e846da..a3097355e 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -26,6 +26,7 @@ export type ErrorCodes =
| "AGENT_INDEX_ONLY"
| "SCRAPE_AUDIO_UNSUPPORTED_URL"
| "CRAWL_DENIAL"
+ | "MAP_FAILED"
| "BAD_REQUEST_INVALID_JSON"
| "BAD_REQUEST";
@@ -127,6 +128,25 @@ export class MapTimeoutError extends TransportableError {
}
}
+export class MapFailedError extends TransportableError {
+ constructor(message: string) {
+ super("MAP_FAILED", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _code: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new MapFailedError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class RacedRedirectError extends TransportableError {
constructor() {
super(