Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/map): use avgrab resolve (#3231)

Gergő Móricz87e6f172d4fd
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index a46871b4d..5983db766 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -15,6 +15,7 @@ import { getMapResults, MapResult } from "../../lib/map-utils";
import { v7 as uuidv7 } from "uuid";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { resolveViaAvgrab } from "../../lib/avgrab-resolve";
configDotenv();
@@ -68,6 +69,63 @@ export async function mapController(
api_key_id: req.acuc?.api_key_id ?? null,
});
+ // Short-circuit: if the URL matches avgrab's resolve pattern, delegate entirely
+ try {
+ const avgrabResults = await resolveViaAvgrab(
+ req.body.url,
+ req.body.limit,
+ logger,
+ );
+
+ if (avgrabResults !== null) {
+ const creditsCost = avgrabResults.length;
+
+ billTeam(
+ req.auth.team_id,
+ req.acuc?.sub_id ?? undefined,
+ creditsCost,
+ req.acuc?.api_key_id ?? null,
+ { endpoint: "map", jobId: mapId },
+ ).catch(error => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for ${creditsCost} credits: ${error}`,
+ );
+ });
+
+ logMap({
+ id: mapId,
+ request_id: mapId,
+ url: req.body.url,
+ team_id: req.auth.team_id,
+ options: {
+ search: req.body.search,
+ sitemap: req.body.sitemap,
+ includeSubdomains: req.body.includeSubdomains,
+ ignoreQueryParameters: req.body.ignoreQueryParameters,
+ limit: req.body.limit,
+ timeout: req.body.timeout,
+ location: req.body.location,
+ },
+ results: avgrabResults,
+ credits_cost: creditsCost,
+ zeroDataRetention: false,
+ }).catch(error => {
+ logger.error(
+ `Failed to log job for team ${req.auth.team_id}: ${error}`,
+ );
+ });
+
+ return res.status(200).json({
+ success: true,
+ links: avgrabResults,
+ });
+ }
+ } catch (error) {
+ logger.warn("avgrab resolve failed, falling back to standard map", {
+ error,
+ });
+ }
+
let result: MapResult;
let timeoutHandle: NodeJS.Timeout | null = null;
diff --git a/apps/api/src/lib/avgrab-resolve.ts b/apps/api/src/lib/avgrab-resolve.ts
new file mode 100644
index 000000000..516d97524
--- /dev/null
+++ b/apps/api/src/lib/avgrab-resolve.ts
@@ -0,0 +1,99 @@
+import { config } from "../config";
+import { MapDocument } from "../controllers/v2/types";
+import * as winston from "winston";
+
+let cachedResolveRegex: RegExp | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+async function getResolveRegex(): Promise<RegExp | null> {
+ if (!config.AVGRAB_SERVICE_URL) return null;
+
+ if (cachedResolveRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+ return cachedResolveRegex;
+ }
+
+ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);
+ if (!res.ok) {
+ throw new Error(
+ "Failed to fetch supported URL patterns from avgrab service",
+ );
+ }
+
+ const data = await res.json().catch(() => null);
+ if (!data || typeof data.resolve_regex !== "string") {
+ throw new Error("avgrab service returned invalid resolve URL pattern");
+ }
+
+ cachedResolveRegex = new RegExp(data.resolve_regex);
+ cacheTimestamp = Date.now();
+ return cachedResolveRegex;
+}
+
+async function matchesResolveRegex(url: string): Promise<boolean> {
+ if (!config.AVGRAB_SERVICE_URL) return false;
+
+ try {
+ const regex = await getResolveRegex();
+ return regex !== null && regex.test(url);
+ } catch {
+ return false;
+ }
+}
+
+interface AvgrabResolvedPost {
+ url: string;
+ title: string;
+ date: string;
+ type: string;
+ media: string[];
+}
+
+interface AvgrabResolveResponse {
+ username: string;
+ posts: AvgrabResolvedPost[];
+}
+
+export async function resolveViaAvgrab(
+ url: string,
+ limit: number,
+ logger: winston.Logger,
+): Promise<MapDocument[] | null> {
+ if (!config.AVGRAB_SERVICE_URL) return null;
+
+ const matches = await matchesResolveRegex(url);
+ if (!matches) return null;
+
+ logger.info("URL matches avgrab resolve pattern, delegating to avgrab", {
+ url,
+ });
+
+ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/resolve`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ url, limit }),
+ });
+
+ if (!response.ok) {
+ const error = await response
+ .json()
+ .catch(() => ({ detail: "Unknown error" }));
+ logger.error("avgrab resolve failed", {
+ url,
+ status: response.status,
+ error,
+ });
+ return null;
+ }
+
+ const data: AvgrabResolveResponse = await response.json();
+
+ return data.posts.map(post => {
+ const { url: _url, title: _title, ...meta } = post;
+ return {
+ url: post.url,
+ title: post.title || undefined,
+ description: JSON.stringify(meta),
+ };
+ });
+}