Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/map): use avgrab resolve (#3231)
Gergő Móricz87e6f172d4fd
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex a46871b4d..5983db766 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -15,6 +15,7 @@ import { getMapResults, MapResult } from "../../lib/map-utils";import { v7 as uuidv7 } from "uuid";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { resolveViaAvgrab } from "../../lib/avgrab-resolve";configDotenv();@@ -68,6 +69,63 @@ export async function mapController(api_key_id: req.acuc?.api_key_id ?? null,});+ // Short-circuit: if the URL matches avgrab's resolve pattern, delegate entirely+ try {+ const avgrabResults = await resolveViaAvgrab(+ req.body.url,+ req.body.limit,+ logger,+ );++ if (avgrabResults !== null) {+ const creditsCost = avgrabResults.length;++ billTeam(+ req.auth.team_id,+ req.acuc?.sub_id ?? undefined,+ creditsCost,+ req.acuc?.api_key_id ?? null,+ { endpoint: "map", jobId: mapId },+ ).catch(error => {+ logger.error(+ `Failed to bill team ${req.auth.team_id} for ${creditsCost} credits: ${error}`,+ );+ });++ logMap({+ id: mapId,+ request_id: mapId,+ url: req.body.url,+ team_id: req.auth.team_id,+ options: {+ search: req.body.search,+ sitemap: req.body.sitemap,+ includeSubdomains: req.body.includeSubdomains,+ ignoreQueryParameters: req.body.ignoreQueryParameters,+ limit: req.body.limit,+ timeout: req.body.timeout,+ location: req.body.location,+ },+ results: avgrabResults,+ credits_cost: creditsCost,+ zeroDataRetention: false,+ }).catch(error => {+ logger.error(+ `Failed to log job for team ${req.auth.team_id}: ${error}`,+ );+ });++ return res.status(200).json({+ success: true,+ links: avgrabResults,+ });+ }+ } catch (error) {+ logger.warn("avgrab resolve failed, falling back to standard map", {+ error,+ });+ }+let result: MapResult;let timeoutHandle: NodeJS.Timeout | null = null;diff --git a/apps/api/src/lib/avgrab-resolve.ts b/apps/api/src/lib/avgrab-resolve.tsnew file mode 100644index 000000000..516d97524--- /dev/null+++ b/apps/api/src/lib/avgrab-resolve.ts@@ -0,0 +1,99 @@+import { config } from "../config";+import { MapDocument } from "../controllers/v2/types";+import * as winston from "winston";++let cachedResolveRegex: RegExp | null = null;+let cacheTimestamp = 0;+const CACHE_TTL_MS = 5 * 60 * 1000;++async function getResolveRegex(): Promise<RegExp | null> {+ if (!config.AVGRAB_SERVICE_URL) return null;++ if (cachedResolveRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {+ return cachedResolveRegex;+ }++ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);+ if (!res.ok) {+ throw new Error(+ "Failed to fetch supported URL patterns from avgrab service",+ );+ }++ const data = await res.json().catch(() => null);+ if (!data || typeof data.resolve_regex !== "string") {+ throw new Error("avgrab service returned invalid resolve URL pattern");+ }++ cachedResolveRegex = new RegExp(data.resolve_regex);+ cacheTimestamp = Date.now();+ return cachedResolveRegex;+}++async function matchesResolveRegex(url: string): Promise<boolean> {+ if (!config.AVGRAB_SERVICE_URL) return false;++ try {+ const regex = await getResolveRegex();+ return regex !== null && regex.test(url);+ } catch {+ return false;+ }+}++interface AvgrabResolvedPost {+ url: string;+ title: string;+ date: string;+ type: string;+ media: string[];+}++interface AvgrabResolveResponse {+ username: string;+ posts: AvgrabResolvedPost[];+}++export async function resolveViaAvgrab(+ url: string,+ limit: number,+ logger: winston.Logger,+): Promise<MapDocument[] | null> {+ if (!config.AVGRAB_SERVICE_URL) return null;++ const matches = await matchesResolveRegex(url);+ if (!matches) return null;++ logger.info("URL matches avgrab resolve pattern, delegating to avgrab", {+ url,+ });++ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/resolve`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ url, limit }),+ });++ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ logger.error("avgrab resolve failed", {+ url,+ status: response.status,+ error,+ });+ return null;+ }++ const data: AvgrabResolveResponse = await response.json();++ return data.posts.map(post => {+ const { url: _url, title: _title, ...meta } = post;+ return {+ url: post.url,+ title: post.title || undefined,+ description: JSON.stringify(meta),+ };+ });+}