Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): read the index in v1 map the same way as v2 map (#5272)
Erik Engervallfcaeb59a645d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex c093aec49..486ced3af 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -24,11 +24,7 @@ import { externalRequestId } from "../../lib/external-request-id";import { performCosineSimilarity } from "../../lib/map-cosine";import { logger } from "../../lib/logger";import Redis from "ioredis";-import {- generateURLSplits,- queryIndexAtDomainSplitLevel,- queryIndexAtSplitLevel,-} from "../../services/index";+import { queryIndex } from "../../lib/map-utils";import { MapTimeoutError } from "../../lib/error";import { checkPermissions } from "../../lib/permissions";import { resolveSafeMode } from "../../lib/safe-mode";@@ -54,39 +50,6 @@ interface MapResult {mapResults: MapDocument[];}-async function queryIndex(- url: string,- limit: number,- useIndex: boolean,- includeSubdomains: boolean,-): Promise<string[]> {- if (!useIndex) {- return [];- }-- const urlSplits = generateURLSplits(url);- if (urlSplits.length === 1) {- const urlObj = new URL(url);- const hostname = urlObj.hostname;-- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery- const [domainLinks, splitLinks] = await Promise.all([- includeSubdomains- ? queryIndexAtDomainSplitLevel(- hostname,- limit,- 14 * 24 * 60 * 60 * 1000,- )- : [],- queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000),- ]);-- return Array.from(new Set([...domainLinks, ...splitLinks]));- } else {- return await queryIndexAtSplitLevel(url, limit);- }-}-export async function getMapResults({url,search,@@ -257,7 +220,7 @@ export async function getMapResults({]);if (indexResults.length > 0) {- links.push(...indexResults);+ links.push(...indexResults.map(x => x.url));}// If sitemap is not ignored, fetch sitemapdiff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex a58beadce..8bf02e853 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -125,17 +125,6 @@ export function queryIndexAtSplitLevel();}-export function queryIndexAtDomainSplitLevel(- i_level: number,- i_domain_hash: Buffer,- i_newer_than: string,-): Promise<{ resolved_url: string }[]> {- return execRows(- dbIndex,- sql`select * from query_index_at_domain_split_level(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,- );-}-export function queryOmceSignatures(i_domain_hash: Buffer,i_newer_than: string,diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 55ba4c0b5..9727c1b86 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -51,7 +51,7 @@ function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {return Array.from(urlMap.values());}-async function queryIndex(+export async function queryIndex(url: string,limit: number,useIndex: boolean,diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex 3a047e6ee..d36606da6 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -4,7 +4,6 @@ import * as schema from "../db/schema";import {insertOmceJobIfNeeded,queryIndexAtSplitLevel as rpcQueryIndexAtSplitLevel,- queryIndexAtDomainSplitLevel as rpcQueryIndexAtDomainSplitLevel,queryOmceSignatures as rpcQueryOmceSignatures,queryEngpickerVerdict as rpcQueryEngpickerVerdict,queryIndexAtSplitLevelWithMeta as rpcQueryIndexAtSplitLevelWithMeta,@@ -469,39 +468,6 @@ export async function queryIndexAtSplitLevel(}}-export async function queryIndexAtDomainSplitLevel(- hostname: string,- limit: number,- maxAge = 2 * 24 * 60 * 60 * 1000,-): Promise<string[]> {- if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {- return [];- }-- const domainSplitsHash = generateDomainSplits(hostname).map(x => hashURL(x));-- const level = domainSplitsHash.length - 1;- if (domainSplitsHash.length === 0) {- return [];- }-- // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the- // previous .range() pagination loop is no longer needed.- try {- const data = await rpcQueryIndexAtDomainSplitLevel(- level,- domainSplitsHash[level],- new Date(Date.now() - maxAge).toISOString(),- );- const links = new Set<string>();- data.forEach(x => links.add(x.resolved_url));- return [...links].slice(0, limit);- } catch (error) {- _logger.warn("Error querying index", { error, hostname, limit });- return [];- }-}-export async function queryOMCESignatures(hostname: string,maxAge = 2 * 24 * 60 * 60 * 1000,