Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): read the index in v1 map the same way as v2 map (#5272)

Erik Engervallfcaeb59a645d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index c093aec49..486ced3af 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -24,11 +24,7 @@ import { externalRequestId } from "../../lib/external-request-id";
import { performCosineSimilarity } from "../../lib/map-cosine";
import { logger } from "../../lib/logger";
import Redis from "ioredis";
-import {
- generateURLSplits,
- queryIndexAtDomainSplitLevel,
- queryIndexAtSplitLevel,
-} from "../../services/index";
+import { queryIndex } from "../../lib/map-utils";
import { MapTimeoutError } from "../../lib/error";
import { checkPermissions } from "../../lib/permissions";
import { resolveSafeMode } from "../../lib/safe-mode";
@@ -54,39 +50,6 @@ interface MapResult {
mapResults: MapDocument[];
}
-async function queryIndex(
- url: string,
- limit: number,
- useIndex: boolean,
- includeSubdomains: boolean,
-): Promise<string[]> {
- if (!useIndex) {
- return [];
- }
-
- const urlSplits = generateURLSplits(url);
- if (urlSplits.length === 1) {
- const urlObj = new URL(url);
- const hostname = urlObj.hostname;
-
- // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
- const [domainLinks, splitLinks] = await Promise.all([
- includeSubdomains
- ? queryIndexAtDomainSplitLevel(
- hostname,
- limit,
- 14 * 24 * 60 * 60 * 1000,
- )
- : [],
- queryIndexAtSplitLevel(url, limit, 14 * 24 * 60 * 60 * 1000),
- ]);
-
- return Array.from(new Set([...domainLinks, ...splitLinks]));
- } else {
- return await queryIndexAtSplitLevel(url, limit);
- }
-}
-
export async function getMapResults({
url,
search,
@@ -257,7 +220,7 @@ export async function getMapResults({
]);
if (indexResults.length > 0) {
- links.push(...indexResults);
+ links.push(...indexResults.map(x => x.url));
}
// If sitemap is not ignored, fetch sitemap
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index a58beadce..8bf02e853 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -125,17 +125,6 @@ export function queryIndexAtSplitLevel(
);
}
-export function queryIndexAtDomainSplitLevel(
- i_level: number,
- i_domain_hash: Buffer,
- i_newer_than: string,
-): Promise<{ resolved_url: string }[]> {
- return execRows(
- dbIndex,
- sql`select * from query_index_at_domain_split_level(i_level => ${i_level}, i_domain_hash => ${i_domain_hash}, i_newer_than => ${i_newer_than}::timestamptz)`,
- );
-}
-
export function queryOmceSignatures(
i_domain_hash: Buffer,
i_newer_than: string,
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 55ba4c0b5..9727c1b86 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -51,7 +51,7 @@ function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
return Array.from(urlMap.values());
}
-async function queryIndex(
+export async function queryIndex(
url: string,
limit: number,
useIndex: boolean,
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index 3a047e6ee..d36606da6 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -4,7 +4,6 @@ import * as schema from "../db/schema";
import {
insertOmceJobIfNeeded,
queryIndexAtSplitLevel as rpcQueryIndexAtSplitLevel,
- queryIndexAtDomainSplitLevel as rpcQueryIndexAtDomainSplitLevel,
queryOmceSignatures as rpcQueryOmceSignatures,
queryEngpickerVerdict as rpcQueryEngpickerVerdict,
queryIndexAtSplitLevelWithMeta as rpcQueryIndexAtSplitLevelWithMeta,
@@ -469,39 +468,6 @@ export async function queryIndexAtSplitLevel(
}
}
-export async function queryIndexAtDomainSplitLevel(
- hostname: string,
- limit: number,
- maxAge = 2 * 24 * 60 * 60 * 1000,
-): Promise<string[]> {
- if (!useIndex || config.FIRECRAWL_INDEX_WRITE_ONLY) {
- return [];
- }
-
- const domainSplitsHash = generateDomainSplits(hostname).map(x => hashURL(x));
-
- const level = domainSplitsHash.length - 1;
- if (domainSplitsHash.length === 0) {
- return [];
- }
-
- // Raw SQL returns the full result set (no PostgREST 1000-row cap), so the
- // previous .range() pagination loop is no longer needed.
- try {
- const data = await rpcQueryIndexAtDomainSplitLevel(
- level,
- domainSplitsHash[level],
- new Date(Date.now() - maxAge).toISOString(),
- );
- const links = new Set<string>();
- data.forEach(x => links.add(x.resolved_url));
- return [...links].slice(0, limit);
- } catch (error) {
- _logger.warn("Error querying index", { error, hostname, limit });
- return [];
- }
-}
-
export async function queryOMCESignatures(
hostname: string,
maxAge = 2 * 24 * 60 * 60 * 1000,