Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(map): full functionality

Gergő Móricz33fb6bb816f9
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index 8be52acc9..eb0941933 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -21,11 +21,11 @@ import { fireEngineMap } from "../../search/fireEngine";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { logger } from "../../lib/logger";
-import Redis from "ioredis";
import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";
+import { redisEvictConnection } from "../../services/redis";
+import { performCosineSimilarityV2 } from "../../lib/map-cosine";
configDotenv();
-const redis = new Redis(process.env.REDIS_URL!);
// Max Links that /map can return
const MAX_MAP_LIMIT = 30000;
@@ -39,7 +39,20 @@ interface MapResult {
mapResults: MapDocument[];
}
-async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<MapDocument[]> {
+function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {
+ let urlSet = new Set<string>(documents.map(x => x.url));
+
+ let newDocuments: MapDocument[] = [
+ ...Array.from(urlSet).map(x => {
+ let localDocs = documents.filter(y => y.url === x);
+ return localDocs.find(x => x.title !== undefined) || localDocs[0];
+ }),
+ ];
+
+ return newDocuments;
+}
+
+async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<MapDocument[]> {
if (!useIndex) {
return [];
}
@@ -49,7 +62,13 @@ async function queryIndex(url: string, limit: number, useIndex: boolean): Promis
const urlObj = new URL(url);
const hostname = urlObj.hostname;
- return (await queryIndexAtDomainSplitLevelWithMeta(hostname, limit));
+ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery
+ const [domainLinks, splitLinks] = await Promise.all([
+ includeSubdomains ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit) : [],
+ queryIndexAtSplitLevelWithMeta(url, limit),
+ ]);
+
+ return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);
} else {
return (await queryIndexAtSplitLevelWithMeta(url, limit));
}
@@ -86,6 +105,7 @@ export async function getMapResults({
}): Promise<MapResult> {
const id = uuidv4();
let mapResults: MapDocument[] = [];
+ const zeroDataRetention = flags?.forceZDR ?? false;
const sc: StoredCrawl = {
originUrl: url,
@@ -98,6 +118,7 @@ export async function getMapResults({
internalOptions: { teamId },
team_id: teamId,
createdAt: Date.now(),
+ zeroDataRetention,
};
const crawler = crawlToCrawler(id, sc, flags);
@@ -107,114 +128,176 @@ export async function getMapResults({
crawler.importRobotsTxt(sc.robots);
} catch (_) {}
- let urlWithoutWww = url.replace("www.", "");
-
- let mapUrl =
- search && allowExternalLinks
- ? `${search} ${urlWithoutWww}`
- : search
- ? `${search} site:${urlWithoutWww}`
- : `site:${url}`;
-
- const resultsPerPage = 100;
- const maxPages = Math.ceil(
- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,
- );
-
- const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = await redis.get(cacheKey);
-
- let allResults: any[] = [];
- let pagePromises: Promise<any>[] = [];
-
- if (cachedResult) {
- allResults = JSON.parse(cachedResult);
+ // If sitemapOnly is true, only get links from sitemap
+ if (crawlerOptions.sitemapOnly) {
+ const sitemap = await crawler.tryGetSitemap(
+ (urls) => {
+ urls.forEach((x) => {
+ mapResults.push({
+ url: x,
+ });
+ });
+ },
+ true,
+ true,
+ crawlerOptions.timeout ?? 30000,
+ abort,
+ crawlerOptions.useMock,
+ );
+ if (sitemap > 0) {
+ mapResults = mapResults
+ .slice(1)
+ .map((x) => {
+ try {
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(x.url).url.trim(),
+ };
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter((x) => x !== null) as MapDocument[];
+ }
} else {
- const fetchPage = async (page: number) => {
- return fireEngineMap(mapUrl, {
- numResults: resultsPerPage,
- page: page,
- }, abort);
- };
-
- pagePromises = Array.from({ length: maxPages }, (_, i) =>
- fetchPage(i + 1),
+ let urlWithoutWww = url.replace("www.", "");
+
+ let mapUrl =
+ search && allowExternalLinks
+ ? `${search} ${urlWithoutWww}`
+ : search
+ ? `${search} site:${urlWithoutWww}`
+ : `site:${url}`;
+
+ const resultsPerPage = 100;
+ const maxPages = Math.ceil(
+ Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,
);
- allResults = await Promise.all(pagePromises);
- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours
- }
+ const cacheKey = `fireEngineMap:${mapUrl}`;
+ const cachedResult = await redisEvictConnection.get(cacheKey);
- // Parallelize sitemap index query with search results
- const [indexResults, ...searchResults] = await Promise.all([
- queryIndex(url, limit, useIndex),
- ...(cachedResult ? [] : pagePromises),
- ]);
+ let pagePromises: (Promise<any> | any)[];
- if (indexResults.length > 0) {
- mapResults.push(...indexResults);
- }
+ if (cachedResult) {
+ pagePromises = JSON.parse(cachedResult);
+ } else {
+ const fetchPage = async (page: number) => {
+ return fireEngineMap(mapUrl, {
+ numResults: resultsPerPage,
+ page: page,
+ }, abort);
+ };
+
+ pagePromises = Array.from({ length: maxPages }, (_, i) =>
+ fetchPage(i + 1),
+ );
+ }
- console.log("searchResults", searchResults);
+ // Parallelize sitemap index query with search results
+ const [indexResults, searchResults] = await Promise.all([
+ queryIndex(url, limit, useIndex, includeSubdomains),
+ Promise.all(pagePromises),
+ ]);
- mapResults = mapResults.concat(searchResults.map(x => ({
- url: x.url,
- title: x.title,
- description: x.description,
- })).flat());
+ if (!zeroDataRetention) {
+ await redisEvictConnection.set(cacheKey, JSON.stringify(searchResults), "EX", 48 * 60 * 60); // Cache for 48 hours
+ }
- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
- if (mapResults.length > minumumCutoff) {
- mapResults = mapResults.slice(0, minumumCutoff);
- }
+ if (indexResults.length > 0) {
+ mapResults.push(...indexResults);
+ }
- mapResults = mapResults
- .map((x) => {
+ // If sitemap is not ignored, fetch sitemap
+ // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days
+ if (!crawlerOptions.ignoreSitemap) {
try {
- return {
- ...x,
- url: checkAndUpdateURLForMap(x.url).url.trim(),
- };
- } catch (_) {
- return null;
+ await crawler.tryGetSitemap(
+ (urls) => {
+ mapResults.push(...urls.map(x => ({
+ url: x,
+ })));
+ },
+ true,
+ false,
+ crawlerOptions.timeout ?? 30000,
+ abort,
+ );
+ } catch (e) {
+ logger.warn("tryGetSitemap threw an error", { error: e });
}
- })
- .filter((x) => x !== null) as MapDocument[];
+ }
- // allows for subdomains to be included
- mapResults = mapResults.filter((x) => isSameDomain(x.url, url));
+ mapResults = mapResults.concat(searchResults.flat().map(x => ({
+ url: x.url,
+ title: x.title,
+ description: x.description,
+ })));
- // if includeSubdomains is false, filter out subdomains
- if (!includeSubdomains) {
- mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));
- }
+ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
+ if (mapResults.length > minumumCutoff) {
+ mapResults = mapResults.slice(0, minumumCutoff);
+ }
- // Filter by path if enabled
- if (filterByPath && !allowExternalLinks) {
- try {
- const urlObj = new URL(url);
- const urlPath = urlObj.pathname;
- // Only apply path filtering if the URL has a significant path (not just '/' or empty)
- // This means we only filter by path if the user has not selected a root domain
- if (urlPath && urlPath !== '/' && urlPath.length > 1) {
- mapResults = mapResults.filter(x => {
- try {
- const linkObj = new URL(x.url);
- return linkObj.pathname.startsWith(urlPath);
- } catch (e) {
- return false;
- }
- });
+ if (search) {
+ const searchQuery = search.toLowerCase();
+ mapResults = performCosineSimilarityV2(mapResults, searchQuery);
+ }
+
+ mapResults = mapResults
+ .map((x) => {
+ try {
+ return {
+ ...x,
+ url: checkAndUpdateURLForMap(x.url).url.trim(),
+ };
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter((x) => x !== null) as MapDocument[];
+
+ // allows for subdomains to be included
+ mapResults = mapResults.filter((x) => isSameDomain(x.url, url));
+
+ // if includeSubdomains is false, filter out subdomains
+ if (!includeSubdomains) {
+ mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));
+ }
+
+ // Filter by path if enabled
+ if (filterByPath && !allowExternalLinks) {
+ try {
+ const urlObj = new URL(url);
+ const urlPath = urlObj.pathname;
+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)
+ // This means we only filter by path if the user has not selected a root domain
+ if (urlPath && urlPath !== '/' && urlPath.length > 1) {
+ mapResults = mapResults.filter(x => {
+ try {
+ const linkObj = new URL(x.url);
+ return linkObj.pathname.startsWith(urlPath);
+ } catch (e) {
+ return false;
+ }
+ });
+ }
+ } catch (e) {
+ // If URL parsing fails, continue without path filtering
+ logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });
}
- } catch (e) {
- // If URL parsing fails, continue without path filtering
- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });
}
+
+ mapResults = dedupeMapDocumentArray(mapResults);
}
+ mapResults = crawlerOptions.sitemapOnly
+ ? mapResults
+ : mapResults.slice(0, limit);
+
return {
success: true,
- mapResults: mapResults,
+ mapResults,
job_id: id,
time_taken: (new Date().getTime() - Date.now()) / 1000,
};
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 47b2c0aa1..76abebcad 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -817,11 +817,6 @@ export type MapResponse =
| {
success: true;
links?: MapDocument[];
- metadata?: {
- totalCount: number;
- hasMore: boolean;
- searchQuery?: string;
- };
};
export type CrawlStatusParams = {
diff --git a/apps/api/src/lib/map-cosine.ts b/apps/api/src/lib/map-cosine.ts
index a6c06e270..5b215ed75 100644
--- a/apps/api/src/lib/map-cosine.ts
+++ b/apps/api/src/lib/map-cosine.ts
@@ -1,4 +1,5 @@
import { logger } from "./logger";
+import { MapDocument } from "../controllers/v2/types";
export function performCosineSimilarity(links: string[], searchQuery: string) {
try {
@@ -44,3 +45,49 @@ export function performCosineSimilarity(links: string[], searchQuery: string) {
return links;
}
}
+
+export function performCosineSimilarityV2(links: MapDocument[], searchQuery: string) {
+ try {
+ // Function to calculate cosine similarity
+ const cosineSimilarity = (vec1: number[], vec2: number[]): number => {
+ const dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);
+ const magnitude1 = Math.sqrt(
+ vec1.reduce((sum, val) => sum + val * val, 0),
+ );
+ const magnitude2 = Math.sqrt(
+ vec2.reduce((sum, val) => sum + val * val, 0),
+ );
+ if (magnitude1 === 0 || magnitude2 === 0) return 0;
+ return dotProduct / (magnitude1 * magnitude2);
+ };
+
+ // Function to convert text to vector
+ const textToVector = (text: string): number[] => {
+ const words = searchQuery.toLowerCase().split(/\W+/);
+ return words.map((word) => {
+ const count = (text.toLowerCase().match(new RegExp(word, "g")) || [])
+ .length;
+ return count / text.length;
+ });
+ };
+
+ // Calculate similarity scores
+ const similarityScores = links.map((link) => {
+ const linkVector = textToVector(link.url); // TODO: we can use title/description here as well!
+ const searchVector = textToVector(searchQuery);
+ return cosineSimilarity(linkVector, searchVector);
+ });
+
+ // Sort links based on similarity scores and print scores
+ const a = links
+ .map((link, index) => ({ link, score: similarityScores[index] }))
+ .sort((a, b) => b.score - a.score);
+
+ links = a.map((item) => item.link);
+ return links;
+ } catch (error) {
+ logger.error(`Error performing cosine similarity: ${error}`);
+ return links;
+ }
+}
+
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 34e7adcd2..08a0848dc 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -74,7 +74,7 @@ export async function getLinksFromSitemap(
mode,
sitemapUrl,
error: response.success
- ? response.document
+ ? response.document.metadata.statusCode
: response.error,
},
);