Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(map): full functionality
Gergő Móricz33fb6bb816f9
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex 8be52acc9..eb0941933 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -21,11 +21,11 @@ import { fireEngineMap } from "../../search/fireEngine";import { billTeam } from "../../services/billing/credit_billing";import { logJob } from "../../services/logging/log_job";import { logger } from "../../lib/logger";-import Redis from "ioredis";import { generateURLSplits, queryIndexAtDomainSplitLevelWithMeta, queryIndexAtSplitLevelWithMeta } from "../../services/index";+import { redisEvictConnection } from "../../services/redis";+import { performCosineSimilarityV2 } from "../../lib/map-cosine";configDotenv();-const redis = new Redis(process.env.REDIS_URL!);// Max Links that /map can returnconst MAX_MAP_LIMIT = 30000;@@ -39,7 +39,20 @@ interface MapResult {mapResults: MapDocument[];}-async function queryIndex(url: string, limit: number, useIndex: boolean): Promise<MapDocument[]> {+function dedupeMapDocumentArray(documents: MapDocument[]): MapDocument[] {+ let urlSet = new Set<string>(documents.map(x => x.url));++ let newDocuments: MapDocument[] = [+ ...Array.from(urlSet).map(x => {+ let localDocs = documents.filter(y => y.url === x);+ return localDocs.find(x => x.title !== undefined) || localDocs[0];+ }),+ ];++ return newDocuments;+}++async function queryIndex(url: string, limit: number, useIndex: boolean, includeSubdomains: boolean): Promise<MapDocument[]> {if (!useIndex) {return [];}@@ -49,7 +62,13 @@ async function queryIndex(url: string, limit: number, useIndex: boolean): Promisconst urlObj = new URL(url);const hostname = urlObj.hostname;- return (await queryIndexAtDomainSplitLevelWithMeta(hostname, limit));+ // TEMP: this should be altered on June 15th 2025 7AM PT - mogery+ const [domainLinks, splitLinks] = await Promise.all([+ includeSubdomains ? queryIndexAtDomainSplitLevelWithMeta(hostname, limit) : [],+ queryIndexAtSplitLevelWithMeta(url, limit),+ ]);++ return dedupeMapDocumentArray([...domainLinks, ...splitLinks]);} else {return (await queryIndexAtSplitLevelWithMeta(url, limit));}@@ -86,6 +105,7 @@ export async function getMapResults({}): Promise<MapResult> {const id = uuidv4();let mapResults: MapDocument[] = [];+ const zeroDataRetention = flags?.forceZDR ?? false;const sc: StoredCrawl = {originUrl: url,@@ -98,6 +118,7 @@ export async function getMapResults({internalOptions: { teamId },team_id: teamId,createdAt: Date.now(),+ zeroDataRetention,};const crawler = crawlToCrawler(id, sc, flags);@@ -107,114 +128,176 @@ export async function getMapResults({crawler.importRobotsTxt(sc.robots);} catch (_) {}- let urlWithoutWww = url.replace("www.", "");-- let mapUrl =- search && allowExternalLinks- ? `${search} ${urlWithoutWww}`- : search- ? `${search} site:${urlWithoutWww}`- : `site:${url}`;-- const resultsPerPage = 100;- const maxPages = Math.ceil(- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,- );-- const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = await redis.get(cacheKey);-- let allResults: any[] = [];- let pagePromises: Promise<any>[] = [];-- if (cachedResult) {- allResults = JSON.parse(cachedResult);+ // If sitemapOnly is true, only get links from sitemap+ if (crawlerOptions.sitemapOnly) {+ const sitemap = await crawler.tryGetSitemap(+ (urls) => {+ urls.forEach((x) => {+ mapResults.push({+ url: x,+ });+ });+ },+ true,+ true,+ crawlerOptions.timeout ?? 30000,+ abort,+ crawlerOptions.useMock,+ );+ if (sitemap > 0) {+ mapResults = mapResults+ .slice(1)+ .map((x) => {+ try {+ return {+ ...x,+ url: checkAndUpdateURLForMap(x.url).url.trim(),+ };+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as MapDocument[];+ }} else {- const fetchPage = async (page: number) => {- return fireEngineMap(mapUrl, {- numResults: resultsPerPage,- page: page,- }, abort);- };-- pagePromises = Array.from({ length: maxPages }, (_, i) =>- fetchPage(i + 1),+ let urlWithoutWww = url.replace("www.", "");++ let mapUrl =+ search && allowExternalLinks+ ? `${search} ${urlWithoutWww}`+ : search+ ? `${search} site:${urlWithoutWww}`+ : `site:${url}`;++ const resultsPerPage = 100;+ const maxPages = Math.ceil(+ Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage,);- allResults = await Promise.all(pagePromises);- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 48 * 60 * 60); // Cache for 48 hours- }+ const cacheKey = `fireEngineMap:${mapUrl}`;+ const cachedResult = await redisEvictConnection.get(cacheKey);- // Parallelize sitemap index query with search results- const [indexResults, ...searchResults] = await Promise.all([- queryIndex(url, limit, useIndex),- ...(cachedResult ? [] : pagePromises),- ]);+ let pagePromises: (Promise<any> | any)[];- if (indexResults.length > 0) {- mapResults.push(...indexResults);- }+ if (cachedResult) {+ pagePromises = JSON.parse(cachedResult);+ } else {+ const fetchPage = async (page: number) => {+ return fireEngineMap(mapUrl, {+ numResults: resultsPerPage,+ page: page,+ }, abort);+ };++ pagePromises = Array.from({ length: maxPages }, (_, i) =>+ fetchPage(i + 1),+ );+ }- console.log("searchResults", searchResults);+ // Parallelize sitemap index query with search results+ const [indexResults, searchResults] = await Promise.all([+ queryIndex(url, limit, useIndex, includeSubdomains),+ Promise.all(pagePromises),+ ]);- mapResults = mapResults.concat(searchResults.map(x => ({- url: x.url,- title: x.title,- description: x.description,- })).flat());+ if (!zeroDataRetention) {+ await redisEvictConnection.set(cacheKey, JSON.stringify(searchResults), "EX", 48 * 60 * 60); // Cache for 48 hours+ }- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);- if (mapResults.length > minumumCutoff) {- mapResults = mapResults.slice(0, minumumCutoff);- }+ if (indexResults.length > 0) {+ mapResults.push(...indexResults);+ }- mapResults = mapResults- .map((x) => {+ // If sitemap is not ignored, fetch sitemap+ // This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days+ if (!crawlerOptions.ignoreSitemap) {try {- return {- ...x,- url: checkAndUpdateURLForMap(x.url).url.trim(),- };- } catch (_) {- return null;+ await crawler.tryGetSitemap(+ (urls) => {+ mapResults.push(...urls.map(x => ({+ url: x,+ })));+ },+ true,+ false,+ crawlerOptions.timeout ?? 30000,+ abort,+ );+ } catch (e) {+ logger.warn("tryGetSitemap threw an error", { error: e });}- })- .filter((x) => x !== null) as MapDocument[];+ }- // allows for subdomains to be included- mapResults = mapResults.filter((x) => isSameDomain(x.url, url));+ mapResults = mapResults.concat(searchResults.flat().map(x => ({+ url: x.url,+ title: x.title,+ description: x.description,+ })));- // if includeSubdomains is false, filter out subdomains- if (!includeSubdomains) {- mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));- }+ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);+ if (mapResults.length > minumumCutoff) {+ mapResults = mapResults.slice(0, minumumCutoff);+ }- // Filter by path if enabled- if (filterByPath && !allowExternalLinks) {- try {- const urlObj = new URL(url);- const urlPath = urlObj.pathname;- // Only apply path filtering if the URL has a significant path (not just '/' or empty)- // This means we only filter by path if the user has not selected a root domain- if (urlPath && urlPath !== '/' && urlPath.length > 1) {- mapResults = mapResults.filter(x => {- try {- const linkObj = new URL(x.url);- return linkObj.pathname.startsWith(urlPath);- } catch (e) {- return false;- }- });+ if (search) {+ const searchQuery = search.toLowerCase();+ mapResults = performCosineSimilarityV2(mapResults, searchQuery);+ }++ mapResults = mapResults+ .map((x) => {+ try {+ return {+ ...x,+ url: checkAndUpdateURLForMap(x.url).url.trim(),+ };+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as MapDocument[];++ // allows for subdomains to be included+ mapResults = mapResults.filter((x) => isSameDomain(x.url, url));++ // if includeSubdomains is false, filter out subdomains+ if (!includeSubdomains) {+ mapResults = mapResults.filter((x) => isSameSubdomain(x.url, url));+ }++ // Filter by path if enabled+ if (filterByPath && !allowExternalLinks) {+ try {+ const urlObj = new URL(url);+ const urlPath = urlObj.pathname;+ // Only apply path filtering if the URL has a significant path (not just '/' or empty)+ // This means we only filter by path if the user has not selected a root domain+ if (urlPath && urlPath !== '/' && urlPath.length > 1) {+ mapResults = mapResults.filter(x => {+ try {+ const linkObj = new URL(x.url);+ return linkObj.pathname.startsWith(urlPath);+ } catch (e) {+ return false;+ }+ });+ }+ } catch (e) {+ // If URL parsing fails, continue without path filtering+ logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });}- } catch (e) {- // If URL parsing fails, continue without path filtering- logger.warn(`Failed to parse URL for path filtering: ${url}`, { error: e });}++ mapResults = dedupeMapDocumentArray(mapResults);}+ mapResults = crawlerOptions.sitemapOnly+ ? mapResults+ : mapResults.slice(0, limit);+return {success: true,- mapResults: mapResults,+ mapResults,job_id: id,time_taken: (new Date().getTime() - Date.now()) / 1000,};diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 47b2c0aa1..76abebcad 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -817,11 +817,6 @@ export type MapResponse =| {success: true;links?: MapDocument[];- metadata?: {- totalCount: number;- hasMore: boolean;- searchQuery?: string;- };};export type CrawlStatusParams = {diff --git a/apps/api/src/lib/map-cosine.ts b/apps/api/src/lib/map-cosine.tsindex a6c06e270..5b215ed75 100644--- a/apps/api/src/lib/map-cosine.ts+++ b/apps/api/src/lib/map-cosine.ts@@ -1,4 +1,5 @@import { logger } from "./logger";+import { MapDocument } from "../controllers/v2/types";export function performCosineSimilarity(links: string[], searchQuery: string) {try {@@ -44,3 +45,49 @@ export function performCosineSimilarity(links: string[], searchQuery: string) {return links;}}++export function performCosineSimilarityV2(links: MapDocument[], searchQuery: string) {+ try {+ // Function to calculate cosine similarity+ const cosineSimilarity = (vec1: number[], vec2: number[]): number => {+ const dotProduct = vec1.reduce((sum, val, i) => sum + val * vec2[i], 0);+ const magnitude1 = Math.sqrt(+ vec1.reduce((sum, val) => sum + val * val, 0),+ );+ const magnitude2 = Math.sqrt(+ vec2.reduce((sum, val) => sum + val * val, 0),+ );+ if (magnitude1 === 0 || magnitude2 === 0) return 0;+ return dotProduct / (magnitude1 * magnitude2);+ };++ // Function to convert text to vector+ const textToVector = (text: string): number[] => {+ const words = searchQuery.toLowerCase().split(/\W+/);+ return words.map((word) => {+ const count = (text.toLowerCase().match(new RegExp(word, "g")) || [])+ .length;+ return count / text.length;+ });+ };++ // Calculate similarity scores+ const similarityScores = links.map((link) => {+ const linkVector = textToVector(link.url); // TODO: we can use title/description here as well!+ const searchVector = textToVector(searchQuery);+ return cosineSimilarity(linkVector, searchVector);+ });++ // Sort links based on similarity scores and print scores+ const a = links+ .map((link, index) => ({ link, score: similarityScores[index] }))+ .sort((a, b) => b.score - a.score);++ links = a.map((item) => item.link);+ return links;+ } catch (error) {+ logger.error(`Error performing cosine similarity: ${error}`);+ return links;+ }+}+diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 34e7adcd2..08a0848dc 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -74,7 +74,7 @@ export async function getLinksFromSitemap(mode,sitemapUrl,error: response.success- ? response.document+ ? response.document.metadata.statusCode: response.error,},);