Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Nick: improved map ranking algorithm"
Nicolasd2344aa14bc8
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 112c06b44..5ed3dd51c 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -2,7 +2,6 @@ import { Response } from "express";import { v4 as uuidv4 } from "uuid";import {legacyCrawlerOptions,- LinkInfo,mapRequestSchema,RequestWithAuth,} from "./types";@@ -110,10 +109,6 @@ export async function mapController(mapResults = mapResults.slice(0, minumumCutoff);}--- let linkInfos: LinkInfo[] = [];-if (mapResults.length > 0) {if (req.body.search) {// Ensure all map results are first, maintaining their order@@ -122,12 +117,6 @@ export async function mapController(...mapResults.slice(1).map((x) => x.url),...links,];-- linkInfos = [- mapResults[0],- ...mapResults.slice(1),- ...links.map((x) => ({ url: x })),- ]} else {mapResults.map((x) => {links.push(x.url);@@ -139,7 +128,7 @@ export async function mapController(if (req.body.search) {const searchQuery = req.body.search.toLowerCase();- links = performCosineSimilarity(linkInfos, searchQuery);+ links = performCosineSimilarity(links, searchQuery);}links = linksdiff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 91618e1a8..033de6e0c 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -478,11 +478,3 @@ export function legacyDocumentConverter(doc: any): Document {},};}----export interface LinkInfo {- url: string;- title?: string;- description?: string;-}\ No newline at end of filediff --git a/apps/api/src/lib/map-cosine.ts b/apps/api/src/lib/map-cosine.tsindex 8804aa58f..db2491a93 100644--- a/apps/api/src/lib/map-cosine.ts+++ b/apps/api/src/lib/map-cosine.ts@@ -1,7 +1,6 @@import { Logger } from "./logger";-import { LinkInfo } from "../controllers/v1/types";-export function performCosineSimilarity(links: LinkInfo[], searchQuery: string) {+export function performCosineSimilarity(links: string[], searchQuery: string) {try {// Function to calculate cosine similarityconst cosineSimilarity = (vec1: number[], vec2: number[]): number => {@@ -28,20 +27,20 @@ export function performCosineSimilarity(links: LinkInfo[], searchQuery: string)// Calculate similarity scoresconst similarityScores = links.map((link) => {- const linkText = `${link.url} ${link.title || ''} ${link.description || ''}`.trim();- const linkVector = textToVector(linkText);+ const linkVector = textToVector(link);const searchVector = textToVector(searchQuery);return cosineSimilarity(linkVector, searchVector);});- // Sort links based on similarity scores- const sortedLinks = links+ // Sort links based on similarity scores and print scores+ const a = links.map((link, index) => ({ link, score: similarityScores[index] })).sort((a, b) => b.score - a.score);- return sortedLinks.map((item) => item.link.url);+ links = a.map((item) => item.link);+ return links;} catch (error) {Logger.error(`Error performing cosine similarity: ${error}`);- return links.map(link => link.url);+ return links;}}