Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: improved map ranking algorithm

Nicolas7acd8d2edb6a
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 5ed3dd51c..112c06b44 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -2,6 +2,7 @@ import { Response } from "express";
import { v4 as uuidv4 } from "uuid";
import {
legacyCrawlerOptions,
+ LinkInfo,
mapRequestSchema,
RequestWithAuth,
} from "./types";
@@ -109,6 +110,10 @@ export async function mapController(
mapResults = mapResults.slice(0, minumumCutoff);
}
+
+
+ let linkInfos: LinkInfo[] = [];
+
if (mapResults.length > 0) {
if (req.body.search) {
// Ensure all map results are first, maintaining their order
@@ -117,6 +122,12 @@ export async function mapController(
...mapResults.slice(1).map((x) => x.url),
...links,
];
+
+ linkInfos = [
+ mapResults[0],
+ ...mapResults.slice(1),
+ ...links.map((x) => ({ url: x })),
+ ]
} else {
mapResults.map((x) => {
links.push(x.url);
@@ -128,7 +139,7 @@ export async function mapController(
if (req.body.search) {
const searchQuery = req.body.search.toLowerCase();
- links = performCosineSimilarity(links, searchQuery);
+ links = performCosineSimilarity(linkInfos, searchQuery);
}
links = links
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 033de6e0c..91618e1a8 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -478,3 +478,11 @@ export function legacyDocumentConverter(doc: any): Document {
},
};
}
+
+
+
+export interface LinkInfo {
+ url: string;
+ title?: string;
+ description?: string;
+}
\ No newline at end of file
diff --git a/apps/api/src/lib/map-cosine.ts b/apps/api/src/lib/map-cosine.ts
index db2491a93..8804aa58f 100644
--- a/apps/api/src/lib/map-cosine.ts
+++ b/apps/api/src/lib/map-cosine.ts
@@ -1,6 +1,7 @@
import { Logger } from "./logger";
+import { LinkInfo } from "../controllers/v1/types";
-export function performCosineSimilarity(links: string[], searchQuery: string) {
+export function performCosineSimilarity(links: LinkInfo[], searchQuery: string) {
try {
// Function to calculate cosine similarity
const cosineSimilarity = (vec1: number[], vec2: number[]): number => {
@@ -27,20 +28,20 @@ export function performCosineSimilarity(links: string[], searchQuery: string) {
// Calculate similarity scores
const similarityScores = links.map((link) => {
- const linkVector = textToVector(link);
+ const linkText = `${link.url} ${link.title || ''} ${link.description || ''}`.trim();
+ const linkVector = textToVector(linkText);
const searchVector = textToVector(searchQuery);
return cosineSimilarity(linkVector, searchVector);
});
- // Sort links based on similarity scores and print scores
- const a = links
+ // Sort links based on similarity scores
+ const sortedLinks = links
.map((link, index) => ({ link, score: similarityScores[index] }))
.sort((a, b) => b.score - a.score);
- links = a.map((item) => item.link);
- return links;
+ return sortedLinks.map((item) => item.link.url);
} catch (error) {
Logger.error(`Error performing cosine similarity: ${error}`);
- return links;
+ return links.map(link => link.url);
}
}