Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasebe9de2ac582
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 71b1f9eb4..0a94289ce 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -28,8 +28,8 @@ configDotenv();const redis = new Redis(process.env.REDIS_URL!);const MAX_EXTRACT_LIMIT = 100;-const MAX_RANKING_LIMIT = 5;-const SCORE_THRESHOLD = 0.75;+const MAX_RANKING_LIMIT = 10;+const SCORE_THRESHOLD = 0.70;export async function extractController(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,@@ -64,28 +64,34 @@ export async function extractController(allowExternalLinks,origin: req.body.origin,limit: req.body.limit,- ignoreSitemap: false,+ ignoreSitemap: true,includeMetadata: true,includeSubdomains: req.body.includeSubdomains,});- let mappedLinks = mapResults.links.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);+ let mappedLinks = mapResults.links as MapDocument[];+ // Limit number of links to MAX_EXTRACT_LIMIT+ mappedLinks = mappedLinks.slice(0, MAX_EXTRACT_LIMIT);++ let mappedLinksRerank = mappedLinks.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);// Filter by path prefix if presentif (pathPrefix) {- mappedLinks = mappedLinks.filter(x => x.includes(`/${pathPrefix}/`));+ mappedLinks = mappedLinks.filter(x => x.url && x.url.includes(`/${pathPrefix}/`));}if (req.body.prompt) {- const linksAndScores = await performRanking(mappedLinks, mapUrl);+ const linksAndScores : { link: string, linkWithContext: string, score: number, originalIndex: number }[] = await performRanking(mappedLinksRerank, mappedLinks.map(l => l.url), mapUrl);mappedLinks = linksAndScores.filter(x => x.score > SCORE_THRESHOLD)- .map(x => x.link.split("url: ")[1].split(",")[0])- .filter(x => !isUrlBlocked(x))+ .map(x => mappedLinks.find(link => link.url === x.link))+ .filter((x): x is MapDocument => x !== undefined && x.url !== undefined && !isUrlBlocked(x.url)).slice(0, MAX_RANKING_LIMIT);+ console.log("linksAndScores", linksAndScores);+ console.log("linksAndScores", linksAndScores.length);}- return mappedLinks;+ return mappedLinks.map(x => x.url) as string[];} else {// Handle direct URLs without glob pattern@@ -100,6 +106,8 @@ export async function extractController(const processedUrls = await Promise.all(urlPromises);links.push(...processedUrls.flat());+ console.log("links", links.length);+ console// Scrape all links in parallelconst scrapePromises = links.map(async (url) => {const origin = req.body.origin || "api";diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex f2e9453a6..ba7be01f2 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -1,6 +1,7 @@import { Response } from "express";import { v4 as uuidv4 } from "uuid";import {+ MapDocument,mapRequestSchema,RequestWithAuth,scrapeOptions,@@ -86,7 +87,7 @@ export async function getMapResults({? `${search} ${urlWithoutWww}`: search ? `${search} site:${urlWithoutWww}`: `site:${url}`;-+const resultsPerPage = 100;const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);@@ -129,7 +130,7 @@ export async function getMapResults({});}- let mapResults = allResults+ let mapResults : MapDocument[] = allResults.flat().filter((result) => result !== null && result !== undefined);diff --git a/apps/api/src/lib/ranker.ts b/apps/api/src/lib/ranker.tsindex 7cd39820e..9a200f499 100644--- a/apps/api/src/lib/ranker.ts+++ b/apps/api/src/lib/ranker.ts@@ -40,24 +40,33 @@ const textToVector = (searchQuery: string, text: string): number[] => {});};-async function performRanking(links: string[], searchQuery: string) {+async function performRanking(linksWithContext: string[], links: string[], searchQuery: string) {try {// Generate embeddings for the search queryconst queryEmbedding = await getEmbedding(searchQuery);// Generate embeddings for each link and calculate similarity- const linksAndScores = await Promise.all(links.map(async (link) => {- const linkEmbedding = await getEmbedding(link);+ const linksAndScores = await Promise.all(linksWithContext.map(async (linkWithContext, index) => {+ const linkEmbedding = await getEmbedding(linkWithContext);// console.log("linkEmbedding", linkEmbedding);- // const linkVector = textToVector(searchQuery, link);+ // const linkVector = textToVector(searchQuery, linkWithContext);const score = cosineSimilarity(queryEmbedding, linkEmbedding);// console.log("score", score);- return { link, score };+ return {+ link: links[index], // Use corresponding link from links array+ linkWithContext,+ score,+ originalIndex: index // Store original position+ };}));- // Sort links based on similarity scores- linksAndScores.sort((a, b) => b.score - a.score);+ // Sort links based on similarity scores while preserving original order for equal scores+ linksAndScores.sort((a, b) => {+ const scoreDiff = b.score - a.score;+ // If scores are equal, maintain original order+ return scoreDiff === 0 ? a.originalIndex - b.originalIndex : scoreDiff;+ });return linksAndScores;} catch (error) {diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 0b82478ee..1eb2419fc 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -29,8 +29,6 @@ export async function fireEngineMap(page: options.page ?? 1,});- console.log("data", data);-if (!process.env.FIRE_ENGINE_BETA_URL) {console.warn("(v1/map Beta) Results might differ from cloud offering currently."