Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed re-ranker and enabled url cache of 2hrs
Nicolasc3fd13a82ba1
diff --git a/.gitignore b/.gitignoreindex fc5274903..311ee4df3 100644--- a/.gitignore+++ b/.gitignore@@ -33,3 +33,6 @@ apps/js-sdk/firecrawl/dist/examples/haiku_web_crawler/firecrawl_env/examples/sonnet_web_crawler/firecrawl_env/examples/internal_link_assitant/firecrawl_env++/apps/api/logs/*+/apps/api/debug/*\ No newline at end of filediff --git a/apps/api/src/lib/cache.ts b/apps/api/src/lib/cache.tsindex cbab4e053..ff91fa884 100644--- a/apps/api/src/lib/cache.ts+++ b/apps/api/src/lib/cache.ts@@ -42,7 +42,7 @@ export async function saveEntryToCache(key: string, entry: CacheEntry) {if (!cacheRedis) return;try {- await cacheRedis.set(key, JSON.stringify(entry));+ await cacheRedis.set(key, JSON.stringify(entry), "EX", 3600); // 1 hour in seconds} catch (error) {logger.warn("Failed to save to cache", { key, error });}diff --git a/apps/api/src/lib/extract/config.ts b/apps/api/src/lib/extract/config.tsnew file mode 100644index 000000000..f8333b3c6--- /dev/null+++ b/apps/api/src/lib/extract/config.ts@@ -0,0 +1,7 @@+export const extractConfig = {+ MAX_INITIAL_RANKING_LIMIT: 1000,+ MAX_RANKING_LIMIT: 20,+ INITIAL_SCORE_THRESHOLD: 0.75,+ FALLBACK_SCORE_THRESHOLD: 0.5,+ MIN_REQUIRED_LINKS: 1,+};\ No newline at end of filediff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsindex 2a4e2f62a..e5b61741f 100644--- a/apps/api/src/lib/extract/reranker.ts+++ b/apps/api/src/lib/extract/reranker.ts@@ -3,15 +3,13 @@ import { performRanking } from "../ranker";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { logger } from "../logger";import { CohereClient } from "cohere-ai";+import { extractConfig } from "./config";const cohere = new CohereClient({token: process.env.COHERE_API_KEY,});-const MAX_RANKING_LIMIT = 10;-const INITIAL_SCORE_THRESHOLD = 0.75;-const FALLBACK_SCORE_THRESHOLD = 0.5;-const MIN_REQUIRED_LINKS = 1;+interface RankingResult {mappedLinks: MapDocument[];@@ -61,32 +59,35 @@ export async function rerankLinks(searchQuery,);+// First try with high thresholdlet filteredLinks = filterAndProcessLinks(mappedLinks,linksAndScores,- INITIAL_SCORE_THRESHOLD,+ extractConfig.INITIAL_SCORE_THRESHOLD,);++// If we don't have enough high-quality links, try with lower threshold- if (filteredLinks.length < MIN_REQUIRED_LINKS) {+ if (filteredLinks.length < extractConfig.MIN_REQUIRED_LINKS) {logger.info(- `Only found ${filteredLinks.length} links with score > ${INITIAL_SCORE_THRESHOLD}. Trying lower threshold...`,+ `Only found ${filteredLinks.length} links with score > ${extractConfig.INITIAL_SCORE_THRESHOLD}. Trying lower threshold...`,);filteredLinks = filterAndProcessLinks(mappedLinks,linksAndScores,- FALLBACK_SCORE_THRESHOLD,+ extractConfig.FALLBACK_SCORE_THRESHOLD,);if (filteredLinks.length === 0) {// If still no results, take top N results regardless of scorelogger.warn(- `No links found with score > ${FALLBACK_SCORE_THRESHOLD}. Taking top ${MIN_REQUIRED_LINKS} results.`,+ `No links found with score > ${extractConfig.FALLBACK_SCORE_THRESHOLD}. Taking top ${extractConfig.MIN_REQUIRED_LINKS} results.`,);filteredLinks = linksAndScores.sort((a, b) => b.score - a.score)- .slice(0, MIN_REQUIRED_LINKS)+ .slice(0, extractConfig.MIN_REQUIRED_LINKS).map((x) => mappedLinks.find((link) => link.url === x.link)).filter((x): x is MapDocument =>@@ -108,7 +109,7 @@ export async function rerankLinks(}});- const rankedLinks = filteredLinks.slice(0, MAX_RANKING_LIMIT);+ const rankedLinks = filteredLinks.slice(0, extractConfig.MAX_RANKING_LIMIT);// Mark URLs that will be used in completionrankedLinks.forEach(link => {@@ -119,7 +120,7 @@ export async function rerankLinks(});// Mark URLs that were dropped due to ranking limit- filteredLinks.slice(MAX_RANKING_LIMIT).forEach(link => {+ filteredLinks.slice(extractConfig.MAX_RANKING_LIMIT).forEach(link => {const trace = urlTraces.find(t => t.url === link.url);if (trace) {trace.warning = 'Excluded due to ranking limit';diff --git a/apps/api/src/lib/extract/url-processor.ts b/apps/api/src/lib/extract/url-processor.tsindex 4d61a8d35..9f255ad7b 100644--- a/apps/api/src/lib/extract/url-processor.ts+++ b/apps/api/src/lib/extract/url-processor.ts@@ -7,8 +7,7 @@ import { generateBasicCompletion } from "../LLM-extraction";import { buildRefrasedPrompt } from "./build-prompts";import { logger } from "../logger";import { rerankLinks } from "./reranker";--const MAX_EXTRACT_LIMIT = 100;+import { extractConfig } from "./config";interface ProcessUrlOptions {url: string;@@ -96,8 +95,8 @@ export async function processUrl(options: ProcessUrlOptions, urlTraces: URLTracemappedLinks = [{ url: baseUrl, title: "", description: "" }];}- // Limit initial set of links- mappedLinks = mappedLinks.slice(0, MAX_EXTRACT_LIMIT);+ // Limit initial set of links (1000)+ mappedLinks = mappedLinks.slice(0, extractConfig.MAX_INITIAL_RANKING_LIMIT);// Perform reranking if prompt is providedif (options.prompt) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex bb0c485c0..bf51ac946 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -38,7 +38,7 @@ const useCache =process.env.CACHE_REDIS_URL !== undefined;export const engines: Engine[] = [- // ...(useCache ? [ "cache" as const ] : []),+ ...(useCache ? [ "cache" as const ] : []),...(useFireEngine? ["fire-engine;chrome-cdp" as const,