Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: better filtering for urls that should be scraped
Nicolasac187452c3b7
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex f3e94b77c..c0e06a2d8 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -26,6 +26,7 @@ import { getMapResults } from "./map";import { buildDocument } from "../../lib/extract/build-document";import { generateBasicCompletion } from "../../lib/LLM-extraction";import { buildRefrasedPrompt } from "../../lib/extract/build-prompts";+import { removeDuplicateUrls } from "../../lib/validateUrl";configDotenv();const redis = new Redis(process.env.REDIS_URL!);@@ -88,8 +89,22 @@ export async function extractController(includeSubdomains: req.body.includeSubdomains,});- let mappedLinks = mapResults.links as MapDocument[];-+ let mappedLinks = mapResults.mapResults as MapDocument[];++ // Remove duplicates between mapResults.links and mappedLinks+ const allUrls = [...mappedLinks.map(m => m.url), ...mapResults.links];+ const uniqueUrls = removeDuplicateUrls(allUrls);++ // Only add URLs from mapResults.links that aren't already in mappedLinks+ const existingUrls = new Set(mappedLinks.map(m => m.url));+ const newUrls = uniqueUrls.filter(url => !existingUrls.has(url));++ mappedLinks = [+ ...mappedLinks,+ ...newUrls.map(url => ({ url, title: "", description: "" }))+ ];++if (mappedLinks.length === 0) {mappedLinks = [{ url: baseUrl, title: "", description: "" }];}@@ -102,6 +117,7 @@ export async function extractController(`url: ${x.url}, title: ${x.title}, description: ${x.description}`,);+if (req.body.prompt) {let searchQuery =req.body.prompt && allowExternalLinksdiff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 393933135..27a926fc8 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -32,10 +32,11 @@ const MAX_FIRE_ENGINE_RESULTS = 500;interface MapResult {success: boolean;- links: string[] | any[];+ links: string[];scrape_id?: string;job_id: string;time_taken: number;+ mapResults: MapDocument[];}export async function getMapResults({@@ -215,7 +216,8 @@ export async function getMapResults({return {success: true,- links: includeMetadata ? mapResults : linksToReturn,+ links: linksToReturn,+ mapResults: mapResults,scrape_id: origin?.includes("website") ? id : undefined,job_id: id,time_taken: (new Date().getTime() - Date.now()) / 1000,diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.tsindex 1ab117c21..f554eadc4 100644--- a/apps/api/src/lib/extract/build-prompts.ts+++ b/apps/api/src/lib/extract/build-prompts.ts@@ -4,11 +4,13 @@ export function buildRefrasedPrompt(prompt: string, url: string): string {Original prompt: "${prompt}"Provide a rephrased search query that:-1. Maintains the core intent of the original prompt+1. Maintains the core intent of the original prompt with ONLY the keywords2. Uses relevant keywords3. Is optimized for search engine results4. Is concise and focused5. Short is better than long+6. It is a search engine, not a chatbot+7. ConciseReturn only the rephrased search query, without any explanation or additional text.`;}