Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: fixed extract issues
Nicolas79e335636a14
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex ed3f149cc..48cba6063 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -61,7 +61,7 @@ export async function extractController(const baseUrl = url.replace("/*", "");// const pathPrefix = baseUrl.split('/').slice(3).join('/'); // Get path after domain if any- const allowExternalLinks = req.body.allowExternalLinks ?? true;+ const allowExternalLinks = req.body.allowExternalLinks;let urlWithoutWww = baseUrl.replace("www.", "");let mapUrl =req.body.prompt && allowExternalLinks@@ -84,6 +84,8 @@ export async function extractController(includeSubdomains: req.body.includeSubdomains,});+ // console.log("mapResults", mapResults);+let mappedLinks = mapResults.links as MapDocument[];// Limit number of links to MAX_EXTRACT_LIMITmappedLinks = mappedLinks.slice(0, MAX_EXTRACT_LIMIT);@@ -92,6 +94,7 @@ export async function extractController((x) =>`url: ${x.url}, title: ${x.title}, description: ${x.description}`,);+ // console.log("mappedLinksRerank", mappedLinksRerank);// Filter by path prefix if present// wrong@@ -150,15 +153,20 @@ export async function extractController(} else {// Handle direct URLs without glob patternif (!isUrlBlocked(url)) {+ // console.log("url", url);return [url];}return [];}});+ // console.log("urlPromises", urlPromises.length);+// Wait for all URL processing to complete and flatten resultsconst processedUrls = await Promise.all(urlPromises);- links.push(...processedUrls.flat());+ const flattenedUrls = processedUrls.flat().filter(url => url); // Filter out any null/undefined values+ links.push(...flattenedUrls);+ // console.log("links", links.length, "flattenedUrls", flattenedUrls.length);if (links.length === 0) {return res.status(400).json({diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2c0545608..5ba36ea72 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -198,7 +198,7 @@ export const extractV1Options = zlimit: z.number().int().positive().finite().safe().optional(),ignoreSitemap: z.boolean().default(false),includeSubdomains: z.boolean().default(true),- allowExternalLinks: z.boolean().default(true),+ allowExternalLinks: z.boolean().default(false),origin: z.string().optional().default("api"),timeout: z.number().int().positive().finite().safe().default(60000),})