Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolas91f4fd815fa5
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 1513365de..6d9b4a2c1 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -44,7 +44,7 @@ export async function extractController(// Process all URLs in parallelconst urlPromises = req.body.urls.map(async (url) => {- if (url.includes('/*')) {+ if (url.includes('/*') || req.body.allowExternalLinks) {// Handle glob pattern URLsconst baseUrl = url.replace('/*', '');const pathPrefix = baseUrl.split('/').slice(3).join('/'); // Get path after domain if any