Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update index.ts
Nicolas50d2335451b9
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex fbd7ec8bf..06364cb6a 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -12,6 +12,12 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {&& meta.winnerEngine !== "index"&& meta.winnerEngine !== "index;documents"&& !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers))+ && !meta.options.parsers?.some(parser => {+ if (typeof parser === "object" && parser !== null && "maxPages" in parser) {+ return true;+ }+ return false;+ })&& (meta.internalOptions.teamId === "sitemap"|| (