Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update index.ts

Nicolas50d2335451b9
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index fbd7ec8bf..06364cb6a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -12,6 +12,12 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
&& meta.winnerEngine !== "index"
&& meta.winnerEngine !== "index;documents"
&& !(meta.winnerEngine === "pdf" && !shouldParsePDF(meta.options.parsers))
+ && !meta.options.parsers?.some(parser => {
+ if (typeof parser === "object" && parser !== null && "maxPages" in parser) {
+ return true;
+ }
+ return false;
+ })
&& (
meta.internalOptions.teamId === "sitemap"
|| (