Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update index.ts
Nicolas0e8fd9285356
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex cc2529479..3f9d8f05e 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -14,6 +14,7 @@ import { indexMaxReasonableTime, scrapeURLWithIndex } from "./index/index";import { useIndex } from "../../../services";import { hasFormatOfType } from "../../../lib/format-utils";import { PDFMetadata } from "../../../lib/pdf-parser";+import { getPDFMaxPages } from "../../../controllers/v2/types";export type Engine =| "fire-engine;chrome-cdp"@@ -446,6 +447,8 @@ export function buildFallbackList(meta: Meta): {&& process.env.FIRECRAWL_INDEX_WRITE_ONLY !== "true"&& meta.options.waitFor === 0&& !hasFormatOfType(meta.options.formats, "changeTracking")+ // Skip index if a non-default PDF maxPages is specified+ && getPDFMaxPages(meta.options.parsers) === undefined&& meta.options.maxAge !== 0&& (meta.options.headers === undefined