Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
stop document engine looping on non-document pages (#4217)
rowinsg01be9e58213c
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex c7fc89e8b..b3ac85a68 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -3,7 +3,11 @@ import { EngineScrapeResult } from "..";import { fetchFileToBuffer } from "../utils/downloadFile";import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";import type { Response } from "undici";-import { DocumentAntibotError, DocumentPrefetchFailed } from "../../error";+import {+ DocumentAntibotError,+ DocumentPrefetchFailed,+ EngineUnsuccessfulError,+} from "../../error";import { readFile, unlink } from "node:fs/promises";const converter = new DocumentConverter();@@ -146,6 +150,21 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {// Validate content type only when fetching directly (not using prefetch)const ct = response.headers.get("Content-Type");if (ct && !isValidDocumentContentType(ct)) {+ // The server declared a non-document type and nothing asked for a+ // document (no `document` flag means neither a document URL extension nor+ // a caller-requested format), so this engine is just the tail of the+ // stealth fallback list running against an ordinary HTML page.+ //+ // Escalating here is unrecoverable: DocumentAntibotError's remedy is to+ // clear the `document` flag and re-run the waterfall, which is a no-op+ // when the flag was never set, so the next attempt fetches the same page,+ // raises the same error, and burns attempts until SCRAPE_MAX_ATTEMPTS.+ // Fail this engine instead and let the waterfall move on. Mirrors the pdf+ // engine's guard, which is why pdf never developed this loop.+ if (!meta.featureFlags.has("document")) {+ throw new EngineUnsuccessfulError("document");+ }+// if downloaded file wasn't a valid document, throw antibot errorthrow new DocumentAntibotError();}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex fa13ae67d..029336c21 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -834,6 +834,15 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error: error.error,},);+ } else if (error.error instanceof EngineUnsuccessfulError) {+ // Deliberately silent. An engine declining the page is a normal+ // waterfall outcome and is already recorded elsewhere: the+ // success-factor check logs "deemed unsuccessful" with its reasoning,+ // and engines that recognise the body as none of their business+ // (pdf/document finding HTML) are preceded by "Scraping via X...".+ // Logging again only duplicated that, ~48k lines/hour across all+ // engines. Recognised here purely so it doesn't fall through to the+ // catch-all branch and get reported as an unexpected error.} else if (error.error instanceof AddFeatureError ||error.error instanceof RemoveFeatureError ||