Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

stop document engine looping on non-document pages (#4217)

rowinsg01be9e58213c
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index c7fc89e8b..b3ac85a68 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -3,7 +3,11 @@ import { EngineScrapeResult } from "..";
import { fetchFileToBuffer } from "../utils/downloadFile";
import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
import type { Response } from "undici";
-import { DocumentAntibotError, DocumentPrefetchFailed } from "../../error";
+import {
+ DocumentAntibotError,
+ DocumentPrefetchFailed,
+ EngineUnsuccessfulError,
+} from "../../error";
import { readFile, unlink } from "node:fs/promises";
const converter = new DocumentConverter();
@@ -146,6 +150,21 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
// Validate content type only when fetching directly (not using prefetch)
const ct = response.headers.get("Content-Type");
if (ct && !isValidDocumentContentType(ct)) {
+ // The server declared a non-document type and nothing asked for a
+ // document (no `document` flag means neither a document URL extension nor
+ // a caller-requested format), so this engine is just the tail of the
+ // stealth fallback list running against an ordinary HTML page.
+ //
+ // Escalating here is unrecoverable: DocumentAntibotError's remedy is to
+ // clear the `document` flag and re-run the waterfall, which is a no-op
+ // when the flag was never set, so the next attempt fetches the same page,
+ // raises the same error, and burns attempts until SCRAPE_MAX_ATTEMPTS.
+ // Fail this engine instead and let the waterfall move on. Mirrors the pdf
+ // engine's guard, which is why pdf never developed this loop.
+ if (!meta.featureFlags.has("document")) {
+ throw new EngineUnsuccessfulError("document");
+ }
+
// if downloaded file wasn't a valid document, throw antibot error
throw new DocumentAntibotError();
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index fa13ae67d..029336c21 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -834,6 +834,15 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error: error.error,
},
);
+ } else if (error.error instanceof EngineUnsuccessfulError) {
+ // Deliberately silent. An engine declining the page is a normal
+ // waterfall outcome and is already recorded elsewhere: the
+ // success-factor check logs "deemed unsuccessful" with its reasoning,
+ // and engines that recognise the body as none of their business
+ // (pdf/document finding HTML) are preceded by "Scraping via X...".
+ // Logging again only duplicated that, ~48k lines/hour across all
+ // engines. Recognised here purely so it doesn't fall through to the
+ // catch-all branch and get reported as an unexpected error.
} else if (
error.error instanceof AddFeatureError ||
error.error instanceof RemoveFeatureError ||