Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): handle non-PDF URL errors (#2147)
tom4cc0b4ed20a6
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex c34b5d6fa..0f6442e53 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -12,6 +12,7 @@ import {PDFInsufficientTimeError,PDFPrefetchFailed,RemoveFeatureError,+ EngineUnsuccessfulError,} from "../../error";import { readFile, unlink } from "node:fs/promises";import path from "node:path";@@ -200,7 +201,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (ct && !ct.includes("application/pdf")) {// if downloaded file wasn't a PDFif (meta.pdfPrefetch === undefined) {- throw new PDFAntibotError();+ // for non-PDF URLs, this is expected, not anti-bot+ if (!meta.featureFlags.has("pdf")) {+ throw new EngineUnsuccessfulError("pdf");+ } else {+ throw new PDFAntibotError();+ }} else {throw new PDFPrefetchFailed();}@@ -233,7 +239,12 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {if (ct && !ct.includes("application/pdf")) {// if downloaded file wasn't a PDFif (meta.pdfPrefetch === undefined) {- throw new PDFAntibotError();+ // for non-PDF URLs, this is expected, not anti-bot+ if (!meta.featureFlags.has("pdf")) {+ throw new EngineUnsuccessfulError("pdf");+ } else {+ throw new PDFAntibotError();+ }} else {throw new PDFPrefetchFailed();}