Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: handle 415 Unsupported Media Type without retrying (#2737)
Abimael Martell4096f7f6af24
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex 5cfebe5f9..dce97e034 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -165,6 +165,14 @@ export async function fireEngineCheckStatus(const failedParse = failedSchema.safeParse(status);if (successParse.success) {+ // Check if this is an unsupported media type error (e.g., binary file)+ if (+ successParse.data.pageStatusCode === 415 &&+ successParse.data.pageError?.startsWith("Unsupported Media Type:")+ ) {+ throw new UnsupportedFileError(successParse.data.pageError);+ }+logger.debug("Scrape succeeded!", { jobId });return successParse.data;} else if (processingParse.success) {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 378c444fd..beef8c46a 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -217,6 +217,14 @@ export async function fireEngineScrape<const failedParse = failedSchema.safeParse(status);if (successParse.success) {+ // Check if this is an unsupported media type error (e.g., binary file)+ if (+ successParse.data.pageStatusCode === 415 &&+ successParse.data.pageError?.startsWith("Unsupported Media Type:")+ ) {+ throw new UnsupportedFileError(successParse.data.pageError);+ }+logger.debug("Scrape succeeded!");// Schedule A/B comparison if enabled (fire-and-forget)