Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: handle 415 Unsupported Media Type without retrying (#2737)

Abimael Martell4096f7f6af24
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index 5cfebe5f9..dce97e034 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -165,6 +165,14 @@ export async function fireEngineCheckStatus(
const failedParse = failedSchema.safeParse(status);
if (successParse.success) {
+ // Check if this is an unsupported media type error (e.g., binary file)
+ if (
+ successParse.data.pageStatusCode === 415 &&
+ successParse.data.pageError?.startsWith("Unsupported Media Type:")
+ ) {
+ throw new UnsupportedFileError(successParse.data.pageError);
+ }
+
logger.debug("Scrape succeeded!", { jobId });
return successParse.data;
} else if (processingParse.success) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 378c444fd..beef8c46a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -217,6 +217,14 @@ export async function fireEngineScrape<
const failedParse = failedSchema.safeParse(status);
if (successParse.success) {
+ // Check if this is an unsupported media type error (e.g., binary file)
+ if (
+ successParse.data.pageStatusCode === 415 &&
+ successParse.data.pageError?.startsWith("Unsupported Media Type:")
+ ) {
+ throw new UnsupportedFileError(successParse.data.pageError);
+ }
+
logger.debug("Scrape succeeded!");
// Schedule A/B comparison if enabled (fire-and-forget)