Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scraper): reject unsupported binary content types in specialtyScrapeCheck
firecrawl-spring[bot]d2f03818f177
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 9233008f5..3e31e0a72 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -1,5 +1,5 @@import { Logger } from "winston";-import { AddFeatureError } from "../../error";+import { AddFeatureError, UnsupportedFileError } from "../../error";import { FireEngineCheckStatusSuccess } from "../fire-engine/checkStatus";import path from "path";import os from "os";@@ -155,4 +155,25 @@ export async function specialtyScrapeCheck() {throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));}++ // Reject unsupported binary content types (images, video, audio, archives, etc.)+ const unsupportedBinaryPrefixes = [+ "image/",+ "video/",+ "audio/",+ "application/zip",+ "application/x-tar",+ "application/gzip",+ "application/x-rar",+ "application/x-7z",+ "application/wasm",+ "application/x-executable",+ "application/x-sharedlib",+ "application/java-archive",+ ];+ if (+ unsupportedBinaryPrefixes.some(prefix => contentType.startsWith(prefix))+ ) {+ throw new UnsupportedFileError(contentType);+ }}