Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: audio format (ENG-4708) (#3226)
Gergő Móricz37889a332726
diff --git a/CLAUDE.md b/CLAUDE.mdindex 8b3a89a96..991b19f37 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -1,6 +1,6 @@Firecrawl is a web scraper API. The directory you have access to is a monorepo:- `apps/api` has the actual API and worker code- - `apps/js-sdk`, `apps/python-sdk`, and `apps/rust-sdk` are various SDKs+ - `apps/js-sdk`, `apps/python-sdk`, `apps/rust-sdk` and `apps/java-sdk` are various SDKsWhen making changes to the API, here are the general steps you should take:1. Write some end-to-end tests that assert your win conditions, if they don't already existdiff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 16e537629..ba1c03d1a 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -2012,6 +2012,42 @@ describe("Attribute formats", () => {);});+ describeIf(!TEST_SELF_HOST)("Audio format", () => {+ it.concurrent(+ "should return audio field with signed GCS URL for a YouTube video",+ async () => {+ const data = await scrape(+ {+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["audio"],+ },+ identity,+ );++ expect(data.audio).toBeDefined();+ expect(typeof data.audio).toBe("string");+ expect(data.audio).toMatch(/^https:\/\//);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "should reject non-YouTube URL with audio format",+ async () => {+ const result = await scrapeWithFailure(+ {+ url: "https://example.com",+ formats: ["audio"],+ },+ identity,+ );++ expect(result.error).toMatch(/audio.*youtube/i);+ },+ scrapeTimeout,+ );+ });+describe("UUID validation", () => {it.concurrent("should reject invalid UUID 'None' for scrape status",diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 7dbb2b5d1..6b04dc208 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -250,6 +250,9 @@ const configSchema = z.object({BROWSER_SERVICE_API_KEY: z.string().optional(),BROWSER_SERVICE_WEBHOOK_SECRET: z.string().optional(),+ // Audio (avgrab)+ AVGRAB_SERVICE_URL: z.string().optional(),+NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),});diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 7bbcbb888..c94d24b93 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -27,6 +27,24 @@ import {} from "../../services/webhook/schema";import { BrandingProfile } from "../../types/branding";+function isYouTubeUrl(url: string): boolean {+ try {+ const parsed = new globalThis.URL(url);+ if (+ parsed.hostname === "youtube.com" ||+ parsed.hostname.endsWith(".youtube.com")+ ) {+ return parsed.pathname === "/watch" && !!parsed.searchParams.get("v");+ }+ if (parsed.hostname === "youtu.be") {+ return parsed.pathname !== "/";+ }+ return false;+ } catch {+ return false;+ }+}+// Base URL schema with common validation logicexport const URL = z.preprocess(x => {@@ -419,7 +437,8 @@ export type FormatObject =| ScreenshotFormatWithOptions| AttributesFormatWithOptions| QueryFormatWithOptions- | { type: "branding" };+ | { type: "branding" }+ | { type: "audio" };const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);@@ -528,6 +547,7 @@ const baseScrapeOptions = z.strictObject({attributesFormatWithOptions,z.strictObject({ type: z.literal("branding") }),queryFormatWithOptions,+ z.strictObject({ type: z.literal("audio") }),]).array().optional()@@ -812,6 +832,10 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase).refine(waitForRefine, waitForRefineOpts)+ .refine(+ x => !x.formats?.some(f => f.type === "audio") || isYouTubeUrl(x.url),+ "The audio format is only supported for YouTube URLs (youtube.com/watch?v=... or youtu.be/...)",+ ).transform(extractTransformRequired);export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;@@ -1005,6 +1029,7 @@ export type Document = {links?: string[];images?: string[];screenshot?: string;+ audio?: string;extract?: any;json?: any;summary?: string;diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 3beca1fcb..9ada777d2 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -59,10 +59,6 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = 5;}- if (hasFormatOfType(options.formats, "query")) {- creditsToBeBilled = 5;- }-if (internalOptions.v1Agent?.model === "fire-1" ||internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"@@ -70,6 +66,14 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);}+ if (hasFormatOfType(options.formats, "query")) {+ creditsToBeBilled += 4;+ }++ if (hasFormatOfType(options.formats, "audio")) {+ creditsToBeBilled += 4;+ }+if (internalOptions.zeroDataRetention) {creditsToBeBilled += flags?.zdrCost ?? 1;}diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.tsnew file mode 100644index 000000000..3798fe01c--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts@@ -0,0 +1,45 @@+import { Meta } from "..";+import { Document } from "../../../controllers/v2/types";+import { config } from "../../../config";+import { hasFormatOfType } from "../../../lib/format-utils";++export async function fetchAudio(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (!hasFormatOfType(meta.options.formats, "audio")) {+ return document;+ }++ if (!config.AVGRAB_SERVICE_URL) {+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");+ document.warning =+ "Audio format is not available (service not configured)." ++ (document.warning ? " " + document.warning : "");+ return document;+ }++ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ url: meta.url }),+ });++ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ throw new Error(`Audio download failed: ${error.detail}`);+ }++ const data = await response.json().catch(() => null);++ if (!data || !data.public_url || typeof data.public_url !== "string") {+ throw new Error(+ "Audio download failed: avgrab service returned an invalid response (missing public_url)",+ );+ }++ document.audio = data.public_url;+ return document;+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex c8916c66c..93d14e9a0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -17,6 +17,7 @@ import { performAgent } from "./agent";import { performAttributes } from "./performAttributes";import { deriveDiff } from "./diff";+import { fetchAudio } from "./audio";import { useIndex, useSearchIndex } from "../../../services/index";import { sendDocumentToIndex } from "../engines/index/index";import { sendDocumentToSearchIndex } from "./sendToSearchIndex";@@ -450,6 +451,15 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}+ const hasAudio = hasFormatOfType(meta.options.formats, "audio");+ if (!hasAudio && document.audio !== undefined) {+ delete document.audio;+ } else if (hasAudio && document.audio === undefined) {+ meta.logger.warn(+ "Request had format: audio, but there was no audio field in the result.",+ );+ }+if (!hasChangeTracking && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",@@ -522,6 +532,7 @@ const transformerStack: Transformer[] = [performAttributes,performAgent,deriveDiff,+ fetchAudio,coerceFieldsToFormats,removeBase64Images,];diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex e87f9f923..3d4232847 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.1.1"+version = "1.2.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.javaindex beb41fb0f..0fc75145d 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java@@ -19,6 +19,7 @@ public class Document {private List<String> links;private List<String> images;private String screenshot;+ private String audio;private List<Map<String, Object>> attributes;private Map<String, Object> actions;private String warning;@@ -34,6 +35,7 @@ public class Document {public List<String> getLinks() { return links; }public List<String> getImages() { return images; }public String getScreenshot() { return screenshot; }+ public String getAudio() { return audio; }public List<Map<String, Object>> getAttributes() { return attributes; }public Map<String, Object> getActions() { return actions; }public String getWarning() { return warning; }diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.javaindex b7ab815d2..a8560bc76 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java@@ -104,8 +104,8 @@ public class ScrapeOptions {/*** Output formats to request. Accepts strings like "markdown", "html", "rawHtml",- * "links", "screenshot", "json", etc., or format configuration maps for advanced- * formats (e.g., JsonFormat, ScreenshotFormat).+ * "links", "screenshot", "json", "audio", etc., or format configuration maps for+ * advanced formats (e.g., JsonFormat, ScreenshotFormat).*/public Builder formats(List<Object> formats) { this.formats = formats; return this; }diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex f0be3bb04..07ae84c7a 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.17.0",+ "version": "4.18.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex bd974a7b1..d8dcd88b1 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -12,7 +12,8 @@ export type FormatString =| 'changeTracking'| 'json'| 'attributes'- | 'branding';+ | 'branding'+ | 'audio';export interface Viewport {width: number;@@ -392,6 +393,7 @@ export interface Document {links?: string[];images?: string[];screenshot?: string;+ audio?: string;attributes?: Array<{selector: string;attribute: string;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 04db4894c..473aa174a 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.20.0"+__version__ = "4.21.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex e19653262..bd1a316a2 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -274,6 +274,7 @@ class Document(BaseModel):links: Optional[List[str]] = Noneimages: Optional[List[str]] = Nonescreenshot: Optional[str] = None+ audio: Optional[str] = Noneactions: Optional[Dict[str, Any]] = Noneanswer: Optional[str] = Nonewarning: Optional[str] = None@@ -386,6 +387,7 @@ FormatString = Literal["attributes","branding","query",+ "audio",# snake_case versions (user-friendly)"raw_html","change_tracking",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 5d86ffe2f..c47284047 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,7 +1,7 @@[package]name = "firecrawl"author= "Mendable.ai"-version = "1.3.0"+version = "1.4.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rsindex 94db9c8fa..ffbe8fe44 100644--- a/apps/rust-sdk/src/v2/scrape.rs+++ b/apps/rust-sdk/src/v2/scrape.rs@@ -338,7 +338,10 @@ impl Client {options: ScrapeExecuteOptions,) -> Result<ScrapeExecuteResponse, FirecrawlError> {let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());- let has_prompt = options.prompt.as_ref().is_some_and(|p| !p.trim().is_empty());+ let has_prompt = options+ .prompt+ .as_ref()+ .is_some_and(|p| !p.trim().is_empty());if !has_code && !has_prompt {return Err(FirecrawlError::Missuse("Either 'code' or 'prompt' must be provided".into(),@@ -393,8 +396,7 @@ impl Client {)})?;- self.handle_response(response, "stop interaction")- .await+ self.handle_response(response, "stop interaction").await}/// Deprecated alias for [`Client::interact`].@@ -653,9 +655,18 @@ mod tests {.unwrap();assert!(response.success);- assert_eq!(response.output, Some("Clicked the login button".to_string()));- assert_eq!(response.live_view_url, Some("https://live.example.com/view".to_string()));- assert_eq!(response.interactive_live_view_url, Some("https://live.example.com/interactive".to_string()));+ assert_eq!(+ response.output,+ Some("Clicked the login button".to_string())+ );+ assert_eq!(+ response.live_view_url,+ Some("https://live.example.com/view".to_string())+ );+ assert_eq!(+ response.interactive_live_view_url,+ Some("https://live.example.com/interactive".to_string())+ );mock.assert();}diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rsindex 06736f0b0..d1bb976c7 100644--- a/apps/rust-sdk/src/v2/types.rs+++ b/apps/rust-sdk/src/v2/types.rs@@ -32,6 +32,8 @@ pub enum Format {Attributes,/// Brand analysis of the page.Branding,+ /// Audio extraction (MP3) from YouTube videos.+ Audio,}/// Viewport dimensions for screenshots.@@ -434,6 +436,8 @@ pub struct Document {pub images: Option<Vec<String>>,/// Screenshot URL or base64 data.pub screenshot: Option<String>,+ /// Audio download URL (signed GCS link for MP3).+ pub audio: Option<String>,/// Extracted attributes.pub attributes: Option<Vec<AttributeResult>>,/// Action results.