Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: audio format (ENG-4708) (#3226)

Gergő Móricz37889a332726
diff --git a/CLAUDE.md b/CLAUDE.md
index 8b3a89a96..991b19f37 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1,6 +1,6 @@
Firecrawl is a web scraper API. The directory you have access to is a monorepo:
- `apps/api` has the actual API and worker code
- - `apps/js-sdk`, `apps/python-sdk`, and `apps/rust-sdk` are various SDKs
+ - `apps/js-sdk`, `apps/python-sdk`, `apps/rust-sdk` and `apps/java-sdk` are various SDKs
When making changes to the API, here are the general steps you should take:
1. Write some end-to-end tests that assert your win conditions, if they don't already exist
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 16e537629..ba1c03d1a 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -2012,6 +2012,42 @@ describe("Attribute formats", () => {
);
});
+ describeIf(!TEST_SELF_HOST)("Audio format", () => {
+ it.concurrent(
+ "should return audio field with signed GCS URL for a YouTube video",
+ async () => {
+ const data = await scrape(
+ {
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["audio"],
+ },
+ identity,
+ );
+
+ expect(data.audio).toBeDefined();
+ expect(typeof data.audio).toBe("string");
+ expect(data.audio).toMatch(/^https:\/\//);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "should reject non-YouTube URL with audio format",
+ async () => {
+ const result = await scrapeWithFailure(
+ {
+ url: "https://example.com",
+ formats: ["audio"],
+ },
+ identity,
+ );
+
+ expect(result.error).toMatch(/audio.*youtube/i);
+ },
+ scrapeTimeout,
+ );
+ });
+
describe("UUID validation", () => {
it.concurrent(
"should reject invalid UUID 'None' for scrape status",
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 7dbb2b5d1..6b04dc208 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -250,6 +250,9 @@ const configSchema = z.object({
BROWSER_SERVICE_API_KEY: z.string().optional(),
BROWSER_SERVICE_WEBHOOK_SECRET: z.string().optional(),
+ // Audio (avgrab)
+ AVGRAB_SERVICE_URL: z.string().optional(),
+
NUQ_PREFETCH_WORKER_HEARTBEAT_URL: z.string().optional(),
});
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 7bbcbb888..c94d24b93 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -27,6 +27,24 @@ import {
} from "../../services/webhook/schema";
import { BrandingProfile } from "../../types/branding";
+function isYouTubeUrl(url: string): boolean {
+ try {
+ const parsed = new globalThis.URL(url);
+ if (
+ parsed.hostname === "youtube.com" ||
+ parsed.hostname.endsWith(".youtube.com")
+ ) {
+ return parsed.pathname === "/watch" && !!parsed.searchParams.get("v");
+ }
+ if (parsed.hostname === "youtu.be") {
+ return parsed.pathname !== "/";
+ }
+ return false;
+ } catch {
+ return false;
+ }
+}
+
// Base URL schema with common validation logic
export const URL = z.preprocess(
x => {
@@ -419,7 +437,8 @@ export type FormatObject =
| ScreenshotFormatWithOptions
| AttributesFormatWithOptions
| QueryFormatWithOptions
- | { type: "branding" };
+ | { type: "branding" }
+ | { type: "audio" };
const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);
@@ -528,6 +547,7 @@ const baseScrapeOptions = z.strictObject({
attributesFormatWithOptions,
z.strictObject({ type: z.literal("branding") }),
queryFormatWithOptions,
+ z.strictObject({ type: z.literal("audio") }),
])
.array()
.optional()
@@ -812,6 +832,10 @@ const scrapeRequestSchemaBase = baseScrapeOptions.extend({
export const scrapeRequestSchema = strictWithMessage(scrapeRequestSchemaBase)
.refine(waitForRefine, waitForRefineOpts)
+ .refine(
+ x => !x.formats?.some(f => f.type === "audio") || isYouTubeUrl(x.url),
+ "The audio format is only supported for YouTube URLs (youtube.com/watch?v=... or youtu.be/...)",
+ )
.transform(extractTransformRequired);
export type ScrapeRequest = z.infer<typeof scrapeRequestSchema>;
@@ -1005,6 +1029,7 @@ export type Document = {
links?: string[];
images?: string[];
screenshot?: string;
+ audio?: string;
extract?: any;
json?: any;
summary?: string;
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 3beca1fcb..9ada777d2 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -59,10 +59,6 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = 5;
}
- if (hasFormatOfType(options.formats, "query")) {
- creditsToBeBilled = 5;
- }
-
if (
internalOptions.v1Agent?.model === "fire-1" ||
internalOptions.v1JSONAgent?.model?.toLowerCase() === "fire-1"
@@ -70,6 +66,14 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled = Math.ceil((costTrackingJSON.totalCost ?? 1) * 1800);
}
+ if (hasFormatOfType(options.formats, "query")) {
+ creditsToBeBilled += 4;
+ }
+
+ if (hasFormatOfType(options.formats, "audio")) {
+ creditsToBeBilled += 4;
+ }
+
if (internalOptions.zeroDataRetention) {
creditsToBeBilled += flags?.zdrCost ?? 1;
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/audio.ts b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
new file mode 100644
index 000000000..3798fe01c
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/audio.ts
@@ -0,0 +1,45 @@
+import { Meta } from "..";
+import { Document } from "../../../controllers/v2/types";
+import { config } from "../../../config";
+import { hasFormatOfType } from "../../../lib/format-utils";
+
+export async function fetchAudio(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (!hasFormatOfType(meta.options.formats, "audio")) {
+ return document;
+ }
+
+ if (!config.AVGRAB_SERVICE_URL) {
+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
+ document.warning =
+ "Audio format is not available (service not configured)." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
+ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ url: meta.url }),
+ });
+
+ if (!response.ok) {
+ const error = await response
+ .json()
+ .catch(() => ({ detail: "Unknown error" }));
+ throw new Error(`Audio download failed: ${error.detail}`);
+ }
+
+ const data = await response.json().catch(() => null);
+
+ if (!data || !data.public_url || typeof data.public_url !== "string") {
+ throw new Error(
+ "Audio download failed: avgrab service returned an invalid response (missing public_url)",
+ );
+ }
+
+ document.audio = data.public_url;
+ return document;
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index c8916c66c..93d14e9a0 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -17,6 +17,7 @@ import { performAgent } from "./agent";
import { performAttributes } from "./performAttributes";
import { deriveDiff } from "./diff";
+import { fetchAudio } from "./audio";
import { useIndex, useSearchIndex } from "../../../services/index";
import { sendDocumentToIndex } from "../engines/index/index";
import { sendDocumentToSearchIndex } from "./sendToSearchIndex";
@@ -450,6 +451,15 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
);
}
+ const hasAudio = hasFormatOfType(meta.options.formats, "audio");
+ if (!hasAudio && document.audio !== undefined) {
+ delete document.audio;
+ } else if (hasAudio && document.audio === undefined) {
+ meta.logger.warn(
+ "Request had format: audio, but there was no audio field in the result.",
+ );
+ }
+
if (!hasChangeTracking && document.changeTracking !== undefined) {
meta.logger.warn(
"Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
@@ -522,6 +532,7 @@ const transformerStack: Transformer[] = [
performAttributes,
performAgent,
deriveDiff,
+ fetchAudio,
coerceFieldsToFormats,
removeBase64Images,
];
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index e87f9f923..3d4232847 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.1.1"
+version = "1.2.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
index beb41fb0f..0fc75145d 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
@@ -19,6 +19,7 @@ public class Document {
private List<String> links;
private List<String> images;
private String screenshot;
+ private String audio;
private List<Map<String, Object>> attributes;
private Map<String, Object> actions;
private String warning;
@@ -34,6 +35,7 @@ public class Document {
public List<String> getLinks() { return links; }
public List<String> getImages() { return images; }
public String getScreenshot() { return screenshot; }
+ public String getAudio() { return audio; }
public List<Map<String, Object>> getAttributes() { return attributes; }
public Map<String, Object> getActions() { return actions; }
public String getWarning() { return warning; }
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
index b7ab815d2..a8560bc76 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
@@ -104,8 +104,8 @@ public class ScrapeOptions {
/**
* Output formats to request. Accepts strings like "markdown", "html", "rawHtml",
- * "links", "screenshot", "json", etc., or format configuration maps for advanced
- * formats (e.g., JsonFormat, ScreenshotFormat).
+ * "links", "screenshot", "json", "audio", etc., or format configuration maps for
+ * advanced formats (e.g., JsonFormat, ScreenshotFormat).
*/
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index f0be3bb04..07ae84c7a 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.17.0",
+ "version": "4.18.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index bd974a7b1..d8dcd88b1 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -12,7 +12,8 @@ export type FormatString =
| 'changeTracking'
| 'json'
| 'attributes'
- | 'branding';
+ | 'branding'
+ | 'audio';
export interface Viewport {
width: number;
@@ -392,6 +393,7 @@ export interface Document {
links?: string[];
images?: string[];
screenshot?: string;
+ audio?: string;
attributes?: Array<{
selector: string;
attribute: string;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 04db4894c..473aa174a 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.20.0"
+__version__ = "4.21.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index e19653262..bd1a316a2 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -274,6 +274,7 @@ class Document(BaseModel):
links: Optional[List[str]] = None
images: Optional[List[str]] = None
screenshot: Optional[str] = None
+ audio: Optional[str] = None
actions: Optional[Dict[str, Any]] = None
answer: Optional[str] = None
warning: Optional[str] = None
@@ -386,6 +387,7 @@ FormatString = Literal[
"attributes",
"branding",
"query",
+ "audio",
# snake_case versions (user-friendly)
"raw_html",
"change_tracking",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 5d86ffe2f..c47284047 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,7 +1,7 @@
[package]
name = "firecrawl"
author= "Mendable.ai"
-version = "1.3.0"
+version = "1.4.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rs
index 94db9c8fa..ffbe8fe44 100644
--- a/apps/rust-sdk/src/v2/scrape.rs
+++ b/apps/rust-sdk/src/v2/scrape.rs
@@ -338,7 +338,10 @@ impl Client {
options: ScrapeExecuteOptions,
) -> Result<ScrapeExecuteResponse, FirecrawlError> {
let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
- let has_prompt = options.prompt.as_ref().is_some_and(|p| !p.trim().is_empty());
+ let has_prompt = options
+ .prompt
+ .as_ref()
+ .is_some_and(|p| !p.trim().is_empty());
if !has_code && !has_prompt {
return Err(FirecrawlError::Missuse(
"Either 'code' or 'prompt' must be provided".into(),
@@ -393,8 +396,7 @@ impl Client {
)
})?;
- self.handle_response(response, "stop interaction")
- .await
+ self.handle_response(response, "stop interaction").await
}
/// Deprecated alias for [`Client::interact`].
@@ -653,9 +655,18 @@ mod tests {
.unwrap();
assert!(response.success);
- assert_eq!(response.output, Some("Clicked the login button".to_string()));
- assert_eq!(response.live_view_url, Some("https://live.example.com/view".to_string()));
- assert_eq!(response.interactive_live_view_url, Some("https://live.example.com/interactive".to_string()));
+ assert_eq!(
+ response.output,
+ Some("Clicked the login button".to_string())
+ );
+ assert_eq!(
+ response.live_view_url,
+ Some("https://live.example.com/view".to_string())
+ );
+ assert_eq!(
+ response.interactive_live_view_url,
+ Some("https://live.example.com/interactive".to_string())
+ );
mock.assert();
}
diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rs
index 06736f0b0..d1bb976c7 100644
--- a/apps/rust-sdk/src/v2/types.rs
+++ b/apps/rust-sdk/src/v2/types.rs
@@ -32,6 +32,8 @@ pub enum Format {
Attributes,
/// Brand analysis of the page.
Branding,
+ /// Audio extraction (MP3) from YouTube videos.
+ Audio,
}
/// Viewport dimensions for screenshots.
@@ -434,6 +436,8 @@ pub struct Document {
pub images: Option<Vec<String>>,
/// Screenshot URL or base64 data.
pub screenshot: Option<String>,
+ /// Audio download URL (signed GCS link for MP3).
+ pub audio: Option<String>,
/// Extracted attributes.
pub attributes: Option<Vec<AttributeResult>>,
/// Action results.