Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): video format (#3516)
Gergő Móriczaf8adaf67b92
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.tsindex 834d06bab..ce17ebb07 100644--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts@@ -84,6 +84,30 @@ describe("Audio format engine routing (buildFallbackList)", () => {expect(engines).not.toContain("fetch");});+ it("routes video format to chrome-cdp before tlsclient", async () => {+ const fallback = await buildFallbackList(buildStubMeta(["video"]));+ const engines = fallback.map(f => f.engine);++ expect(engines).toEqual([+ "fire-engine;chrome-cdp",+ "fire-engine(retry);chrome-cdp",+ "fire-engine;tlsclient",+ ]);+ });++ it("excludes index and non-browser engines when video format is requested", async () => {+ const fallback = await buildFallbackList(buildStubMeta(["video"]));+ const engines = fallback.map(f => f.engine);++ expect(engines).toContain("fire-engine;chrome-cdp");+ expect(engines).toContain("fire-engine(retry);chrome-cdp");+ expect(engines).not.toContain("index");+ expect(engines).not.toContain("index;documents");+ expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");+ expect(engines).not.toContain("fetch");+ });+it("still allows chrome-cdp for non-audio requests", async () => {const fallback = await buildFallbackList(buildStubMeta([]));const engines = fallback.map(f => f.engine);diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.tsindex b8b29b1dc..e8fca4d52 100644--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts@@ -99,10 +99,10 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {);test(- "should serve cache and skip audio fetch even when audio format is requested",+ "should serve cache and skip media fetch even when audio and video formats are requested",async () => {const id = crypto.randomUUID();- const url = "https://firecrawl.dev/?testAudioGate=" + id;+ const url = "https://firecrawl.dev/?testMediaGate=" + id;const seed = await scrape({ url }, identity);expect(seed).toBeDefined();@@ -110,15 +110,15 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {await new Promise(resolve => setTimeout(resolve, 20000));- // Without the lockdown audio gate this would either throw- // AudioUnsupportedUrlError (firecrawl.dev is not an audio source) or+ // Without the lockdown media gate this would either throw an unsupported+ // media URL error (firecrawl.dev is not an avgrab source) or// POST to AVGRAB_SERVICE_URL with the target URL. Success here implies// the gate short-circuited before any outbound call.const data = await scrape({url,lockdown: true,- formats: ["markdown", "audio"],+ formats: ["markdown", "audio", "video"],},identity,);@@ -126,6 +126,7 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {expect(data).toBeDefined();expect(data.metadata.cacheState).toBe("hit");expect(data.audio).toBeUndefined();+ expect(data.video).toBeUndefined();},scrapeTimeout * 2 + 20000,);diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 53c58c5c4..df41959bf 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -2065,6 +2065,42 @@ describe("Attribute formats", () => {);});+ describeIf(!TEST_SELF_HOST)("Video format", () => {+ it.concurrent(+ "should return video field with signed GCS URL for a supported video URL",+ async () => {+ const data = await scrape(+ {+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["video"],+ },+ identity,+ );++ expect(data.video).toBeDefined();+ expect(typeof data.video).toBe("string");+ expect(data.video).toMatch(/^https:\/\//);+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "should reject unsupported URL with video format",+ async () => {+ const result = await scrapeWithFailure(+ {+ url: "https://example.com",+ formats: ["video"],+ },+ identity,+ );++ expect(result.error).toMatch(/video/i);+ },+ scrapeTimeout,+ );+ });+describe("UUID validation", () => {it.concurrent("should reject invalid UUID 'None' for scrape status",diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex aef9c2067..b29bcf03e 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -60,6 +60,24 @@ describe("V2 Types Validation", () => {expect(result.formats).toEqual([{ type: "markdown" }, { type: "html" }]);});+ it("should accept video format as string and object", () => {+ const stringInput: ScrapeRequestInput = {+ url: "https://example.com",+ formats: ["video"],+ };+ const objectInput: ScrapeRequestInput = {+ url: "https://example.com",+ formats: [{ type: "video" }],+ };++ expect(scrapeRequestSchema.parse(stringInput).formats).toEqual([+ { type: "video" },+ ]);+ expect(scrapeRequestSchema.parse(objectInput).formats).toEqual([+ { type: "video" },+ ]);+ });+it("should accept valid scrape request with json format options", () => {const input: ScrapeRequestInput = {url: "https://example.com",diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2b838f087..d2b8238e5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -438,7 +438,8 @@ export type FormatObject =| HighlightsFormatWithOptions| QueryFormatWithOptions| { type: "branding" }- | { type: "audio" };+ | { type: "audio" }+ | { type: "video" };const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);@@ -550,6 +551,7 @@ const baseScrapeOptions = z.strictObject({highlightsFormatWithOptions,queryFormatWithOptions,z.strictObject({ type: z.literal("audio") }),+ z.strictObject({ type: z.literal("video") }),]).array().optional()@@ -1097,6 +1099,7 @@ export type Document = {images?: string[];screenshot?: string;audio?: string;+ video?: string;extract?: any;json?: any;summary?: string;diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 1a6c31afb..cdf4ddf17 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -32,6 +32,7 @@ import {ScrapeRetryLimitError,BrandingNotSupportedError,AudioUnsupportedUrlError,+ VideoUnsupportedUrlError,XTwitterConfigurationError,} from "../scraper/scrapeURL/error";@@ -65,6 +66,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_SITEMAP_ERROR: SitemapError,CRAWL_DENIAL: CrawlDenialError,SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,+ SCRAPE_VIDEO_UNSUPPORTED_URL: VideoUnsupportedUrlError,SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,MAP_FAILED: MapFailedError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 838eb3167..d45ffdd5d 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -26,6 +26,7 @@ export type ErrorCodes =| "SCRAPE_BRANDING_NOT_SUPPORTED"| "AGENT_INDEX_ONLY"| "SCRAPE_AUDIO_UNSUPPORTED_URL"+ | "SCRAPE_VIDEO_UNSUPPORTED_URL"| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"| "PARSE_UNSUPPORTED_OPTIONS"| "CRAWL_DENIAL"diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 166a91eab..dd5bee7d3 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -90,6 +90,10 @@ export async function calculateCreditsToBeBilled(creditsToBeBilled += 4;}+ if (hasFormatOfType(options.formats, "video")) {+ creditsToBeBilled += 4;+ }+if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {creditsToBeBilled += xTwitterCostBonus;}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 95f188389..605b33c4c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -265,6 +265,7 @@ export async function scrapeURLWithFireEngineChromeCDP(});const hasBranding = hasFormatOfType(meta.options.formats, "branding");const hasAudio = hasFormatOfType(meta.options.formats, "audio");+ const hasVideo = hasFormatOfType(meta.options.formats, "video");const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(meta,new URL(meta.rewrittenUrl ?? meta.url),@@ -321,7 +322,7 @@ export async function scrapeURLWithFireEngineChromeCDP(},]: []),- ...(hasAudio || shouldRunYoutubePostprocessor+ ...(hasAudio || hasVideo || shouldRunYoutubePostprocessor? ([{type: "getCookies",@@ -464,7 +465,9 @@ export async function scrapeURLWithFireEngineChromeCDP(proxyUsed: response.usedMobileProxy ? "stealth" : "basic",youtubeTranscriptContent: response.youtubeTranscriptContent,timezone: response.timezone,- ...(hasAudio || shouldRunYoutubePostprocessor ? { audioCookies } : {}),+ ...(hasAudio || hasVideo || shouldRunYoutubePostprocessor+ ? { audioCookies }+ : {}),};});}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex bcb71c34b..2c1818ae2 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -90,6 +90,7 @@ const featureFlags = ["pdf","document","audio",+ "video","atsv","location","mobile",@@ -114,6 +115,7 @@ const featureFlagOptions: {pdf: { priority: 100 },document: { priority: 100 },audio: { priority: 100 },+ video: { priority: 100 },atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if notuseFastMode: { priority: 90 },location: { priority: 10 },@@ -224,6 +226,7 @@ const engineOptions: {pdf: false,document: false,audio: false,+ video: false,atsv: false,mobile: true,location: true,@@ -244,6 +247,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: false,location: true,mobile: true,@@ -264,6 +268,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: false,location: true,mobile: true,@@ -284,6 +289,7 @@ const engineOptions: {pdf: true,document: true,audio: false,+ video: false,atsv: false,location: true,mobile: true,@@ -304,6 +310,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: false,location: true,mobile: true,@@ -324,6 +331,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: false,location: true,mobile: true,@@ -344,6 +352,7 @@ const engineOptions: {pdf: false,document: false,audio: false,+ video: false,atsv: false,location: false,mobile: false,@@ -364,6 +373,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: true,location: true,mobile: false,@@ -384,6 +394,7 @@ const engineOptions: {pdf: false,document: false,audio: true,+ video: true,atsv: true,location: true,mobile: false,@@ -404,6 +415,7 @@ const engineOptions: {pdf: false,document: false,audio: false,+ video: false,atsv: false,location: false,mobile: false,@@ -424,6 +436,7 @@ const engineOptions: {pdf: true,document: false,audio: false,+ video: false,atsv: false,location: false,mobile: false,@@ -444,6 +457,7 @@ const engineOptions: {pdf: false,document: true,audio: false,+ video: false,atsv: false,location: false,mobile: false,@@ -464,6 +478,7 @@ const engineOptions: {pdf: false,document: false,audio: false,+ video: false,atsv: false,location: false,mobile: false,@@ -484,6 +499,7 @@ const engineOptions: {pdf: false,document: false,audio: false,+ video: false,atsv: false,location: false,mobile: false,diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex acfcebc2f..5f90f25cc 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -535,6 +535,28 @@ export class AudioUnsupportedUrlError extends TransportableError {}}+export class VideoUnsupportedUrlError extends TransportableError {+ constructor(message?: string) {+ super(+ "SCRAPE_VIDEO_UNSUPPORTED_URL",+ message ?? "The video format does not support the provided URL",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new VideoUnsupportedUrlError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class BrandingNotSupportedError extends TransportableError {constructor(message: string) {super("SCRAPE_BRANDING_NOT_SUPPORTED", message);diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0b3a02f61..6589f6737 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -189,6 +189,10 @@ function buildFeatureFlags(flags.add("audio");}+ if (hasFormatOfType(options.formats, "video")) {+ flags.add("video");+ }+if (options.waitFor !== 0) {flags.add("waitFor");}diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.tsnew file mode 100644index 000000000..c7d847767--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts@@ -0,0 +1,178 @@+import { fetchVideo } from "../video";+import { config } from "../../../../config";++describe("fetchVideo", () => {+ const originalFetch = global.fetch;+ const originalAvgrabServiceUrl = config.AVGRAB_SERVICE_URL;++ afterEach(() => {+ global.fetch = originalFetch;+ config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;+ jest.clearAllMocks();+ });++ function mockSuccessfulAvgrab() {+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {+ if (url.endsWith("/supported-urls")) {+ return {+ ok: true,+ json: async () => ({ regex: "https://example\\.com/video" }),+ };+ }++ return {+ ok: true,+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),+ };+ });++ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";+ return fetchSpy;+ }++ it("does not issue any fetch when lockdown is true, even if video format is requested", async () => {+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as any;++ const meta: any = {+ url: "https://example.com/video",+ options: {+ lockdown: true,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = { markdown: "cached" };++ const result = await fetchVideo(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(result).toBe(document);+ expect(result.video).toBeUndefined();+ });++ it("returns early when video format is not requested regardless of lockdown", async () => {+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as any;++ const meta: any = {+ url: "https://example.com/video",+ options: {+ lockdown: false,+ formats: [{ type: "markdown" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = { markdown: "cached" };++ await fetchVideo(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ });++ it("returns a warning when avgrab is not configured", async () => {+ const fetchSpy = jest.fn();+ global.fetch = fetchSpy as any;+ config.AVGRAB_SERVICE_URL = undefined;++ const meta: any = {+ url: "https://example.com/video",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ const result = await fetchVideo(meta, document);++ expect(fetchSpy).not.toHaveBeenCalled();+ expect(meta.logger.warn).toHaveBeenCalledWith(+ "AVGRAB_SERVICE_URL is not configured",+ );+ expect(result.warning).toMatch(/Video format is not available/);+ });++ it("forwards browser cookies to avgrab when video cookies are available", async () => {+ const fetchSpy = mockSuccessfulAvgrab();+ const cookies = [+ {+ name: "VISITOR_INFO1_LIVE",+ value: "visitor",+ domain: ".youtube.com",+ path: "/",+ secure: true,+ httpOnly: true,+ },+ ];++ const meta: any = {+ url: "https://example.com/video",+ audioCookies: cookies,+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchVideo(meta, document);++ const downloadCall = fetchSpy.mock.calls.find(([url]) =>+ String(url).endsWith("/download-video"),+ );+ expect(downloadCall).toBeDefined();+ const body = downloadCall![1]?.body;+ expect(typeof body).toBe("string");+ expect(JSON.parse(body as string)).toEqual({+ url: "https://example.com/video",+ cookies,+ });+ expect(document.video).toBe("https://storage.example/video.mp4");+ });++ it("omits cookies from the avgrab request when no video cookies are available", async () => {+ const fetchSpy = mockSuccessfulAvgrab();++ const meta: any = {+ url: "https://example.com/video",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await fetchVideo(meta, document);++ const downloadCall = fetchSpy.mock.calls.find(([url]) =>+ String(url).endsWith("/download-video"),+ );+ expect(downloadCall).toBeDefined();+ const body = downloadCall![1]?.body;+ expect(typeof body).toBe("string");+ expect(JSON.parse(body as string)).toEqual({+ url: "https://example.com/video",+ });+ });++ it("rejects unsupported URLs", async () => {+ mockSuccessfulAvgrab();++ const meta: any = {+ url: "https://example.com/unsupported",+ options: {+ lockdown: false,+ formats: [{ type: "video" }],+ },+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },+ };+ const document: any = {};++ await expect(fetchVideo(meta, document)).rejects.toThrow(/video/i);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 883ec5873..9af263688 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -18,6 +18,7 @@ import { performAttributes } from "./performAttributes";import { deriveDiff } from "./diff";import { fetchAudio } from "./audio";+import { fetchVideo } from "./video";import { useIndex, useSearchIndex } from "../../../services/index";import { sendDocumentToIndex } from "../engines/index/index";import { sendDocumentToSearchIndex } from "./sendToSearchIndex";@@ -484,6 +485,15 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {);}+ const hasVideo = hasFormatOfType(meta.options.formats, "video");+ if (!hasVideo && document.video !== undefined) {+ delete document.video;+ } else if (hasVideo && document.video === undefined) {+ meta.logger.warn(+ "Request had format: video, but there was no video field in the result.",+ );+ }+if (!hasChangeTracking && document.changeTracking !== undefined) {meta.logger.warn("Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",@@ -558,6 +568,7 @@ const transformerStack: Transformer[] = [removeBase64Images,deriveDiff,fetchAudio,+ fetchVideo,coerceFieldsToFormats,];diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.tsnew file mode 100644index 000000000..5a61b59e0--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts@@ -0,0 +1,94 @@+import { Meta } from "..";+import { Document } from "../../../controllers/v2/types";+import { config } from "../../../config";+import { hasFormatOfType } from "../../../lib/format-utils";+import { VideoUnsupportedUrlError } from "../error";++let cachedUrlRegex: RegExp | null = null;+let cacheTimestamp = 0;+const CACHE_TTL_MS = 5 * 60 * 1000;++async function getSupportedUrlRegex(): Promise<RegExp> {+ if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {+ return cachedUrlRegex;+ }++ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);+ if (!res.ok) {+ throw new Error(+ "Failed to fetch supported URL patterns from video service",+ );+ }++ const data = await res.json().catch(() => null);+ if (!data || typeof data.regex !== "string") {+ throw new Error("Video service returned invalid supported URL patterns");+ }++ try {+ cachedUrlRegex = new RegExp(data.regex);+ } catch {+ throw new Error("Video service returned invalid supported URL patterns");+ }+ cacheTimestamp = Date.now();+ return cachedUrlRegex;+}++export async function fetchVideo(+ meta: Meta,+ document: Document,+): Promise<Document> {+ if (!hasFormatOfType(meta.options.formats, "video")) {+ return document;+ }++ // Lockdown forbids outbound requests that touch the target URL. avgrab+ // fetches the source on our behalf, so skip it here.+ if (meta.options.lockdown) {+ return document;+ }++ if (!config.AVGRAB_SERVICE_URL) {+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");+ document.warning =+ "Video format is not available (service not configured)." ++ (document.warning ? " " + document.warning : "");+ return document;+ }++ const urlRegex = await getSupportedUrlRegex();+ if (!urlRegex.test(meta.url)) {+ throw new VideoUnsupportedUrlError();+ }++ const requestBody = {+ url: meta.url,+ ...(meta.audioCookies && meta.audioCookies.length > 0+ ? { cookies: meta.audioCookies }+ : {}),+ };++ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download-video`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify(requestBody),+ });++ if (!response.ok) {+ const error = await response+ .json()+ .catch(() => ({ detail: "Unknown error" }));+ throw new Error(`Video download failed: ${error.detail}`);+ }++ const data = await response.json().catch(() => null);++ if (!data || !data.public_url || typeof data.public_url !== "string") {+ throw new Error(+ "Video download failed: avgrab service returned an invalid response (missing public_url)",+ );+ }++ document.video = data.public_url;+ return document;+}diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.csindex 492f4c22d..acf959b82 100644--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs@@ -119,6 +119,7 @@ public class ModelsTests{"markdown": "# Hello World","html": "<h1>Hello World</h1>",+ "video": "https://storage.googleapis.com/firecrawl/video.mp4","metadata": {"title": "Test","sourceURL": "https://example.com"@@ -131,6 +132,7 @@ public class ModelsTestsAssert.NotNull(doc);Assert.Equal("# Hello World", doc.Markdown);Assert.Equal("<h1>Hello World</h1>", doc.Html);+ Assert.Equal("https://storage.googleapis.com/firecrawl/video.mp4", doc.Video);Assert.NotNull(doc.Metadata);Assert.Null(doc.Warning);}diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex efdeaea3d..da297be8f 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.3.1</Version>+ <Version>1.4.0</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/Models/Document.cs b/apps/dot-net-sdk/Firecrawl/Models/Document.csindex 5dd68eb44..ba601fbf2 100644--- a/apps/dot-net-sdk/Firecrawl/Models/Document.cs+++ b/apps/dot-net-sdk/Firecrawl/Models/Document.cs@@ -37,6 +37,9 @@ public class Document[JsonPropertyName("audio")]public object? Audio { get; set; }+ [JsonPropertyName("video")]+ public string? Video { get; set; }+[JsonPropertyName("actions")]public object? Actions { get; set; }diff --git a/apps/dot-net-sdk/README.md b/apps/dot-net-sdk/README.mdindex 70560fec1..3c61ee7fe 100644--- a/apps/dot-net-sdk/README.md+++ b/apps/dot-net-sdk/README.md@@ -73,6 +73,17 @@ Console.WriteLine(doc.Markdown);Console.WriteLine(doc.Html);```+### Video Extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `Video` property is a signed URL to the extracted video file.++```csharp+var doc = await client.ScrapeAsync("https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ new ScrapeOptions { Formats = new List<object> { "video" } });++Console.WriteLine(doc.Video);+```+### Crawl```csharp@@ -133,7 +144,7 @@ foreach (var doc in job.Data!)Upload and parse a local file (HTML, PDF, DOCX, etc.) through the `/v2/parse`endpoint. Parse does not support browser-rendering options like `actions`,`waitFor`, `location`, `mobile`, or the `screenshot` / `branding` /-`changeTracking` formats.+`changeTracking` / `audio` / `video` formats.```csharpusing Firecrawl;diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.mdindex ec3efb68c..8a1cfd2aa 100644--- a/apps/elixir-sdk/README.md+++ b/apps/elixir-sdk/README.md@@ -11,7 +11,7 @@ Add `firecrawl` to your list of dependencies in `mix.exs`:```elixirdef deps do[- {:firecrawl, "~> 1.0"}+ {:firecrawl, "~> 1.4"}]end```@@ -43,6 +43,12 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missiformats: ["markdown"])+# Extract video from supported video URLs+{:ok, response} = Firecrawl.scrape_and_extract_from_url(+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ formats: ["video"]+)+# Crawl a site{:ok, response} = Firecrawl.crawl_urls(url: "https://example.com",diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex f66938714..97e56e952 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.3.1"+ @version "1.4.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdindex ad5cce194..99661dec8 100644--- a/apps/go-sdk/README.md+++ b/apps/go-sdk/README.md@@ -94,6 +94,20 @@ doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{})```+### Video Extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `Video` field is a signed URL to the extracted video file.++```go+doc, err := client.Scrape(ctx, "https://www.youtube.com/watch?v=dQw4w9WgXcQ", &firecrawl.ScrapeOptions{+ Formats: []string{"video"},+})+if err != nil {+ return err+}+fmt.Println(doc.Video)+```+#### Interactive BrowserExecute code in a scrape-bound browser session:@@ -112,8 +126,8 @@ deleteResp, err := client.StopInteractiveBrowser(ctx, scrapeJobID)Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, actions, waitFor,-location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.+features such as change tracking, screenshot, branding, audio, video, actions,+waitFor, location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.```go// From disk@@ -341,10 +355,10 @@ The Go SDK lives in a monorepo subdirectory, so releases follow Go's**must** be prefixed with the module subdirectory path:```-apps/go-sdk/v1.0.0+apps/go-sdk/v1.3.0```-A bare `v1.0.0` tag will not be resolvable by the Go module proxy.+A bare `v1.3.0` tag will not be resolvable by the Go module proxy.### Release workflow@@ -352,7 +366,7 @@ The SDK version is the single source of truth in[`version.go`](./version.go):```go-const Version = "1.0.0"+const Version = "1.3.0"```To cut a release:@@ -371,7 +385,7 @@ The workflow is idempotent: if the tag already exists, it is a no-op.### Consuming a specific version```bash-go get github.com/firecrawl/firecrawl/apps/[email protected]+go get github.com/firecrawl/firecrawl/apps/[email protected]```Users pin via the semantic version suffix; they never reference thediff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex a4c14b0ce..33b1df63e 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -14,6 +14,7 @@ type Document struct {Images []string `json:"images,omitempty"`Screenshot string `json:"screenshot,omitempty"`Audio string `json:"audio,omitempty"`+ Video string `json:"video,omitempty"`Attributes []map[string]interface{} `json:"attributes,omitempty"`Actions map[string]interface{} `json:"actions,omitempty"`Answer string `json:"answer,omitempty"`diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.goindex 9416ec487..07892be5a 100644--- a/apps/go-sdk/options_test.go+++ b/apps/go-sdk/options_test.go@@ -53,13 +53,13 @@ func TestScrapeOptionsSerializesQuestionAndHighlightsFormats(t *testing.T) {func TestScrapeOptionsPreservesStringFormats(t *testing.T) {payload, err := json.Marshal(ScrapeOptions{- Formats: []string{"markdown"},+ Formats: []string{"markdown", "video"},})if err != nil {t.Fatalf("Marshal ScrapeOptions: %v", err)}- if !strings.Contains(string(payload), `"formats":["markdown"]`) {+ if !strings.Contains(string(payload), `"formats":["markdown","video"]`) {t.Fatalf("serialized string formats = %s", payload)}}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 702528172..a6ab7a966 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.2.2"+const Version = "1.3.0"diff --git a/apps/java-sdk/README.md b/apps/java-sdk/README.mdindex 2555b1f2a..0a83d6576 100644--- a/apps/java-sdk/README.md+++ b/apps/java-sdk/README.md@@ -82,13 +82,13 @@ Before using the Java SDK, ensure you have the following installed:### Gradle (Kotlin DSL)```kotlin-implementation("com.firecrawl:firecrawl-java:1.1.1")+implementation("com.firecrawl:firecrawl-java:1.6.0")```### Gradle (Groovy)```groovy-implementation 'com.firecrawl:firecrawl-java:1.1.1'+implementation 'com.firecrawl:firecrawl-java:1.6.0'```### Maven@@ -97,7 +97,7 @@ implementation 'com.firecrawl:firecrawl-java:1.1.1'<dependency><groupId>com.firecrawl</groupId><artifactId>firecrawl-java</artifactId>- <version>1.1.1</version>+ <version>1.6.0</version></dependency>```@@ -147,10 +147,23 @@ System.out.println(doc.getMarkdown());System.out.println(doc.getMetadata().get("title"));```+### Video Extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```java+Document doc = client.scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ ScrapeOptions.builder()+ .formats(List.of("video"))+ .build());++System.out.println(doc.getVideo());+```+### Parse Uploaded FilesUpload local files (`html`, `pdf`, `docx`, etc.) via multipart form data and parse them synchronously.-Parse options intentionally exclude browser-only features like change tracking, screenshot, branding, actions, waitFor, location, and mobile.+Parse options intentionally exclude browser-only features like change tracking, screenshot, branding, audio, video, actions, waitFor, location, and mobile.```javaParseFile file = ParseFile.builder()@@ -391,14 +404,14 @@ gradle build```bashgradle jar-# Output: build/libs/firecrawl-java-1.1.1.jar+# Output: build/libs/firecrawl-java-1.6.0.jar```### Install Locally```bashgradle publishToMavenLocal-# Now available as: com.firecrawl:firecrawl-java:1.1.1 in local Maven repository+# Now available as: com.firecrawl:firecrawl-java:1.6.0 in local Maven repository```## Running Testsdiff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 8df199421..b0a7ffab0 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.5.1"+version = "1.6.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.javaindex b61670d2a..6d4eee5c9 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java@@ -20,6 +20,7 @@ public class Document {private List<String> images;private String screenshot;private String audio;+ private String video;private List<Map<String, Object>> attributes;private Map<String, Object> actions;private String answer;@@ -38,6 +39,7 @@ public class Document {public List<String> getImages() { return images; }public String getScreenshot() { return screenshot; }public String getAudio() { return audio; }+ public String getVideo() { return video; }public List<Map<String, Object>> getAttributes() { return attributes; }public Map<String, Object> getActions() { return actions; }public String getAnswer() { return answer; }diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.javaindex daf137521..5e881d425 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java@@ -10,8 +10,8 @@ import java.util.Map;/*** Options for parsing uploaded files via /v2/parse.*- * <p>Parse does not support browser-rendering formats/options such as- * change tracking, screenshot, branding, actions, waitFor, location, or mobile.+ * <p>Parse does not support browser-rendering formats/options such as+ * change tracking, screenshot, branding, audio, video, actions, waitFor, location, or mobile.*/@JsonInclude(JsonInclude.Include.NON_NULL)public class ParseOptions {@@ -86,7 +86,9 @@ public class ParseOptions {|| normalized.equals("change_tracking")|| normalized.equals("screenshot")|| normalized.equals("screenshot@fullPage")- || normalized.equals("branding");+ || normalized.equals("branding")+ || normalized.equals("audio")+ || normalized.equals("video");}public static final class Builder {diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.javaindex 700e29179..3bd3df5f2 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java@@ -108,7 +108,7 @@ public class ScrapeOptions {/*** Output formats to request. Accepts strings like "markdown", "html", "rawHtml",- * "links", "screenshot", "json", "audio", etc., or format configuration maps/objects for+ * "links", "screenshot", "json", "audio", "video", etc., or format configuration maps/objects for* advanced formats (e.g., JsonFormat, QuestionFormat, HighlightsFormat).*/public Builder formats(List<Object> formats) { this.formats = formats; return this; }diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.javaindex 1d7cc74af..44b25247e 100644--- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java+++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java@@ -66,13 +66,13 @@ class FirecrawlClientTest {.build();ScrapeOptions options = ScrapeOptions.builder()- .formats(List.of("markdown", "html", queryFormat))+ .formats(List.of("markdown", "html", "video", queryFormat)).onlyMainContent(true).timeout(30000).mobile(false).build();- assertEquals(List.of("markdown", "html", queryFormat), options.getFormats());+ assertEquals(List.of("markdown", "html", "video", queryFormat), options.getFormats());assertEquals("query", queryFormat.getType());assertEquals(QueryFormat.Mode.DIRECT_QUOTE, queryFormat.getMode());assertTrue(options.getOnlyMainContent());@@ -230,6 +230,15 @@ class FirecrawlClientTest {);}+ @Test+ void testParseOptionsRejectsVideoFormat() {+ assertThrows(IllegalArgumentException.class, () ->+ ParseOptions.builder()+ .formats(List.of("video"))+ .build()+ );+ }+// ================================================================// E2E TESTS (require FIRECRAWL_API_KEY)// ================================================================diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex ab1fbc382..d312ff118 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -46,10 +46,22 @@ const url = 'https://example.com';const scrapedData = await app.scrape(url);```+### Video extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```js+const doc = await app.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', {+ formats: ['video'],+});++console.log(doc.video);+```+### Parsing uploaded filesUse `parse` to upload a file (`html`, `pdf`, `docx`, etc.) as multipart form data and process it through the same parsing pipeline.-Parse does not support browser-only formats/options like `changeTracking`, `screenshot`, `branding`, `actions`, `waitFor`, `location`, or `mobile`.+Parse does not support browser-only formats/options like `changeTracking`, `screenshot`, `branding`, `audio`, `video`, `actions`, `waitFor`, `location`, or `mobile`.```jsconst parsed = await app.parse(diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 2ec28d6e4..3a6f436c6 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.22.2",+ "version": "4.23.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.tsindex cd0f16a2b..32fddf77a 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts@@ -38,6 +38,24 @@ describe("v2.parse unit", () => {).rejects.toThrow("parse does not support changeTracking format");});+ test("rejects video format before making requests", async () => {+ const client = new FirecrawlClient({+ apiKey: "test-key",+ apiUrl: "https://localhost:3002",+ });++ await expect(+ client.parse(+ {+ data: Buffer.from("<html></html>"),+ filename: "upload.html",+ contentType: "text/html",+ },+ { formats: ["video" as any] },+ ),+ ).rejects.toThrow("parse does not support video format");+ });+test("rejects lockdown option before making requests", async () => {const client = new FirecrawlClient({apiKey: "test-key",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.tsindex 83b8795fb..bcac9d4c4 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts@@ -9,6 +9,11 @@ describe("v2 utils: validation", () => {expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);});+ test("ensureValidFormats: accepts video string format", () => {+ const formats: FormatOption[] = ["markdown", "video"];+ expect(() => ensureValidFormats(formats)).not.toThrow();+ });+test("ensureValidFormats: json format requires prompt or schema", () => {// Valid cases - should not throwconst valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex f375bb457..ccd93d8a1 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -204,6 +204,7 @@ export class FirecrawlClient {* @param file File payload (data, filename, optional contentType).* @param options Optional parse options (formats, parsers, etc.).* Note: parse does not support changeTracking, screenshot, branding,+ * audio, video,* actions, waitFor, location, or mobile options.* @returns Parsed document with requested formats.*/diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 14f5cc322..2ceae804d 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -13,7 +13,8 @@ export type FormatString =| 'json'| 'attributes'| 'branding'- | 'audio';+ | 'audio'+ | 'video';export interface Viewport {width: number;@@ -82,7 +83,7 @@ export type FormatOption =export type ParseFormatString = Exclude<FormatString,- 'screenshot' | 'changeTracking' | 'branding'+ 'screenshot' | 'changeTracking' | 'branding' | 'audio' | 'video'>;export interface ParseFormat {@@ -458,6 +459,7 @@ export interface Document {images?: string[];screenshot?: string;audio?: string;+ video?: string;attributes?: Array<{selector: string;attribute: string;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.tsindex b8082268f..e42e2242c 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts@@ -114,6 +114,9 @@ export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {if (fmt === "branding") {throw new Error("parse does not support branding format");}+ if (fmt === "audio" || fmt === "video") {+ throw new Error(`parse does not support ${fmt} format`);+ }continue;}@@ -127,6 +130,9 @@ export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {if (type === "branding") {throw new Error("parse does not support branding format");}+ if (type === "audio" || type === "video") {+ throw new Error(`parse does not support ${type} format`);+ }if ((fmt as JsonFormat).type === "json") {const j = fmt as JsonFormat;diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.mdindex c9b1dd29a..a7bb660e7 100644--- a/apps/php-sdk/CHANGELOG.md+++ b/apps/php-sdk/CHANGELOG.md@@ -5,6 +5,11 @@ All notable changes to the Firecrawl PHP SDK will be documented in this file.The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).+## [1.3.0] - 2026-05-12++### Added+- Added `video` scrape format support and `Document::getVideo()` for video extraction results.+## [1.1.0] - 2026-04-21### Addeddiff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.mdindex 5c5b6fdc9..a36fb6301 100644--- a/apps/php-sdk/README.md+++ b/apps/php-sdk/README.md@@ -83,12 +83,24 @@ $doc = $client->scrape('https://example.com/product', ScrapeOptions::with(echo $doc->getJson(); // Structured data```+### Video Extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```php+$doc = $client->scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', ScrapeOptions::with(+ formats: ['video'],+));++echo $doc->getVideo();+```+### ParseUpload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, actions, waitFor,-location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.+features such as change tracking, screenshot, branding, audio, video, actions,+waitFor, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.```phpuse Firecrawl\Models\ParseFile;diff --git a/apps/php-sdk/src/Models/Document.php b/apps/php-sdk/src/Models/Document.phpindex 994de0661..5b4f02c98 100644--- a/apps/php-sdk/src/Models/Document.php+++ b/apps/php-sdk/src/Models/Document.php@@ -26,6 +26,7 @@ final class Documentprivate readonly ?array $images = null,private readonly ?string $screenshot = null,private readonly ?string $audio = null,+ private readonly ?string $video = null,private readonly ?array $attributes = null,private readonly ?array $actions = null,private readonly ?string $answer = null,@@ -49,6 +50,7 @@ final class Documentimages: $data['images'] ?? null,screenshot: $data['screenshot'] ?? null,audio: $data['audio'] ?? null,+ video: $data['video'] ?? null,attributes: $data['attributes'] ?? null,actions: $data['actions'] ?? null,answer: $data['answer'] ?? null,@@ -112,6 +114,11 @@ final class Documentreturn $this->audio;}+ public function getVideo(): ?string+ {+ return $this->video;+ }+/** @return list<array<string, mixed>>|null */public function getAttributes(): ?array{diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.phpindex 89e03faff..e9cff9030 100644--- a/apps/php-sdk/src/Models/ParseOptions.php+++ b/apps/php-sdk/src/Models/ParseOptions.php@@ -10,7 +10,7 @@ use Firecrawl\Exceptions\FirecrawlException;* Options for parsing uploaded files via `/v2/parse`.** Parse does not support browser-rendering features (actions, waitFor,- * location, mobile) nor the screenshot, branding, or changeTracking formats.+ * location, mobile) nor the screenshot, branding, audio, video, or changeTracking formats.* The proxy field only accepts "auto" or "basic".*/final class ParseOptions@@ -20,6 +20,8 @@ final class ParseOptions'screenshot','screenshot@fullPage','branding',+ 'audio',+ 'video',];/**diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.phpindex b0d8d50d7..f79b01306 100644--- a/apps/php-sdk/src/Models/ScrapeOptions.php+++ b/apps/php-sdk/src/Models/ScrapeOptions.php@@ -7,7 +7,7 @@ namespace Firecrawl\Models;final class ScrapeOptions{/**- * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats Supports strings like "markdown", "audio", and "video".* @param array<string, string>|null $headers* @param list<string>|null $includeTags* @param list<string>|null $excludeTags@@ -41,7 +41,7 @@ final class ScrapeOptions) {}/**- * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats Supports strings like "markdown", "audio", and "video".* @param array<string, string>|null $headers* @param list<string>|null $includeTags* @param list<string>|null $excludeTagsdiff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex aefc62765..1ed1826b6 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.2.1';+ public const SDK_VERSION = '1.3.0';}diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.phpindex b36a04d18..f9f6d8ecd 100644--- a/apps/php-sdk/tests/Unit/ModelsTest.php+++ b/apps/php-sdk/tests/Unit/ModelsTest.php@@ -3,6 +3,7 @@declare(strict_types=1);use Firecrawl\Models\CreditUsage;+use Firecrawl\Models\Document;use Firecrawl\Models\MapData;use Firecrawl\Models\BatchScrapeJob;use Firecrawl\Models\CrawlJob;@@ -93,6 +94,16 @@ it('preserves null creditsUsed in CrawlJob', function (): void {expect($job->getCreditsUsed())->toBeNull();});+it('hydrates video URL in Document', function (): void {+ $doc = Document::fromArray([+ 'markdown' => '# Video',+ 'video' => 'https://storage.googleapis.com/firecrawl/video.mp4',+ ]);++ expect($doc->getMarkdown())->toBe('# Video');+ expect($doc->getVideo())->toBe('https://storage.googleapis.com/firecrawl/video.mp4');+});+it('preserves positional integration in ScrapeOptions::with', function (): void {$options = ScrapeOptions::with(null,diff --git a/apps/php-sdk/tests/Unit/ParseTest.php b/apps/php-sdk/tests/Unit/ParseTest.phpindex 6cf083c46..3f00e2de8 100644--- a/apps/php-sdk/tests/Unit/ParseTest.php+++ b/apps/php-sdk/tests/Unit/ParseTest.php@@ -39,6 +39,10 @@ it('rejects unsupported parse formats', function (): void {ParseOptions::with(formats: ['screenshot']);})->throws(FirecrawlException::class);+it('rejects video parse format', function (): void {+ ParseOptions::with(formats: ['video']);+})->throws(FirecrawlException::class);+it('rejects invalid proxy values', function (): void {ParseOptions::with(proxy: 'stealth');})->throws(FirecrawlException::class);diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 9e7fdc636..bdef88a52 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -49,10 +49,19 @@ scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'print(scrape_result)```+### Video extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```python+doc = firecrawl.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', formats=['video'])+print(doc.video)+```+### Parsing uploaded filesUse `parse` to upload local bytes/files (`html`, `pdf`, `docx`, etc.) as multipart form data and return the parsed document.-`parse` does not support change tracking or browser-only options (actions, wait_for, location, mobile, screenshot, branding).+`parse` does not support change tracking or browser-only options (actions, wait_for, location, mobile, screenshot, branding, audio, video).```pythonfrom firecrawl import Firecrawldiff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 8e097fa9c..aeabd25e1 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.25.2"+__version__ = "4.26.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.pyindex ccf24e99f..a49a0764d 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py@@ -66,6 +66,16 @@ class TestParseRequestPreparation:content_type="text/html",)+ def test_prepare_parse_request_rejects_video_format(self):+ options = ParseOptions(formats=["video"])+ with pytest.raises(ValueError, match="do not support video output"):+ _prepare_parse_request(+ b"<html><body><h1>Parse</h1></body></html>",+ options,+ filename="upload.html",+ content_type="text/html",+ )+def test_prepare_parse_request_strips_lockdown(self):options = ParseOptions(formats=["markdown"], lockdown=True)fields, _ = _prepare_parse_request(diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.pyindex 2988d3972..387ceb355 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py@@ -102,7 +102,7 @@ class TestPrepareScrapeOptions:def test_prepare_basic_options(self):"""Test preparation with basic options."""options = ScrapeOptions(- formats=["markdown"],+ formats=["markdown", "video"],timeout=30000,wait_for=2000)@@ -112,6 +112,7 @@ class TestPrepareScrapeOptions:assert "formats" in resultassert "timeout" in resultassert "waitFor" in result+ assert result["formats"] == ["markdown", "video"]assert result["timeout"] == 30000assert result["waitFor"] == 2000diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.pyindex 9886a7dbd..f40188d0e 100644--- a/apps/python-sdk/firecrawl/v2/methods/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py@@ -15,7 +15,7 @@ from ..utils.get_version import get_versionversion = get_version()ParseFileInput = Union[str, bytes, bytearray, Path, BinaryIO]-UNSUPPORTED_PARSE_FORMATS = {"changeTracking", "screenshot", "branding"}+UNSUPPORTED_PARSE_FORMATS = {"changeTracking", "screenshot", "branding", "audio", "video"}def _extract_format_type(format_item: Any) -> Optional[str]:@@ -53,6 +53,8 @@ def _validate_parse_options_payload(options_payload: Dict[str, Any]) -> None:raise ValueError("Parse uploads do not support change tracking.")if fmt_type == "screenshot":raise ValueError("Parse uploads do not support screenshot output.")+ if fmt_type in {"audio", "video"}:+ raise ValueError(f"Parse uploads do not support {fmt_type} output.")raise ValueError("Parse uploads do not support branding output.")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 00fde19a7..0ef08b9e2 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -275,6 +275,7 @@ class Document(BaseModel):images: Optional[List[str]] = Nonescreenshot: Optional[str] = Noneaudio: Optional[str] = None+ video: Optional[str] = Noneactions: Optional[Dict[str, Any]] = Noneanswer: Optional[str] = Nonehighlights: Optional[str] = None@@ -389,6 +390,7 @@ FormatString = Literal["branding","query","audio",+ "video",# snake_case versions (user-friendly)"raw_html","change_tracking",diff --git a/apps/ruby-sdk/README.md b/apps/ruby-sdk/README.mdindex 0f79fae21..032ccaa53 100644--- a/apps/ruby-sdk/README.md+++ b/apps/ruby-sdk/README.md@@ -11,7 +11,7 @@ Ruby SDK for the [Firecrawl](https://firecrawl.dev) v2 web scraping API.Add to your `Gemfile`:```ruby-gem "firecrawl-sdk", "~> 1.0"+gem "firecrawl-sdk", "~> 1.5"```Or install directly:@@ -63,12 +63,23 @@ doc = client.scrape("https://example.com",puts doc.html```+### Video Extraction++Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```ruby+doc = client.scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ",+ Firecrawl::Models::ScrapeOptions.new(formats: ["video"]))++puts doc.video+```+### ParseUpload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data andparse it synchronously. Parse options intentionally exclude browser-only-features such as change tracking, screenshot, branding, actions, wait_for,-location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.+features such as change tracking, screenshot, branding, audio, video, actions,+wait_for, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.```ruby# From diskdiff --git a/apps/ruby-sdk/lib/firecrawl/models/document.rb b/apps/ruby-sdk/lib/firecrawl/models/document.rbindex 578c095b9..0b0200dc7 100644--- a/apps/ruby-sdk/lib/firecrawl/models/document.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/document.rb@@ -6,8 +6,8 @@ module Firecrawlclass Documentattr_reader :markdown, :html, :raw_html, :json, :summary,:metadata, :links, :images, :screenshot, :audio,- :attributes, :actions, :answer, :highlights, :warning,- :change_tracking, :branding+ :video, :attributes, :actions, :answer, :highlights,+ :warning, :change_tracking, :brandingdef initialize(data)@markdown = data["markdown"]@@ -20,6 +20,7 @@ module Firecrawl@images = data["images"]@screenshot = data["screenshot"]@audio = data["audio"]+ @video = data["video"]@attributes = data["attributes"]@actions = data["actions"]@answer = data["answer"]diff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rbindex 1d67aa3be..ef183e140 100644--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb@@ -5,10 +5,10 @@ module Firecrawl# Options for parsing uploaded files via `/v2/parse`.## Parse does not support browser-rendering features (actions, waitFor,- # location, mobile) nor screenshot/branding/changeTracking formats. The+ # location, mobile) nor screenshot/branding/audio/video/changeTracking formats. The# proxy field only accepts "auto" or "basic".class ParseOptions- UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding].freeze+ UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding audio video].freezeFIELDS = %i[formats headers include_tags exclude_tags only_main_contentdiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 7b806711f..7bb66e07a 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.4.1"+ VERSION = "1.5.0"enddiff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rbindex 7b4997d2c..97ff8f537 100644--- a/apps/ruby-sdk/test/firecrawl/client_test.rb+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb@@ -74,13 +74,14 @@ class ClientTest < Minitest::Test).to_return(status: 200,- body: JSON.generate(data: { markdown: "# Hello", metadata: { title: "Example", sourceURL: "https://example.com" } }),+ body: JSON.generate(data: { markdown: "# Hello", video: "https://storage.googleapis.com/firecrawl/video.mp4", metadata: { title: "Example", sourceURL: "https://example.com" } }),headers: { "Content-Type" => "application/json" })doc = @client.scrape("https://example.com")assert_instance_of Firecrawl::Models::Document, docassert_equal "# Hello", doc.markdown+ assert_equal "https://storage.googleapis.com/firecrawl/video.mp4", doc.videoassert_equal "Example", doc.metadata["title"]end@@ -677,6 +678,12 @@ class ClientTest < Minitest::Testendend+ def test_parse_options_rejects_video_format+ assert_raises(ArgumentError) do+ Firecrawl::Models::ParseOptions.new(formats: ["video"])+ end+ end+def test_parse_options_rejects_invalid_proxyassert_raises(ArgumentError) doFirecrawl::Models::ParseOptions.new(proxy: "stealth")diff --git a/apps/rust-sdk/CHANGELOG.md b/apps/rust-sdk/CHANGELOG.mdindex 8342b9fa8..e30e1f68d 100644--- a/apps/rust-sdk/CHANGELOG.md+++ b/apps/rust-sdk/CHANGELOG.md@@ -1,5 +1,11 @@## CHANGELOG+## [2.5.0] - 2026-05-12++### Added++- Added `Format::Video` and `Document.video` support for video extraction results.+## [0.1]### Addeddiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 48cda122f..2c3cb7cc3 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.4.1"+version = "2.5.0"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 8e6bb84c0..2c48e2edd 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.4.1"+version = "2.5.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.mdindex 36c6d1ff5..171b82df1 100644--- a/apps/rust-sdk/README.md+++ b/apps/rust-sdk/README.md@@ -7,7 +7,7 @@ To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`:```toml[dependencies]-firecrawl = "2.1.0"+firecrawl = "2.5.0"tokio = { version = "^1", features = ["full"] }```@@ -40,6 +40,24 @@ match scrape_result {}```+### Video extraction++Use `Format::Video` on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.++```rust+use firecrawl::{Format, ScrapeOptions};++let options = ScrapeOptions {+ formats: Some(vec![Format::Video]),+ ..Default::default()+};++let doc = client+ .scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ", options)+ .await?;+println!("{:?}", doc.video);+```+### Parsing uploaded files (v2)Use the v2 client `parse` method to upload local files (`html`, `pdf`, `docx`, etc.) as multipart form data.diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rsindex 5cb1cdea0..4e8d03cde 100644--- a/apps/rust-sdk/src/types.rs+++ b/apps/rust-sdk/src/types.rs@@ -33,6 +33,8 @@ pub enum Format {Branding,/// Audio extraction (MP3) from YouTube videos.Audio,+ /// Video extraction from supported video URLs.+ Video,/// Question answer generated from the page content.Question(QuestionFormat),/// Direct highlights selected from the page content.@@ -59,6 +61,7 @@ impl Serialize for Format {Format::Attributes => serializer.serialize_str("attributes"),Format::Branding => serializer.serialize_str("branding"),Format::Audio => serializer.serialize_str("audio"),+ Format::Video => serializer.serialize_str("video"),Format::Question(question) => question.serialize(serializer),Format::Highlights(highlights) => highlights.serialize(serializer),Format::Query(query) => query.serialize(serializer),@@ -86,6 +89,7 @@ impl<'de> Deserialize<'de> for Format {"attributes" => Ok(Format::Attributes),"branding" => Ok(Format::Branding),"audio" => Ok(Format::Audio),+ "video" => Ok(Format::Video),_ => Err(de::Error::custom(format!("unknown format: {}", format))),},Value::Object(_) => match value.get("type").and_then(Value::as_str) {@@ -658,6 +662,8 @@ pub struct Document {pub screenshot: Option<String>,/// Audio download URL (signed GCS link for MP3).pub audio: Option<String>,+ /// Video download URL (signed GCS link).+ pub video: Option<String>,/// Extracted attributes.pub attributes: Option<Vec<AttributeResult>>,/// Action results.@@ -793,6 +799,7 @@ mod tests {fn test_full_document_with_array_metadata() {let json = json!({"markdown": "# Hello",+ "video": "https://storage.googleapis.com/firecrawl/video.mp4","metadata": {"sourceURL": "https://example.com","statusCode": 200,@@ -806,6 +813,10 @@ mod tests {});let doc: Document = serde_json::from_value(json).unwrap();assert_eq!(doc.markdown, Some("# Hello".to_string()));+ assert_eq!(+ doc.video,+ Some("https://storage.googleapis.com/firecrawl/video.mp4".to_string())+ );let meta = doc.metadata.unwrap();assert_eq!(meta.title, Some("Example Page".to_string()));assert_eq!(