Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): video format (#3516)

Gergő Móriczaf8adaf67b92
diff --git a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
index 834d06bab..ce17ebb07 100644
--- a/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/audio-routing.test.ts
@@ -84,6 +84,30 @@ describe("Audio format engine routing (buildFallbackList)", () => {
expect(engines).not.toContain("fetch");
});
+ it("routes video format to chrome-cdp before tlsclient", async () => {
+ const fallback = await buildFallbackList(buildStubMeta(["video"]));
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).toEqual([
+ "fire-engine;chrome-cdp",
+ "fire-engine(retry);chrome-cdp",
+ "fire-engine;tlsclient",
+ ]);
+ });
+
+ it("excludes index and non-browser engines when video format is requested", async () => {
+ const fallback = await buildFallbackList(buildStubMeta(["video"]));
+ const engines = fallback.map(f => f.engine);
+
+ expect(engines).toContain("fire-engine;chrome-cdp");
+ expect(engines).toContain("fire-engine(retry);chrome-cdp");
+ expect(engines).not.toContain("index");
+ expect(engines).not.toContain("index;documents");
+ expect(engines).not.toContain("fire-engine;chrome-cdp;stealth");
+ expect(engines).not.toContain("fire-engine(retry);chrome-cdp;stealth");
+ expect(engines).not.toContain("fetch");
+ });
+
it("still allows chrome-cdp for non-audio requests", async () => {
const fallback = await buildFallbackList(buildStubMeta([]));
const engines = fallback.map(f => f.engine);
diff --git a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
index b8b29b1dc..e8fca4d52 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-lockdown.test.ts
@@ -99,10 +99,10 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
);
test(
- "should serve cache and skip audio fetch even when audio format is requested",
+ "should serve cache and skip media fetch even when audio and video formats are requested",
async () => {
const id = crypto.randomUUID();
- const url = "https://firecrawl.dev/?testAudioGate=" + id;
+ const url = "https://firecrawl.dev/?testMediaGate=" + id;
const seed = await scrape({ url }, identity);
expect(seed).toBeDefined();
@@ -110,15 +110,15 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
await new Promise(resolve => setTimeout(resolve, 20000));
- // Without the lockdown audio gate this would either throw
- // AudioUnsupportedUrlError (firecrawl.dev is not an audio source) or
+ // Without the lockdown media gate this would either throw an unsupported
+ // media URL error (firecrawl.dev is not an avgrab source) or
// POST to AVGRAB_SERVICE_URL with the target URL. Success here implies
// the gate short-circuited before any outbound call.
const data = await scrape(
{
url,
lockdown: true,
- formats: ["markdown", "audio"],
+ formats: ["markdown", "audio", "video"],
},
identity,
);
@@ -126,6 +126,7 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Lockdown Mode", () => {
expect(data).toBeDefined();
expect(data.metadata.cacheState).toBe("hit");
expect(data.audio).toBeUndefined();
+ expect(data.video).toBeUndefined();
},
scrapeTimeout * 2 + 20000,
);
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 53c58c5c4..df41959bf 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -2065,6 +2065,42 @@ describe("Attribute formats", () => {
);
});
+ describeIf(!TEST_SELF_HOST)("Video format", () => {
+ it.concurrent(
+ "should return video field with signed GCS URL for a supported video URL",
+ async () => {
+ const data = await scrape(
+ {
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["video"],
+ },
+ identity,
+ );
+
+ expect(data.video).toBeDefined();
+ expect(typeof data.video).toBe("string");
+ expect(data.video).toMatch(/^https:\/\//);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "should reject unsupported URL with video format",
+ async () => {
+ const result = await scrapeWithFailure(
+ {
+ url: "https://example.com",
+ formats: ["video"],
+ },
+ identity,
+ );
+
+ expect(result.error).toMatch(/video/i);
+ },
+ scrapeTimeout,
+ );
+ });
+
describe("UUID validation", () => {
it.concurrent(
"should reject invalid UUID 'None' for scrape status",
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index aef9c2067..b29bcf03e 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -60,6 +60,24 @@ describe("V2 Types Validation", () => {
expect(result.formats).toEqual([{ type: "markdown" }, { type: "html" }]);
});
+ it("should accept video format as string and object", () => {
+ const stringInput: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: ["video"],
+ };
+ const objectInput: ScrapeRequestInput = {
+ url: "https://example.com",
+ formats: [{ type: "video" }],
+ };
+
+ expect(scrapeRequestSchema.parse(stringInput).formats).toEqual([
+ { type: "video" },
+ ]);
+ expect(scrapeRequestSchema.parse(objectInput).formats).toEqual([
+ { type: "video" },
+ ]);
+ });
+
it("should accept valid scrape request with json format options", () => {
const input: ScrapeRequestInput = {
url: "https://example.com",
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2b838f087..d2b8238e5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -438,7 +438,8 @@ export type FormatObject =
| HighlightsFormatWithOptions
| QueryFormatWithOptions
| { type: "branding" }
- | { type: "audio" };
+ | { type: "audio" }
+ | { type: "video" };
const pdfModeSchema = z.enum(["fast", "auto", "ocr"]);
@@ -550,6 +551,7 @@ const baseScrapeOptions = z.strictObject({
highlightsFormatWithOptions,
queryFormatWithOptions,
z.strictObject({ type: z.literal("audio") }),
+ z.strictObject({ type: z.literal("video") }),
])
.array()
.optional()
@@ -1097,6 +1099,7 @@ export type Document = {
images?: string[];
screenshot?: string;
audio?: string;
+ video?: string;
extract?: any;
json?: any;
summary?: string;
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 1a6c31afb..cdf4ddf17 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -32,6 +32,7 @@ import {
ScrapeRetryLimitError,
BrandingNotSupportedError,
AudioUnsupportedUrlError,
+ VideoUnsupportedUrlError,
XTwitterConfigurationError,
} from "../scraper/scrapeURL/error";
@@ -65,6 +66,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_SITEMAP_ERROR: SitemapError,
CRAWL_DENIAL: CrawlDenialError,
SCRAPE_AUDIO_UNSUPPORTED_URL: AudioUnsupportedUrlError,
+ SCRAPE_VIDEO_UNSUPPORTED_URL: VideoUnsupportedUrlError,
SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
MAP_FAILED: MapFailedError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 838eb3167..d45ffdd5d 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -26,6 +26,7 @@ export type ErrorCodes =
| "SCRAPE_BRANDING_NOT_SUPPORTED"
| "AGENT_INDEX_ONLY"
| "SCRAPE_AUDIO_UNSUPPORTED_URL"
+ | "SCRAPE_VIDEO_UNSUPPORTED_URL"
| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"
| "PARSE_UNSUPPORTED_OPTIONS"
| "CRAWL_DENIAL"
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 166a91eab..dd5bee7d3 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -90,6 +90,10 @@ export async function calculateCreditsToBeBilled(
creditsToBeBilled += 4;
}
+ if (hasFormatOfType(options.formats, "video")) {
+ creditsToBeBilled += 4;
+ }
+
if (document.metadata?.postprocessorsUsed?.includes("x-twitter")) {
creditsToBeBilled += xTwitterCostBonus;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 95f188389..605b33c4c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -265,6 +265,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
});
const hasBranding = hasFormatOfType(meta.options.formats, "branding");
const hasAudio = hasFormatOfType(meta.options.formats, "audio");
+ const hasVideo = hasFormatOfType(meta.options.formats, "video");
const shouldRunYoutubePostprocessor = youtubePostprocessor.shouldRun(
meta,
new URL(meta.rewrittenUrl ?? meta.url),
@@ -321,7 +322,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
},
]
: []),
- ...(hasAudio || shouldRunYoutubePostprocessor
+ ...(hasAudio || hasVideo || shouldRunYoutubePostprocessor
? ([
{
type: "getCookies",
@@ -464,7 +465,9 @@ export async function scrapeURLWithFireEngineChromeCDP(
proxyUsed: response.usedMobileProxy ? "stealth" : "basic",
youtubeTranscriptContent: response.youtubeTranscriptContent,
timezone: response.timezone,
- ...(hasAudio || shouldRunYoutubePostprocessor ? { audioCookies } : {}),
+ ...(hasAudio || hasVideo || shouldRunYoutubePostprocessor
+ ? { audioCookies }
+ : {}),
};
});
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index bcb71c34b..2c1818ae2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -90,6 +90,7 @@ const featureFlags = [
"pdf",
"document",
"audio",
+ "video",
"atsv",
"location",
"mobile",
@@ -114,6 +115,7 @@ const featureFlagOptions: {
pdf: { priority: 100 },
document: { priority: 100 },
audio: { priority: 100 },
+ video: { priority: 100 },
atsv: { priority: 90 }, // NOTE: should atsv force to tlsclient? adjust priority if not
useFastMode: { priority: 90 },
location: { priority: 10 },
@@ -224,6 +226,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: false,
+ video: false,
atsv: false,
mobile: true,
location: true,
@@ -244,6 +247,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: false,
location: true,
mobile: true,
@@ -264,6 +268,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: false,
location: true,
mobile: true,
@@ -284,6 +289,7 @@ const engineOptions: {
pdf: true,
document: true,
audio: false,
+ video: false,
atsv: false,
location: true,
mobile: true,
@@ -304,6 +310,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: false,
location: true,
mobile: true,
@@ -324,6 +331,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: false,
location: true,
mobile: true,
@@ -344,6 +352,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
@@ -364,6 +373,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: true,
location: true,
mobile: false,
@@ -384,6 +394,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: true,
+ video: true,
atsv: true,
location: true,
mobile: false,
@@ -404,6 +415,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
@@ -424,6 +436,7 @@ const engineOptions: {
pdf: true,
document: false,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
@@ -444,6 +457,7 @@ const engineOptions: {
pdf: false,
document: true,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
@@ -464,6 +478,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
@@ -484,6 +499,7 @@ const engineOptions: {
pdf: false,
document: false,
audio: false,
+ video: false,
atsv: false,
location: false,
mobile: false,
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index acfcebc2f..5f90f25cc 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -535,6 +535,28 @@ export class AudioUnsupportedUrlError extends TransportableError {
}
}
+export class VideoUnsupportedUrlError extends TransportableError {
+ constructor(message?: string) {
+ super(
+ "SCRAPE_VIDEO_UNSUPPORTED_URL",
+ message ?? "The video format does not support the provided URL",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new VideoUnsupportedUrlError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class BrandingNotSupportedError extends TransportableError {
constructor(message: string) {
super("SCRAPE_BRANDING_NOT_SUPPORTED", message);
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0b3a02f61..6589f6737 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -189,6 +189,10 @@ function buildFeatureFlags(
flags.add("audio");
}
+ if (hasFormatOfType(options.formats, "video")) {
+ flags.add("video");
+ }
+
if (options.waitFor !== 0) {
flags.add("waitFor");
}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
new file mode 100644
index 000000000..c7d847767
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/__tests__/video.test.ts
@@ -0,0 +1,178 @@
+import { fetchVideo } from "../video";
+import { config } from "../../../../config";
+
+describe("fetchVideo", () => {
+ const originalFetch = global.fetch;
+ const originalAvgrabServiceUrl = config.AVGRAB_SERVICE_URL;
+
+ afterEach(() => {
+ global.fetch = originalFetch;
+ config.AVGRAB_SERVICE_URL = originalAvgrabServiceUrl;
+ jest.clearAllMocks();
+ });
+
+ function mockSuccessfulAvgrab() {
+ const fetchSpy = jest.fn(async (url: string, _init?: RequestInit) => {
+ if (url.endsWith("/supported-urls")) {
+ return {
+ ok: true,
+ json: async () => ({ regex: "https://example\\.com/video" }),
+ };
+ }
+
+ return {
+ ok: true,
+ json: async () => ({ public_url: "https://storage.example/video.mp4" }),
+ };
+ });
+
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = "https://avgrab.example";
+ return fetchSpy;
+ }
+
+ it("does not issue any fetch when lockdown is true, even if video format is requested", async () => {
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as any;
+
+ const meta: any = {
+ url: "https://example.com/video",
+ options: {
+ lockdown: true,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = { markdown: "cached" };
+
+ const result = await fetchVideo(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(result).toBe(document);
+ expect(result.video).toBeUndefined();
+ });
+
+ it("returns early when video format is not requested regardless of lockdown", async () => {
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as any;
+
+ const meta: any = {
+ url: "https://example.com/video",
+ options: {
+ lockdown: false,
+ formats: [{ type: "markdown" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = { markdown: "cached" };
+
+ await fetchVideo(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+
+ it("returns a warning when avgrab is not configured", async () => {
+ const fetchSpy = jest.fn();
+ global.fetch = fetchSpy as any;
+ config.AVGRAB_SERVICE_URL = undefined;
+
+ const meta: any = {
+ url: "https://example.com/video",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ const result = await fetchVideo(meta, document);
+
+ expect(fetchSpy).not.toHaveBeenCalled();
+ expect(meta.logger.warn).toHaveBeenCalledWith(
+ "AVGRAB_SERVICE_URL is not configured",
+ );
+ expect(result.warning).toMatch(/Video format is not available/);
+ });
+
+ it("forwards browser cookies to avgrab when video cookies are available", async () => {
+ const fetchSpy = mockSuccessfulAvgrab();
+ const cookies = [
+ {
+ name: "VISITOR_INFO1_LIVE",
+ value: "visitor",
+ domain: ".youtube.com",
+ path: "/",
+ secure: true,
+ httpOnly: true,
+ },
+ ];
+
+ const meta: any = {
+ url: "https://example.com/video",
+ audioCookies: cookies,
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchVideo(meta, document);
+
+ const downloadCall = fetchSpy.mock.calls.find(([url]) =>
+ String(url).endsWith("/download-video"),
+ );
+ expect(downloadCall).toBeDefined();
+ const body = downloadCall![1]?.body;
+ expect(typeof body).toBe("string");
+ expect(JSON.parse(body as string)).toEqual({
+ url: "https://example.com/video",
+ cookies,
+ });
+ expect(document.video).toBe("https://storage.example/video.mp4");
+ });
+
+ it("omits cookies from the avgrab request when no video cookies are available", async () => {
+ const fetchSpy = mockSuccessfulAvgrab();
+
+ const meta: any = {
+ url: "https://example.com/video",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await fetchVideo(meta, document);
+
+ const downloadCall = fetchSpy.mock.calls.find(([url]) =>
+ String(url).endsWith("/download-video"),
+ );
+ expect(downloadCall).toBeDefined();
+ const body = downloadCall![1]?.body;
+ expect(typeof body).toBe("string");
+ expect(JSON.parse(body as string)).toEqual({
+ url: "https://example.com/video",
+ });
+ });
+
+ it("rejects unsupported URLs", async () => {
+ mockSuccessfulAvgrab();
+
+ const meta: any = {
+ url: "https://example.com/unsupported",
+ options: {
+ lockdown: false,
+ formats: [{ type: "video" }],
+ },
+ logger: { warn: jest.fn(), info: jest.fn(), debug: jest.fn() },
+ };
+ const document: any = {};
+
+ await expect(fetchVideo(meta, document)).rejects.toThrow(/video/i);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.ts
index 883ec5873..9af263688 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts
@@ -18,6 +18,7 @@ import { performAttributes } from "./performAttributes";
import { deriveDiff } from "./diff";
import { fetchAudio } from "./audio";
+import { fetchVideo } from "./video";
import { useIndex, useSearchIndex } from "../../../services/index";
import { sendDocumentToIndex } from "../engines/index/index";
import { sendDocumentToSearchIndex } from "./sendToSearchIndex";
@@ -484,6 +485,15 @@ function coerceFieldsToFormats(meta: Meta, document: Document): Document {
);
}
+ const hasVideo = hasFormatOfType(meta.options.formats, "video");
+ if (!hasVideo && document.video !== undefined) {
+ delete document.video;
+ } else if (hasVideo && document.video === undefined) {
+ meta.logger.warn(
+ "Request had format: video, but there was no video field in the result.",
+ );
+ }
+
if (!hasChangeTracking && document.changeTracking !== undefined) {
meta.logger.warn(
"Removed changeTracking from Document because it wasn't in formats -- this is extremely wasteful and indicates a bug.",
@@ -558,6 +568,7 @@ const transformerStack: Transformer[] = [
removeBase64Images,
deriveDiff,
fetchAudio,
+ fetchVideo,
coerceFieldsToFormats,
];
diff --git a/apps/api/src/scraper/scrapeURL/transformers/video.ts b/apps/api/src/scraper/scrapeURL/transformers/video.ts
new file mode 100644
index 000000000..5a61b59e0
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/transformers/video.ts
@@ -0,0 +1,94 @@
+import { Meta } from "..";
+import { Document } from "../../../controllers/v2/types";
+import { config } from "../../../config";
+import { hasFormatOfType } from "../../../lib/format-utils";
+import { VideoUnsupportedUrlError } from "../error";
+
+let cachedUrlRegex: RegExp | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+async function getSupportedUrlRegex(): Promise<RegExp> {
+ if (cachedUrlRegex && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+ return cachedUrlRegex;
+ }
+
+ const res = await fetch(`${config.AVGRAB_SERVICE_URL}/supported-urls`);
+ if (!res.ok) {
+ throw new Error(
+ "Failed to fetch supported URL patterns from video service",
+ );
+ }
+
+ const data = await res.json().catch(() => null);
+ if (!data || typeof data.regex !== "string") {
+ throw new Error("Video service returned invalid supported URL patterns");
+ }
+
+ try {
+ cachedUrlRegex = new RegExp(data.regex);
+ } catch {
+ throw new Error("Video service returned invalid supported URL patterns");
+ }
+ cacheTimestamp = Date.now();
+ return cachedUrlRegex;
+}
+
+export async function fetchVideo(
+ meta: Meta,
+ document: Document,
+): Promise<Document> {
+ if (!hasFormatOfType(meta.options.formats, "video")) {
+ return document;
+ }
+
+ // Lockdown forbids outbound requests that touch the target URL. avgrab
+ // fetches the source on our behalf, so skip it here.
+ if (meta.options.lockdown) {
+ return document;
+ }
+
+ if (!config.AVGRAB_SERVICE_URL) {
+ meta.logger.warn("AVGRAB_SERVICE_URL is not configured");
+ document.warning =
+ "Video format is not available (service not configured)." +
+ (document.warning ? " " + document.warning : "");
+ return document;
+ }
+
+ const urlRegex = await getSupportedUrlRegex();
+ if (!urlRegex.test(meta.url)) {
+ throw new VideoUnsupportedUrlError();
+ }
+
+ const requestBody = {
+ url: meta.url,
+ ...(meta.audioCookies && meta.audioCookies.length > 0
+ ? { cookies: meta.audioCookies }
+ : {}),
+ };
+
+ const response = await fetch(`${config.AVGRAB_SERVICE_URL}/download-video`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(requestBody),
+ });
+
+ if (!response.ok) {
+ const error = await response
+ .json()
+ .catch(() => ({ detail: "Unknown error" }));
+ throw new Error(`Video download failed: ${error.detail}`);
+ }
+
+ const data = await response.json().catch(() => null);
+
+ if (!data || !data.public_url || typeof data.public_url !== "string") {
+ throw new Error(
+ "Video download failed: avgrab service returned an invalid response (missing public_url)",
+ );
+ }
+
+ document.video = data.public_url;
+ return document;
+}
diff --git a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
index 492f4c22d..acf959b82 100644
--- a/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
+++ b/apps/dot-net-sdk/Firecrawl.Tests/ModelsTests.cs
@@ -119,6 +119,7 @@ public class ModelsTests
{
"markdown": "# Hello World",
"html": "<h1>Hello World</h1>",
+ "video": "https://storage.googleapis.com/firecrawl/video.mp4",
"metadata": {
"title": "Test",
"sourceURL": "https://example.com"
@@ -131,6 +132,7 @@ public class ModelsTests
Assert.NotNull(doc);
Assert.Equal("# Hello World", doc.Markdown);
Assert.Equal("<h1>Hello World</h1>", doc.Html);
+ Assert.Equal("https://storage.googleapis.com/firecrawl/video.mp4", doc.Video);
Assert.NotNull(doc.Metadata);
Assert.Null(doc.Warning);
}
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index efdeaea3d..da297be8f 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.3.1</Version>
+ <Version>1.4.0</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/Models/Document.cs b/apps/dot-net-sdk/Firecrawl/Models/Document.cs
index 5dd68eb44..ba601fbf2 100644
--- a/apps/dot-net-sdk/Firecrawl/Models/Document.cs
+++ b/apps/dot-net-sdk/Firecrawl/Models/Document.cs
@@ -37,6 +37,9 @@ public class Document
[JsonPropertyName("audio")]
public object? Audio { get; set; }
+ [JsonPropertyName("video")]
+ public string? Video { get; set; }
+
[JsonPropertyName("actions")]
public object? Actions { get; set; }
diff --git a/apps/dot-net-sdk/README.md b/apps/dot-net-sdk/README.md
index 70560fec1..3c61ee7fe 100644
--- a/apps/dot-net-sdk/README.md
+++ b/apps/dot-net-sdk/README.md
@@ -73,6 +73,17 @@ Console.WriteLine(doc.Markdown);
Console.WriteLine(doc.Html);
```
+### Video Extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `Video` property is a signed URL to the extracted video file.
+
+```csharp
+var doc = await client.ScrapeAsync("https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ new ScrapeOptions { Formats = new List<object> { "video" } });
+
+Console.WriteLine(doc.Video);
+```
+
### Crawl
```csharp
@@ -133,7 +144,7 @@ foreach (var doc in job.Data!)
Upload and parse a local file (HTML, PDF, DOCX, etc.) through the `/v2/parse`
endpoint. Parse does not support browser-rendering options like `actions`,
`waitFor`, `location`, `mobile`, or the `screenshot` / `branding` /
-`changeTracking` formats.
+`changeTracking` / `audio` / `video` formats.
```csharp
using Firecrawl;
diff --git a/apps/elixir-sdk/README.md b/apps/elixir-sdk/README.md
index ec3efb68c..8a1cfd2aa 100644
--- a/apps/elixir-sdk/README.md
+++ b/apps/elixir-sdk/README.md
@@ -11,7 +11,7 @@ Add `firecrawl` to your list of dependencies in `mix.exs`:
```elixir
def deps do
[
- {:firecrawl, "~> 1.0"}
+ {:firecrawl, "~> 1.4"}
]
end
```
@@ -43,6 +43,12 @@ All params are passed as keyword lists with snake_case keys. Invalid keys, missi
formats: ["markdown"]
)
+# Extract video from supported video URLs
+{:ok, response} = Firecrawl.scrape_and_extract_from_url(
+ url: "https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ formats: ["video"]
+)
+
# Crawl a site
{:ok, response} = Firecrawl.crawl_urls(
url: "https://example.com",
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index f66938714..97e56e952 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.3.1"
+ @version "1.4.0"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.md
index ad5cce194..99661dec8 100644
--- a/apps/go-sdk/README.md
+++ b/apps/go-sdk/README.md
@@ -94,6 +94,20 @@ doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{
})
```
+### Video Extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `Video` field is a signed URL to the extracted video file.
+
+```go
+doc, err := client.Scrape(ctx, "https://www.youtube.com/watch?v=dQw4w9WgXcQ", &firecrawl.ScrapeOptions{
+ Formats: []string{"video"},
+})
+if err != nil {
+ return err
+}
+fmt.Println(doc.Video)
+```
+
#### Interactive Browser
Execute code in a scrape-bound browser session:
@@ -112,8 +126,8 @@ deleteResp, err := client.StopInteractiveBrowser(ctx, scrapeJobID)
Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data and
parse it synchronously. Parse options intentionally exclude browser-only
-features such as change tracking, screenshot, branding, actions, waitFor,
-location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.
+features such as change tracking, screenshot, branding, audio, video, actions,
+waitFor, location, and mobile. The `Proxy` option only accepts `"auto"` or `"basic"`.
```go
// From disk
@@ -341,10 +355,10 @@ The Go SDK lives in a monorepo subdirectory, so releases follow Go's
**must** be prefixed with the module subdirectory path:
```
-apps/go-sdk/v1.0.0
+apps/go-sdk/v1.3.0
```
-A bare `v1.0.0` tag will not be resolvable by the Go module proxy.
+A bare `v1.3.0` tag will not be resolvable by the Go module proxy.
### Release workflow
@@ -352,7 +366,7 @@ The SDK version is the single source of truth in
[`version.go`](./version.go):
```go
-const Version = "1.0.0"
+const Version = "1.3.0"
```
To cut a release:
@@ -371,7 +385,7 @@ The workflow is idempotent: if the tag already exists, it is a no-op.
### Consuming a specific version
```bash
-go get github.com/firecrawl/firecrawl/apps/[email protected]
+go get github.com/firecrawl/firecrawl/apps/[email protected]
```
Users pin via the semantic version suffix; they never reference the
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index a4c14b0ce..33b1df63e 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -14,6 +14,7 @@ type Document struct {
Images []string `json:"images,omitempty"`
Screenshot string `json:"screenshot,omitempty"`
Audio string `json:"audio,omitempty"`
+ Video string `json:"video,omitempty"`
Attributes []map[string]interface{} `json:"attributes,omitempty"`
Actions map[string]interface{} `json:"actions,omitempty"`
Answer string `json:"answer,omitempty"`
diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go
index 9416ec487..07892be5a 100644
--- a/apps/go-sdk/options_test.go
+++ b/apps/go-sdk/options_test.go
@@ -53,13 +53,13 @@ func TestScrapeOptionsSerializesQuestionAndHighlightsFormats(t *testing.T) {
func TestScrapeOptionsPreservesStringFormats(t *testing.T) {
payload, err := json.Marshal(ScrapeOptions{
- Formats: []string{"markdown"},
+ Formats: []string{"markdown", "video"},
})
if err != nil {
t.Fatalf("Marshal ScrapeOptions: %v", err)
}
- if !strings.Contains(string(payload), `"formats":["markdown"]`) {
+ if !strings.Contains(string(payload), `"formats":["markdown","video"]`) {
t.Fatalf("serialized string formats = %s", payload)
}
}
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index 702528172..a6ab7a966 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.2.2"
+const Version = "1.3.0"
diff --git a/apps/java-sdk/README.md b/apps/java-sdk/README.md
index 2555b1f2a..0a83d6576 100644
--- a/apps/java-sdk/README.md
+++ b/apps/java-sdk/README.md
@@ -82,13 +82,13 @@ Before using the Java SDK, ensure you have the following installed:
### Gradle (Kotlin DSL)
```kotlin
-implementation("com.firecrawl:firecrawl-java:1.1.1")
+implementation("com.firecrawl:firecrawl-java:1.6.0")
```
### Gradle (Groovy)
```groovy
-implementation 'com.firecrawl:firecrawl-java:1.1.1'
+implementation 'com.firecrawl:firecrawl-java:1.6.0'
```
### Maven
@@ -97,7 +97,7 @@ implementation 'com.firecrawl:firecrawl-java:1.1.1'
<dependency>
<groupId>com.firecrawl</groupId>
<artifactId>firecrawl-java</artifactId>
- <version>1.1.1</version>
+ <version>1.6.0</version>
</dependency>
```
@@ -147,10 +147,23 @@ System.out.println(doc.getMarkdown());
System.out.println(doc.getMetadata().get("title"));
```
+### Video Extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```java
+Document doc = client.scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ ScrapeOptions.builder()
+ .formats(List.of("video"))
+ .build());
+
+System.out.println(doc.getVideo());
+```
+
### Parse Uploaded Files
Upload local files (`html`, `pdf`, `docx`, etc.) via multipart form data and parse them synchronously.
-Parse options intentionally exclude browser-only features like change tracking, screenshot, branding, actions, waitFor, location, and mobile.
+Parse options intentionally exclude browser-only features like change tracking, screenshot, branding, audio, video, actions, waitFor, location, and mobile.
```java
ParseFile file = ParseFile.builder()
@@ -391,14 +404,14 @@ gradle build
```bash
gradle jar
-# Output: build/libs/firecrawl-java-1.1.1.jar
+# Output: build/libs/firecrawl-java-1.6.0.jar
```
### Install Locally
```bash
gradle publishToMavenLocal
-# Now available as: com.firecrawl:firecrawl-java:1.1.1 in local Maven repository
+# Now available as: com.firecrawl:firecrawl-java:1.6.0 in local Maven repository
```
## Running Tests
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 8df199421..b0a7ffab0 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.5.1"
+version = "1.6.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
index b61670d2a..6d4eee5c9 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Document.java
@@ -20,6 +20,7 @@ public class Document {
private List<String> images;
private String screenshot;
private String audio;
+ private String video;
private List<Map<String, Object>> attributes;
private Map<String, Object> actions;
private String answer;
@@ -38,6 +39,7 @@ public class Document {
public List<String> getImages() { return images; }
public String getScreenshot() { return screenshot; }
public String getAudio() { return audio; }
+ public String getVideo() { return video; }
public List<Map<String, Object>> getAttributes() { return attributes; }
public Map<String, Object> getActions() { return actions; }
public String getAnswer() { return answer; }
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
index daf137521..5e881d425 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ParseOptions.java
@@ -10,8 +10,8 @@ import java.util.Map;
/**
* Options for parsing uploaded files via /v2/parse.
*
- * <p>Parse does not support browser-rendering formats/options such as
- * change tracking, screenshot, branding, actions, waitFor, location, or mobile.
+ * <p>Parse does not support browser-rendering formats/options such as
+ * change tracking, screenshot, branding, audio, video, actions, waitFor, location, or mobile.
*/
@JsonInclude(JsonInclude.Include.NON_NULL)
public class ParseOptions {
@@ -86,7 +86,9 @@ public class ParseOptions {
|| normalized.equals("change_tracking")
|| normalized.equals("screenshot")
|| normalized.equals("screenshot@fullPage")
- || normalized.equals("branding");
+ || normalized.equals("branding")
+ || normalized.equals("audio")
+ || normalized.equals("video");
}
public static final class Builder {
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
index 700e29179..3bd3df5f2 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java
@@ -108,7 +108,7 @@ public class ScrapeOptions {
/**
* Output formats to request. Accepts strings like "markdown", "html", "rawHtml",
- * "links", "screenshot", "json", "audio", etc., or format configuration maps/objects for
+ * "links", "screenshot", "json", "audio", "video", etc., or format configuration maps/objects for
* advanced formats (e.g., JsonFormat, QuestionFormat, HighlightsFormat).
*/
public Builder formats(List<Object> formats) { this.formats = formats; return this; }
diff --git a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
index 1d7cc74af..44b25247e 100644
--- a/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
+++ b/apps/java-sdk/src/test/java/com/firecrawl/FirecrawlClientTest.java
@@ -66,13 +66,13 @@ class FirecrawlClientTest {
.build();
ScrapeOptions options = ScrapeOptions.builder()
- .formats(List.of("markdown", "html", queryFormat))
+ .formats(List.of("markdown", "html", "video", queryFormat))
.onlyMainContent(true)
.timeout(30000)
.mobile(false)
.build();
- assertEquals(List.of("markdown", "html", queryFormat), options.getFormats());
+ assertEquals(List.of("markdown", "html", "video", queryFormat), options.getFormats());
assertEquals("query", queryFormat.getType());
assertEquals(QueryFormat.Mode.DIRECT_QUOTE, queryFormat.getMode());
assertTrue(options.getOnlyMainContent());
@@ -230,6 +230,15 @@ class FirecrawlClientTest {
);
}
+ @Test
+ void testParseOptionsRejectsVideoFormat() {
+ assertThrows(IllegalArgumentException.class, () ->
+ ParseOptions.builder()
+ .formats(List.of("video"))
+ .build()
+ );
+ }
+
// ================================================================
// E2E TESTS (require FIRECRAWL_API_KEY)
// ================================================================
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md
index ab1fbc382..d312ff118 100644
--- a/apps/js-sdk/firecrawl/README.md
+++ b/apps/js-sdk/firecrawl/README.md
@@ -46,10 +46,22 @@ const url = 'https://example.com';
const scrapedData = await app.scrape(url);
```
+### Video extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```js
+const doc = await app.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', {
+ formats: ['video'],
+});
+
+console.log(doc.video);
+```
+
### Parsing uploaded files
Use `parse` to upload a file (`html`, `pdf`, `docx`, etc.) as multipart form data and process it through the same parsing pipeline.
-Parse does not support browser-only formats/options like `changeTracking`, `screenshot`, `branding`, `actions`, `waitFor`, `location`, or `mobile`.
+Parse does not support browser-only formats/options like `changeTracking`, `screenshot`, `branding`, `audio`, `video`, `actions`, `waitFor`, `location`, or `mobile`.
```js
const parsed = await app.parse(
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 2ec28d6e4..3a6f436c6 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.22.2",
+ "version": "4.23.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
index cd0f16a2b..32fddf77a 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/parse.unit.test.ts
@@ -38,6 +38,24 @@ describe("v2.parse unit", () => {
).rejects.toThrow("parse does not support changeTracking format");
});
+ test("rejects video format before making requests", async () => {
+ const client = new FirecrawlClient({
+ apiKey: "test-key",
+ apiUrl: "https://localhost:3002",
+ });
+
+ await expect(
+ client.parse(
+ {
+ data: Buffer.from("<html></html>"),
+ filename: "upload.html",
+ contentType: "text/html",
+ },
+ { formats: ["video" as any] },
+ ),
+ ).rejects.toThrow("parse does not support video format");
+ });
+
test("rejects lockdown option before making requests", async () => {
const client = new FirecrawlClient({
apiKey: "test-key",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
index 83b8795fb..bcac9d4c4 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/validation.test.ts
@@ -9,6 +9,11 @@ describe("v2 utils: validation", () => {
expect(() => ensureValidFormats(formats)).toThrow(/json format must be an object/i);
});
+ test("ensureValidFormats: accepts video string format", () => {
+ const formats: FormatOption[] = ["markdown", "video"];
+ expect(() => ensureValidFormats(formats)).not.toThrow();
+ });
+
test("ensureValidFormats: json format requires prompt or schema", () => {
// Valid cases - should not throw
const valid1: FormatOption[] = [{ type: "json", prompt: "p" } as any];
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index f375bb457..ccd93d8a1 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -204,6 +204,7 @@ export class FirecrawlClient {
* @param file File payload (data, filename, optional contentType).
* @param options Optional parse options (formats, parsers, etc.).
* Note: parse does not support changeTracking, screenshot, branding,
+ * audio, video,
* actions, waitFor, location, or mobile options.
* @returns Parsed document with requested formats.
*/
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 14f5cc322..2ceae804d 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -13,7 +13,8 @@ export type FormatString =
| 'json'
| 'attributes'
| 'branding'
- | 'audio';
+ | 'audio'
+ | 'video';
export interface Viewport {
width: number;
@@ -82,7 +83,7 @@ export type FormatOption =
export type ParseFormatString = Exclude<
FormatString,
- 'screenshot' | 'changeTracking' | 'branding'
+ 'screenshot' | 'changeTracking' | 'branding' | 'audio' | 'video'
>;
export interface ParseFormat {
@@ -458,6 +459,7 @@ export interface Document {
images?: string[];
screenshot?: string;
audio?: string;
+ video?: string;
attributes?: Array<{
selector: string;
attribute: string;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
index b8082268f..e42e2242c 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/validation.ts
@@ -114,6 +114,9 @@ export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {
if (fmt === "branding") {
throw new Error("parse does not support branding format");
}
+ if (fmt === "audio" || fmt === "video") {
+ throw new Error(`parse does not support ${fmt} format`);
+ }
continue;
}
@@ -127,6 +130,9 @@ export function ensureValidParseFormats(formats?: ParseFormatOption[]): void {
if (type === "branding") {
throw new Error("parse does not support branding format");
}
+ if (type === "audio" || type === "video") {
+ throw new Error(`parse does not support ${type} format`);
+ }
if ((fmt as JsonFormat).type === "json") {
const j = fmt as JsonFormat;
diff --git a/apps/php-sdk/CHANGELOG.md b/apps/php-sdk/CHANGELOG.md
index c9b1dd29a..a7bb660e7 100644
--- a/apps/php-sdk/CHANGELOG.md
+++ b/apps/php-sdk/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to the Firecrawl PHP SDK will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [1.3.0] - 2026-05-12
+
+### Added
+- Added `video` scrape format support and `Document::getVideo()` for video extraction results.
+
## [1.1.0] - 2026-04-21
### Added
diff --git a/apps/php-sdk/README.md b/apps/php-sdk/README.md
index 5c5b6fdc9..a36fb6301 100644
--- a/apps/php-sdk/README.md
+++ b/apps/php-sdk/README.md
@@ -83,12 +83,24 @@ $doc = $client->scrape('https://example.com/product', ScrapeOptions::with(
echo $doc->getJson(); // Structured data
```
+### Video Extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```php
+$doc = $client->scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', ScrapeOptions::with(
+ formats: ['video'],
+));
+
+echo $doc->getVideo();
+```
+
### Parse
Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data and
parse it synchronously. Parse options intentionally exclude browser-only
-features such as change tracking, screenshot, branding, actions, waitFor,
-location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.
+features such as change tracking, screenshot, branding, audio, video, actions,
+waitFor, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.
```php
use Firecrawl\Models\ParseFile;
diff --git a/apps/php-sdk/src/Models/Document.php b/apps/php-sdk/src/Models/Document.php
index 994de0661..5b4f02c98 100644
--- a/apps/php-sdk/src/Models/Document.php
+++ b/apps/php-sdk/src/Models/Document.php
@@ -26,6 +26,7 @@ final class Document
private readonly ?array $images = null,
private readonly ?string $screenshot = null,
private readonly ?string $audio = null,
+ private readonly ?string $video = null,
private readonly ?array $attributes = null,
private readonly ?array $actions = null,
private readonly ?string $answer = null,
@@ -49,6 +50,7 @@ final class Document
images: $data['images'] ?? null,
screenshot: $data['screenshot'] ?? null,
audio: $data['audio'] ?? null,
+ video: $data['video'] ?? null,
attributes: $data['attributes'] ?? null,
actions: $data['actions'] ?? null,
answer: $data['answer'] ?? null,
@@ -112,6 +114,11 @@ final class Document
return $this->audio;
}
+ public function getVideo(): ?string
+ {
+ return $this->video;
+ }
+
/** @return list<array<string, mixed>>|null */
public function getAttributes(): ?array
{
diff --git a/apps/php-sdk/src/Models/ParseOptions.php b/apps/php-sdk/src/Models/ParseOptions.php
index 89e03faff..e9cff9030 100644
--- a/apps/php-sdk/src/Models/ParseOptions.php
+++ b/apps/php-sdk/src/Models/ParseOptions.php
@@ -10,7 +10,7 @@ use Firecrawl\Exceptions\FirecrawlException;
* Options for parsing uploaded files via `/v2/parse`.
*
* Parse does not support browser-rendering features (actions, waitFor,
- * location, mobile) nor the screenshot, branding, or changeTracking formats.
+ * location, mobile) nor the screenshot, branding, audio, video, or changeTracking formats.
* The proxy field only accepts "auto" or "basic".
*/
final class ParseOptions
@@ -20,6 +20,8 @@ final class ParseOptions
'screenshot',
'screenshot@fullPage',
'branding',
+ 'audio',
+ 'video',
];
/**
diff --git a/apps/php-sdk/src/Models/ScrapeOptions.php b/apps/php-sdk/src/Models/ScrapeOptions.php
index b0d8d50d7..f79b01306 100644
--- a/apps/php-sdk/src/Models/ScrapeOptions.php
+++ b/apps/php-sdk/src/Models/ScrapeOptions.php
@@ -7,7 +7,7 @@ namespace Firecrawl\Models;
final class ScrapeOptions
{
/**
- * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats Supports strings like "markdown", "audio", and "video".
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
@@ -41,7 +41,7 @@ final class ScrapeOptions
) {}
/**
- * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats
+ * @param list<string|JsonFormat|ScreenshotFormat|QuestionFormat|HighlightsFormat|QueryFormat>|null $formats Supports strings like "markdown", "audio", and "video".
* @param array<string, string>|null $headers
* @param list<string>|null $includeTags
* @param list<string>|null $excludeTags
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index aefc62765..1ed1826b6 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.2.1';
+ public const SDK_VERSION = '1.3.0';
}
diff --git a/apps/php-sdk/tests/Unit/ModelsTest.php b/apps/php-sdk/tests/Unit/ModelsTest.php
index b36a04d18..f9f6d8ecd 100644
--- a/apps/php-sdk/tests/Unit/ModelsTest.php
+++ b/apps/php-sdk/tests/Unit/ModelsTest.php
@@ -3,6 +3,7 @@
declare(strict_types=1);
use Firecrawl\Models\CreditUsage;
+use Firecrawl\Models\Document;
use Firecrawl\Models\MapData;
use Firecrawl\Models\BatchScrapeJob;
use Firecrawl\Models\CrawlJob;
@@ -93,6 +94,16 @@ it('preserves null creditsUsed in CrawlJob', function (): void {
expect($job->getCreditsUsed())->toBeNull();
});
+it('hydrates video URL in Document', function (): void {
+ $doc = Document::fromArray([
+ 'markdown' => '# Video',
+ 'video' => 'https://storage.googleapis.com/firecrawl/video.mp4',
+ ]);
+
+ expect($doc->getMarkdown())->toBe('# Video');
+ expect($doc->getVideo())->toBe('https://storage.googleapis.com/firecrawl/video.mp4');
+});
+
it('preserves positional integration in ScrapeOptions::with', function (): void {
$options = ScrapeOptions::with(
null,
diff --git a/apps/php-sdk/tests/Unit/ParseTest.php b/apps/php-sdk/tests/Unit/ParseTest.php
index 6cf083c46..3f00e2de8 100644
--- a/apps/php-sdk/tests/Unit/ParseTest.php
+++ b/apps/php-sdk/tests/Unit/ParseTest.php
@@ -39,6 +39,10 @@ it('rejects unsupported parse formats', function (): void {
ParseOptions::with(formats: ['screenshot']);
})->throws(FirecrawlException::class);
+it('rejects video parse format', function (): void {
+ ParseOptions::with(formats: ['video']);
+})->throws(FirecrawlException::class);
+
it('rejects invalid proxy values', function (): void {
ParseOptions::with(proxy: 'stealth');
})->throws(FirecrawlException::class);
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md
index 9e7fdc636..bdef88a52 100644
--- a/apps/python-sdk/README.md
+++ b/apps/python-sdk/README.md
@@ -49,10 +49,19 @@ scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', '
print(scrape_result)
```
+### Video extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```python
+doc = firecrawl.scrape('https://www.youtube.com/watch?v=dQw4w9WgXcQ', formats=['video'])
+print(doc.video)
+```
+
### Parsing uploaded files
Use `parse` to upload local bytes/files (`html`, `pdf`, `docx`, etc.) as multipart form data and return the parsed document.
-`parse` does not support change tracking or browser-only options (actions, wait_for, location, mobile, screenshot, branding).
+`parse` does not support change tracking or browser-only options (actions, wait_for, location, mobile, screenshot, branding, audio, video).
```python
from firecrawl import Firecrawl
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 8e097fa9c..aeabd25e1 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.25.2"
+__version__ = "4.26.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
index ccf24e99f..a49a0764d 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_request_preparation.py
@@ -66,6 +66,16 @@ class TestParseRequestPreparation:
content_type="text/html",
)
+ def test_prepare_parse_request_rejects_video_format(self):
+ options = ParseOptions(formats=["video"])
+ with pytest.raises(ValueError, match="do not support video output"):
+ _prepare_parse_request(
+ b"<html><body><h1>Parse</h1></body></html>",
+ options,
+ filename="upload.html",
+ content_type="text/html",
+ )
+
def test_prepare_parse_request_strips_lockdown(self):
options = ParseOptions(formats=["markdown"], lockdown=True)
fields, _ = _prepare_parse_request(
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
index 2988d3972..387ceb355 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_validation.py
@@ -102,7 +102,7 @@ class TestPrepareScrapeOptions:
def test_prepare_basic_options(self):
"""Test preparation with basic options."""
options = ScrapeOptions(
- formats=["markdown"],
+ formats=["markdown", "video"],
timeout=30000,
wait_for=2000
)
@@ -112,6 +112,7 @@ class TestPrepareScrapeOptions:
assert "formats" in result
assert "timeout" in result
assert "waitFor" in result
+ assert result["formats"] == ["markdown", "video"]
assert result["timeout"] == 30000
assert result["waitFor"] == 2000
diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.py
index 9886a7dbd..f40188d0e 100644
--- a/apps/python-sdk/firecrawl/v2/methods/parse.py
+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py
@@ -15,7 +15,7 @@ from ..utils.get_version import get_version
version = get_version()
ParseFileInput = Union[str, bytes, bytearray, Path, BinaryIO]
-UNSUPPORTED_PARSE_FORMATS = {"changeTracking", "screenshot", "branding"}
+UNSUPPORTED_PARSE_FORMATS = {"changeTracking", "screenshot", "branding", "audio", "video"}
def _extract_format_type(format_item: Any) -> Optional[str]:
@@ -53,6 +53,8 @@ def _validate_parse_options_payload(options_payload: Dict[str, Any]) -> None:
raise ValueError("Parse uploads do not support change tracking.")
if fmt_type == "screenshot":
raise ValueError("Parse uploads do not support screenshot output.")
+ if fmt_type in {"audio", "video"}:
+ raise ValueError(f"Parse uploads do not support {fmt_type} output.")
raise ValueError("Parse uploads do not support branding output.")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 00fde19a7..0ef08b9e2 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -275,6 +275,7 @@ class Document(BaseModel):
images: Optional[List[str]] = None
screenshot: Optional[str] = None
audio: Optional[str] = None
+ video: Optional[str] = None
actions: Optional[Dict[str, Any]] = None
answer: Optional[str] = None
highlights: Optional[str] = None
@@ -389,6 +390,7 @@ FormatString = Literal[
"branding",
"query",
"audio",
+ "video",
# snake_case versions (user-friendly)
"raw_html",
"change_tracking",
diff --git a/apps/ruby-sdk/README.md b/apps/ruby-sdk/README.md
index 0f79fae21..032ccaa53 100644
--- a/apps/ruby-sdk/README.md
+++ b/apps/ruby-sdk/README.md
@@ -11,7 +11,7 @@ Ruby SDK for the [Firecrawl](https://firecrawl.dev) v2 web scraping API.
Add to your `Gemfile`:
```ruby
-gem "firecrawl-sdk", "~> 1.0"
+gem "firecrawl-sdk", "~> 1.5"
```
Or install directly:
@@ -63,12 +63,23 @@ doc = client.scrape("https://example.com",
puts doc.html
```
+### Video Extraction
+
+Use the `video` format on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```ruby
+doc = client.scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ",
+ Firecrawl::Models::ScrapeOptions.new(formats: ["video"]))
+
+puts doc.video
+```
+
### Parse
Upload a local file (`html`, `pdf`, `docx`, etc.) via multipart form data and
parse it synchronously. Parse options intentionally exclude browser-only
-features such as change tracking, screenshot, branding, actions, wait_for,
-location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.
+features such as change tracking, screenshot, branding, audio, video, actions,
+wait_for, location, and mobile. The `proxy` option only accepts `"auto"` or `"basic"`.
```ruby
# From disk
diff --git a/apps/ruby-sdk/lib/firecrawl/models/document.rb b/apps/ruby-sdk/lib/firecrawl/models/document.rb
index 578c095b9..0b0200dc7 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/document.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/document.rb
@@ -6,8 +6,8 @@ module Firecrawl
class Document
attr_reader :markdown, :html, :raw_html, :json, :summary,
:metadata, :links, :images, :screenshot, :audio,
- :attributes, :actions, :answer, :highlights, :warning,
- :change_tracking, :branding
+ :video, :attributes, :actions, :answer, :highlights,
+ :warning, :change_tracking, :branding
def initialize(data)
@markdown = data["markdown"]
@@ -20,6 +20,7 @@ module Firecrawl
@images = data["images"]
@screenshot = data["screenshot"]
@audio = data["audio"]
+ @video = data["video"]
@attributes = data["attributes"]
@actions = data["actions"]
@answer = data["answer"]
diff --git a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
index 1d67aa3be..ef183e140 100644
--- a/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
+++ b/apps/ruby-sdk/lib/firecrawl/models/parse_options.rb
@@ -5,10 +5,10 @@ module Firecrawl
# Options for parsing uploaded files via `/v2/parse`.
#
# Parse does not support browser-rendering features (actions, waitFor,
- # location, mobile) nor screenshot/branding/changeTracking formats. The
+ # location, mobile) nor screenshot/branding/audio/video/changeTracking formats. The
# proxy field only accepts "auto" or "basic".
class ParseOptions
- UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding].freeze
+ UNSUPPORTED_FORMATS = %w[changeTracking screenshot screenshot@fullPage branding audio video].freeze
FIELDS = %i[
formats headers include_tags exclude_tags only_main_content
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index 7b806711f..7bb66e07a 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.4.1"
+ VERSION = "1.5.0"
end
diff --git a/apps/ruby-sdk/test/firecrawl/client_test.rb b/apps/ruby-sdk/test/firecrawl/client_test.rb
index 7b4997d2c..97ff8f537 100644
--- a/apps/ruby-sdk/test/firecrawl/client_test.rb
+++ b/apps/ruby-sdk/test/firecrawl/client_test.rb
@@ -74,13 +74,14 @@ class ClientTest < Minitest::Test
)
.to_return(
status: 200,
- body: JSON.generate(data: { markdown: "# Hello", metadata: { title: "Example", sourceURL: "https://example.com" } }),
+ body: JSON.generate(data: { markdown: "# Hello", video: "https://storage.googleapis.com/firecrawl/video.mp4", metadata: { title: "Example", sourceURL: "https://example.com" } }),
headers: { "Content-Type" => "application/json" }
)
doc = @client.scrape("https://example.com")
assert_instance_of Firecrawl::Models::Document, doc
assert_equal "# Hello", doc.markdown
+ assert_equal "https://storage.googleapis.com/firecrawl/video.mp4", doc.video
assert_equal "Example", doc.metadata["title"]
end
@@ -677,6 +678,12 @@ class ClientTest < Minitest::Test
end
end
+ def test_parse_options_rejects_video_format
+ assert_raises(ArgumentError) do
+ Firecrawl::Models::ParseOptions.new(formats: ["video"])
+ end
+ end
+
def test_parse_options_rejects_invalid_proxy
assert_raises(ArgumentError) do
Firecrawl::Models::ParseOptions.new(proxy: "stealth")
diff --git a/apps/rust-sdk/CHANGELOG.md b/apps/rust-sdk/CHANGELOG.md
index 8342b9fa8..e30e1f68d 100644
--- a/apps/rust-sdk/CHANGELOG.md
+++ b/apps/rust-sdk/CHANGELOG.md
@@ -1,5 +1,11 @@
## CHANGELOG
+## [2.5.0] - 2026-05-12
+
+### Added
+
+- Added `Format::Video` and `Document.video` support for video extraction results.
+
## [0.1]
### Added
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 48cda122f..2c3cb7cc3 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.4.1"
+version = "2.5.0"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 8e6bb84c0..2c48e2edd 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.4.1"
+version = "2.5.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/README.md b/apps/rust-sdk/README.md
index 36c6d1ff5..171b82df1 100644
--- a/apps/rust-sdk/README.md
+++ b/apps/rust-sdk/README.md
@@ -7,7 +7,7 @@ To install the Firecrawl Rust SDK, add the following to your `Cargo.toml`:
```toml
[dependencies]
-firecrawl = "2.1.0"
+firecrawl = "2.5.0"
tokio = { version = "^1", features = ["full"] }
```
@@ -40,6 +40,24 @@ match scrape_result {
}
```
+### Video extraction
+
+Use `Format::Video` on supported video URLs, including YouTube and TikTok. The returned `video` field is a signed URL to the extracted video file.
+
+```rust
+use firecrawl::{Format, ScrapeOptions};
+
+let options = ScrapeOptions {
+ formats: Some(vec![Format::Video]),
+ ..Default::default()
+};
+
+let doc = client
+ .scrape("https://www.youtube.com/watch?v=dQw4w9WgXcQ", options)
+ .await?;
+println!("{:?}", doc.video);
+```
+
### Parsing uploaded files (v2)
Use the v2 client `parse` method to upload local files (`html`, `pdf`, `docx`, etc.) as multipart form data.
diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rs
index 5cb1cdea0..4e8d03cde 100644
--- a/apps/rust-sdk/src/types.rs
+++ b/apps/rust-sdk/src/types.rs
@@ -33,6 +33,8 @@ pub enum Format {
Branding,
/// Audio extraction (MP3) from YouTube videos.
Audio,
+ /// Video extraction from supported video URLs.
+ Video,
/// Question answer generated from the page content.
Question(QuestionFormat),
/// Direct highlights selected from the page content.
@@ -59,6 +61,7 @@ impl Serialize for Format {
Format::Attributes => serializer.serialize_str("attributes"),
Format::Branding => serializer.serialize_str("branding"),
Format::Audio => serializer.serialize_str("audio"),
+ Format::Video => serializer.serialize_str("video"),
Format::Question(question) => question.serialize(serializer),
Format::Highlights(highlights) => highlights.serialize(serializer),
Format::Query(query) => query.serialize(serializer),
@@ -86,6 +89,7 @@ impl<'de> Deserialize<'de> for Format {
"attributes" => Ok(Format::Attributes),
"branding" => Ok(Format::Branding),
"audio" => Ok(Format::Audio),
+ "video" => Ok(Format::Video),
_ => Err(de::Error::custom(format!("unknown format: {}", format))),
},
Value::Object(_) => match value.get("type").and_then(Value::as_str) {
@@ -658,6 +662,8 @@ pub struct Document {
pub screenshot: Option<String>,
/// Audio download URL (signed GCS link for MP3).
pub audio: Option<String>,
+ /// Video download URL (signed GCS link).
+ pub video: Option<String>,
/// Extracted attributes.
pub attributes: Option<Vec<AttributeResult>>,
/// Action results.
@@ -793,6 +799,7 @@ mod tests {
fn test_full_document_with_array_metadata() {
let json = json!({
"markdown": "# Hello",
+ "video": "https://storage.googleapis.com/firecrawl/video.mp4",
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200,
@@ -806,6 +813,10 @@ mod tests {
});
let doc: Document = serde_json::from_value(json).unwrap();
assert_eq!(doc.markdown, Some("# Hello".to_string()));
+ assert_eq!(
+ doc.video,
+ Some("https://storage.googleapis.com/firecrawl/video.mp4".to_string())
+ );
let meta = doc.metadata.unwrap();
assert_eq!(meta.title, Some("Example Page".to_string()));
assert_eq!(