Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(rust-sdk): add v2 API namespace with agent support (#2778)

Abimael Martell43f61e7fe5c8
diff --git a/.github/workflows/test-rust-sdk.yml b/.github/workflows/test-rust-sdk.yml
new file mode 100644
index 000000000..ed85ac685
--- /dev/null
+++ b/.github/workflows/test-rust-sdk.yml
@@ -0,0 +1,51 @@
+name: Rust SDK Test Suite
+
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - apps/rust-sdk/**
+
+jobs:
+ test:
+ name: Build and Test
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ components: clippy
+
+ - name: Cache cargo registry
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ apps/rust-sdk/target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-
+
+ - name: Check formatting
+ working-directory: ./apps/rust-sdk
+ run: cargo fmt --check
+
+ - name: Build
+ working-directory: ./apps/rust-sdk
+ run: cargo build --all-targets
+
+ - name: Run clippy
+ working-directory: ./apps/rust-sdk
+ run: cargo clippy --all-targets -- -D warnings -A clippy::needless_borrows_for_generic_args -A clippy::redundant_closure -A clippy::large_enum_variant -A clippy::extra_unused_lifetimes
+
+ - name: Run unit tests (mocked)
+ working-directory: ./apps/rust-sdk
+ run: cargo test --lib
+
+ - name: Build examples
+ working-directory: ./apps/rust-sdk
+ run: cargo build --examples
diff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rs
index 51592f3f1..37a3df7d2 100644
--- a/apps/rust-sdk/examples/example.rs
+++ b/apps/rust-sdk/examples/example.rs
@@ -1,6 +1,6 @@
use firecrawl::{
crawl::CrawlOptions,
- scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions},
+ scrape::{JsonOptions, ScrapeFormats, ScrapeOptions},
FirecrawlApp,
};
use serde_json::json;
@@ -62,8 +62,8 @@ async fn main() {
});
let llm_extraction_options = ScrapeOptions {
- formats: vec![ScrapeFormats::Extract].into(),
- extract: ExtractOptions {
+ formats: vec![ScrapeFormats::Json].into(),
+ json_options: JsonOptions {
schema: json_schema.into(),
..Default::default()
}
diff --git a/apps/rust-sdk/examples/v2_example.rs b/apps/rust-sdk/examples/v2_example.rs
new file mode 100644
index 000000000..a5401c928
--- /dev/null
+++ b/apps/rust-sdk/examples/v2_example.rs
@@ -0,0 +1,216 @@
+//! Example usage of Firecrawl v2 API.
+//!
+//! Run with: cargo run --example v2_example
+
+use firecrawl::v2::{
+ AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,
+ ScrapeOptions, SearchOptions, SitemapMode,
+};
+use serde::Deserialize;
+use serde_json::json;
+
+#[tokio::main]
+async fn main() {
+ // Initialize the v2 Client with the API key
+ let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");
+
+ // Or, connect to a self-hosted instance:
+ // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)
+ // .expect("Failed to initialize Client");
+
+ // Example 1: Simple Scrape
+ println!("=== Example 1: Simple Scrape ===");
+ let result = client.scrape("https://example.com", None).await;
+ match result {
+ Ok(doc) => println!("Markdown: {:?}", doc.markdown),
+ Err(e) => eprintln!("Scrape failed: {}", e),
+ }
+
+ // Example 2: Scrape with Options
+ println!("\n=== Example 2: Scrape with Options ===");
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ only_main_content: Some(true),
+ ..Default::default()
+ };
+ let result = client.scrape("https://example.com", options).await;
+ match result {
+ Ok(doc) => {
+ println!("Links: {:?}", doc.links);
+ }
+ Err(e) => eprintln!("Scrape with options failed: {}", e),
+ }
+
+ // Example 3: Scrape with JSON Schema Extraction
+ println!("\n=== Example 3: JSON Schema Extraction ===");
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "description": { "type": "string" }
+ }
+ });
+ let result = client
+ .scrape_with_schema(
+ "https://example.com",
+ schema,
+ Some("Extract title and description"),
+ )
+ .await;
+ match result {
+ Ok(data) => println!(
+ "Extracted: {}",
+ serde_json::to_string_pretty(&data).unwrap()
+ ),
+ Err(e) => eprintln!("Schema extraction failed: {}", e),
+ }
+
+ // Example 4: Search
+ println!("\n=== Example 4: Search ===");
+ let options = SearchOptions {
+ limit: Some(5),
+ ..Default::default()
+ };
+ let result = client.search("rust programming", options).await;
+ match result {
+ Ok(response) => {
+ if let Some(web) = response.data.web {
+ for item in web {
+ match item {
+ firecrawl::v2::SearchResultOrDocument::WebResult(r) => {
+ println!("Result: {} - {}", r.url, r.title.unwrap_or_default());
+ }
+ firecrawl::v2::SearchResultOrDocument::Document(d) => {
+ if let Some(meta) = d.metadata {
+ println!("Document: {:?}", meta.title);
+ }
+ }
+ }
+ }
+ }
+ }
+ Err(e) => eprintln!("Search failed: {}", e),
+ }
+
+ // Example 5: Map a Website
+ println!("\n=== Example 5: Map ===");
+ let options = MapOptions {
+ sitemap: Some(SitemapMode::Include),
+ limit: Some(20),
+ ..Default::default()
+ };
+ let result = client.map("https://example.com", options).await;
+ match result {
+ Ok(response) => {
+ println!("Found {} links", response.links.len());
+ for link in response.links.iter().take(5) {
+ println!(" - {}", link.url);
+ }
+ }
+ Err(e) => eprintln!("Map failed: {}", e),
+ }
+
+ // Example 6: Crawl a Website
+ println!("\n=== Example 6: Crawl ===");
+ let options = CrawlOptions {
+ limit: Some(5),
+ sitemap: Some(SitemapMode::Include),
+ poll_interval: Some(3000),
+ ..Default::default()
+ };
+ let result = client.crawl("https://example.com", options).await;
+ match result {
+ Ok(job) => {
+ println!("Crawled {} pages", job.data.len());
+ for doc in job.data.iter().take(3) {
+ if let Some(meta) = &doc.metadata {
+ println!(" - {:?}", meta.source_url);
+ }
+ }
+ }
+ Err(e) => eprintln!("Crawl failed: {}", e),
+ }
+
+ // Example 7: Batch Scrape
+ println!("\n=== Example 7: Batch Scrape ===");
+ let urls = vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ];
+ let options = BatchScrapeOptions {
+ options: Some(ScrapeOptions {
+ formats: Some(vec![Format::Markdown]),
+ ..Default::default()
+ }),
+ poll_interval: Some(2000),
+ ..Default::default()
+ };
+ let result = client.batch_scrape(urls, options).await;
+ match result {
+ Ok(job) => {
+ println!("Batch scraped {} pages", job.data.len());
+ }
+ Err(e) => eprintln!("Batch scrape failed: {}", e),
+ }
+
+ // Example 8: Agent (Autonomous Web Browsing)
+ println!("\n=== Example 8: Agent ===");
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Describe what this website is about and list any key features mentioned"
+ .to_string(),
+ model: Some(AgentModel::Spark1Pro),
+ timeout: Some(60),
+ ..Default::default()
+ };
+ let result = client.agent(options).await;
+ match result {
+ Ok(response) => {
+ println!("Agent status: {:?}", response.status);
+ if let Some(data) = response.data {
+ println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());
+ }
+ }
+ Err(e) => eprintln!("Agent failed: {}", e),
+ }
+
+ // Example 9: Agent with Typed Schema
+ println!("\n=== Example 9: Agent with Typed Schema ===");
+
+ #[derive(Debug, Deserialize)]
+ struct CompanyInfo {
+ name: String,
+ description: Option<String>,
+ industry: Option<String>,
+ }
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "description": { "type": "string" },
+ "industry": { "type": "string" }
+ },
+ "required": ["name"]
+ });
+
+ let result: Result<Option<CompanyInfo>, _> = client
+ .agent_with_schema(
+ vec!["https://firecrawl.dev".to_string()],
+ "Extract company information from this website",
+ schema,
+ )
+ .await;
+
+ match result {
+ Ok(Some(info)) => {
+ println!("Company: {}", info.name);
+ println!("Description: {:?}", info.description);
+ println!("Industry: {:?}", info.industry);
+ }
+ Ok(None) => println!("No data extracted"),
+ Err(e) => eprintln!("Agent with schema failed: {}", e),
+ }
+
+ println!("\n=== All examples completed ===");
+}
diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rs
index 6d27ff283..bd3b4273a 100644
--- a/apps/rust-sdk/src/crawl.rs
+++ b/apps/rust-sdk/src/crawl.rs
@@ -197,17 +197,6 @@ struct CrawlRequestBody {
options: CrawlOptions,
}
-#[derive(Deserialize, Serialize, Debug, Default)]
-#[serde(rename_all = "camelCase")]
-struct CrawlResponse {
- /// This will always be `true` due to `FirecrawlApp::handle_response`.
- /// No need to expose.
- success: bool,
-
- /// The resulting document.
- data: Document,
-}
-
#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]
#[serde(rename_all = "camelCase")]
pub enum CrawlStatusTypes {
diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs
index 920900a25..9f69ac226 100644
--- a/apps/rust-sdk/src/lib.rs
+++ b/apps/rust-sdk/src/lib.rs
@@ -1,17 +1,55 @@
+//! Firecrawl Rust SDK
+//!
+//! This SDK provides access to the Firecrawl API for web scraping and crawling.
+//!
+//! # Quick Start
+//!
+//! ## v1 API (existing)
+//!
+//! ```no_run
+//! use firecrawl::FirecrawlApp;
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let app = FirecrawlApp::new("your-api-key")?;
+//! let document = app.scrape_url("https://example.com", None).await?;
+//! println!("{:?}", document.markdown);
+//! Ok(())
+//! }
+//! ```
+//!
+//! ## v2 API (new)
+//!
+//! ```no_run
+//! use firecrawl::v2::Client;
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let client = Client::new("your-api-key")?;
+//! let document = client.scrape("https://example.com", None).await?;
+//! println!("{:?}", document.markdown);
+//! Ok(())
+//! }
+//! ```
+
use reqwest::{Client, Response};
use serde::de::DeserializeOwned;
use serde_json::Value;
+// v1 modules (existing API)
pub mod batch_scrape;
pub mod crawl;
pub mod document;
-mod error;
+pub mod error;
pub mod extract;
pub mod llmstxt;
pub mod map;
pub mod scrape;
pub mod search;
+// v2 module (new API)
+pub mod v2;
+
use error::FirecrawlAPIError;
pub use error::FirecrawlError;
diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rs
index 386cb098e..035785bf6 100644
--- a/apps/rust-sdk/src/map.rs
+++ b/apps/rust-sdk/src/map.rs
@@ -8,7 +8,7 @@ use crate::{FirecrawlApp, FirecrawlError, API_VERSION};
pub struct Location {
/// Country code (ISO 3166-1 alpha-2)
pub country: Option<String>,
-
+
/// List of language codes
pub languages: Option<Vec<String>>,
}
diff --git a/apps/rust-sdk/src/v2/agent.rs b/apps/rust-sdk/src/v2/agent.rs
new file mode 100644
index 000000000..47b4f734f
--- /dev/null
+++ b/apps/rust-sdk/src/v2/agent.rs
@@ -0,0 +1,674 @@
+//! Agent endpoint for Firecrawl API v2.
+//!
+//! The Agent endpoint provides autonomous web browsing capabilities using AI
+//! to accomplish complex tasks that may require multiple page interactions.
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use super::client::Client;
+use super::types::{AgentModel, AgentWebhookConfig};
+use crate::FirecrawlError;
+
+/// Options for running an agent task.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AgentOptions {
+ /// Starting URLs for the agent to explore.
+ pub urls: Option<Vec<String>>,
+
+ /// The prompt describing what the agent should accomplish.
+ pub prompt: String,
+
+ /// JSON schema for the expected output structure.
+ pub schema: Option<Value>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Maximum credits the agent can use.
+ pub max_credits: Option<u32>,
+
+ /// Strictly constrain the agent to the provided URLs.
+ pub strict_constrain_to_urls: Option<bool>,
+
+ /// Agent model to use.
+ pub model: Option<AgentModel>,
+
+ /// Webhook configuration for agent notifications.
+ pub webhook: Option<AgentWebhookConfig>,
+
+ /// Poll interval for synchronous agent execution (milliseconds).
+ #[serde(skip)]
+ pub poll_interval: Option<u64>,
+
+ /// Timeout for synchronous agent execution (seconds).
+ #[serde(skip)]
+ pub timeout: Option<u64>,
+}
+
+/// Response from starting an agent task.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AgentResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// The agent task ID.
+ pub id: String,
+ /// Error message if the request failed.
+ pub error: Option<String>,
+}
+
+/// Agent task status.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub enum AgentStatus {
+ /// The agent is still processing.
+ Processing,
+ /// The agent has completed its task.
+ Completed,
+ /// The agent task failed.
+ Failed,
+ /// The agent task was cancelled.
+ Cancelled,
+}
+
+/// Status response from an agent task.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AgentStatusResponse {
+ /// Whether the status check was successful.
+ pub success: bool,
+ /// Current status of the agent task.
+ pub status: AgentStatus,
+ /// Error message if the task failed.
+ pub error: Option<String>,
+ /// Extracted data (if schema was provided) or task results.
+ pub data: Option<Value>,
+ /// Model used for the agent task.
+ pub model: Option<AgentModel>,
+ /// Expiry time of the task data.
+ pub expires_at: Option<String>,
+ /// Credits used by the agent task.
+ pub credits_used: Option<u32>,
+}
+
+impl Client {
+ /// Starts an agent task asynchronously.
+ ///
+ /// Returns immediately with a task ID that can be used to check status.
+ ///
+ /// # Arguments
+ ///
+ /// * `options` - Agent task configuration including the prompt.
+ ///
+ /// # Returns
+ ///
+ /// An `AgentResponse` containing the task ID.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, AgentOptions};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let options = AgentOptions {
+ /// urls: Some(vec!["https://example.com".to_string()]),
+ /// prompt: "Find the pricing information on this website".to_string(),
+ /// ..Default::default()
+ /// };
+ ///
+ /// let response = client.start_agent(options).await?;
+ /// println!("Agent task started: {}", response.id);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn start_agent(
+ &self,
+ options: AgentOptions,
+ ) -> Result<AgentResponse, FirecrawlError> {
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(self.url("/agent"))
+ .headers(headers)
+ .json(&options)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Starting agent task".to_string(), e))?;
+
+ self.handle_response(response, "start agent").await
+ }
+
+ /// Gets the status of an agent task.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The agent task ID.
+ ///
+ /// # Returns
+ ///
+ /// An `AgentStatusResponse` containing the current status and any results.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let status = client.get_agent_status("task-id").await?;
+ /// println!("Status: {:?}", status.status);
+ ///
+ /// if let Some(data) = status.data {
+ /// println!("Result: {}", data);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_agent_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<AgentStatusResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/agent/{}", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Getting agent status {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, format!("agent status {}", id.as_ref()))
+ .await
+ }
+
+ /// Runs an agent task and waits for completion.
+ ///
+ /// This method starts an agent task and polls until it completes, fails, or times out.
+ ///
+ /// # Arguments
+ ///
+ /// * `options` - Agent task configuration including the prompt.
+ ///
+ /// # Returns
+ ///
+ /// An `AgentStatusResponse` containing the final status and results.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, AgentOptions, AgentModel};
+ /// use serde_json::json;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let options = AgentOptions {
+ /// urls: Some(vec!["https://example.com/pricing".to_string()]),
+ /// prompt: "Extract the pricing tiers and their features".to_string(),
+ /// schema: Some(json!({
+ /// "type": "object",
+ /// "properties": {
+ /// "tiers": {
+ /// "type": "array",
+ /// "items": {
+ /// "type": "object",
+ /// "properties": {
+ /// "name": { "type": "string" },
+ /// "price": { "type": "number" },
+ /// "features": { "type": "array", "items": { "type": "string" } }
+ /// }
+ /// }
+ /// }
+ /// }
+ /// })),
+ /// model: Some(AgentModel::Spark1Pro),
+ /// poll_interval: Some(3000),
+ /// timeout: Some(300),
+ /// ..Default::default()
+ /// };
+ ///
+ /// let result = client.agent(options).await?;
+ ///
+ /// if let Some(data) = result.data {
+ /// println!("Extracted pricing: {}", serde_json::to_string_pretty(&data)?);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn agent(
+ &self,
+ options: AgentOptions,
+ ) -> Result<AgentStatusResponse, FirecrawlError> {
+ let poll_interval = options.poll_interval.unwrap_or(2000);
+ let timeout = options.timeout;
+
+ let response = self.start_agent(options).await?;
+ self.wait_for_agent(&response.id, poll_interval, timeout)
+ .await
+ }
+
+ /// Waits for an agent task to complete.
+ async fn wait_for_agent(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ timeout: Option<u64>,
+ ) -> Result<AgentStatusResponse, FirecrawlError> {
+ let start = std::time::Instant::now();
+
+ loop {
+ let status = self.get_agent_status(id).await?;
+
+ match status.status {
+ AgentStatus::Completed | AgentStatus::Failed | AgentStatus::Cancelled => {
+ return Ok(status);
+ }
+ AgentStatus::Processing => {
+ // Check timeout
+ if let Some(timeout_secs) = timeout {
+ if start.elapsed().as_secs() > timeout_secs {
+ return Ok(status);
+ }
+ }
+
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ }
+ }
+ }
+
+ /// Cancels a running agent task.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The agent task ID to cancel.
+ ///
+ /// # Returns
+ ///
+ /// `true` if the cancellation was successful.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let cancelled = client.cancel_agent("task-id").await?;
+ /// println!("Cancelled: {}", cancelled);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn cancel_agent(&self, id: impl AsRef<str>) -> Result<bool, FirecrawlError> {
+ let response = self
+ .client
+ .delete(self.url(&format!("/agent/{}", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Cancelling agent {}", id.as_ref()), e)
+ })?;
+
+ #[derive(Deserialize)]
+ struct CancelResponse {
+ success: bool,
+ }
+
+ let result: CancelResponse = self
+ .handle_response(response, format!("cancel agent {}", id.as_ref()))
+ .await?;
+
+ Ok(result.success)
+ }
+
+ /// Runs an agent with a typed schema for structured output.
+ ///
+ /// This is a convenience method that automatically converts the result
+ /// to the specified type.
+ ///
+ /// # Arguments
+ ///
+ /// * `urls` - Starting URLs for the agent.
+ /// * `prompt` - The task description.
+ /// * `schema` - JSON schema for the expected output.
+ ///
+ /// # Returns
+ ///
+ /// The extracted data as the specified type, or `None` if extraction failed.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ /// use serde::Deserialize;
+ /// use serde_json::json;
+ ///
+ /// #[derive(Debug, Deserialize)]
+ /// struct ProductInfo {
+ /// name: String,
+ /// price: f64,
+ /// description: Option<String>,
+ /// }
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let schema = json!({
+ /// "type": "object",
+ /// "properties": {
+ /// "name": { "type": "string" },
+ /// "price": { "type": "number" },
+ /// "description": { "type": "string" }
+ /// },
+ /// "required": ["name", "price"]
+ /// });
+ ///
+ /// let result: Option<ProductInfo> = client.agent_with_schema(
+ /// vec!["https://example.com/product".to_string()],
+ /// "Extract the product information",
+ /// schema,
+ /// ).await?;
+ ///
+ /// if let Some(product) = result {
+ /// println!("Product: {} - ${}", product.name, product.price);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn agent_with_schema<T: serde::de::DeserializeOwned>(
+ &self,
+ urls: Vec<String>,
+ prompt: impl AsRef<str>,
+ schema: Value,
+ ) -> Result<Option<T>, FirecrawlError> {
+ let options = AgentOptions {
+ urls: Some(urls),
+ prompt: prompt.as_ref().to_string(),
+ schema: Some(schema),
+ ..Default::default()
+ };
+
+ let result = self.agent(options).await?;
+
+ if result.status != AgentStatus::Completed {
+ return Ok(None);
+ }
+
+ match result.data {
+ Some(data) => {
+ let typed: T =
+ serde_json::from_value(data).map_err(FirecrawlError::ResponseParseError)?;
+ Ok(Some(typed))
+ }
+ None => Ok(None),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_start_agent_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/agent")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "agent-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Find the contact information".to_string(),
+ ..Default::default()
+ };
+
+ let response = client.start_agent(options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "agent-123");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_get_agent_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v2/agent/agent-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "data": {
+ "email": "[email protected]",
+ "phone": "555-1234"
+ },
+ "creditsUsed": 5,
+ "expiresAt": "2024-12-31T23:59:59Z"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = client.get_agent_status("agent-123").await.unwrap();
+
+ assert!(status.success);
+ assert_eq!(status.status, AgentStatus::Completed);
+ assert!(status.data.is_some());
+ assert_eq!(status.credits_used, Some(5));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_agent_sync_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/agent")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "agent-456"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Mock the status endpoint (completed immediately)
+ let status_mock = server
+ .mock("GET", "/v2/agent/agent-456")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "data": {
+ "result": "Task completed successfully"
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Test task".to_string(),
+ ..Default::default()
+ };
+
+ let result = client.agent(options).await.unwrap();
+
+ assert_eq!(result.status, AgentStatus::Completed);
+ assert!(result.data.is_some());
+ start_mock.assert();
+ status_mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_cancel_agent_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("DELETE", "/v2/agent/agent-789")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let cancelled = client.cancel_agent("agent-789").await.unwrap();
+
+ assert!(cancelled);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_agent_with_schema() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/agent")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "agent-schema"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Mock the status endpoint
+ let status_mock = server
+ .mock("GET", "/v2/agent/agent-schema")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "status": "completed",
+ "data": {
+ "name": "Test Product",
+ "price": 29.99
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ #[derive(Debug, serde::Deserialize, PartialEq)]
+ struct Product {
+ name: String,
+ price: f64,
+ }
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "name": { "type": "string" },
+ "price": { "type": "number" }
+ }
+ });
+
+ let result: Option<Product> = client
+ .agent_with_schema(
+ vec!["https://example.com".to_string()],
+ "Extract product info",
+ schema,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ result,
+ Some(Product {
+ name: "Test Product".to_string(),
+ price: 29.99
+ })
+ );
+ start_mock.assert();
+ status_mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_agent_with_model_option() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/agent")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "agent-model"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Task with specific model".to_string(),
+ model: Some(AgentModel::Spark1Pro),
+ max_credits: Some(100),
+ ..Default::default()
+ };
+
+ let response = client.start_agent(options).await.unwrap();
+
+ assert!(response.success);
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/batch_scrape.rs b/apps/rust-sdk/src/v2/batch_scrape.rs
new file mode 100644
index 000000000..cb831e787
--- /dev/null
+++ b/apps/rust-sdk/src/v2/batch_scrape.rs
@@ -0,0 +1,591 @@
+//! Batch scrape endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+
+use super::client::Client;
+use super::scrape::ScrapeOptions;
+use super::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};
+use crate::FirecrawlError;
+
+/// Options for batch scraping.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeOptions {
+ /// Scrape options to apply to all URLs.
+ #[serde(flatten)]
+ pub options: Option<ScrapeOptions>,
+
+ /// Webhook configuration for job notifications.
+ pub webhook: Option<WebhookConfig>,
+
+ /// ID of an existing batch job to append URLs to.
+ pub append_to_id: Option<String>,
+
+ /// Whether to ignore invalid URLs instead of failing.
+ #[serde(rename = "ignoreInvalidURLs")]
+ pub ignore_invalid_urls: Option<bool>,
+
+ /// Maximum concurrent requests.
+ pub max_concurrency: Option<u32>,
+
+ /// Enable zero data retention mode.
+ pub zero_data_retention: Option<bool>,
+
+ /// Idempotency key for the request.
+ #[serde(skip)]
+ pub idempotency_key: Option<String>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Poll interval for synchronous batch scrape (milliseconds).
+ #[serde(skip)]
+ pub poll_interval: Option<u64>,
+}
+
+/// Request body for batch scrape endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct BatchScrapeRequest {
+ urls: Vec<String>,
+ #[serde(flatten)]
+ options: BatchScrapeOptions,
+}
+
+/// Response from starting a batch scrape job.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// The batch scrape job ID.
+ pub id: String,
+ /// URL to check the batch scrape status.
+ pub url: String,
+ /// URLs that were invalid and ignored.
+ #[serde(rename = "invalidURLs")]
+ pub invalid_urls: Option<Vec<String>>,
+}
+
+/// Status of a batch scrape job.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct BatchScrapeJob {
+ /// Current status of the batch scrape job.
+ pub status: JobStatus,
+ /// Number of URLs completed.
+ pub completed: u32,
+ /// Total number of URLs to scrape.
+ pub total: u32,
+ /// Credits used by the batch scrape.
+ pub credits_used: Option<u32>,
+ /// Expiry time of the batch data.
+ pub expires_at: Option<String>,
+ /// URL for the next page of results.
+ pub next: Option<String>,
+ /// Scraped documents.
+ pub data: Vec<Document>,
+}
+
+impl Client {
+ /// Starts a batch scrape job asynchronously.
+ ///
+ /// Returns immediately with a job ID that can be used to check status.
+ ///
+ /// # Arguments
+ ///
+ /// * `urls` - List of URLs to scrape.
+ /// * `options` - Optional batch scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeResponse` containing the job ID.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, BatchScrapeOptions};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = vec![
+ /// "https://example.com".to_string(),
+ /// "https://example.org".to_string(),
+ /// ];
+ ///
+ /// let response = client.start_batch_scrape(urls, None).await?;
+ /// println!("Batch job started: {}", response.id);
+ ///
+ /// // Check status later
+ /// let status = client.get_batch_scrape_status(&response.id).await?;
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn start_batch_scrape(
+ &self,
+ urls: Vec<String>,
+ options: impl Into<Option<BatchScrapeOptions>>,
+ ) -> Result<BatchScrapeResponse, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let body = BatchScrapeRequest {
+ urls,
+ options: options.clone(),
+ };
+
+ let headers = self.prepare_headers(options.idempotency_key.as_ref());
+
+ let response = self
+ .client
+ .post(self.url("/batch/scrape"))
+ .headers(headers)
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;
+
+ self.handle_response(response, "start batch scrape").await
+ }
+
+ /// Gets the status of a batch scrape job.
+ ///
+ /// If the job is completed, this will automatically fetch all pages of results.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The batch scrape job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeJob` containing the current status and any available documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let status = client.get_batch_scrape_status("job-id").await?;
+ /// println!("Status: {:?}", status.status);
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ /// println!("Documents: {}", status.data.len());
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_batch_scrape_status(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/batch/scrape/{}", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(
+ format!("Checking batch scrape status {}", id.as_ref()),
+ e,
+ )
+ })?;
+
+ let mut status: BatchScrapeJob = self
+ .handle_response(response, format!("batch scrape status {}", id.as_ref()))
+ .await?;
+
+ // Auto-paginate if completed
+ if status.status == JobStatus::Completed {
+ while let Some(next) = status.next.take() {
+ let next_status = self.get_batch_scrape_status_next(&next).await?;
+ status.data.extend(next_status.data);
+ status.next = next_status.next;
+ }
+ }
+
+ Ok(status)
+ }
+
+ /// Fetches the next page of batch scrape results.
+ async fn get_batch_scrape_status_next(
+ &self,
+ next: &str,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
+ let response = self
+ .client
+ .get(next)
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)
+ })?;
+
+ self.handle_response(response, "batch scrape pagination")
+ .await
+ }
+
+ /// Scrapes multiple URLs and waits for completion.
+ ///
+ /// This method starts a batch scrape and polls until it completes or fails.
+ ///
+ /// # Arguments
+ ///
+ /// * `urls` - List of URLs to scrape.
+ /// * `options` - Optional batch scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `BatchScrapeJob` containing all scraped documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, BatchScrapeOptions, ScrapeOptions, Format};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = vec![
+ /// "https://example.com/page1".to_string(),
+ /// "https://example.com/page2".to_string(),
+ /// "https://example.com/page3".to_string(),
+ /// ];
+ ///
+ /// let options = BatchScrapeOptions {
+ /// options: Some(ScrapeOptions {
+ /// formats: Some(vec![Format::Markdown, Format::Links]),
+ /// ..Default::default()
+ /// }),
+ /// ignore_invalid_urls: Some(true),
+ /// poll_interval: Some(3000),
+ /// ..Default::default()
+ /// };
+ ///
+ /// let result = client.batch_scrape(urls, options).await?;
+ /// println!("Scraped {} pages", result.data.len());
+ ///
+ /// for doc in result.data {
+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn batch_scrape(
+ &self,
+ urls: Vec<String>,
+ options: impl Into<Option<BatchScrapeOptions>>,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let poll_interval = options.poll_interval.unwrap_or(2000);
+
+ let response = self.start_batch_scrape(urls, options).await?;
+ self.wait_for_batch_scrape(&response.id, poll_interval)
+ .await
+ }
+
+ /// Waits for a batch scrape job to complete.
+ async fn wait_for_batch_scrape(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ ) -> Result<BatchScrapeJob, FirecrawlError> {
+ loop {
+ let status = self.get_batch_scrape_status(id).await?;
+
+ match status.status {
+ JobStatus::Completed => return Ok(status),
+ JobStatus::Scraping => {
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ JobStatus::Failed => {
+ return Err(FirecrawlError::CrawlJobFailed(
+ "Batch scrape job failed".to_string(),
+ convert_batch_job_to_crawl_status(status),
+ ));
+ }
+ JobStatus::Cancelled => {
+ return Err(FirecrawlError::CrawlJobFailed(
+ "Batch scrape job was cancelled".to_string(),
+ convert_batch_job_to_crawl_status(status),
+ ));
+ }
+ }
+ }
+ }
+
+ /// Gets errors from a batch scrape job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The batch scrape job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlErrorsResponse` containing error details.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let errors = client.get_batch_scrape_errors("job-id").await?;
+ /// for error in errors.errors {
+ /// println!("Error on {}: {}", error.url, error.error);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_batch_scrape_errors(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "batch scrape errors").await
+ }
+}
+
+/// Converts a BatchScrapeJob to CrawlStatus for error compatibility.
+fn convert_batch_job_to_crawl_status(job: BatchScrapeJob) -> crate::crawl::CrawlStatus {
+ crate::crawl::CrawlStatus {
+ status: match job.status {
+ JobStatus::Completed => crate::crawl::CrawlStatusTypes::Completed,
+ JobStatus::Scraping => crate::crawl::CrawlStatusTypes::Scraping,
+ JobStatus::Failed => crate::crawl::CrawlStatusTypes::Failed,
+ JobStatus::Cancelled => crate::crawl::CrawlStatusTypes::Cancelled,
+ },
+ total: job.total,
+ completed: job.completed,
+ credits_used: job.credits_used.unwrap_or(0),
+ expires_at: job.expires_at.unwrap_or_default(),
+ next: job.next,
+ data: job
+ .data
+ .into_iter()
+ .map(|d| super::crawl::convert_v2_document_to_v1_pub(d))
+ .collect(),
+ }
+}
+
+impl super::crawl::CrawlJob {
+ /// Helper function to convert v2 document to v1 format.
+ pub(crate) fn _convert_document(doc: Document) -> crate::document::Document {
+ super::crawl::convert_v2_document_to_v1_pub(doc)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_start_batch_scrape_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/batch/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-123",
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ];
+
+ let response = client.start_batch_scrape(urls, None).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "batch-123");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_get_batch_scrape_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v2/batch/scrape/batch-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "completed",
+ "total": 3,
+ "completed": 3,
+ "creditsUsed": 3,
+ "data": [
+ {
+ "markdown": "# Page 1",
+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
+ },
+ {
+ "markdown": "# Page 2",
+ "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = client.get_batch_scrape_status("batch-123").await.unwrap();
+
+ assert_eq!(status.status, JobStatus::Completed);
+ assert_eq!(status.total, 3);
+ assert_eq!(status.completed, 3);
+ assert_eq!(status.data.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_batch_scrape_with_invalid_urls() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/batch/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-456",
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",
+ "invalidURLs": ["not-a-url"]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];
+
+ let options = BatchScrapeOptions {
+ ignore_invalid_urls: Some(true),
+ ..Default::default()
+ };
+
+ let response = client.start_batch_scrape(urls, options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_batch_scrape_sync() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/batch/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "batch-789",
+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Mock the status endpoint (completed immediately)
+ let status_mock = server
+ .mock("GET", "/v2/batch/scrape/batch-789")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "completed",
+ "total": 2,
+ "completed": 2,
+ "data": [
+ {
+ "markdown": "# Content",
+ "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = vec!["https://example.com".to_string()];
+
+ let result = client.batch_scrape(urls, None).await.unwrap();
+
+ assert_eq!(result.status, JobStatus::Completed);
+ assert_eq!(result.data.len(), 1);
+ start_mock.assert();
+ status_mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_get_batch_scrape_errors() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v2/batch/scrape/batch-123/errors")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "errors": [
+ {
+ "id": "err-1",
+ "url": "https://example.com/broken",
+ "error": "Connection timeout"
+ }
+ ],
+ "robotsBlocked": []
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();
+
+ assert_eq!(errors.errors.len(), 1);
+ assert_eq!(errors.errors[0].error, "Connection timeout");
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/client.rs b/apps/rust-sdk/src/v2/client.rs
new file mode 100644
index 000000000..f3881d83d
--- /dev/null
+++ b/apps/rust-sdk/src/v2/client.rs
@@ -0,0 +1,273 @@
+//! Firecrawl API v2 client.
+
+use reqwest::Response;
+use serde::de::DeserializeOwned;
+use serde_json::Value;
+
+use crate::error::{FirecrawlAPIError, FirecrawlError};
+
+pub(crate) const API_VERSION: &str = "/v2";
+const CLOUD_API_URL: &str = "https://api.firecrawl.dev";
+
+/// Firecrawl API v2 client.
+///
+/// This client provides access to all v2 API endpoints including scrape, crawl,
+/// search, map, batch scrape, and agent operations.
+///
+/// # Example
+///
+/// ```no_run
+/// use firecrawl::v2::Client;
+///
+/// #[tokio::main]
+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+/// // Create a client for the Firecrawl cloud service
+/// let client = Client::new("your-api-key")?;
+///
+/// // Or create a client for a self-hosted instance
+/// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key"))?;
+///
+/// Ok(())
+/// }
+/// ```
+#[derive(Clone, Debug)]
+pub struct Client {
+ pub(crate) api_key: Option<String>,
+ pub(crate) api_url: String,
+ pub(crate) client: reqwest::Client,
+}
+
+impl Client {
+ /// Creates a new client for the Firecrawl cloud service.
+ ///
+ /// # Arguments
+ ///
+ /// * `api_key` - Your Firecrawl API key.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the API key is empty.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// let client = Client::new("your-api-key").unwrap();
+ /// ```
+ pub fn new(api_key: impl AsRef<str>) -> Result<Self, FirecrawlError> {
+ Client::new_selfhosted(CLOUD_API_URL, Some(api_key))
+ }
+
+ /// Creates a new client for a self-hosted Firecrawl instance.
+ ///
+ /// # Arguments
+ ///
+ /// * `api_url` - The base URL of your Firecrawl instance.
+ /// * `api_key` - Optional API key (required for cloud, optional for self-hosted).
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if using the cloud service without an API key.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// // Self-hosted without authentication
+ /// let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();
+ ///
+ /// // Self-hosted with authentication
+ /// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();
+ /// ```
+ pub fn new_selfhosted(
+ api_url: impl AsRef<str>,
+ api_key: Option<impl AsRef<str>>,
+ ) -> Result<Self, FirecrawlError> {
+ // Normalize URL by trimming trailing slashes for consistent comparison
+ let url = api_url.as_ref().trim_end_matches('/').to_string();
+ let api_key = api_key.map(|k| k.as_ref().to_string());
+
+ // Reject empty or missing API key for cloud service
+ if url == CLOUD_API_URL {
+ match &api_key {
+ None => {
+ return Err(FirecrawlError::APIError(
+ "Configuration".to_string(),
+ FirecrawlAPIError {
+ success: false,
+ error: "API key is required for cloud service".to_string(),
+ details: None,
+ },
+ ));
+ }
+ Some(key) if key.trim().is_empty() => {
+ return Err(FirecrawlError::APIError(
+ "Configuration".to_string(),
+ FirecrawlAPIError {
+ success: false,
+ error: "API key cannot be empty for cloud service".to_string(),
+ details: None,
+ },
+ ));
+ }
+ _ => {}
+ }
+ }
+
+ Ok(Client {
+ api_key,
+ api_url: url,
+ client: reqwest::Client::new(),
+ })
+ }
+
+ /// Prepares headers for API requests.
+ pub(crate) fn prepare_headers(
+ &self,
+ idempotency_key: Option<&String>,
+ ) -> reqwest::header::HeaderMap {
+ use reqwest::header::HeaderValue;
+
+ let mut headers = reqwest::header::HeaderMap::new();
+ // Static string is always valid ASCII
+ headers.insert("Content-Type", HeaderValue::from_static("application/json"));
+ if let Some(api_key) = self.api_key.as_ref() {
+ // API key is validated at client creation, so this should always succeed.
+ // Use if-let to gracefully handle edge cases without panicking.
+ if let Ok(value) = format!("Bearer {}", api_key).parse() {
+ headers.insert("Authorization", value);
+ }
+ }
+ if let Some(key) = idempotency_key {
+ // Gracefully skip invalid idempotency keys instead of panicking
+ if let Ok(value) = key.parse() {
+ headers.insert("x-idempotency-key", value);
+ }
+ }
+ headers
+ }
+
+ /// Handles API responses, parsing JSON and handling errors.
+ pub(crate) async fn handle_response<T: DeserializeOwned>(
+ &self,
+ response: Response,
+ action: impl AsRef<str>,
+ ) -> Result<T, FirecrawlError> {
+ let (is_success, status) = (response.status().is_success(), response.status());
+
+ let response = response
+ .text()
+ .await
+ .map_err(FirecrawlError::ResponseParseErrorText)
+ .and_then(|response_json| {
+ serde_json::from_str::<Value>(&response_json)
+ .map_err(FirecrawlError::ResponseParseError)
+ .inspect(|data| {
+ tracing::debug!("Response JSON: {:#?}", data);
+ })
+ })
+ .and_then(|response_value| {
+ // Check for success field, or allow responses without it for status checks
+ if action.as_ref().contains("status")
+ || action.as_ref().contains("cancel")
+ || response_value["success"].as_bool().unwrap_or(false)
+ || response_value.get("success").is_none()
+ {
+ serde_json::from_value::<T>(response_value)
+ .map_err(FirecrawlError::ResponseParseError)
+ } else {
+ Err(FirecrawlError::APIError(
+ action.as_ref().to_string(),
+ serde_json::from_value(response_value)
+ .map_err(FirecrawlError::ResponseParseError)?,
+ ))
+ }
+ });
+
+ match &response {
+ Ok(_) => response,
+ Err(FirecrawlError::ResponseParseError(_))
+ | Err(FirecrawlError::ResponseParseErrorText(_)) => {
+ if is_success {
+ response
+ } else {
+ Err(FirecrawlError::HttpRequestFailed(
+ action.as_ref().to_string(),
+ status.as_u16(),
+ status.as_str().to_string(),
+ ))
+ }
+ }
+ Err(_) => response,
+ }
+ }
+
+ /// Builds the full URL for an API endpoint.
+ pub(crate) fn url(&self, path: &str) -> String {
+ format!("{}{}{}", self.api_url, API_VERSION, path)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_new_client() {
+ let client = Client::new("test-api-key").unwrap();
+ assert_eq!(client.api_key, Some("test-api-key".to_string()));
+ assert_eq!(client.api_url, CLOUD_API_URL);
+ }
+
+ #[test]
+ fn test_new_client_requires_api_key_for_cloud() {
+ let result = Client::new_selfhosted(CLOUD_API_URL, None::<&str>);
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn test_new_client_rejects_empty_api_key_for_cloud() {
+ let result = Client::new_selfhosted(CLOUD_API_URL, Some(""));
+ assert!(result.is_err());
+
+ let result = Client::new_selfhosted(CLOUD_API_URL, Some(" "));
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn test_new_selfhosted_client() {
+ let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();
+ assert_eq!(client.api_key, Some("api-key".to_string()));
+ assert_eq!(client.api_url, "http://localhost:3000");
+ }
+
+ #[test]
+ fn test_selfhosted_without_api_key() {
+ let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();
+ assert_eq!(client.api_key, None);
+ assert_eq!(client.api_url, "http://localhost:3000");
+ }
+
+ #[test]
+ fn test_url_builder() {
+ let client = Client::new("test-key").unwrap();
+ assert_eq!(client.url("/scrape"), "https://api.firecrawl.dev/v2/scrape");
+ }
+
+ #[test]
+ fn test_url_normalization_trailing_slash() {
+ // Cloud URL with trailing slash should still require API key
+ let result = Client::new_selfhosted("https://api.firecrawl.dev/", None::<&str>);
+ assert!(result.is_err());
+
+ // Should work with API key
+ let client = Client::new_selfhosted("https://api.firecrawl.dev/", Some("key")).unwrap();
+ assert_eq!(client.api_url, "https://api.firecrawl.dev");
+
+ // Self-hosted URL normalization
+ let client = Client::new_selfhosted("http://localhost:3000/", None::<&str>).unwrap();
+ assert_eq!(client.api_url, "http://localhost:3000");
+ }
+}
diff --git a/apps/rust-sdk/src/v2/crawl.rs b/apps/rust-sdk/src/v2/crawl.rs
new file mode 100644
index 000000000..4b773abe3
--- /dev/null
+++ b/apps/rust-sdk/src/v2/crawl.rs
@@ -0,0 +1,688 @@
+//! Crawl endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+
+use super::client::Client;
+use super::scrape::ScrapeOptions;
+use super::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};
+use crate::FirecrawlError;
+
+/// Options for crawling a website.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlOptions {
+ /// Natural language prompt to guide crawl behavior.
+ pub prompt: Option<String>,
+
+ /// URL path patterns to exclude from crawling.
+ pub exclude_paths: Option<Vec<String>>,
+
+ /// URL path patterns to include in crawling.
+ pub include_paths: Option<Vec<String>>,
+
+ /// Maximum depth of links to follow from the initial URL.
+ pub max_discovery_depth: Option<u32>,
+
+ /// How to handle the sitemap.
+ pub sitemap: Option<SitemapMode>,
+
+ /// Ignore query parameters when deduplicating URLs.
+ pub ignore_query_parameters: Option<bool>,
+
+ /// Maximum number of pages to crawl.
+ pub limit: Option<u32>,
+
+ /// Crawl the entire domain regardless of path structure.
+ pub crawl_entire_domain: Option<bool>,
+
+ /// Allow following links to external domains.
+ pub allow_external_links: Option<bool>,
+
+ /// Allow following links to subdomains.
+ pub allow_subdomains: Option<bool>,
+
+ /// Delay between requests in seconds.
+ pub delay: Option<u32>,
+
+ /// Maximum concurrent requests.
+ pub max_concurrency: Option<u32>,
+
+ /// Webhook configuration for job notifications.
+ pub webhook: Option<WebhookConfig>,
+
+ /// Scrape options to apply to each page.
+ pub scrape_options: Option<ScrapeOptions>,
+
+ /// Enable zero data retention mode.
+ pub zero_data_retention: Option<bool>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Idempotency key for the request.
+ #[serde(skip)]
+ pub idempotency_key: Option<String>,
+
+ /// Poll interval for synchronous crawl (milliseconds).
+ #[serde(skip)]
+ pub poll_interval: Option<u64>,
+}
+
+/// Request body for crawl endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct CrawlRequest {
+ url: String,
+ #[serde(flatten)]
+ options: CrawlOptions,
+}
+
+/// Response from starting a crawl job.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// The crawl job ID.
+ pub id: String,
+ /// URL to check the crawl status.
+ pub url: String,
+}
+
+/// Status of a crawl job.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlJob {
+ /// Current status of the crawl job.
+ pub status: JobStatus,
+ /// Total number of pages to crawl.
+ pub total: u32,
+ /// Number of pages completed.
+ pub completed: u32,
+ /// Credits used by the crawl.
+ pub credits_used: Option<u32>,
+ /// Expiry time of the crawl data.
+ pub expires_at: Option<String>,
+ /// URL for the next page of results.
+ pub next: Option<String>,
+ /// Crawled documents.
+ pub data: Vec<Document>,
+}
+
+/// Response from canceling a crawl.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CancelCrawlResponse {
+ /// Status of the cancellation.
+ pub status: String,
+}
+
+impl Client {
+ /// Starts a crawl job asynchronously.
+ ///
+ /// Returns immediately with a job ID that can be used to check status.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to start crawling from.
+ /// * `options` - Optional crawl configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlResponse` containing the job ID.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, CrawlOptions};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let response = client.start_crawl("https://example.com", None).await?;
+ /// println!("Crawl job started: {}", response.id);
+ ///
+ /// // Check status later
+ /// let status = client.get_crawl_status(&response.id).await?;
+ /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn start_crawl(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<CrawlOptions>>,
+ ) -> Result<CrawlResponse, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let body = CrawlRequest {
+ url: url.as_ref().to_string(),
+ options: options.clone(),
+ };
+
+ let headers = self.prepare_headers(options.idempotency_key.as_ref());
+
+ let response = self
+ .client
+ .post(self.url("/crawl"))
+ .headers(headers)
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "start crawl").await
+ }
+
+ /// Gets the status of a crawl job.
+ ///
+ /// If the job is completed, this will automatically fetch all pages of results.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlJob` containing the current status and any available documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let status = client.get_crawl_status("job-id").await?;
+ /// println!("Status: {:?}", status.status);
+ /// println!("Completed: {}/{}", status.completed, status.total);
+ /// println!("Documents: {}", status.data.len());
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/crawl/{}", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)
+ })?;
+
+ let mut status: CrawlJob = self
+ .handle_response(response, format!("crawl status {}", id.as_ref()))
+ .await?;
+
+ // Auto-paginate if completed
+ if status.status == JobStatus::Completed {
+ while let Some(next) = status.next.take() {
+ let next_status = self.get_crawl_status_next(&next).await?;
+ status.data.extend(next_status.data);
+ status.next = next_status.next;
+ }
+ }
+
+ Ok(status)
+ }
+
+ /// Fetches the next page of crawl results.
+ async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {
+ let response = self
+ .client
+ .get(next)
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;
+
+ self.handle_response(response, "crawl pagination").await
+ }
+
+ /// Crawls a website and waits for completion.
+ ///
+ /// This method starts a crawl and polls until it completes or fails.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to start crawling from.
+ /// * `options` - Optional crawl configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlJob` containing all crawled documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, CrawlOptions, SitemapMode};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let options = CrawlOptions {
+ /// sitemap: Some(SitemapMode::Include),
+ /// limit: Some(100),
+ /// poll_interval: Some(5000), // Check every 5 seconds
+ /// ..Default::default()
+ /// };
+ ///
+ /// let result = client.crawl("https://example.com", options).await?;
+ /// println!("Crawled {} pages", result.data.len());
+ ///
+ /// for doc in result.data {
+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn crawl(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<CrawlOptions>>,
+ ) -> Result<CrawlJob, FirecrawlError> {
+ let options = options.into().unwrap_or_default();
+ let poll_interval = options.poll_interval.unwrap_or(2000);
+
+ let response = self.start_crawl(url, options).await?;
+ self.wait_for_crawl(&response.id, poll_interval).await
+ }
+
+ /// Waits for a crawl job to complete.
+ async fn wait_for_crawl(
+ &self,
+ id: &str,
+ poll_interval: u64,
+ ) -> Result<CrawlJob, FirecrawlError> {
+ loop {
+ let status = self.get_crawl_status(id).await?;
+
+ match status.status {
+ JobStatus::Completed => return Ok(status),
+ JobStatus::Scraping => {
+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;
+ }
+ JobStatus::Failed => {
+ return Err(FirecrawlError::CrawlJobFailed(
+ "Crawl job failed".to_string(),
+ crate::crawl::CrawlStatus {
+ status: crate::crawl::CrawlStatusTypes::Failed,
+ total: status.total,
+ completed: status.completed,
+ credits_used: status.credits_used.unwrap_or(0),
+ expires_at: status.expires_at.unwrap_or_default(),
+ next: status.next,
+ data: status
+ .data
+ .into_iter()
+ .map(|d| convert_v2_document_to_v1(d))
+ .collect(),
+ },
+ ));
+ }
+ JobStatus::Cancelled => {
+ return Err(FirecrawlError::CrawlJobFailed(
+ "Crawl job was cancelled".to_string(),
+ crate::crawl::CrawlStatus {
+ status: crate::crawl::CrawlStatusTypes::Cancelled,
+ total: status.total,
+ completed: status.completed,
+ credits_used: status.credits_used.unwrap_or(0),
+ expires_at: status.expires_at.unwrap_or_default(),
+ next: status.next,
+ data: status
+ .data
+ .into_iter()
+ .map(|d| convert_v2_document_to_v1(d))
+ .collect(),
+ },
+ ));
+ }
+ }
+ }
+ }
+
+ /// Cancels a running crawl job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID to cancel.
+ ///
+ /// # Returns
+ ///
+ /// A `CancelCrawlResponse` indicating the cancellation status.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let response = client.cancel_crawl("job-id").await?;
+ /// println!("Cancellation status: {}", response.status);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn cancel_crawl(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CancelCrawlResponse, FirecrawlError> {
+ let response = self
+ .client
+ .delete(self.url(&format!("/crawl/{}", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "cancel crawl").await
+ }
+
+ /// Gets errors from a crawl job.
+ ///
+ /// # Arguments
+ ///
+ /// * `id` - The crawl job ID.
+ ///
+ /// # Returns
+ ///
+ /// A `CrawlErrorsResponse` containing error details.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let errors = client.get_crawl_errors("job-id").await?;
+ /// for error in errors.errors {
+ /// println!("Error on {}: {}", error.url, error.error);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn get_crawl_errors(
+ &self,
+ id: impl AsRef<str>,
+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/crawl/{}/errors", id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "crawl errors").await
+ }
+}
+
+/// Converts a v2 Document to a v1 Document for error compatibility.
+pub(crate) fn convert_v2_document_to_v1_pub(doc: Document) -> crate::document::Document {
+ convert_v2_document_to_v1(doc)
+}
+
+/// Converts a v2 Document to a v1 Document for error compatibility.
+fn convert_v2_document_to_v1(doc: Document) -> crate::document::Document {
+ let metadata = doc.metadata.unwrap_or_default();
+ crate::document::Document {
+ markdown: doc.markdown,
+ html: doc.html,
+ raw_html: doc.raw_html,
+ screenshot: doc.screenshot,
+ links: doc.links,
+ extract: doc.json,
+ metadata: crate::document::DocumentMetadata {
+ source_url: metadata.source_url.unwrap_or_default(),
+ status_code: metadata.status_code.unwrap_or(0),
+ error: metadata.error,
+ title: metadata.title,
+ description: metadata.description,
+ language: metadata.language,
+ keywords: metadata.keywords,
+ robots: metadata.robots,
+ og_title: metadata.og_title,
+ og_description: metadata.og_description,
+ og_url: metadata.og_url,
+ og_image: metadata.og_image,
+ og_audio: metadata.og_audio,
+ og_determiner: metadata.og_determiner,
+ og_locale: metadata.og_locale,
+ og_locale_alternate: metadata.og_locale_alternate,
+ og_site_name: metadata.og_site_name,
+ og_video: metadata.og_video,
+ article_section: metadata.article_section,
+ article_tag: metadata.article_tag,
+ published_time: metadata.published_time,
+ modified_time: metadata.modified_time,
+ dcterms_keywords: metadata.dcterms_keywords,
+ dc_description: metadata.dc_description,
+ dc_subject: metadata.dc_subject,
+ dcterms_subject: metadata.dcterms_subject,
+ dcterms_audience: metadata.dcterms_audience,
+ dc_type: metadata.dc_type,
+ dcterms_type: metadata.dcterms_type,
+ dc_date: metadata.dc_date,
+ dc_date_created: metadata.dc_date_created,
+ dcterms_created: metadata.dcterms_created,
+ },
+ warning: doc.warning,
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_start_crawl_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/crawl")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "crawl-123",
+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client
+ .start_crawl("https://example.com", None)
+ .await
+ .unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.id, "crawl-123");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_get_crawl_status_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v2/crawl/crawl-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "completed",
+ "total": 5,
+ "completed": 5,
+ "creditsUsed": 5,
+ "expiresAt": "2024-12-31T23:59:59Z",
+ "data": [
+ {
+ "markdown": "# Page 1",
+ "metadata": {
+ "sourceURL": "https://example.com/page1",
+ "statusCode": 200
+ }
+ },
+ {
+ "markdown": "# Page 2",
+ "metadata": {
+ "sourceURL": "https://example.com/page2",
+ "statusCode": 200
+ }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let status = client.get_crawl_status("crawl-123").await.unwrap();
+
+ assert_eq!(status.status, JobStatus::Completed);
+ assert_eq!(status.total, 5);
+ assert_eq!(status.completed, 5);
+ assert_eq!(status.data.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_cancel_crawl_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("DELETE", "/v2/crawl/crawl-123")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "cancelled"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.cancel_crawl("crawl-123").await.unwrap();
+
+ assert_eq!(response.status, "cancelled");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_get_crawl_errors_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("GET", "/v2/crawl/crawl-123/errors")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "errors": [
+ {
+ "id": "error-1",
+ "timestamp": "2024-01-01T00:00:00Z",
+ "url": "https://example.com/broken",
+ "error": "404 Not Found"
+ }
+ ],
+ "robotsBlocked": [
+ "https://example.com/admin"
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let errors = client.get_crawl_errors("crawl-123").await.unwrap();
+
+ assert_eq!(errors.errors.len(), 1);
+ assert_eq!(errors.errors[0].url, "https://example.com/broken");
+ assert_eq!(errors.robots_blocked.len(), 1);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_crawl_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ // Mock the start endpoint
+ let start_mock = server
+ .mock("POST", "/v2/crawl")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "id": "crawl-456",
+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"
+ })
+ .to_string(),
+ )
+ .create();
+
+ // Mock the status endpoint (completed immediately)
+ let status_mock = server
+ .mock("GET", "/v2/crawl/crawl-456")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "status": "completed",
+ "total": 2,
+ "completed": 2,
+ "data": [
+ {
+ "markdown": "# Page 1",
+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = CrawlOptions {
+ limit: Some(10),
+ sitemap: Some(SitemapMode::Include),
+ ..Default::default()
+ };
+
+ let result = client.crawl("https://example.com", options).await.unwrap();
+
+ assert_eq!(result.status, JobStatus::Completed);
+ assert_eq!(result.data.len(), 1);
+ start_mock.assert();
+ status_mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/map.rs b/apps/rust-sdk/src/v2/map.rs
new file mode 100644
index 000000000..ce2dc54ca
--- /dev/null
+++ b/apps/rust-sdk/src/v2/map.rs
@@ -0,0 +1,333 @@
+//! Map endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+
+use super::client::Client;
+use super::types::{LocationConfig, SearchResultWeb, SitemapMode};
+use crate::FirecrawlError;
+
+/// Options for mapping a URL.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MapOptions {
+ /// Search query to filter discovered links.
+ pub search: Option<String>,
+
+ /// How to handle the sitemap.
+ pub sitemap: Option<SitemapMode>,
+
+ /// Include subdomains in the mapping.
+ pub include_subdomains: Option<bool>,
+
+ /// Ignore query parameters when deduplicating URLs.
+ pub ignore_query_parameters: Option<bool>,
+
+ /// Maximum number of links to return.
+ pub limit: Option<u32>,
+
+ /// Timeout in milliseconds.
+ pub timeout: Option<u32>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// Location configuration for proxy routing.
+ pub location: Option<LocationConfig>,
+}
+
+/// Request body for map endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct MapRequest {
+ url: String,
+ #[serde(flatten)]
+ options: MapOptions,
+}
+
+/// Response from map endpoint.
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct MapResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// Discovered links with metadata.
+ pub links: Vec<SearchResultWeb>,
+ /// Warning message if any.
+ pub warning: Option<String>,
+}
+
+impl Client {
+ /// Maps a URL to discover all associated links.
+ ///
+ /// This endpoint discovers links from a website's sitemap, page content,
+ /// and other sources without fully scraping each page.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to map.
+ /// * `options` - Optional mapping configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `MapResponse` containing the discovered links.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, MapOptions, SitemapMode};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple map
+ /// let response = client.map("https://example.com", None).await?;
+ /// println!("Found {} links", response.links.len());
+ ///
+ /// // Map with options
+ /// let options = MapOptions {
+ /// sitemap: Some(SitemapMode::Include),
+ /// include_subdomains: Some(true),
+ /// limit: Some(1000),
+ /// ..Default::default()
+ /// };
+ /// let response = client.map("https://example.com", options).await?;
+ ///
+ /// for link in response.links {
+ /// println!("URL: {}, Title: {:?}", link.url, link.title);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn map(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<MapOptions>>,
+ ) -> Result<MapResponse, FirecrawlError> {
+ let body = MapRequest {
+ url: url.as_ref().to_string(),
+ options: options.into().unwrap_or_default(),
+ };
+
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(self.url("/map"))
+ .headers(headers)
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;
+
+ self.handle_response(response, "map").await
+ }
+
+ /// Maps a URL and returns just the list of URLs.
+ ///
+ /// This is a convenience method that returns only the URL strings.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to map.
+ /// * `options` - Optional mapping configuration.
+ ///
+ /// # Returns
+ ///
+ /// A vector of discovered URL strings.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let urls = client.map_urls("https://example.com", None).await?;
+ /// for url in urls {
+ /// println!("{}", url);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn map_urls(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<MapOptions>>,
+ ) -> Result<Vec<String>, FirecrawlError> {
+ let response = self.map(url, options).await?;
+ Ok(response.links.into_iter().map(|link| link.url).collect())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_map_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ {
+ "url": "https://example.com/",
+ "title": "Example Domain",
+ "description": "Home page"
+ },
+ {
+ "url": "https://example.com/about",
+ "title": "About Us"
+ },
+ {
+ "url": "https://example.com/contact"
+ }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.map("https://example.com", None).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 3);
+ assert_eq!(response.links[0].url, "https://example.com/");
+ assert_eq!(response.links[0].title, Some("Example Domain".to_string()));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/page1" },
+ { "url": "https://example.com/page2" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = MapOptions {
+ sitemap: Some(SitemapMode::Include),
+ include_subdomains: Some(true),
+ limit: Some(100),
+ ..Default::default()
+ };
+
+ let response = client.map("https://example.com", options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_urls() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/page1" },
+ { "url": "https://example.com/page2" },
+ { "url": "https://example.com/page3" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let urls = client.map_urls("https://example.com", None).await.unwrap();
+
+ assert_eq!(urls.len(), 3);
+ assert_eq!(urls[0], "https://example.com/page1");
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_with_search_filter() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "links": [
+ { "url": "https://example.com/blog/post1" },
+ { "url": "https://example.com/blog/post2" }
+ ]
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = MapOptions {
+ search: Some("blog".to_string()),
+ ..Default::default()
+ };
+
+ let response = client.map("https://example.com", options).await.unwrap();
+
+ assert!(response.success);
+ assert_eq!(response.links.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_map_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/map")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid URL"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.map("invalid-url", None).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/mod.rs b/apps/rust-sdk/src/v2/mod.rs
new file mode 100644
index 000000000..caa1a4fcb
--- /dev/null
+++ b/apps/rust-sdk/src/v2/mod.rs
@@ -0,0 +1,40 @@
+//! Firecrawl API v2 client module.
+//!
+//! This module provides access to the v2 API endpoints while maintaining
+//! backward compatibility with v1. The v2 API includes new features like
+//! the Agent endpoint and improved options.
+//!
+//! # Example
+//!
+//! ```no_run
+//! use firecrawl::v2::{Client, ScrapeOptions};
+//!
+//! #[tokio::main]
+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let client = Client::new("your-api-key")?;
+//!
+//! // Scrape a URL
+//! let document = client.scrape("https://example.com", None).await?;
+//! println!("Content: {:?}", document.markdown);
+//!
+//! Ok(())
+//! }
+//! ```
+
+mod agent;
+mod batch_scrape;
+mod client;
+mod crawl;
+mod map;
+mod scrape;
+mod search;
+mod types;
+
+pub use agent::*;
+pub use batch_scrape::*;
+pub use client::Client;
+pub use crawl::*;
+pub use map::*;
+pub use scrape::*;
+pub use search::*;
+pub use types::*;
diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rs
new file mode 100644
index 000000000..83c934b04
--- /dev/null
+++ b/apps/rust-sdk/src/v2/scrape.rs
@@ -0,0 +1,404 @@
+//! Scrape endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+
+use super::client::Client;
+use super::types::{
+ Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,
+ LocationConfig, ProxyType, ScreenshotOptions,
+};
+use crate::FirecrawlError;
+
+/// Options for scraping a URL.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScrapeOptions {
+ /// Output formats to include in the response.
+ pub formats: Option<Vec<Format>>,
+
+ /// Additional HTTP headers to send with the request.
+ pub headers: Option<HashMap<String, String>>,
+
+ /// HTML tags to exclusively include in the output.
+ pub include_tags: Option<Vec<String>>,
+
+ /// HTML tags to exclude from the output.
+ pub exclude_tags: Option<Vec<String>>,
+
+ /// Only extract the main content of the page.
+ pub only_main_content: Option<bool>,
+
+ /// Timeout in milliseconds before returning an error.
+ pub timeout: Option<u32>,
+
+ /// Time to wait after page load before scraping (milliseconds).
+ pub wait_for: Option<u32>,
+
+ /// Emulate a mobile device.
+ pub mobile: Option<bool>,
+
+ /// Parser configurations (e.g., for PDFs).
+ pub parsers: Option<Vec<ParserConfig>>,
+
+ /// Browser automation actions to perform before scraping.
+ pub actions: Option<Vec<Action>>,
+
+ /// Location configuration for proxy routing.
+ pub location: Option<LocationConfig>,
+
+ /// Skip TLS certificate verification.
+ pub skip_tls_verification: Option<bool>,
+
+ /// Remove base64-encoded images from the output.
+ pub remove_base64_images: Option<bool>,
+
+ /// Enable fast mode for quicker scrapes with reduced accuracy.
+ pub fast_mode: Option<bool>,
+
+ /// Block advertisements on the page.
+ pub block_ads: Option<bool>,
+
+ /// Proxy type to use.
+ pub proxy: Option<ProxyType>,
+
+ /// Maximum age of cached content to accept (seconds).
+ pub max_age: Option<u32>,
+
+ /// Minimum age of cached content to accept (seconds).
+ pub min_age: Option<u32>,
+
+ /// Store the result in cache for future requests.
+ pub store_in_cache: Option<bool>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+
+ /// JSON extraction options.
+ pub json_options: Option<JsonOptions>,
+
+ /// Screenshot options.
+ pub screenshot_options: Option<ScreenshotOptions>,
+
+ /// Change tracking options.
+ pub change_tracking_options: Option<ChangeTrackingOptions>,
+
+ /// Attribute selectors for extraction.
+ pub attribute_selectors: Option<Vec<AttributeSelector>>,
+}
+
+/// Parser configuration for document parsing.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum ParserConfig {
+ /// Simple parser type string.
+ Simple(String),
+ /// PDF parser with options.
+ Pdf {
+ #[serde(rename = "type")]
+ parser_type: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ max_pages: Option<u32>,
+ },
+}
+
+/// Request body for scrape endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct ScrapeRequest {
+ url: String,
+ #[serde(flatten)]
+ options: ScrapeOptions,
+}
+
+/// Response from scrape endpoint.
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct ScrapeResponse {
+ success: bool,
+ data: Document,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ warning: Option<String>,
+}
+
+impl Client {
+ /// Scrapes a URL and returns the content in the requested formats.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to scrape.
+ /// * `options` - Optional scrape configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `Document` containing the scraped content.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, ScrapeOptions, Format};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple scrape
+ /// let document = client.scrape("https://example.com", None).await?;
+ /// println!("Markdown: {:?}", document.markdown);
+ ///
+ /// // Scrape with options
+ /// let options = ScrapeOptions {
+ /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ /// only_main_content: Some(true),
+ /// ..Default::default()
+ /// };
+ /// let document = client.scrape("https://example.com", options).await?;
+ /// println!("Links: {:?}", document.links);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn scrape(
+ &self,
+ url: impl AsRef<str>,
+ options: impl Into<Option<ScrapeOptions>>,
+ ) -> Result<Document, FirecrawlError> {
+ let body = ScrapeRequest {
+ url: url.as_ref().to_string(),
+ options: options.into().unwrap_or_default(),
+ };
+
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(self.url("/scrape"))
+ .headers(headers)
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
+
+ let response: ScrapeResponse = self.handle_response(response, "scrape").await?;
+
+ Ok(response.data)
+ }
+
+ /// Scrapes a URL with a JSON schema for structured extraction.
+ ///
+ /// This is a convenience method that combines scraping with JSON extraction.
+ ///
+ /// # Arguments
+ ///
+ /// * `url` - The URL to scrape.
+ /// * `schema` - JSON schema for the extraction.
+ /// * `prompt` - Optional extraction prompt.
+ ///
+ /// # Returns
+ ///
+ /// The extracted JSON value matching the schema.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ /// use serde_json::json;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let schema = json!({
+ /// "type": "object",
+ /// "properties": {
+ /// "title": { "type": "string" },
+ /// "price": { "type": "number" }
+ /// }
+ /// });
+ ///
+ /// let data = client.scrape_with_schema(
+ /// "https://example.com/product",
+ /// schema,
+ /// Some("Extract the product title and price")
+ /// ).await?;
+ ///
+ /// println!("Extracted: {}", data);
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn scrape_with_schema(
+ &self,
+ url: impl AsRef<str>,
+ schema: Value,
+ prompt: Option<impl AsRef<str>>,
+ ) -> Result<Value, FirecrawlError> {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Json]),
+ json_options: Some(JsonOptions {
+ schema: Some(schema),
+ prompt: prompt.map(|p| p.as_ref().to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let document = self.scrape(url, options).await?;
+ Ok(document.json.unwrap_or(Value::Null))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_scrape_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "markdown": "# Example Domain\n\nThis is an example.",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example Domain"
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let document = client.scrape("https://example.com", None).await.unwrap();
+
+ assert!(document.markdown.is_some());
+ assert!(document.markdown.unwrap().contains("Example Domain"));
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "markdown": "# Test",
+ "html": "<h1>Test</h1>",
+ "links": ["https://example.com/page1", "https://example.com/page2"],
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ only_main_content: Some(true),
+ ..Default::default()
+ };
+
+ let document = client.scrape("https://example.com", options).await.unwrap();
+
+ assert!(document.markdown.is_some());
+ assert!(document.html.is_some());
+ assert!(document.links.is_some());
+ assert_eq!(document.links.unwrap().len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_with_schema() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "json": {
+ "title": "Product Name",
+ "price": 99.99
+ },
+ "metadata": {
+ "sourceURL": "https://example.com/product",
+ "statusCode": 200
+ }
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "price": { "type": "number" }
+ }
+ });
+
+ let data = client
+ .scrape_with_schema(
+ "https://example.com/product",
+ schema,
+ Some("Extract product info"),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(data["title"], "Product Name");
+ assert_eq!(data["price"], 99.99);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_scrape_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/scrape")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid URL"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.scrape("invalid-url", None).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/search.rs b/apps/rust-sdk/src/v2/search.rs
new file mode 100644
index 000000000..ccca680f2
--- /dev/null
+++ b/apps/rust-sdk/src/v2/search.rs
@@ -0,0 +1,466 @@
+//! Search endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+
+use super::client::Client;
+use super::scrape::ScrapeOptions;
+use super::types::{
+ Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,
+};
+use crate::FirecrawlError;
+
+/// Options for search requests.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchOptions {
+ /// Maximum number of results to return. Default: 5, Max: 20.
+ pub limit: Option<u32>,
+
+ /// Search sources to query (web, news, images).
+ pub sources: Option<Vec<SearchSource>>,
+
+ /// Categories to filter results (github, research, pdf).
+ pub categories: Option<Vec<SearchCategory>>,
+
+ /// Time-based search filter (e.g., "qdr:d" for past day).
+ pub tbs: Option<String>,
+
+ /// Geographic location string for local search results.
+ pub location: Option<String>,
+
+ /// Whether to ignore invalid URLs in results.
+ pub ignore_invalid_urls: Option<bool>,
+
+ /// Timeout in milliseconds.
+ pub timeout: Option<u32>,
+
+ /// Scrape options to apply to each search result.
+ pub scrape_options: Option<ScrapeOptions>,
+
+ /// Integration identifier for tracking.
+ pub integration: Option<String>,
+}
+
+/// Request body for search endpoint.
+#[derive(Deserialize, Serialize, Debug, Default)]
+#[serde(rename_all = "camelCase")]
+struct SearchRequest {
+ query: String,
+ #[serde(flatten)]
+ options: SearchOptions,
+}
+
+/// Search results data structure.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchData {
+ /// Web search results (may include scraped documents).
+ pub web: Option<Vec<SearchResultOrDocument>>,
+ /// News search results.
+ pub news: Option<Vec<SearchResultNews>>,
+ /// Image search results.
+ pub images: Option<Vec<SearchResultImage>>,
+}
+
+/// A search result that may be a simple result or a full document.
+///
+/// Uses custom deserialization to properly distinguish between web results
+/// and scraped documents by checking for document-specific fields.
+#[derive(Serialize, Debug, Clone)]
+#[serde(untagged)]
+pub enum SearchResultOrDocument {
+ /// Simple web search result.
+ WebResult(SearchResultWeb),
+ /// Full scraped document.
+ Document(Document),
+}
+
+impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+ where
+ D: serde::Deserializer<'de>,
+ {
+ use serde_json::Value;
+
+ let value = Value::deserialize(deserializer)?;
+
+ // Check for document-specific fields that indicate scraped content
+ // If any of these exist, it's a Document, not a simple WebResult
+ let is_document = value.get("markdown").is_some()
+ || value.get("html").is_some()
+ || value.get("rawHtml").is_some()
+ || value.get("metadata").is_some();
+
+ if is_document {
+ Document::deserialize(value)
+ .map(SearchResultOrDocument::Document)
+ .map_err(serde::de::Error::custom)
+ } else {
+ SearchResultWeb::deserialize(value)
+ .map(SearchResultOrDocument::WebResult)
+ .map_err(serde::de::Error::custom)
+ }
+ }
+}
+
+/// Response from search endpoint.
+#[derive(Deserialize, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchResponse {
+ /// Whether the request was successful.
+ pub success: bool,
+ /// Search results data.
+ pub data: SearchData,
+ /// Warning message if any.
+ pub warning: Option<String>,
+}
+
+impl Client {
+ /// Searches the web and optionally scrapes the results.
+ ///
+ /// # Arguments
+ ///
+ /// * `query` - The search query string.
+ /// * `options` - Optional search configuration.
+ ///
+ /// # Returns
+ ///
+ /// A `SearchResponse` containing the search results.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::{Client, SearchOptions, SearchSource};
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// // Simple search
+ /// let results = client.search("rust programming", None).await?;
+ /// for result in results.data.web.unwrap_or_default() {
+ /// match result {
+ /// firecrawl::v2::SearchResultOrDocument::WebResult(r) => {
+ /// println!("URL: {}", r.url);
+ /// }
+ /// firecrawl::v2::SearchResultOrDocument::Document(d) => {
+ /// println!("Content: {:?}", d.markdown);
+ /// }
+ /// }
+ /// }
+ ///
+ /// // Search with options
+ /// let options = SearchOptions {
+ /// limit: Some(10),
+ /// sources: Some(vec![SearchSource::Web, SearchSource::News]),
+ /// ..Default::default()
+ /// };
+ /// let results = client.search("rust programming", options).await?;
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn search(
+ &self,
+ query: impl AsRef<str>,
+ options: impl Into<Option<SearchOptions>>,
+ ) -> Result<SearchResponse, FirecrawlError> {
+ let body = SearchRequest {
+ query: query.as_ref().to_string(),
+ options: options.into().unwrap_or_default(),
+ };
+
+ let headers = self.prepare_headers(None);
+
+ let response = self
+ .client
+ .post(self.url("/search"))
+ .headers(headers)
+ .json(&body)
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)
+ })?;
+
+ self.handle_response(response, "search").await
+ }
+
+ /// Searches the web and scrapes the results.
+ ///
+ /// This is a convenience method that enables scraping for all results.
+ ///
+ /// # Arguments
+ ///
+ /// * `query` - The search query string.
+ /// * `limit` - Maximum number of results to return.
+ ///
+ /// # Returns
+ ///
+ /// A vector of scraped documents.
+ ///
+ /// # Example
+ ///
+ /// ```no_run
+ /// use firecrawl::v2::Client;
+ ///
+ /// #[tokio::main]
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
+ /// let client = Client::new("your-api-key")?;
+ ///
+ /// let documents = client.search_and_scrape("rust programming", 5).await?;
+ /// for doc in documents {
+ /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));
+ /// println!("Content: {:?}", doc.markdown);
+ /// }
+ ///
+ /// Ok(())
+ /// }
+ /// ```
+ pub async fn search_and_scrape(
+ &self,
+ query: impl AsRef<str>,
+ limit: u32,
+ ) -> Result<Vec<Document>, FirecrawlError> {
+ let options = SearchOptions {
+ limit: Some(limit),
+ scrape_options: Some(ScrapeOptions::default()),
+ ..Default::default()
+ };
+
+ let response = self.search(query, options).await?;
+
+ let documents: Vec<Document> = response
+ .data
+ .web
+ .unwrap_or_default()
+ .into_iter()
+ .filter_map(|result| match result {
+ SearchResultOrDocument::Document(doc) => Some(doc),
+ SearchResultOrDocument::WebResult(_) => None,
+ })
+ .collect();
+
+ Ok(documents)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[tokio::test]
+ async fn test_search_with_mock() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ {
+ "url": "https://example.com",
+ "title": "Example Domain",
+ "description": "This domain is for examples"
+ },
+ {
+ "url": "https://example.org",
+ "title": "Another Example",
+ "description": "More examples"
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.search("test query", None).await.unwrap();
+
+ assert!(response.success);
+ let web_results = response.data.web.unwrap();
+ assert_eq!(web_results.len(), 2);
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_with_options() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ {
+ "url": "https://example.com",
+ "title": "Test Result",
+ "description": "A test result"
+ }
+ ],
+ "news": [
+ {
+ "title": "Breaking News",
+ "url": "https://news.example.com",
+ "snippet": "Something happened",
+ "date": "2024-01-01"
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let options = SearchOptions {
+ limit: Some(10),
+ sources: Some(vec![SearchSource::Web, SearchSource::News]),
+ ..Default::default()
+ };
+
+ let response = client.search("test", options).await.unwrap();
+
+ assert!(response.success);
+ assert!(response.data.web.is_some());
+ assert!(response.data.news.is_some());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_and_scrape() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ {
+ "markdown": "# Example\n\nThis is the scraped content.",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example"
+ }
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let documents = client.search_and_scrape("test", 5).await.unwrap();
+
+ assert_eq!(documents.len(), 1);
+ assert!(documents[0].markdown.is_some());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_error_response() {
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(400)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": false,
+ "error": "Invalid query"
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let result = client.search("", None).await;
+
+ assert!(result.is_err());
+ mock.assert();
+ }
+
+ #[tokio::test]
+ async fn test_search_mixed_results_deserialization() {
+ // Test that results with markdown/metadata are correctly identified as Documents
+ // and simple results are identified as WebResults
+ let mut server = mockito::Server::new_async().await;
+
+ let mock = server
+ .mock("POST", "/v2/search")
+ .with_status(200)
+ .with_header("content-type", "application/json")
+ .with_body(
+ json!({
+ "success": true,
+ "data": {
+ "web": [
+ // Simple web result (no markdown/metadata)
+ {
+ "url": "https://example.com/simple",
+ "title": "Simple Result",
+ "description": "Just a search result"
+ },
+ // Scraped document with url AND markdown
+ {
+ "url": "https://example.com/scraped",
+ "markdown": "# Scraped Content\n\nThis has content.",
+ "metadata": {
+ "sourceURL": "https://example.com/scraped",
+ "statusCode": 200,
+ "title": "Scraped Page"
+ }
+ }
+ ]
+ }
+ })
+ .to_string(),
+ )
+ .create();
+
+ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
+ let response = client.search("test", None).await.unwrap();
+
+ let web_results = response.data.web.unwrap();
+ assert_eq!(web_results.len(), 2);
+
+ // First should be WebResult
+ match &web_results[0] {
+ SearchResultOrDocument::WebResult(r) => {
+ assert_eq!(r.url, "https://example.com/simple");
+ assert_eq!(r.title, Some("Simple Result".to_string()));
+ }
+ SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),
+ }
+
+ // Second should be Document (has markdown)
+ match &web_results[1] {
+ SearchResultOrDocument::Document(d) => {
+ assert!(d.markdown.is_some());
+ assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));
+ }
+ SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),
+ }
+
+ mock.assert();
+ }
+}
diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rs
new file mode 100644
index 000000000..89ffe68b8
--- /dev/null
+++ b/apps/rust-sdk/src/v2/types.rs
@@ -0,0 +1,510 @@
+//! Type definitions for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+
+/// Available output formats for scraping operations.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub enum Format {
+ /// Markdown content of the page.
+ Markdown,
+ /// Filtered, content-only HTML.
+ Html,
+ /// Original, untouched HTML.
+ RawHtml,
+ /// List of URLs found on the page.
+ Links,
+ /// List of image URLs found on the page.
+ Images,
+ /// Screenshot of the visible viewport.
+ Screenshot,
+ /// AI-generated summary of the page content.
+ Summary,
+ /// Change tracking information.
+ ChangeTracking,
+ /// Structured JSON extraction via LLM.
+ Json,
+ /// Custom attribute extraction.
+ Attributes,
+ /// Brand analysis of the page.
+ Branding,
+}
+
+/// Viewport dimensions for screenshots.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+pub struct Viewport {
+ pub width: u32,
+ pub height: u32,
+}
+
+/// Screenshot format options.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ScreenshotOptions {
+ /// Take a full-page screenshot instead of just the visible viewport.
+ pub full_page: Option<bool>,
+ /// Quality of the screenshot (1-100).
+ pub quality: Option<u8>,
+ /// Custom viewport dimensions.
+ pub viewport: Option<Viewport>,
+}
+
+/// Change tracking format options.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ChangeTrackingOptions {
+ /// Modes for change tracking output.
+ pub modes: Option<Vec<ChangeTrackingMode>>,
+ /// JSON schema for structured change output.
+ pub schema: Option<Value>,
+ /// Prompt for LLM-based change analysis.
+ pub prompt: Option<String>,
+ /// Tag to identify this tracking session.
+ pub tag: Option<String>,
+}
+
+/// Available change tracking modes.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub enum ChangeTrackingMode {
+ GitDiff,
+ Json,
+}
+
+/// Attribute extraction selector.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+pub struct AttributeSelector {
+ /// CSS selector for the element.
+ pub selector: String,
+ /// Attribute name to extract.
+ pub attribute: String,
+}
+
+/// JSON extraction options.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct JsonOptions {
+ /// JSON schema the output should adhere to.
+ pub schema: Option<Value>,
+ /// System prompt for the LLM agent.
+ pub system_prompt: Option<String>,
+ /// Extraction prompt for the LLM agent.
+ pub prompt: Option<String>,
+}
+
+/// Location configuration for proxy routing.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct LocationConfig {
+ /// Country code (ISO 3166-1 alpha-2).
+ pub country: Option<String>,
+ /// List of preferred language codes.
+ pub languages: Option<Vec<String>>,
+}
+
+/// Proxy type for scraping.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ProxyType {
+ Basic,
+ Stealth,
+ Enhanced,
+ Auto,
+}
+
+/// Browser action types for automation.
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(tag = "type", rename_all = "camelCase")]
+pub enum Action {
+ /// Wait for a specified time or element.
+ Wait {
+ /// Milliseconds to wait.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ milliseconds: Option<u32>,
+ /// CSS selector to wait for.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ selector: Option<String>,
+ },
+ /// Take a screenshot.
+ Screenshot {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ full_page: Option<bool>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ quality: Option<u8>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ viewport: Option<Viewport>,
+ },
+ /// Click an element.
+ Click {
+ /// CSS selector of the element to click.
+ selector: String,
+ },
+ /// Write text to the focused input.
+ Write {
+ /// Text to write.
+ text: String,
+ },
+ /// Press a keyboard key.
+ Press {
+ /// Key name to press.
+ key: String,
+ },
+ /// Scroll the page.
+ Scroll {
+ /// Direction to scroll.
+ direction: ScrollDirection,
+ /// Optional selector to scroll within.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ selector: Option<String>,
+ },
+ /// Trigger a scrape action.
+ Scrape,
+ /// Execute custom JavaScript.
+ #[serde(rename = "executeJavascript")]
+ ExecuteJavascript {
+ /// JavaScript code to execute.
+ script: String,
+ },
+ /// Generate a PDF.
+ Pdf {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ format: Option<PdfFormat>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ landscape: Option<bool>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ scale: Option<f32>,
+ },
+}
+
+/// Scroll direction for scroll actions.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum ScrollDirection {
+ Up,
+ Down,
+}
+
+/// PDF format options.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PdfFormat {
+ A0,
+ A1,
+ A2,
+ A3,
+ A4,
+ A5,
+ A6,
+ Letter,
+ Legal,
+ Tabloid,
+ Ledger,
+}
+
+/// Webhook configuration for async operations.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct WebhookConfig {
+ /// URL to send webhook notifications to.
+ pub url: String,
+ /// Custom headers to include in webhook requests.
+ pub headers: Option<HashMap<String, String>>,
+ /// Custom metadata to include in webhook payloads.
+ pub metadata: Option<HashMap<String, String>>,
+ /// Event types to receive notifications for.
+ pub events: Option<Vec<WebhookEvent>>,
+}
+
+impl From<String> for WebhookConfig {
+ fn from(url: String) -> Self {
+ Self {
+ url,
+ ..Default::default()
+ }
+ }
+}
+
+impl From<&str> for WebhookConfig {
+ fn from(url: &str) -> Self {
+ Self {
+ url: url.to_string(),
+ ..Default::default()
+ }
+ }
+}
+
+/// Webhook event types for crawl/batch operations.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub enum WebhookEvent {
+ Completed,
+ Failed,
+ Page,
+ Started,
+}
+
+/// Agent-specific webhook event types.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub enum AgentWebhookEvent {
+ Started,
+ Action,
+ Completed,
+ Failed,
+ Cancelled,
+}
+
+/// Agent webhook configuration.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct AgentWebhookConfig {
+ /// URL to send webhook notifications to.
+ pub url: String,
+ /// Custom headers to include in webhook requests.
+ pub headers: Option<HashMap<String, String>>,
+ /// Custom metadata to include in webhook payloads.
+ pub metadata: Option<HashMap<String, String>>,
+ /// Event types to receive notifications for.
+ pub events: Option<Vec<AgentWebhookEvent>>,
+}
+
+impl From<String> for AgentWebhookConfig {
+ fn from(url: String) -> Self {
+ Self {
+ url,
+ ..Default::default()
+ }
+ }
+}
+
+impl From<&str> for AgentWebhookConfig {
+ fn from(url: &str) -> Self {
+ Self {
+ url: url.to_string(),
+ ..Default::default()
+ }
+ }
+}
+
+/// Document metadata returned from scrape operations.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DocumentMetadata {
+ // Firecrawl specific
+ #[serde(rename = "sourceURL")]
+ pub source_url: Option<String>,
+ pub status_code: Option<u16>,
+ pub error: Option<String>,
+
+ // Basic meta tags
+ pub title: Option<String>,
+ pub description: Option<String>,
+ pub language: Option<String>,
+ pub keywords: Option<String>,
+ pub robots: Option<String>,
+
+ // OpenGraph namespace
+ pub og_title: Option<String>,
+ pub og_description: Option<String>,
+ pub og_url: Option<String>,
+ pub og_image: Option<String>,
+ pub og_audio: Option<String>,
+ pub og_determiner: Option<String>,
+ pub og_locale: Option<String>,
+ pub og_locale_alternate: Option<Vec<String>>,
+ pub og_site_name: Option<String>,
+ pub og_video: Option<String>,
+
+ // Article namespace
+ pub article_section: Option<String>,
+ pub article_tag: Option<String>,
+ pub published_time: Option<String>,
+ pub modified_time: Option<String>,
+
+ // Dublin Core namespace
+ pub dcterms_keywords: Option<String>,
+ pub dc_description: Option<String>,
+ pub dc_subject: Option<String>,
+ pub dcterms_subject: Option<String>,
+ pub dcterms_audience: Option<String>,
+ pub dc_type: Option<String>,
+ pub dcterms_type: Option<String>,
+ pub dc_date: Option<String>,
+ pub dc_date_created: Option<String>,
+ pub dcterms_created: Option<String>,
+
+ // Response metadata
+ pub scrape_id: Option<String>,
+ pub num_pages: Option<u32>,
+ pub content_type: Option<String>,
+ pub timezone: Option<String>,
+ pub proxy_used: Option<String>,
+ pub cache_state: Option<String>,
+ pub cached_at: Option<String>,
+ pub credits_used: Option<u32>,
+ pub concurrency_limited: Option<bool>,
+}
+
+/// Extracted attribute result.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+pub struct AttributeResult {
+ pub selector: String,
+ pub attribute: String,
+ pub values: Vec<String>,
+}
+
+/// Document returned from scrape operations.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Document {
+ /// Markdown content of the page.
+ pub markdown: Option<String>,
+ /// Filtered HTML content.
+ pub html: Option<String>,
+ /// Raw HTML content.
+ pub raw_html: Option<String>,
+ /// Structured JSON extraction result.
+ pub json: Option<Value>,
+ /// AI-generated summary.
+ pub summary: Option<String>,
+ /// Document metadata.
+ pub metadata: Option<DocumentMetadata>,
+ /// Links found on the page.
+ pub links: Option<Vec<String>>,
+ /// Images found on the page.
+ pub images: Option<Vec<String>>,
+ /// Screenshot URL or base64 data.
+ pub screenshot: Option<String>,
+ /// Extracted attributes.
+ pub attributes: Option<Vec<AttributeResult>>,
+ /// Action results.
+ pub actions: Option<HashMap<String, Value>>,
+ /// Warning message.
+ pub warning: Option<String>,
+ /// Change tracking data.
+ pub change_tracking: Option<Value>,
+ /// Branding analysis.
+ pub branding: Option<Value>,
+}
+
+/// Job status types for crawl and batch operations.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub enum JobStatus {
+ Scraping,
+ Completed,
+ Failed,
+ Cancelled,
+}
+
+/// Sitemap handling mode.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum SitemapMode {
+ /// Skip sitemap entirely.
+ Skip,
+ /// Include sitemap links alongside discovered links.
+ Include,
+ /// Only use links from the sitemap.
+ Only,
+}
+
+/// Agent model types.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub enum AgentModel {
+ #[serde(rename = "spark-1-pro")]
+ Spark1Pro,
+ #[serde(rename = "spark-1-mini")]
+ Spark1Mini,
+}
+
+/// Search source types.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum SearchSource {
+ Web,
+ News,
+ Images,
+}
+
+/// Search category types.
+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum SearchCategory {
+ Github,
+ Research,
+ Pdf,
+}
+
+/// Web search result.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchResultWeb {
+ pub url: String,
+ pub title: Option<String>,
+ pub description: Option<String>,
+ pub category: Option<String>,
+}
+
+/// News search result.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchResultNews {
+ pub title: Option<String>,
+ pub url: Option<String>,
+ pub snippet: Option<String>,
+ pub date: Option<String>,
+ pub image_url: Option<String>,
+ pub position: Option<u32>,
+ pub category: Option<String>,
+}
+
+/// Image search result.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct SearchResultImage {
+ pub title: Option<String>,
+ pub image_url: Option<String>,
+ pub image_width: Option<u32>,
+ pub image_height: Option<u32>,
+ pub url: Option<String>,
+ pub position: Option<u32>,
+}
+
+/// Crawl error information.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlError {
+ pub id: String,
+ pub timestamp: Option<String>,
+ pub url: String,
+ pub code: Option<String>,
+ pub error: String,
+}
+
+/// Crawl errors response.
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CrawlErrorsResponse {
+ pub errors: Vec<CrawlError>,
+ #[serde(rename = "robotsBlocked")]
+ pub robots_blocked: Vec<String>,
+}
diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rs
index 071537f50..05c6d0080 100644
--- a/apps/rust-sdk/tests/e2e_with_auth.rs
+++ b/apps/rust-sdk/tests/e2e_with_auth.rs
@@ -1,5 +1,5 @@
use dotenvy::dotenv;
-use firecrawl::scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions};
+use firecrawl::scrape::{JsonOptions, ScrapeFormats, ScrapeOptions};
use firecrawl::{FirecrawlApp, FirecrawlError};
use serde_json::json;
use std::env;
@@ -123,8 +123,8 @@ async fn test_llm_extraction() {
let api_key = env::var("TEST_API_KEY").ok();
let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();
let options = ScrapeOptions {
- formats: vec! [ ScrapeFormats::Extract ].into(),
- extract: ExtractOptions {
+ formats: vec![ScrapeFormats::Json].into(),
+ json_options: JsonOptions {
prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source".to_string().into(),
schema: json!({
"type": "object",
diff --git a/apps/rust-sdk/tests/v2_e2e.rs b/apps/rust-sdk/tests/v2_e2e.rs
new file mode 100644
index 000000000..f91398206
--- /dev/null
+++ b/apps/rust-sdk/tests/v2_e2e.rs
@@ -0,0 +1,302 @@
+//! End-to-end tests for Firecrawl v2 API.
+//!
+//! These tests require the following environment variables:
+//! - API_URL: The Firecrawl API URL
+//! - TEST_API_KEY: A valid API key (optional for self-hosted)
+//!
+//! Run with: cargo test --test v2_e2e -- --ignored
+
+use dotenvy::dotenv;
+use firecrawl::v2::{
+ AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions, ScrapeOptions,
+ SearchOptions, SitemapMode,
+};
+use serde_json::json;
+use std::env;
+
+fn get_client() -> Client {
+ dotenv().ok();
+ let api_url = env::var("API_URL").expect("API_URL environment variable is required");
+ let api_key = env::var("TEST_API_KEY").ok();
+ Client::new_selfhosted(api_url, api_key).expect("Failed to create client")
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_scrape() {
+ let client = get_client();
+ let doc = client
+ .scrape("https://example.com", None)
+ .await
+ .expect("Scrape should succeed");
+
+ assert!(doc.markdown.is_some(), "Response should contain markdown");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_scrape_with_options() {
+ let client = get_client();
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
+ only_main_content: Some(true),
+ ..Default::default()
+ };
+
+ let doc = client
+ .scrape("https://example.com", options)
+ .await
+ .expect("Scrape with options should succeed");
+
+ assert!(doc.markdown.is_some(), "Response should contain markdown");
+ assert!(doc.html.is_some(), "Response should contain html");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_scrape_with_schema() {
+ let client = get_client();
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "description": { "type": "string" }
+ }
+ });
+
+ let data = client
+ .scrape_with_schema("https://example.com", schema, Some("Extract page info"))
+ .await
+ .expect("Schema scrape should succeed");
+
+ assert!(data.is_object(), "Response should be a JSON object");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_search() {
+ let client = get_client();
+ let response = client
+ .search("rust programming", None)
+ .await
+ .expect("Search should succeed");
+
+ assert!(response.success, "Response should indicate success");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_search_with_options() {
+ let client = get_client();
+ let options = SearchOptions {
+ limit: Some(5),
+ ..Default::default()
+ };
+
+ let response = client
+ .search("firecrawl web scraping", options)
+ .await
+ .expect("Search with options should succeed");
+
+ assert!(response.success, "Response should indicate success");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_map() {
+ let client = get_client();
+ let response = client
+ .map("https://example.com", None)
+ .await
+ .expect("Map should succeed");
+
+ assert!(response.success, "Response should indicate success");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_map_with_options() {
+ let client = get_client();
+ let options = MapOptions {
+ sitemap: Some(SitemapMode::Include),
+ limit: Some(50),
+ ..Default::default()
+ };
+
+ let response = client
+ .map("https://example.com", options)
+ .await
+ .expect("Map with options should succeed");
+
+ assert!(response.success, "Response should indicate success");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_crawl_async() {
+ let client = get_client();
+ let response = client
+ .start_crawl("https://example.com", None)
+ .await
+ .expect("Start crawl should succeed");
+
+ assert!(response.success, "Response should indicate success");
+ assert!(!response.id.is_empty(), "Response should contain job ID");
+
+ // Check status
+ let status = client
+ .get_crawl_status(&response.id)
+ .await
+ .expect("Get crawl status should succeed");
+ assert!(
+ matches!(
+ status.status,
+ firecrawl::v2::JobStatus::Scraping | firecrawl::v2::JobStatus::Completed
+ ),
+ "Status should be scraping or completed"
+ );
+
+ // Cancel the crawl to clean up
+ let _ = client.cancel_crawl(&response.id).await;
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_crawl_sync() {
+ let client = get_client();
+ let options = CrawlOptions {
+ limit: Some(2),
+ poll_interval: Some(2000),
+ ..Default::default()
+ };
+
+ let job = client
+ .crawl("https://example.com", options)
+ .await
+ .expect("Crawl should succeed");
+
+ assert!(
+ job.status == firecrawl::v2::JobStatus::Completed,
+ "Job should be completed"
+ );
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_batch_scrape_async() {
+ let client = get_client();
+ let urls = vec![
+ "https://example.com".to_string(),
+ "https://example.org".to_string(),
+ ];
+
+ let response = client
+ .start_batch_scrape(urls, None)
+ .await
+ .expect("Start batch scrape should succeed");
+
+ assert!(response.success, "Response should indicate success");
+ assert!(!response.id.is_empty(), "Response should contain job ID");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_batch_scrape_sync() {
+ let client = get_client();
+ let urls = vec!["https://example.com".to_string()];
+
+ let options = BatchScrapeOptions {
+ options: Some(ScrapeOptions {
+ formats: Some(vec![Format::Markdown]),
+ ..Default::default()
+ }),
+ poll_interval: Some(2000),
+ ..Default::default()
+ };
+
+ let job = client
+ .batch_scrape(urls, options)
+ .await
+ .expect("Batch scrape should succeed");
+
+ assert!(
+ job.status == firecrawl::v2::JobStatus::Completed,
+ "Job should be completed"
+ );
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_agent_async() {
+ let client = get_client();
+ let options = AgentOptions {
+ urls: Some(vec!["https://example.com".to_string()]),
+ prompt: "Describe what this website is about".to_string(),
+ ..Default::default()
+ };
+
+ let response = client
+ .start_agent(options)
+ .await
+ .expect("Start agent should succeed");
+
+ assert!(response.success, "Response should indicate success");
+ assert!(!response.id.is_empty(), "Response should contain task ID");
+}
+
+#[tokio::test]
+#[ignore = "Requires API access"]
+async fn test_v2_agent_with_schema() {
+ let client = get_client();
+
+ #[derive(Debug, serde::Deserialize)]
+ #[allow(dead_code)]
+ struct WebsiteInfo {
+ title: Option<String>,
+ description: Option<String>,
+ }
+
+ let schema = json!({
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "description": { "type": "string" }
+ }
+ });
+
+ let result: Option<WebsiteInfo> = client
+ .agent_with_schema(
+ vec!["https://example.com".to_string()],
+ "Extract the title and description",
+ schema,
+ )
+ .await
+ .expect("Agent with schema should succeed");
+
+ // Agent may or may not return data depending on the page
+ if let Some(info) = result {
+ println!("Agent extracted: {:?}", info);
+ }
+}
+
+// Test that the v2 client can be created with different configurations
+// This test doesn't require API access
+#[test]
+fn test_v2_client_creation() {
+ // Cloud client requires API key
+ let cloud_result = Client::new("test-key");
+ assert!(cloud_result.is_ok());
+
+ // Cloud client without API key should fail
+ let cloud_no_key = Client::new_selfhosted("https://api.firecrawl.dev", None::<&str>);
+ assert!(cloud_no_key.is_err());
+
+ // Self-hosted client without API key should work
+ let selfhosted = Client::new_selfhosted("http://localhost:3000", None::<&str>);
+ assert!(selfhosted.is_ok());
+
+ // Self-hosted client with API key should work
+ let selfhosted_with_key = Client::new_selfhosted("http://localhost:3000", Some("key"));
+ assert!(selfhosted_with_key.is_ok());
+}