Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(rust-sdk): add v2 API namespace with agent support (#2778)
Abimael Martell43f61e7fe5c8
diff --git a/.github/workflows/test-rust-sdk.yml b/.github/workflows/test-rust-sdk.ymlnew file mode 100644index 000000000..ed85ac685--- /dev/null+++ b/.github/workflows/test-rust-sdk.yml@@ -0,0 +1,51 @@+name: Rust SDK Test Suite++on:+ pull_request:+ branches:+ - main+ paths:+ - apps/rust-sdk/**++jobs:+ test:+ name: Build and Test+ runs-on: blacksmith-4vcpu-ubuntu-2404+ steps:+ - uses: actions/checkout@v4++ - name: Install Rust toolchain+ uses: dtolnay/rust-toolchain@stable+ with:+ components: clippy++ - name: Cache cargo registry+ uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry+ ~/.cargo/git+ apps/rust-sdk/target+ key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}+ restore-keys: |+ ${{ runner.os }}-cargo-++ - name: Check formatting+ working-directory: ./apps/rust-sdk+ run: cargo fmt --check++ - name: Build+ working-directory: ./apps/rust-sdk+ run: cargo build --all-targets++ - name: Run clippy+ working-directory: ./apps/rust-sdk+ run: cargo clippy --all-targets -- -D warnings -A clippy::needless_borrows_for_generic_args -A clippy::redundant_closure -A clippy::large_enum_variant -A clippy::extra_unused_lifetimes++ - name: Run unit tests (mocked)+ working-directory: ./apps/rust-sdk+ run: cargo test --lib++ - name: Build examples+ working-directory: ./apps/rust-sdk+ run: cargo build --examplesdiff --git a/apps/rust-sdk/examples/example.rs b/apps/rust-sdk/examples/example.rsindex 51592f3f1..37a3df7d2 100644--- a/apps/rust-sdk/examples/example.rs+++ b/apps/rust-sdk/examples/example.rs@@ -1,6 +1,6 @@use firecrawl::{crawl::CrawlOptions,- scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions},+ scrape::{JsonOptions, ScrapeFormats, ScrapeOptions},FirecrawlApp,};use serde_json::json;@@ -62,8 +62,8 @@ async fn main() {});let llm_extraction_options = ScrapeOptions {- formats: vec![ScrapeFormats::Extract].into(),- extract: ExtractOptions {+ formats: vec![ScrapeFormats::Json].into(),+ json_options: JsonOptions {schema: json_schema.into(),..Default::default()}diff --git a/apps/rust-sdk/examples/v2_example.rs b/apps/rust-sdk/examples/v2_example.rsnew file mode 100644index 000000000..a5401c928--- /dev/null+++ b/apps/rust-sdk/examples/v2_example.rs@@ -0,0 +1,216 @@+//! Example usage of Firecrawl v2 API.+//!+//! Run with: cargo run --example v2_example++use firecrawl::v2::{+ AgentModel, AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions,+ ScrapeOptions, SearchOptions, SitemapMode,+};+use serde::Deserialize;+use serde_json::json;++#[tokio::main]+async fn main() {+ // Initialize the v2 Client with the API key+ let client = Client::new("fc-YOUR-API-KEY").expect("Failed to initialize Client");++ // Or, connect to a self-hosted instance:+ // let client = Client::new_selfhosted("http://localhost:3002", None::<&str>)+ // .expect("Failed to initialize Client");++ // Example 1: Simple Scrape+ println!("=== Example 1: Simple Scrape ===");+ let result = client.scrape("https://example.com", None).await;+ match result {+ Ok(doc) => println!("Markdown: {:?}", doc.markdown),+ Err(e) => eprintln!("Scrape failed: {}", e),+ }++ // Example 2: Scrape with Options+ println!("\n=== Example 2: Scrape with Options ===");+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ only_main_content: Some(true),+ ..Default::default()+ };+ let result = client.scrape("https://example.com", options).await;+ match result {+ Ok(doc) => {+ println!("Links: {:?}", doc.links);+ }+ Err(e) => eprintln!("Scrape with options failed: {}", e),+ }++ // Example 3: Scrape with JSON Schema Extraction+ println!("\n=== Example 3: JSON Schema Extraction ===");+ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "description": { "type": "string" }+ }+ });+ let result = client+ .scrape_with_schema(+ "https://example.com",+ schema,+ Some("Extract title and description"),+ )+ .await;+ match result {+ Ok(data) => println!(+ "Extracted: {}",+ serde_json::to_string_pretty(&data).unwrap()+ ),+ Err(e) => eprintln!("Schema extraction failed: {}", e),+ }++ // Example 4: Search+ println!("\n=== Example 4: Search ===");+ let options = SearchOptions {+ limit: Some(5),+ ..Default::default()+ };+ let result = client.search("rust programming", options).await;+ match result {+ Ok(response) => {+ if let Some(web) = response.data.web {+ for item in web {+ match item {+ firecrawl::v2::SearchResultOrDocument::WebResult(r) => {+ println!("Result: {} - {}", r.url, r.title.unwrap_or_default());+ }+ firecrawl::v2::SearchResultOrDocument::Document(d) => {+ if let Some(meta) = d.metadata {+ println!("Document: {:?}", meta.title);+ }+ }+ }+ }+ }+ }+ Err(e) => eprintln!("Search failed: {}", e),+ }++ // Example 5: Map a Website+ println!("\n=== Example 5: Map ===");+ let options = MapOptions {+ sitemap: Some(SitemapMode::Include),+ limit: Some(20),+ ..Default::default()+ };+ let result = client.map("https://example.com", options).await;+ match result {+ Ok(response) => {+ println!("Found {} links", response.links.len());+ for link in response.links.iter().take(5) {+ println!(" - {}", link.url);+ }+ }+ Err(e) => eprintln!("Map failed: {}", e),+ }++ // Example 6: Crawl a Website+ println!("\n=== Example 6: Crawl ===");+ let options = CrawlOptions {+ limit: Some(5),+ sitemap: Some(SitemapMode::Include),+ poll_interval: Some(3000),+ ..Default::default()+ };+ let result = client.crawl("https://example.com", options).await;+ match result {+ Ok(job) => {+ println!("Crawled {} pages", job.data.len());+ for doc in job.data.iter().take(3) {+ if let Some(meta) = &doc.metadata {+ println!(" - {:?}", meta.source_url);+ }+ }+ }+ Err(e) => eprintln!("Crawl failed: {}", e),+ }++ // Example 7: Batch Scrape+ println!("\n=== Example 7: Batch Scrape ===");+ let urls = vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ];+ let options = BatchScrapeOptions {+ options: Some(ScrapeOptions {+ formats: Some(vec![Format::Markdown]),+ ..Default::default()+ }),+ poll_interval: Some(2000),+ ..Default::default()+ };+ let result = client.batch_scrape(urls, options).await;+ match result {+ Ok(job) => {+ println!("Batch scraped {} pages", job.data.len());+ }+ Err(e) => eprintln!("Batch scrape failed: {}", e),+ }++ // Example 8: Agent (Autonomous Web Browsing)+ println!("\n=== Example 8: Agent ===");+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Describe what this website is about and list any key features mentioned"+ .to_string(),+ model: Some(AgentModel::Spark1Pro),+ timeout: Some(60),+ ..Default::default()+ };+ let result = client.agent(options).await;+ match result {+ Ok(response) => {+ println!("Agent status: {:?}", response.status);+ if let Some(data) = response.data {+ println!("Result: {}", serde_json::to_string_pretty(&data).unwrap());+ }+ }+ Err(e) => eprintln!("Agent failed: {}", e),+ }++ // Example 9: Agent with Typed Schema+ println!("\n=== Example 9: Agent with Typed Schema ===");++ #[derive(Debug, Deserialize)]+ struct CompanyInfo {+ name: String,+ description: Option<String>,+ industry: Option<String>,+ }++ let schema = json!({+ "type": "object",+ "properties": {+ "name": { "type": "string" },+ "description": { "type": "string" },+ "industry": { "type": "string" }+ },+ "required": ["name"]+ });++ let result: Result<Option<CompanyInfo>, _> = client+ .agent_with_schema(+ vec!["https://firecrawl.dev".to_string()],+ "Extract company information from this website",+ schema,+ )+ .await;++ match result {+ Ok(Some(info)) => {+ println!("Company: {}", info.name);+ println!("Description: {:?}", info.description);+ println!("Industry: {:?}", info.industry);+ }+ Ok(None) => println!("No data extracted"),+ Err(e) => eprintln!("Agent with schema failed: {}", e),+ }++ println!("\n=== All examples completed ===");+}diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rsindex 6d27ff283..bd3b4273a 100644--- a/apps/rust-sdk/src/crawl.rs+++ b/apps/rust-sdk/src/crawl.rs@@ -197,17 +197,6 @@ struct CrawlRequestBody {options: CrawlOptions,}-#[derive(Deserialize, Serialize, Debug, Default)]-#[serde(rename_all = "camelCase")]-struct CrawlResponse {- /// This will always be `true` due to `FirecrawlApp::handle_response`.- /// No need to expose.- success: bool,-- /// The resulting document.- data: Document,-}-#[derive(Deserialize, Serialize, Debug, PartialEq, Eq, Clone, Copy)]#[serde(rename_all = "camelCase")]pub enum CrawlStatusTypes {diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex 920900a25..9f69ac226 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -1,17 +1,55 @@+//! Firecrawl Rust SDK+//!+//! This SDK provides access to the Firecrawl API for web scraping and crawling.+//!+//! # Quick Start+//!+//! ## v1 API (existing)+//!+//! ```no_run+//! use firecrawl::FirecrawlApp;+//!+//! #[tokio::main]+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {+//! let app = FirecrawlApp::new("your-api-key")?;+//! let document = app.scrape_url("https://example.com", None).await?;+//! println!("{:?}", document.markdown);+//! Ok(())+//! }+//! ```+//!+//! ## v2 API (new)+//!+//! ```no_run+//! use firecrawl::v2::Client;+//!+//! #[tokio::main]+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {+//! let client = Client::new("your-api-key")?;+//! let document = client.scrape("https://example.com", None).await?;+//! println!("{:?}", document.markdown);+//! Ok(())+//! }+//! ```+use reqwest::{Client, Response};use serde::de::DeserializeOwned;use serde_json::Value;+// v1 modules (existing API)pub mod batch_scrape;pub mod crawl;pub mod document;-mod error;+pub mod error;pub mod extract;pub mod llmstxt;pub mod map;pub mod scrape;pub mod search;+// v2 module (new API)+pub mod v2;+use error::FirecrawlAPIError;pub use error::FirecrawlError;diff --git a/apps/rust-sdk/src/map.rs b/apps/rust-sdk/src/map.rsindex 386cb098e..035785bf6 100644--- a/apps/rust-sdk/src/map.rs+++ b/apps/rust-sdk/src/map.rs@@ -8,7 +8,7 @@ use crate::{FirecrawlApp, FirecrawlError, API_VERSION};pub struct Location {/// Country code (ISO 3166-1 alpha-2)pub country: Option<String>,-+/// List of language codespub languages: Option<Vec<String>>,}diff --git a/apps/rust-sdk/src/v2/agent.rs b/apps/rust-sdk/src/v2/agent.rsnew file mode 100644index 000000000..47b4f734f--- /dev/null+++ b/apps/rust-sdk/src/v2/agent.rs@@ -0,0 +1,674 @@+//! Agent endpoint for Firecrawl API v2.+//!+//! The Agent endpoint provides autonomous web browsing capabilities using AI+//! to accomplish complex tasks that may require multiple page interactions.++use serde::{Deserialize, Serialize};+use serde_json::Value;++use super::client::Client;+use super::types::{AgentModel, AgentWebhookConfig};+use crate::FirecrawlError;++/// Options for running an agent task.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct AgentOptions {+ /// Starting URLs for the agent to explore.+ pub urls: Option<Vec<String>>,++ /// The prompt describing what the agent should accomplish.+ pub prompt: String,++ /// JSON schema for the expected output structure.+ pub schema: Option<Value>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Maximum credits the agent can use.+ pub max_credits: Option<u32>,++ /// Strictly constrain the agent to the provided URLs.+ pub strict_constrain_to_urls: Option<bool>,++ /// Agent model to use.+ pub model: Option<AgentModel>,++ /// Webhook configuration for agent notifications.+ pub webhook: Option<AgentWebhookConfig>,++ /// Poll interval for synchronous agent execution (milliseconds).+ #[serde(skip)]+ pub poll_interval: Option<u64>,++ /// Timeout for synchronous agent execution (seconds).+ #[serde(skip)]+ pub timeout: Option<u64>,+}++/// Response from starting an agent task.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct AgentResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// The agent task ID.+ pub id: String,+ /// Error message if the request failed.+ pub error: Option<String>,+}++/// Agent task status.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum AgentStatus {+ /// The agent is still processing.+ Processing,+ /// The agent has completed its task.+ Completed,+ /// The agent task failed.+ Failed,+ /// The agent task was cancelled.+ Cancelled,+}++/// Status response from an agent task.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct AgentStatusResponse {+ /// Whether the status check was successful.+ pub success: bool,+ /// Current status of the agent task.+ pub status: AgentStatus,+ /// Error message if the task failed.+ pub error: Option<String>,+ /// Extracted data (if schema was provided) or task results.+ pub data: Option<Value>,+ /// Model used for the agent task.+ pub model: Option<AgentModel>,+ /// Expiry time of the task data.+ pub expires_at: Option<String>,+ /// Credits used by the agent task.+ pub credits_used: Option<u32>,+}++impl Client {+ /// Starts an agent task asynchronously.+ ///+ /// Returns immediately with a task ID that can be used to check status.+ ///+ /// # Arguments+ ///+ /// * `options` - Agent task configuration including the prompt.+ ///+ /// # Returns+ ///+ /// An `AgentResponse` containing the task ID.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, AgentOptions};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let options = AgentOptions {+ /// urls: Some(vec!["https://example.com".to_string()]),+ /// prompt: "Find the pricing information on this website".to_string(),+ /// ..Default::default()+ /// };+ ///+ /// let response = client.start_agent(options).await?;+ /// println!("Agent task started: {}", response.id);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn start_agent(+ &self,+ options: AgentOptions,+ ) -> Result<AgentResponse, FirecrawlError> {+ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(self.url("/agent"))+ .headers(headers)+ .json(&options)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Starting agent task".to_string(), e))?;++ self.handle_response(response, "start agent").await+ }++ /// Gets the status of an agent task.+ ///+ /// # Arguments+ ///+ /// * `id` - The agent task ID.+ ///+ /// # Returns+ ///+ /// An `AgentStatusResponse` containing the current status and any results.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let status = client.get_agent_status("task-id").await?;+ /// println!("Status: {:?}", status.status);+ ///+ /// if let Some(data) = status.data {+ /// println!("Result: {}", data);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_agent_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<AgentStatusResponse, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/agent/{}", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Getting agent status {}", id.as_ref()), e)+ })?;++ self.handle_response(response, format!("agent status {}", id.as_ref()))+ .await+ }++ /// Runs an agent task and waits for completion.+ ///+ /// This method starts an agent task and polls until it completes, fails, or times out.+ ///+ /// # Arguments+ ///+ /// * `options` - Agent task configuration including the prompt.+ ///+ /// # Returns+ ///+ /// An `AgentStatusResponse` containing the final status and results.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, AgentOptions, AgentModel};+ /// use serde_json::json;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let options = AgentOptions {+ /// urls: Some(vec!["https://example.com/pricing".to_string()]),+ /// prompt: "Extract the pricing tiers and their features".to_string(),+ /// schema: Some(json!({+ /// "type": "object",+ /// "properties": {+ /// "tiers": {+ /// "type": "array",+ /// "items": {+ /// "type": "object",+ /// "properties": {+ /// "name": { "type": "string" },+ /// "price": { "type": "number" },+ /// "features": { "type": "array", "items": { "type": "string" } }+ /// }+ /// }+ /// }+ /// }+ /// })),+ /// model: Some(AgentModel::Spark1Pro),+ /// poll_interval: Some(3000),+ /// timeout: Some(300),+ /// ..Default::default()+ /// };+ ///+ /// let result = client.agent(options).await?;+ ///+ /// if let Some(data) = result.data {+ /// println!("Extracted pricing: {}", serde_json::to_string_pretty(&data)?);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn agent(+ &self,+ options: AgentOptions,+ ) -> Result<AgentStatusResponse, FirecrawlError> {+ let poll_interval = options.poll_interval.unwrap_or(2000);+ let timeout = options.timeout;++ let response = self.start_agent(options).await?;+ self.wait_for_agent(&response.id, poll_interval, timeout)+ .await+ }++ /// Waits for an agent task to complete.+ async fn wait_for_agent(+ &self,+ id: &str,+ poll_interval: u64,+ timeout: Option<u64>,+ ) -> Result<AgentStatusResponse, FirecrawlError> {+ let start = std::time::Instant::now();++ loop {+ let status = self.get_agent_status(id).await?;++ match status.status {+ AgentStatus::Completed | AgentStatus::Failed | AgentStatus::Cancelled => {+ return Ok(status);+ }+ AgentStatus::Processing => {+ // Check timeout+ if let Some(timeout_secs) = timeout {+ if start.elapsed().as_secs() > timeout_secs {+ return Ok(status);+ }+ }++ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ }+ }+ }++ /// Cancels a running agent task.+ ///+ /// # Arguments+ ///+ /// * `id` - The agent task ID to cancel.+ ///+ /// # Returns+ ///+ /// `true` if the cancellation was successful.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let cancelled = client.cancel_agent("task-id").await?;+ /// println!("Cancelled: {}", cancelled);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn cancel_agent(&self, id: impl AsRef<str>) -> Result<bool, FirecrawlError> {+ let response = self+ .client+ .delete(self.url(&format!("/agent/{}", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Cancelling agent {}", id.as_ref()), e)+ })?;++ #[derive(Deserialize)]+ struct CancelResponse {+ success: bool,+ }++ let result: CancelResponse = self+ .handle_response(response, format!("cancel agent {}", id.as_ref()))+ .await?;++ Ok(result.success)+ }++ /// Runs an agent with a typed schema for structured output.+ ///+ /// This is a convenience method that automatically converts the result+ /// to the specified type.+ ///+ /// # Arguments+ ///+ /// * `urls` - Starting URLs for the agent.+ /// * `prompt` - The task description.+ /// * `schema` - JSON schema for the expected output.+ ///+ /// # Returns+ ///+ /// The extracted data as the specified type, or `None` if extraction failed.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ /// use serde::Deserialize;+ /// use serde_json::json;+ ///+ /// #[derive(Debug, Deserialize)]+ /// struct ProductInfo {+ /// name: String,+ /// price: f64,+ /// description: Option<String>,+ /// }+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let schema = json!({+ /// "type": "object",+ /// "properties": {+ /// "name": { "type": "string" },+ /// "price": { "type": "number" },+ /// "description": { "type": "string" }+ /// },+ /// "required": ["name", "price"]+ /// });+ ///+ /// let result: Option<ProductInfo> = client.agent_with_schema(+ /// vec!["https://example.com/product".to_string()],+ /// "Extract the product information",+ /// schema,+ /// ).await?;+ ///+ /// if let Some(product) = result {+ /// println!("Product: {} - ${}", product.name, product.price);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn agent_with_schema<T: serde::de::DeserializeOwned>(+ &self,+ urls: Vec<String>,+ prompt: impl AsRef<str>,+ schema: Value,+ ) -> Result<Option<T>, FirecrawlError> {+ let options = AgentOptions {+ urls: Some(urls),+ prompt: prompt.as_ref().to_string(),+ schema: Some(schema),+ ..Default::default()+ };++ let result = self.agent(options).await?;++ if result.status != AgentStatus::Completed {+ return Ok(None);+ }++ match result.data {+ Some(data) => {+ let typed: T =+ serde_json::from_value(data).map_err(FirecrawlError::ResponseParseError)?;+ Ok(Some(typed))+ }+ None => Ok(None),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_start_agent_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/agent")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "agent-123"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Find the contact information".to_string(),+ ..Default::default()+ };++ let response = client.start_agent(options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "agent-123");+ mock.assert();+ }++ #[tokio::test]+ async fn test_get_agent_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v2/agent/agent-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "data": {+ "email": "[email protected]",+ "phone": "555-1234"+ },+ "creditsUsed": 5,+ "expiresAt": "2024-12-31T23:59:59Z"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = client.get_agent_status("agent-123").await.unwrap();++ assert!(status.success);+ assert_eq!(status.status, AgentStatus::Completed);+ assert!(status.data.is_some());+ assert_eq!(status.credits_used, Some(5));+ mock.assert();+ }++ #[tokio::test]+ async fn test_agent_sync_with_mock() {+ let mut server = mockito::Server::new_async().await;++ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/agent")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "agent-456"+ })+ .to_string(),+ )+ .create();++ // Mock the status endpoint (completed immediately)+ let status_mock = server+ .mock("GET", "/v2/agent/agent-456")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "data": {+ "result": "Task completed successfully"+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Test task".to_string(),+ ..Default::default()+ };++ let result = client.agent(options).await.unwrap();++ assert_eq!(result.status, AgentStatus::Completed);+ assert!(result.data.is_some());+ start_mock.assert();+ status_mock.assert();+ }++ #[tokio::test]+ async fn test_cancel_agent_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("DELETE", "/v2/agent/agent-789")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let cancelled = client.cancel_agent("agent-789").await.unwrap();++ assert!(cancelled);+ mock.assert();+ }++ #[tokio::test]+ async fn test_agent_with_schema() {+ let mut server = mockito::Server::new_async().await;++ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/agent")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "agent-schema"+ })+ .to_string(),+ )+ .create();++ // Mock the status endpoint+ let status_mock = server+ .mock("GET", "/v2/agent/agent-schema")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "status": "completed",+ "data": {+ "name": "Test Product",+ "price": 29.99+ }+ })+ .to_string(),+ )+ .create();++ #[derive(Debug, serde::Deserialize, PartialEq)]+ struct Product {+ name: String,+ price: f64,+ }++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let schema = json!({+ "type": "object",+ "properties": {+ "name": { "type": "string" },+ "price": { "type": "number" }+ }+ });++ let result: Option<Product> = client+ .agent_with_schema(+ vec!["https://example.com".to_string()],+ "Extract product info",+ schema,+ )+ .await+ .unwrap();++ assert_eq!(+ result,+ Some(Product {+ name: "Test Product".to_string(),+ price: 29.99+ })+ );+ start_mock.assert();+ status_mock.assert();+ }++ #[tokio::test]+ async fn test_agent_with_model_option() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/agent")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "agent-model"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Task with specific model".to_string(),+ model: Some(AgentModel::Spark1Pro),+ max_credits: Some(100),+ ..Default::default()+ };++ let response = client.start_agent(options).await.unwrap();++ assert!(response.success);+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/batch_scrape.rs b/apps/rust-sdk/src/v2/batch_scrape.rsnew file mode 100644index 000000000..cb831e787--- /dev/null+++ b/apps/rust-sdk/src/v2/batch_scrape.rs@@ -0,0 +1,591 @@+//! Batch scrape endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};++use super::client::Client;+use super::scrape::ScrapeOptions;+use super::types::{CrawlErrorsResponse, Document, JobStatus, WebhookConfig};+use crate::FirecrawlError;++/// Options for batch scraping.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeOptions {+ /// Scrape options to apply to all URLs.+ #[serde(flatten)]+ pub options: Option<ScrapeOptions>,++ /// Webhook configuration for job notifications.+ pub webhook: Option<WebhookConfig>,++ /// ID of an existing batch job to append URLs to.+ pub append_to_id: Option<String>,++ /// Whether to ignore invalid URLs instead of failing.+ #[serde(rename = "ignoreInvalidURLs")]+ pub ignore_invalid_urls: Option<bool>,++ /// Maximum concurrent requests.+ pub max_concurrency: Option<u32>,++ /// Enable zero data retention mode.+ pub zero_data_retention: Option<bool>,++ /// Idempotency key for the request.+ #[serde(skip)]+ pub idempotency_key: Option<String>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Poll interval for synchronous batch scrape (milliseconds).+ #[serde(skip)]+ pub poll_interval: Option<u64>,+}++/// Request body for batch scrape endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct BatchScrapeRequest {+ urls: Vec<String>,+ #[serde(flatten)]+ options: BatchScrapeOptions,+}++/// Response from starting a batch scrape job.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// The batch scrape job ID.+ pub id: String,+ /// URL to check the batch scrape status.+ pub url: String,+ /// URLs that were invalid and ignored.+ #[serde(rename = "invalidURLs")]+ pub invalid_urls: Option<Vec<String>>,+}++/// Status of a batch scrape job.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct BatchScrapeJob {+ /// Current status of the batch scrape job.+ pub status: JobStatus,+ /// Number of URLs completed.+ pub completed: u32,+ /// Total number of URLs to scrape.+ pub total: u32,+ /// Credits used by the batch scrape.+ pub credits_used: Option<u32>,+ /// Expiry time of the batch data.+ pub expires_at: Option<String>,+ /// URL for the next page of results.+ pub next: Option<String>,+ /// Scraped documents.+ pub data: Vec<Document>,+}++impl Client {+ /// Starts a batch scrape job asynchronously.+ ///+ /// Returns immediately with a job ID that can be used to check status.+ ///+ /// # Arguments+ ///+ /// * `urls` - List of URLs to scrape.+ /// * `options` - Optional batch scrape configuration.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeResponse` containing the job ID.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, BatchScrapeOptions};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = vec![+ /// "https://example.com".to_string(),+ /// "https://example.org".to_string(),+ /// ];+ ///+ /// let response = client.start_batch_scrape(urls, None).await?;+ /// println!("Batch job started: {}", response.id);+ ///+ /// // Check status later+ /// let status = client.get_batch_scrape_status(&response.id).await?;+ /// println!("Completed: {}/{}", status.completed, status.total);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn start_batch_scrape(+ &self,+ urls: Vec<String>,+ options: impl Into<Option<BatchScrapeOptions>>,+ ) -> Result<BatchScrapeResponse, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let body = BatchScrapeRequest {+ urls,+ options: options.clone(),+ };++ let headers = self.prepare_headers(options.idempotency_key.as_ref());++ let response = self+ .client+ .post(self.url("/batch/scrape"))+ .headers(headers)+ .json(&body)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Starting batch scrape".to_string(), e))?;++ self.handle_response(response, "start batch scrape").await+ }++ /// Gets the status of a batch scrape job.+ ///+ /// If the job is completed, this will automatically fetch all pages of results.+ ///+ /// # Arguments+ ///+ /// * `id` - The batch scrape job ID.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeJob` containing the current status and any available documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let status = client.get_batch_scrape_status("job-id").await?;+ /// println!("Status: {:?}", status.status);+ /// println!("Completed: {}/{}", status.completed, status.total);+ /// println!("Documents: {}", status.data.len());+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_batch_scrape_status(+ &self,+ id: impl AsRef<str>,+ ) -> Result<BatchScrapeJob, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/batch/scrape/{}", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(+ format!("Checking batch scrape status {}", id.as_ref()),+ e,+ )+ })?;++ let mut status: BatchScrapeJob = self+ .handle_response(response, format!("batch scrape status {}", id.as_ref()))+ .await?;++ // Auto-paginate if completed+ if status.status == JobStatus::Completed {+ while let Some(next) = status.next.take() {+ let next_status = self.get_batch_scrape_status_next(&next).await?;+ status.data.extend(next_status.data);+ status.next = next_status.next;+ }+ }++ Ok(status)+ }++ /// Fetches the next page of batch scrape results.+ async fn get_batch_scrape_status_next(+ &self,+ next: &str,+ ) -> Result<BatchScrapeJob, FirecrawlError> {+ let response = self+ .client+ .get(next)+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Paginating batch scrape at {}", next), e)+ })?;++ self.handle_response(response, "batch scrape pagination")+ .await+ }++ /// Scrapes multiple URLs and waits for completion.+ ///+ /// This method starts a batch scrape and polls until it completes or fails.+ ///+ /// # Arguments+ ///+ /// * `urls` - List of URLs to scrape.+ /// * `options` - Optional batch scrape configuration.+ ///+ /// # Returns+ ///+ /// A `BatchScrapeJob` containing all scraped documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, BatchScrapeOptions, ScrapeOptions, Format};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = vec![+ /// "https://example.com/page1".to_string(),+ /// "https://example.com/page2".to_string(),+ /// "https://example.com/page3".to_string(),+ /// ];+ ///+ /// let options = BatchScrapeOptions {+ /// options: Some(ScrapeOptions {+ /// formats: Some(vec![Format::Markdown, Format::Links]),+ /// ..Default::default()+ /// }),+ /// ignore_invalid_urls: Some(true),+ /// poll_interval: Some(3000),+ /// ..Default::default()+ /// };+ ///+ /// let result = client.batch_scrape(urls, options).await?;+ /// println!("Scraped {} pages", result.data.len());+ ///+ /// for doc in result.data {+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn batch_scrape(+ &self,+ urls: Vec<String>,+ options: impl Into<Option<BatchScrapeOptions>>,+ ) -> Result<BatchScrapeJob, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let poll_interval = options.poll_interval.unwrap_or(2000);++ let response = self.start_batch_scrape(urls, options).await?;+ self.wait_for_batch_scrape(&response.id, poll_interval)+ .await+ }++ /// Waits for a batch scrape job to complete.+ async fn wait_for_batch_scrape(+ &self,+ id: &str,+ poll_interval: u64,+ ) -> Result<BatchScrapeJob, FirecrawlError> {+ loop {+ let status = self.get_batch_scrape_status(id).await?;++ match status.status {+ JobStatus::Completed => return Ok(status),+ JobStatus::Scraping => {+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ JobStatus::Failed => {+ return Err(FirecrawlError::CrawlJobFailed(+ "Batch scrape job failed".to_string(),+ convert_batch_job_to_crawl_status(status),+ ));+ }+ JobStatus::Cancelled => {+ return Err(FirecrawlError::CrawlJobFailed(+ "Batch scrape job was cancelled".to_string(),+ convert_batch_job_to_crawl_status(status),+ ));+ }+ }+ }+ }++ /// Gets errors from a batch scrape job.+ ///+ /// # Arguments+ ///+ /// * `id` - The batch scrape job ID.+ ///+ /// # Returns+ ///+ /// A `CrawlErrorsResponse` containing error details.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let errors = client.get_batch_scrape_errors("job-id").await?;+ /// for error in errors.errors {+ /// println!("Error on {}: {}", error.url, error.error);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_batch_scrape_errors(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/batch/scrape/{}/errors", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Getting batch scrape errors {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "batch scrape errors").await+ }+}++/// Converts a BatchScrapeJob to CrawlStatus for error compatibility.+fn convert_batch_job_to_crawl_status(job: BatchScrapeJob) -> crate::crawl::CrawlStatus {+ crate::crawl::CrawlStatus {+ status: match job.status {+ JobStatus::Completed => crate::crawl::CrawlStatusTypes::Completed,+ JobStatus::Scraping => crate::crawl::CrawlStatusTypes::Scraping,+ JobStatus::Failed => crate::crawl::CrawlStatusTypes::Failed,+ JobStatus::Cancelled => crate::crawl::CrawlStatusTypes::Cancelled,+ },+ total: job.total,+ completed: job.completed,+ credits_used: job.credits_used.unwrap_or(0),+ expires_at: job.expires_at.unwrap_or_default(),+ next: job.next,+ data: job+ .data+ .into_iter()+ .map(|d| super::crawl::convert_v2_document_to_v1_pub(d))+ .collect(),+ }+}++impl super::crawl::CrawlJob {+ /// Helper function to convert v2 document to v1 format.+ pub(crate) fn _convert_document(doc: Document) -> crate::document::Document {+ super::crawl::convert_v2_document_to_v1_pub(doc)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_start_batch_scrape_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/batch/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-123",+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-123"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ];++ let response = client.start_batch_scrape(urls, None).await.unwrap();++ assert!(response.success);+ assert_eq!(response.id, "batch-123");+ mock.assert();+ }++ #[tokio::test]+ async fn test_get_batch_scrape_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v2/batch/scrape/batch-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "completed",+ "total": 3,+ "completed": 3,+ "creditsUsed": 3,+ "data": [+ {+ "markdown": "# Page 1",+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }+ },+ {+ "markdown": "# Page 2",+ "metadata": { "sourceURL": "https://example.com/2", "statusCode": 200 }+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = client.get_batch_scrape_status("batch-123").await.unwrap();++ assert_eq!(status.status, JobStatus::Completed);+ assert_eq!(status.total, 3);+ assert_eq!(status.completed, 3);+ assert_eq!(status.data.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_batch_scrape_with_invalid_urls() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/batch/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-456",+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-456",+ "invalidURLs": ["not-a-url"]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec!["https://example.com".to_string(), "not-a-url".to_string()];++ let options = BatchScrapeOptions {+ ignore_invalid_urls: Some(true),+ ..Default::default()+ };++ let response = client.start_batch_scrape(urls, options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.invalid_urls, Some(vec!["not-a-url".to_string()]));+ mock.assert();+ }++ #[tokio::test]+ async fn test_batch_scrape_sync() {+ let mut server = mockito::Server::new_async().await;++ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/batch/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "batch-789",+ "url": "https://api.firecrawl.dev/v2/batch/scrape/batch-789"+ })+ .to_string(),+ )+ .create();++ // Mock the status endpoint (completed immediately)+ let status_mock = server+ .mock("GET", "/v2/batch/scrape/batch-789")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "completed",+ "total": 2,+ "completed": 2,+ "data": [+ {+ "markdown": "# Content",+ "metadata": { "sourceURL": "https://example.com", "statusCode": 200 }+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = vec!["https://example.com".to_string()];++ let result = client.batch_scrape(urls, None).await.unwrap();++ assert_eq!(result.status, JobStatus::Completed);+ assert_eq!(result.data.len(), 1);+ start_mock.assert();+ status_mock.assert();+ }++ #[tokio::test]+ async fn test_get_batch_scrape_errors() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v2/batch/scrape/batch-123/errors")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "errors": [+ {+ "id": "err-1",+ "url": "https://example.com/broken",+ "error": "Connection timeout"+ }+ ],+ "robotsBlocked": []+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let errors = client.get_batch_scrape_errors("batch-123").await.unwrap();++ assert_eq!(errors.errors.len(), 1);+ assert_eq!(errors.errors[0].error, "Connection timeout");+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/client.rs b/apps/rust-sdk/src/v2/client.rsnew file mode 100644index 000000000..f3881d83d--- /dev/null+++ b/apps/rust-sdk/src/v2/client.rs@@ -0,0 +1,273 @@+//! Firecrawl API v2 client.++use reqwest::Response;+use serde::de::DeserializeOwned;+use serde_json::Value;++use crate::error::{FirecrawlAPIError, FirecrawlError};++pub(crate) const API_VERSION: &str = "/v2";+const CLOUD_API_URL: &str = "https://api.firecrawl.dev";++/// Firecrawl API v2 client.+///+/// This client provides access to all v2 API endpoints including scrape, crawl,+/// search, map, batch scrape, and agent operations.+///+/// # Example+///+/// ```no_run+/// use firecrawl::v2::Client;+///+/// #[tokio::main]+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {+/// // Create a client for the Firecrawl cloud service+/// let client = Client::new("your-api-key")?;+///+/// // Or create a client for a self-hosted instance+/// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key"))?;+///+/// Ok(())+/// }+/// ```+#[derive(Clone, Debug)]+pub struct Client {+ pub(crate) api_key: Option<String>,+ pub(crate) api_url: String,+ pub(crate) client: reqwest::Client,+}++impl Client {+ /// Creates a new client for the Firecrawl cloud service.+ ///+ /// # Arguments+ ///+ /// * `api_key` - Your Firecrawl API key.+ ///+ /// # Errors+ ///+ /// Returns an error if the API key is empty.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// let client = Client::new("your-api-key").unwrap();+ /// ```+ pub fn new(api_key: impl AsRef<str>) -> Result<Self, FirecrawlError> {+ Client::new_selfhosted(CLOUD_API_URL, Some(api_key))+ }++ /// Creates a new client for a self-hosted Firecrawl instance.+ ///+ /// # Arguments+ ///+ /// * `api_url` - The base URL of your Firecrawl instance.+ /// * `api_key` - Optional API key (required for cloud, optional for self-hosted).+ ///+ /// # Errors+ ///+ /// Returns an error if using the cloud service without an API key.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// // Self-hosted without authentication+ /// let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();+ ///+ /// // Self-hosted with authentication+ /// let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();+ /// ```+ pub fn new_selfhosted(+ api_url: impl AsRef<str>,+ api_key: Option<impl AsRef<str>>,+ ) -> Result<Self, FirecrawlError> {+ // Normalize URL by trimming trailing slashes for consistent comparison+ let url = api_url.as_ref().trim_end_matches('/').to_string();+ let api_key = api_key.map(|k| k.as_ref().to_string());++ // Reject empty or missing API key for cloud service+ if url == CLOUD_API_URL {+ match &api_key {+ None => {+ return Err(FirecrawlError::APIError(+ "Configuration".to_string(),+ FirecrawlAPIError {+ success: false,+ error: "API key is required for cloud service".to_string(),+ details: None,+ },+ ));+ }+ Some(key) if key.trim().is_empty() => {+ return Err(FirecrawlError::APIError(+ "Configuration".to_string(),+ FirecrawlAPIError {+ success: false,+ error: "API key cannot be empty for cloud service".to_string(),+ details: None,+ },+ ));+ }+ _ => {}+ }+ }++ Ok(Client {+ api_key,+ api_url: url,+ client: reqwest::Client::new(),+ })+ }++ /// Prepares headers for API requests.+ pub(crate) fn prepare_headers(+ &self,+ idempotency_key: Option<&String>,+ ) -> reqwest::header::HeaderMap {+ use reqwest::header::HeaderValue;++ let mut headers = reqwest::header::HeaderMap::new();+ // Static string is always valid ASCII+ headers.insert("Content-Type", HeaderValue::from_static("application/json"));+ if let Some(api_key) = self.api_key.as_ref() {+ // API key is validated at client creation, so this should always succeed.+ // Use if-let to gracefully handle edge cases without panicking.+ if let Ok(value) = format!("Bearer {}", api_key).parse() {+ headers.insert("Authorization", value);+ }+ }+ if let Some(key) = idempotency_key {+ // Gracefully skip invalid idempotency keys instead of panicking+ if let Ok(value) = key.parse() {+ headers.insert("x-idempotency-key", value);+ }+ }+ headers+ }++ /// Handles API responses, parsing JSON and handling errors.+ pub(crate) async fn handle_response<T: DeserializeOwned>(+ &self,+ response: Response,+ action: impl AsRef<str>,+ ) -> Result<T, FirecrawlError> {+ let (is_success, status) = (response.status().is_success(), response.status());++ let response = response+ .text()+ .await+ .map_err(FirecrawlError::ResponseParseErrorText)+ .and_then(|response_json| {+ serde_json::from_str::<Value>(&response_json)+ .map_err(FirecrawlError::ResponseParseError)+ .inspect(|data| {+ tracing::debug!("Response JSON: {:#?}", data);+ })+ })+ .and_then(|response_value| {+ // Check for success field, or allow responses without it for status checks+ if action.as_ref().contains("status")+ || action.as_ref().contains("cancel")+ || response_value["success"].as_bool().unwrap_or(false)+ || response_value.get("success").is_none()+ {+ serde_json::from_value::<T>(response_value)+ .map_err(FirecrawlError::ResponseParseError)+ } else {+ Err(FirecrawlError::APIError(+ action.as_ref().to_string(),+ serde_json::from_value(response_value)+ .map_err(FirecrawlError::ResponseParseError)?,+ ))+ }+ });++ match &response {+ Ok(_) => response,+ Err(FirecrawlError::ResponseParseError(_))+ | Err(FirecrawlError::ResponseParseErrorText(_)) => {+ if is_success {+ response+ } else {+ Err(FirecrawlError::HttpRequestFailed(+ action.as_ref().to_string(),+ status.as_u16(),+ status.as_str().to_string(),+ ))+ }+ }+ Err(_) => response,+ }+ }++ /// Builds the full URL for an API endpoint.+ pub(crate) fn url(&self, path: &str) -> String {+ format!("{}{}{}", self.api_url, API_VERSION, path)+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_new_client() {+ let client = Client::new("test-api-key").unwrap();+ assert_eq!(client.api_key, Some("test-api-key".to_string()));+ assert_eq!(client.api_url, CLOUD_API_URL);+ }++ #[test]+ fn test_new_client_requires_api_key_for_cloud() {+ let result = Client::new_selfhosted(CLOUD_API_URL, None::<&str>);+ assert!(result.is_err());+ }++ #[test]+ fn test_new_client_rejects_empty_api_key_for_cloud() {+ let result = Client::new_selfhosted(CLOUD_API_URL, Some(""));+ assert!(result.is_err());++ let result = Client::new_selfhosted(CLOUD_API_URL, Some(" "));+ assert!(result.is_err());+ }++ #[test]+ fn test_new_selfhosted_client() {+ let client = Client::new_selfhosted("http://localhost:3000", Some("api-key")).unwrap();+ assert_eq!(client.api_key, Some("api-key".to_string()));+ assert_eq!(client.api_url, "http://localhost:3000");+ }++ #[test]+ fn test_selfhosted_without_api_key() {+ let client = Client::new_selfhosted("http://localhost:3000", None::<&str>).unwrap();+ assert_eq!(client.api_key, None);+ assert_eq!(client.api_url, "http://localhost:3000");+ }++ #[test]+ fn test_url_builder() {+ let client = Client::new("test-key").unwrap();+ assert_eq!(client.url("/scrape"), "https://api.firecrawl.dev/v2/scrape");+ }++ #[test]+ fn test_url_normalization_trailing_slash() {+ // Cloud URL with trailing slash should still require API key+ let result = Client::new_selfhosted("https://api.firecrawl.dev/", None::<&str>);+ assert!(result.is_err());++ // Should work with API key+ let client = Client::new_selfhosted("https://api.firecrawl.dev/", Some("key")).unwrap();+ assert_eq!(client.api_url, "https://api.firecrawl.dev");++ // Self-hosted URL normalization+ let client = Client::new_selfhosted("http://localhost:3000/", None::<&str>).unwrap();+ assert_eq!(client.api_url, "http://localhost:3000");+ }+}diff --git a/apps/rust-sdk/src/v2/crawl.rs b/apps/rust-sdk/src/v2/crawl.rsnew file mode 100644index 000000000..4b773abe3--- /dev/null+++ b/apps/rust-sdk/src/v2/crawl.rs@@ -0,0 +1,688 @@+//! Crawl endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};++use super::client::Client;+use super::scrape::ScrapeOptions;+use super::types::{CrawlErrorsResponse, Document, JobStatus, SitemapMode, WebhookConfig};+use crate::FirecrawlError;++/// Options for crawling a website.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlOptions {+ /// Natural language prompt to guide crawl behavior.+ pub prompt: Option<String>,++ /// URL path patterns to exclude from crawling.+ pub exclude_paths: Option<Vec<String>>,++ /// URL path patterns to include in crawling.+ pub include_paths: Option<Vec<String>>,++ /// Maximum depth of links to follow from the initial URL.+ pub max_discovery_depth: Option<u32>,++ /// How to handle the sitemap.+ pub sitemap: Option<SitemapMode>,++ /// Ignore query parameters when deduplicating URLs.+ pub ignore_query_parameters: Option<bool>,++ /// Maximum number of pages to crawl.+ pub limit: Option<u32>,++ /// Crawl the entire domain regardless of path structure.+ pub crawl_entire_domain: Option<bool>,++ /// Allow following links to external domains.+ pub allow_external_links: Option<bool>,++ /// Allow following links to subdomains.+ pub allow_subdomains: Option<bool>,++ /// Delay between requests in seconds.+ pub delay: Option<u32>,++ /// Maximum concurrent requests.+ pub max_concurrency: Option<u32>,++ /// Webhook configuration for job notifications.+ pub webhook: Option<WebhookConfig>,++ /// Scrape options to apply to each page.+ pub scrape_options: Option<ScrapeOptions>,++ /// Enable zero data retention mode.+ pub zero_data_retention: Option<bool>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Idempotency key for the request.+ #[serde(skip)]+ pub idempotency_key: Option<String>,++ /// Poll interval for synchronous crawl (milliseconds).+ #[serde(skip)]+ pub poll_interval: Option<u64>,+}++/// Request body for crawl endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct CrawlRequest {+ url: String,+ #[serde(flatten)]+ options: CrawlOptions,+}++/// Response from starting a crawl job.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// The crawl job ID.+ pub id: String,+ /// URL to check the crawl status.+ pub url: String,+}++/// Status of a crawl job.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlJob {+ /// Current status of the crawl job.+ pub status: JobStatus,+ /// Total number of pages to crawl.+ pub total: u32,+ /// Number of pages completed.+ pub completed: u32,+ /// Credits used by the crawl.+ pub credits_used: Option<u32>,+ /// Expiry time of the crawl data.+ pub expires_at: Option<String>,+ /// URL for the next page of results.+ pub next: Option<String>,+ /// Crawled documents.+ pub data: Vec<Document>,+}++/// Response from canceling a crawl.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CancelCrawlResponse {+ /// Status of the cancellation.+ pub status: String,+}++impl Client {+ /// Starts a crawl job asynchronously.+ ///+ /// Returns immediately with a job ID that can be used to check status.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to start crawling from.+ /// * `options` - Optional crawl configuration.+ ///+ /// # Returns+ ///+ /// A `CrawlResponse` containing the job ID.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, CrawlOptions};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let response = client.start_crawl("https://example.com", None).await?;+ /// println!("Crawl job started: {}", response.id);+ ///+ /// // Check status later+ /// let status = client.get_crawl_status(&response.id).await?;+ /// println!("Status: {:?}, Completed: {}/{}", status.status, status.completed, status.total);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn start_crawl(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<CrawlOptions>>,+ ) -> Result<CrawlResponse, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let body = CrawlRequest {+ url: url.as_ref().to_string(),+ options: options.clone(),+ };++ let headers = self.prepare_headers(options.idempotency_key.as_ref());++ let response = self+ .client+ .post(self.url("/crawl"))+ .headers(headers)+ .json(&body)+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Starting crawl of {:?}", url.as_ref()), e)+ })?;++ self.handle_response(response, "start crawl").await+ }++ /// Gets the status of a crawl job.+ ///+ /// If the job is completed, this will automatically fetch all pages of results.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID.+ ///+ /// # Returns+ ///+ /// A `CrawlJob` containing the current status and any available documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let status = client.get_crawl_status("job-id").await?;+ /// println!("Status: {:?}", status.status);+ /// println!("Completed: {}/{}", status.completed, status.total);+ /// println!("Documents: {}", status.data.len());+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_crawl_status(&self, id: impl AsRef<str>) -> Result<CrawlJob, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/crawl/{}", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Checking crawl status {}", id.as_ref()), e)+ })?;++ let mut status: CrawlJob = self+ .handle_response(response, format!("crawl status {}", id.as_ref()))+ .await?;++ // Auto-paginate if completed+ if status.status == JobStatus::Completed {+ while let Some(next) = status.next.take() {+ let next_status = self.get_crawl_status_next(&next).await?;+ status.data.extend(next_status.data);+ status.next = next_status.next;+ }+ }++ Ok(status)+ }++ /// Fetches the next page of crawl results.+ async fn get_crawl_status_next(&self, next: &str) -> Result<CrawlJob, FirecrawlError> {+ let response = self+ .client+ .get(next)+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError(format!("Paginating crawl at {}", next), e))?;++ self.handle_response(response, "crawl pagination").await+ }++ /// Crawls a website and waits for completion.+ ///+ /// This method starts a crawl and polls until it completes or fails.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to start crawling from.+ /// * `options` - Optional crawl configuration.+ ///+ /// # Returns+ ///+ /// A `CrawlJob` containing all crawled documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, CrawlOptions, SitemapMode};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let options = CrawlOptions {+ /// sitemap: Some(SitemapMode::Include),+ /// limit: Some(100),+ /// poll_interval: Some(5000), // Check every 5 seconds+ /// ..Default::default()+ /// };+ ///+ /// let result = client.crawl("https://example.com", options).await?;+ /// println!("Crawled {} pages", result.data.len());+ ///+ /// for doc in result.data {+ /// println!("URL: {:?}", doc.metadata.and_then(|m| m.source_url));+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn crawl(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<CrawlOptions>>,+ ) -> Result<CrawlJob, FirecrawlError> {+ let options = options.into().unwrap_or_default();+ let poll_interval = options.poll_interval.unwrap_or(2000);++ let response = self.start_crawl(url, options).await?;+ self.wait_for_crawl(&response.id, poll_interval).await+ }++ /// Waits for a crawl job to complete.+ async fn wait_for_crawl(+ &self,+ id: &str,+ poll_interval: u64,+ ) -> Result<CrawlJob, FirecrawlError> {+ loop {+ let status = self.get_crawl_status(id).await?;++ match status.status {+ JobStatus::Completed => return Ok(status),+ JobStatus::Scraping => {+ tokio::time::sleep(tokio::time::Duration::from_millis(poll_interval)).await;+ }+ JobStatus::Failed => {+ return Err(FirecrawlError::CrawlJobFailed(+ "Crawl job failed".to_string(),+ crate::crawl::CrawlStatus {+ status: crate::crawl::CrawlStatusTypes::Failed,+ total: status.total,+ completed: status.completed,+ credits_used: status.credits_used.unwrap_or(0),+ expires_at: status.expires_at.unwrap_or_default(),+ next: status.next,+ data: status+ .data+ .into_iter()+ .map(|d| convert_v2_document_to_v1(d))+ .collect(),+ },+ ));+ }+ JobStatus::Cancelled => {+ return Err(FirecrawlError::CrawlJobFailed(+ "Crawl job was cancelled".to_string(),+ crate::crawl::CrawlStatus {+ status: crate::crawl::CrawlStatusTypes::Cancelled,+ total: status.total,+ completed: status.completed,+ credits_used: status.credits_used.unwrap_or(0),+ expires_at: status.expires_at.unwrap_or_default(),+ next: status.next,+ data: status+ .data+ .into_iter()+ .map(|d| convert_v2_document_to_v1(d))+ .collect(),+ },+ ));+ }+ }+ }+ }++ /// Cancels a running crawl job.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID to cancel.+ ///+ /// # Returns+ ///+ /// A `CancelCrawlResponse` indicating the cancellation status.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let response = client.cancel_crawl("job-id").await?;+ /// println!("Cancellation status: {}", response.status);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn cancel_crawl(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CancelCrawlResponse, FirecrawlError> {+ let response = self+ .client+ .delete(self.url(&format!("/crawl/{}", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Cancelling crawl {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "cancel crawl").await+ }++ /// Gets errors from a crawl job.+ ///+ /// # Arguments+ ///+ /// * `id` - The crawl job ID.+ ///+ /// # Returns+ ///+ /// A `CrawlErrorsResponse` containing error details.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let errors = client.get_crawl_errors("job-id").await?;+ /// for error in errors.errors {+ /// println!("Error on {}: {}", error.url, error.error);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn get_crawl_errors(+ &self,+ id: impl AsRef<str>,+ ) -> Result<CrawlErrorsResponse, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/crawl/{}/errors", id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Getting crawl errors {}", id.as_ref()), e)+ })?;++ self.handle_response(response, "crawl errors").await+ }+}++/// Converts a v2 Document to a v1 Document for error compatibility.+pub(crate) fn convert_v2_document_to_v1_pub(doc: Document) -> crate::document::Document {+ convert_v2_document_to_v1(doc)+}++/// Converts a v2 Document to a v1 Document for error compatibility.+fn convert_v2_document_to_v1(doc: Document) -> crate::document::Document {+ let metadata = doc.metadata.unwrap_or_default();+ crate::document::Document {+ markdown: doc.markdown,+ html: doc.html,+ raw_html: doc.raw_html,+ screenshot: doc.screenshot,+ links: doc.links,+ extract: doc.json,+ metadata: crate::document::DocumentMetadata {+ source_url: metadata.source_url.unwrap_or_default(),+ status_code: metadata.status_code.unwrap_or(0),+ error: metadata.error,+ title: metadata.title,+ description: metadata.description,+ language: metadata.language,+ keywords: metadata.keywords,+ robots: metadata.robots,+ og_title: metadata.og_title,+ og_description: metadata.og_description,+ og_url: metadata.og_url,+ og_image: metadata.og_image,+ og_audio: metadata.og_audio,+ og_determiner: metadata.og_determiner,+ og_locale: metadata.og_locale,+ og_locale_alternate: metadata.og_locale_alternate,+ og_site_name: metadata.og_site_name,+ og_video: metadata.og_video,+ article_section: metadata.article_section,+ article_tag: metadata.article_tag,+ published_time: metadata.published_time,+ modified_time: metadata.modified_time,+ dcterms_keywords: metadata.dcterms_keywords,+ dc_description: metadata.dc_description,+ dc_subject: metadata.dc_subject,+ dcterms_subject: metadata.dcterms_subject,+ dcterms_audience: metadata.dcterms_audience,+ dc_type: metadata.dc_type,+ dcterms_type: metadata.dcterms_type,+ dc_date: metadata.dc_date,+ dc_date_created: metadata.dc_date_created,+ dcterms_created: metadata.dcterms_created,+ },+ warning: doc.warning,+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_start_crawl_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/crawl")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "crawl-123",+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-123"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client+ .start_crawl("https://example.com", None)+ .await+ .unwrap();++ assert!(response.success);+ assert_eq!(response.id, "crawl-123");+ mock.assert();+ }++ #[tokio::test]+ async fn test_get_crawl_status_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v2/crawl/crawl-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "completed",+ "total": 5,+ "completed": 5,+ "creditsUsed": 5,+ "expiresAt": "2024-12-31T23:59:59Z",+ "data": [+ {+ "markdown": "# Page 1",+ "metadata": {+ "sourceURL": "https://example.com/page1",+ "statusCode": 200+ }+ },+ {+ "markdown": "# Page 2",+ "metadata": {+ "sourceURL": "https://example.com/page2",+ "statusCode": 200+ }+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let status = client.get_crawl_status("crawl-123").await.unwrap();++ assert_eq!(status.status, JobStatus::Completed);+ assert_eq!(status.total, 5);+ assert_eq!(status.completed, 5);+ assert_eq!(status.data.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_cancel_crawl_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("DELETE", "/v2/crawl/crawl-123")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "cancelled"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.cancel_crawl("crawl-123").await.unwrap();++ assert_eq!(response.status, "cancelled");+ mock.assert();+ }++ #[tokio::test]+ async fn test_get_crawl_errors_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("GET", "/v2/crawl/crawl-123/errors")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "errors": [+ {+ "id": "error-1",+ "timestamp": "2024-01-01T00:00:00Z",+ "url": "https://example.com/broken",+ "error": "404 Not Found"+ }+ ],+ "robotsBlocked": [+ "https://example.com/admin"+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let errors = client.get_crawl_errors("crawl-123").await.unwrap();++ assert_eq!(errors.errors.len(), 1);+ assert_eq!(errors.errors[0].url, "https://example.com/broken");+ assert_eq!(errors.robots_blocked.len(), 1);+ mock.assert();+ }++ #[tokio::test]+ async fn test_crawl_with_options() {+ let mut server = mockito::Server::new_async().await;++ // Mock the start endpoint+ let start_mock = server+ .mock("POST", "/v2/crawl")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "id": "crawl-456",+ "url": "https://api.firecrawl.dev/v2/crawl/crawl-456"+ })+ .to_string(),+ )+ .create();++ // Mock the status endpoint (completed immediately)+ let status_mock = server+ .mock("GET", "/v2/crawl/crawl-456")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "status": "completed",+ "total": 2,+ "completed": 2,+ "data": [+ {+ "markdown": "# Page 1",+ "metadata": { "sourceURL": "https://example.com/1", "statusCode": 200 }+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = CrawlOptions {+ limit: Some(10),+ sitemap: Some(SitemapMode::Include),+ ..Default::default()+ };++ let result = client.crawl("https://example.com", options).await.unwrap();++ assert_eq!(result.status, JobStatus::Completed);+ assert_eq!(result.data.len(), 1);+ start_mock.assert();+ status_mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/map.rs b/apps/rust-sdk/src/v2/map.rsnew file mode 100644index 000000000..ce2dc54ca--- /dev/null+++ b/apps/rust-sdk/src/v2/map.rs@@ -0,0 +1,333 @@+//! Map endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};++use super::client::Client;+use super::types::{LocationConfig, SearchResultWeb, SitemapMode};+use crate::FirecrawlError;++/// Options for mapping a URL.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MapOptions {+ /// Search query to filter discovered links.+ pub search: Option<String>,++ /// How to handle the sitemap.+ pub sitemap: Option<SitemapMode>,++ /// Include subdomains in the mapping.+ pub include_subdomains: Option<bool>,++ /// Ignore query parameters when deduplicating URLs.+ pub ignore_query_parameters: Option<bool>,++ /// Maximum number of links to return.+ pub limit: Option<u32>,++ /// Timeout in milliseconds.+ pub timeout: Option<u32>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// Location configuration for proxy routing.+ pub location: Option<LocationConfig>,+}++/// Request body for map endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct MapRequest {+ url: String,+ #[serde(flatten)]+ options: MapOptions,+}++/// Response from map endpoint.+#[derive(Deserialize, Serialize, Debug)]+#[serde(rename_all = "camelCase")]+pub struct MapResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// Discovered links with metadata.+ pub links: Vec<SearchResultWeb>,+ /// Warning message if any.+ pub warning: Option<String>,+}++impl Client {+ /// Maps a URL to discover all associated links.+ ///+ /// This endpoint discovers links from a website's sitemap, page content,+ /// and other sources without fully scraping each page.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to map.+ /// * `options` - Optional mapping configuration.+ ///+ /// # Returns+ ///+ /// A `MapResponse` containing the discovered links.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, MapOptions, SitemapMode};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple map+ /// let response = client.map("https://example.com", None).await?;+ /// println!("Found {} links", response.links.len());+ ///+ /// // Map with options+ /// let options = MapOptions {+ /// sitemap: Some(SitemapMode::Include),+ /// include_subdomains: Some(true),+ /// limit: Some(1000),+ /// ..Default::default()+ /// };+ /// let response = client.map("https://example.com", options).await?;+ ///+ /// for link in response.links {+ /// println!("URL: {}, Title: {:?}", link.url, link.title);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn map(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<MapOptions>>,+ ) -> Result<MapResponse, FirecrawlError> {+ let body = MapRequest {+ url: url.as_ref().to_string(),+ options: options.into().unwrap_or_default(),+ };++ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(self.url("/map"))+ .headers(headers)+ .json(&body)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError(format!("Mapping {:?}", url.as_ref()), e))?;++ self.handle_response(response, "map").await+ }++ /// Maps a URL and returns just the list of URLs.+ ///+ /// This is a convenience method that returns only the URL strings.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to map.+ /// * `options` - Optional mapping configuration.+ ///+ /// # Returns+ ///+ /// A vector of discovered URL strings.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let urls = client.map_urls("https://example.com", None).await?;+ /// for url in urls {+ /// println!("{}", url);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn map_urls(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<MapOptions>>,+ ) -> Result<Vec<String>, FirecrawlError> {+ let response = self.map(url, options).await?;+ Ok(response.links.into_iter().map(|link| link.url).collect())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_map_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ {+ "url": "https://example.com/",+ "title": "Example Domain",+ "description": "Home page"+ },+ {+ "url": "https://example.com/about",+ "title": "About Us"+ },+ {+ "url": "https://example.com/contact"+ }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.map("https://example.com", None).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 3);+ assert_eq!(response.links[0].url, "https://example.com/");+ assert_eq!(response.links[0].title, Some("Example Domain".to_string()));+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_with_options() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/page1" },+ { "url": "https://example.com/page2" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = MapOptions {+ sitemap: Some(SitemapMode::Include),+ include_subdomains: Some(true),+ limit: Some(100),+ ..Default::default()+ };++ let response = client.map("https://example.com", options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_urls() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/page1" },+ { "url": "https://example.com/page2" },+ { "url": "https://example.com/page3" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let urls = client.map_urls("https://example.com", None).await.unwrap();++ assert_eq!(urls.len(), 3);+ assert_eq!(urls[0], "https://example.com/page1");+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_with_search_filter() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "links": [+ { "url": "https://example.com/blog/post1" },+ { "url": "https://example.com/blog/post2" }+ ]+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = MapOptions {+ search: Some("blog".to_string()),+ ..Default::default()+ };++ let response = client.map("https://example.com", options).await.unwrap();++ assert!(response.success);+ assert_eq!(response.links.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_map_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/map")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid URL"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.map("invalid-url", None).await;++ assert!(result.is_err());+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/mod.rs b/apps/rust-sdk/src/v2/mod.rsnew file mode 100644index 000000000..caa1a4fcb--- /dev/null+++ b/apps/rust-sdk/src/v2/mod.rs@@ -0,0 +1,40 @@+//! Firecrawl API v2 client module.+//!+//! This module provides access to the v2 API endpoints while maintaining+//! backward compatibility with v1. The v2 API includes new features like+//! the Agent endpoint and improved options.+//!+//! # Example+//!+//! ```no_run+//! use firecrawl::v2::{Client, ScrapeOptions};+//!+//! #[tokio::main]+//! async fn main() -> Result<(), Box<dyn std::error::Error>> {+//! let client = Client::new("your-api-key")?;+//!+//! // Scrape a URL+//! let document = client.scrape("https://example.com", None).await?;+//! println!("Content: {:?}", document.markdown);+//!+//! Ok(())+//! }+//! ```++mod agent;+mod batch_scrape;+mod client;+mod crawl;+mod map;+mod scrape;+mod search;+mod types;++pub use agent::*;+pub use batch_scrape::*;+pub use client::Client;+pub use crawl::*;+pub use map::*;+pub use scrape::*;+pub use search::*;+pub use types::*;diff --git a/apps/rust-sdk/src/v2/scrape.rs b/apps/rust-sdk/src/v2/scrape.rsnew file mode 100644index 000000000..83c934b04--- /dev/null+++ b/apps/rust-sdk/src/v2/scrape.rs@@ -0,0 +1,404 @@+//! Scrape endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};+use serde_json::Value;+use std::collections::HashMap;++use super::client::Client;+use super::types::{+ Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,+ LocationConfig, ProxyType, ScreenshotOptions,+};+use crate::FirecrawlError;++/// Options for scraping a URL.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeOptions {+ /// Output formats to include in the response.+ pub formats: Option<Vec<Format>>,++ /// Additional HTTP headers to send with the request.+ pub headers: Option<HashMap<String, String>>,++ /// HTML tags to exclusively include in the output.+ pub include_tags: Option<Vec<String>>,++ /// HTML tags to exclude from the output.+ pub exclude_tags: Option<Vec<String>>,++ /// Only extract the main content of the page.+ pub only_main_content: Option<bool>,++ /// Timeout in milliseconds before returning an error.+ pub timeout: Option<u32>,++ /// Time to wait after page load before scraping (milliseconds).+ pub wait_for: Option<u32>,++ /// Emulate a mobile device.+ pub mobile: Option<bool>,++ /// Parser configurations (e.g., for PDFs).+ pub parsers: Option<Vec<ParserConfig>>,++ /// Browser automation actions to perform before scraping.+ pub actions: Option<Vec<Action>>,++ /// Location configuration for proxy routing.+ pub location: Option<LocationConfig>,++ /// Skip TLS certificate verification.+ pub skip_tls_verification: Option<bool>,++ /// Remove base64-encoded images from the output.+ pub remove_base64_images: Option<bool>,++ /// Enable fast mode for quicker scrapes with reduced accuracy.+ pub fast_mode: Option<bool>,++ /// Block advertisements on the page.+ pub block_ads: Option<bool>,++ /// Proxy type to use.+ pub proxy: Option<ProxyType>,++ /// Maximum age of cached content to accept (seconds).+ pub max_age: Option<u32>,++ /// Minimum age of cached content to accept (seconds).+ pub min_age: Option<u32>,++ /// Store the result in cache for future requests.+ pub store_in_cache: Option<bool>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,++ /// JSON extraction options.+ pub json_options: Option<JsonOptions>,++ /// Screenshot options.+ pub screenshot_options: Option<ScreenshotOptions>,++ /// Change tracking options.+ pub change_tracking_options: Option<ChangeTrackingOptions>,++ /// Attribute selectors for extraction.+ pub attribute_selectors: Option<Vec<AttributeSelector>>,+}++/// Parser configuration for document parsing.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(untagged)]+pub enum ParserConfig {+ /// Simple parser type string.+ Simple(String),+ /// PDF parser with options.+ Pdf {+ #[serde(rename = "type")]+ parser_type: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ max_pages: Option<u32>,+ },+}++/// Request body for scrape endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct ScrapeRequest {+ url: String,+ #[serde(flatten)]+ options: ScrapeOptions,+}++/// Response from scrape endpoint.+#[derive(Deserialize, Serialize, Debug)]+#[serde(rename_all = "camelCase")]+struct ScrapeResponse {+ success: bool,+ data: Document,+ #[serde(skip_serializing_if = "Option::is_none")]+ warning: Option<String>,+}++impl Client {+ /// Scrapes a URL and returns the content in the requested formats.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to scrape.+ /// * `options` - Optional scrape configuration.+ ///+ /// # Returns+ ///+ /// A `Document` containing the scraped content.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, ScrapeOptions, Format};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple scrape+ /// let document = client.scrape("https://example.com", None).await?;+ /// println!("Markdown: {:?}", document.markdown);+ ///+ /// // Scrape with options+ /// let options = ScrapeOptions {+ /// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ /// only_main_content: Some(true),+ /// ..Default::default()+ /// };+ /// let document = client.scrape("https://example.com", options).await?;+ /// println!("Links: {:?}", document.links);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn scrape(+ &self,+ url: impl AsRef<str>,+ options: impl Into<Option<ScrapeOptions>>,+ ) -> Result<Document, FirecrawlError> {+ let body = ScrapeRequest {+ url: url.as_ref().to_string(),+ options: options.into().unwrap_or_default(),+ };++ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(self.url("/scrape"))+ .headers(headers)+ .json(&body)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;++ let response: ScrapeResponse = self.handle_response(response, "scrape").await?;++ Ok(response.data)+ }++ /// Scrapes a URL with a JSON schema for structured extraction.+ ///+ /// This is a convenience method that combines scraping with JSON extraction.+ ///+ /// # Arguments+ ///+ /// * `url` - The URL to scrape.+ /// * `schema` - JSON schema for the extraction.+ /// * `prompt` - Optional extraction prompt.+ ///+ /// # Returns+ ///+ /// The extracted JSON value matching the schema.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ /// use serde_json::json;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let schema = json!({+ /// "type": "object",+ /// "properties": {+ /// "title": { "type": "string" },+ /// "price": { "type": "number" }+ /// }+ /// });+ ///+ /// let data = client.scrape_with_schema(+ /// "https://example.com/product",+ /// schema,+ /// Some("Extract the product title and price")+ /// ).await?;+ ///+ /// println!("Extracted: {}", data);+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn scrape_with_schema(+ &self,+ url: impl AsRef<str>,+ schema: Value,+ prompt: Option<impl AsRef<str>>,+ ) -> Result<Value, FirecrawlError> {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Json]),+ json_options: Some(JsonOptions {+ schema: Some(schema),+ prompt: prompt.map(|p| p.as_ref().to_string()),+ ..Default::default()+ }),+ ..Default::default()+ };++ let document = self.scrape(url, options).await?;+ Ok(document.json.unwrap_or(Value::Null))+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_scrape_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "markdown": "# Example Domain\n\nThis is an example.",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example Domain"+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let document = client.scrape("https://example.com", None).await.unwrap();++ assert!(document.markdown.is_some());+ assert!(document.markdown.unwrap().contains("Example Domain"));+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_with_options() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "markdown": "# Test",+ "html": "<h1>Test</h1>",+ "links": ["https://example.com/page1", "https://example.com/page2"],+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ only_main_content: Some(true),+ ..Default::default()+ };++ let document = client.scrape("https://example.com", options).await.unwrap();++ assert!(document.markdown.is_some());+ assert!(document.html.is_some());+ assert!(document.links.is_some());+ assert_eq!(document.links.unwrap().len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_with_schema() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "json": {+ "title": "Product Name",+ "price": 99.99+ },+ "metadata": {+ "sourceURL": "https://example.com/product",+ "statusCode": 200+ }+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();++ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "price": { "type": "number" }+ }+ });++ let data = client+ .scrape_with_schema(+ "https://example.com/product",+ schema,+ Some("Extract product info"),+ )+ .await+ .unwrap();++ assert_eq!(data["title"], "Product Name");+ assert_eq!(data["price"], 99.99);+ mock.assert();+ }++ #[tokio::test]+ async fn test_scrape_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/scrape")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid URL"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.scrape("invalid-url", None).await;++ assert!(result.is_err());+ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/search.rs b/apps/rust-sdk/src/v2/search.rsnew file mode 100644index 000000000..ccca680f2--- /dev/null+++ b/apps/rust-sdk/src/v2/search.rs@@ -0,0 +1,466 @@+//! Search endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};++use super::client::Client;+use super::scrape::ScrapeOptions;+use super::types::{+ Document, SearchCategory, SearchResultImage, SearchResultNews, SearchResultWeb, SearchSource,+};+use crate::FirecrawlError;++/// Options for search requests.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchOptions {+ /// Maximum number of results to return. Default: 5, Max: 20.+ pub limit: Option<u32>,++ /// Search sources to query (web, news, images).+ pub sources: Option<Vec<SearchSource>>,++ /// Categories to filter results (github, research, pdf).+ pub categories: Option<Vec<SearchCategory>>,++ /// Time-based search filter (e.g., "qdr:d" for past day).+ pub tbs: Option<String>,++ /// Geographic location string for local search results.+ pub location: Option<String>,++ /// Whether to ignore invalid URLs in results.+ pub ignore_invalid_urls: Option<bool>,++ /// Timeout in milliseconds.+ pub timeout: Option<u32>,++ /// Scrape options to apply to each search result.+ pub scrape_options: Option<ScrapeOptions>,++ /// Integration identifier for tracking.+ pub integration: Option<String>,+}++/// Request body for search endpoint.+#[derive(Deserialize, Serialize, Debug, Default)]+#[serde(rename_all = "camelCase")]+struct SearchRequest {+ query: String,+ #[serde(flatten)]+ options: SearchOptions,+}++/// Search results data structure.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchData {+ /// Web search results (may include scraped documents).+ pub web: Option<Vec<SearchResultOrDocument>>,+ /// News search results.+ pub news: Option<Vec<SearchResultNews>>,+ /// Image search results.+ pub images: Option<Vec<SearchResultImage>>,+}++/// A search result that may be a simple result or a full document.+///+/// Uses custom deserialization to properly distinguish between web results+/// and scraped documents by checking for document-specific fields.+#[derive(Serialize, Debug, Clone)]+#[serde(untagged)]+pub enum SearchResultOrDocument {+ /// Simple web search result.+ WebResult(SearchResultWeb),+ /// Full scraped document.+ Document(Document),+}++impl<'de> serde::Deserialize<'de> for SearchResultOrDocument {+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>+ where+ D: serde::Deserializer<'de>,+ {+ use serde_json::Value;++ let value = Value::deserialize(deserializer)?;++ // Check for document-specific fields that indicate scraped content+ // If any of these exist, it's a Document, not a simple WebResult+ let is_document = value.get("markdown").is_some()+ || value.get("html").is_some()+ || value.get("rawHtml").is_some()+ || value.get("metadata").is_some();++ if is_document {+ Document::deserialize(value)+ .map(SearchResultOrDocument::Document)+ .map_err(serde::de::Error::custom)+ } else {+ SearchResultWeb::deserialize(value)+ .map(SearchResultOrDocument::WebResult)+ .map_err(serde::de::Error::custom)+ }+ }+}++/// Response from search endpoint.+#[derive(Deserialize, Serialize, Debug)]+#[serde(rename_all = "camelCase")]+pub struct SearchResponse {+ /// Whether the request was successful.+ pub success: bool,+ /// Search results data.+ pub data: SearchData,+ /// Warning message if any.+ pub warning: Option<String>,+}++impl Client {+ /// Searches the web and optionally scrapes the results.+ ///+ /// # Arguments+ ///+ /// * `query` - The search query string.+ /// * `options` - Optional search configuration.+ ///+ /// # Returns+ ///+ /// A `SearchResponse` containing the search results.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::{Client, SearchOptions, SearchSource};+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// // Simple search+ /// let results = client.search("rust programming", None).await?;+ /// for result in results.data.web.unwrap_or_default() {+ /// match result {+ /// firecrawl::v2::SearchResultOrDocument::WebResult(r) => {+ /// println!("URL: {}", r.url);+ /// }+ /// firecrawl::v2::SearchResultOrDocument::Document(d) => {+ /// println!("Content: {:?}", d.markdown);+ /// }+ /// }+ /// }+ ///+ /// // Search with options+ /// let options = SearchOptions {+ /// limit: Some(10),+ /// sources: Some(vec![SearchSource::Web, SearchSource::News]),+ /// ..Default::default()+ /// };+ /// let results = client.search("rust programming", options).await?;+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn search(+ &self,+ query: impl AsRef<str>,+ options: impl Into<Option<SearchOptions>>,+ ) -> Result<SearchResponse, FirecrawlError> {+ let body = SearchRequest {+ query: query.as_ref().to_string(),+ options: options.into().unwrap_or_default(),+ };++ let headers = self.prepare_headers(None);++ let response = self+ .client+ .post(self.url("/search"))+ .headers(headers)+ .json(&body)+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError(format!("Searching for {:?}", query.as_ref()), e)+ })?;++ self.handle_response(response, "search").await+ }++ /// Searches the web and scrapes the results.+ ///+ /// This is a convenience method that enables scraping for all results.+ ///+ /// # Arguments+ ///+ /// * `query` - The search query string.+ /// * `limit` - Maximum number of results to return.+ ///+ /// # Returns+ ///+ /// A vector of scraped documents.+ ///+ /// # Example+ ///+ /// ```no_run+ /// use firecrawl::v2::Client;+ ///+ /// #[tokio::main]+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {+ /// let client = Client::new("your-api-key")?;+ ///+ /// let documents = client.search_and_scrape("rust programming", 5).await?;+ /// for doc in documents {+ /// println!("Title: {:?}", doc.metadata.and_then(|m| m.title));+ /// println!("Content: {:?}", doc.markdown);+ /// }+ ///+ /// Ok(())+ /// }+ /// ```+ pub async fn search_and_scrape(+ &self,+ query: impl AsRef<str>,+ limit: u32,+ ) -> Result<Vec<Document>, FirecrawlError> {+ let options = SearchOptions {+ limit: Some(limit),+ scrape_options: Some(ScrapeOptions::default()),+ ..Default::default()+ };++ let response = self.search(query, options).await?;++ let documents: Vec<Document> = response+ .data+ .web+ .unwrap_or_default()+ .into_iter()+ .filter_map(|result| match result {+ SearchResultOrDocument::Document(doc) => Some(doc),+ SearchResultOrDocument::WebResult(_) => None,+ })+ .collect();++ Ok(documents)+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[tokio::test]+ async fn test_search_with_mock() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ {+ "url": "https://example.com",+ "title": "Example Domain",+ "description": "This domain is for examples"+ },+ {+ "url": "https://example.org",+ "title": "Another Example",+ "description": "More examples"+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.search("test query", None).await.unwrap();++ assert!(response.success);+ let web_results = response.data.web.unwrap();+ assert_eq!(web_results.len(), 2);+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_with_options() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ {+ "url": "https://example.com",+ "title": "Test Result",+ "description": "A test result"+ }+ ],+ "news": [+ {+ "title": "Breaking News",+ "url": "https://news.example.com",+ "snippet": "Something happened",+ "date": "2024-01-01"+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let options = SearchOptions {+ limit: Some(10),+ sources: Some(vec![SearchSource::Web, SearchSource::News]),+ ..Default::default()+ };++ let response = client.search("test", options).await.unwrap();++ assert!(response.success);+ assert!(response.data.web.is_some());+ assert!(response.data.news.is_some());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_and_scrape() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ {+ "markdown": "# Example\n\nThis is the scraped content.",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example"+ }+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let documents = client.search_and_scrape("test", 5).await.unwrap();++ assert_eq!(documents.len(), 1);+ assert!(documents[0].markdown.is_some());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_error_response() {+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(400)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": false,+ "error": "Invalid query"+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let result = client.search("", None).await;++ assert!(result.is_err());+ mock.assert();+ }++ #[tokio::test]+ async fn test_search_mixed_results_deserialization() {+ // Test that results with markdown/metadata are correctly identified as Documents+ // and simple results are identified as WebResults+ let mut server = mockito::Server::new_async().await;++ let mock = server+ .mock("POST", "/v2/search")+ .with_status(200)+ .with_header("content-type", "application/json")+ .with_body(+ json!({+ "success": true,+ "data": {+ "web": [+ // Simple web result (no markdown/metadata)+ {+ "url": "https://example.com/simple",+ "title": "Simple Result",+ "description": "Just a search result"+ },+ // Scraped document with url AND markdown+ {+ "url": "https://example.com/scraped",+ "markdown": "# Scraped Content\n\nThis has content.",+ "metadata": {+ "sourceURL": "https://example.com/scraped",+ "statusCode": 200,+ "title": "Scraped Page"+ }+ }+ ]+ }+ })+ .to_string(),+ )+ .create();++ let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();+ let response = client.search("test", None).await.unwrap();++ let web_results = response.data.web.unwrap();+ assert_eq!(web_results.len(), 2);++ // First should be WebResult+ match &web_results[0] {+ SearchResultOrDocument::WebResult(r) => {+ assert_eq!(r.url, "https://example.com/simple");+ assert_eq!(r.title, Some("Simple Result".to_string()));+ }+ SearchResultOrDocument::Document(_) => panic!("Expected WebResult, got Document"),+ }++ // Second should be Document (has markdown)+ match &web_results[1] {+ SearchResultOrDocument::Document(d) => {+ assert!(d.markdown.is_some());+ assert!(d.markdown.as_ref().unwrap().contains("Scraped Content"));+ }+ SearchResultOrDocument::WebResult(_) => panic!("Expected Document, got WebResult"),+ }++ mock.assert();+ }+}diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rsnew file mode 100644index 000000000..89ffe68b8--- /dev/null+++ b/apps/rust-sdk/src/v2/types.rs@@ -0,0 +1,510 @@+//! Type definitions for Firecrawl API v2.++use serde::{Deserialize, Serialize};+use serde_json::Value;+use std::collections::HashMap;++/// Available output formats for scraping operations.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum Format {+ /// Markdown content of the page.+ Markdown,+ /// Filtered, content-only HTML.+ Html,+ /// Original, untouched HTML.+ RawHtml,+ /// List of URLs found on the page.+ Links,+ /// List of image URLs found on the page.+ Images,+ /// Screenshot of the visible viewport.+ Screenshot,+ /// AI-generated summary of the page content.+ Summary,+ /// Change tracking information.+ ChangeTracking,+ /// Structured JSON extraction via LLM.+ Json,+ /// Custom attribute extraction.+ Attributes,+ /// Brand analysis of the page.+ Branding,+}++/// Viewport dimensions for screenshots.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+pub struct Viewport {+ pub width: u32,+ pub height: u32,+}++/// Screenshot format options.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ScreenshotOptions {+ /// Take a full-page screenshot instead of just the visible viewport.+ pub full_page: Option<bool>,+ /// Quality of the screenshot (1-100).+ pub quality: Option<u8>,+ /// Custom viewport dimensions.+ pub viewport: Option<Viewport>,+}++/// Change tracking format options.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct ChangeTrackingOptions {+ /// Modes for change tracking output.+ pub modes: Option<Vec<ChangeTrackingMode>>,+ /// JSON schema for structured change output.+ pub schema: Option<Value>,+ /// Prompt for LLM-based change analysis.+ pub prompt: Option<String>,+ /// Tag to identify this tracking session.+ pub tag: Option<String>,+}++/// Available change tracking modes.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "kebab-case")]+pub enum ChangeTrackingMode {+ GitDiff,+ Json,+}++/// Attribute extraction selector.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+pub struct AttributeSelector {+ /// CSS selector for the element.+ pub selector: String,+ /// Attribute name to extract.+ pub attribute: String,+}++/// JSON extraction options.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct JsonOptions {+ /// JSON schema the output should adhere to.+ pub schema: Option<Value>,+ /// System prompt for the LLM agent.+ pub system_prompt: Option<String>,+ /// Extraction prompt for the LLM agent.+ pub prompt: Option<String>,+}++/// Location configuration for proxy routing.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct LocationConfig {+ /// Country code (ISO 3166-1 alpha-2).+ pub country: Option<String>,+ /// List of preferred language codes.+ pub languages: Option<Vec<String>>,+}++/// Proxy type for scraping.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum ProxyType {+ Basic,+ Stealth,+ Enhanced,+ Auto,+}++/// Browser action types for automation.+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(tag = "type", rename_all = "camelCase")]+pub enum Action {+ /// Wait for a specified time or element.+ Wait {+ /// Milliseconds to wait.+ #[serde(skip_serializing_if = "Option::is_none")]+ milliseconds: Option<u32>,+ /// CSS selector to wait for.+ #[serde(skip_serializing_if = "Option::is_none")]+ selector: Option<String>,+ },+ /// Take a screenshot.+ Screenshot {+ #[serde(skip_serializing_if = "Option::is_none")]+ full_page: Option<bool>,+ #[serde(skip_serializing_if = "Option::is_none")]+ quality: Option<u8>,+ #[serde(skip_serializing_if = "Option::is_none")]+ viewport: Option<Viewport>,+ },+ /// Click an element.+ Click {+ /// CSS selector of the element to click.+ selector: String,+ },+ /// Write text to the focused input.+ Write {+ /// Text to write.+ text: String,+ },+ /// Press a keyboard key.+ Press {+ /// Key name to press.+ key: String,+ },+ /// Scroll the page.+ Scroll {+ /// Direction to scroll.+ direction: ScrollDirection,+ /// Optional selector to scroll within.+ #[serde(skip_serializing_if = "Option::is_none")]+ selector: Option<String>,+ },+ /// Trigger a scrape action.+ Scrape,+ /// Execute custom JavaScript.+ #[serde(rename = "executeJavascript")]+ ExecuteJavascript {+ /// JavaScript code to execute.+ script: String,+ },+ /// Generate a PDF.+ Pdf {+ #[serde(skip_serializing_if = "Option::is_none")]+ format: Option<PdfFormat>,+ #[serde(skip_serializing_if = "Option::is_none")]+ landscape: Option<bool>,+ #[serde(skip_serializing_if = "Option::is_none")]+ scale: Option<f32>,+ },+}++/// Scroll direction for scroll actions.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum ScrollDirection {+ Up,+ Down,+}++/// PDF format options.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+pub enum PdfFormat {+ A0,+ A1,+ A2,+ A3,+ A4,+ A5,+ A6,+ Letter,+ Legal,+ Tabloid,+ Ledger,+}++/// Webhook configuration for async operations.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct WebhookConfig {+ /// URL to send webhook notifications to.+ pub url: String,+ /// Custom headers to include in webhook requests.+ pub headers: Option<HashMap<String, String>>,+ /// Custom metadata to include in webhook payloads.+ pub metadata: Option<HashMap<String, String>>,+ /// Event types to receive notifications for.+ pub events: Option<Vec<WebhookEvent>>,+}++impl From<String> for WebhookConfig {+ fn from(url: String) -> Self {+ Self {+ url,+ ..Default::default()+ }+ }+}++impl From<&str> for WebhookConfig {+ fn from(url: &str) -> Self {+ Self {+ url: url.to_string(),+ ..Default::default()+ }+ }+}++/// Webhook event types for crawl/batch operations.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum WebhookEvent {+ Completed,+ Failed,+ Page,+ Started,+}++/// Agent-specific webhook event types.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum AgentWebhookEvent {+ Started,+ Action,+ Completed,+ Failed,+ Cancelled,+}++/// Agent webhook configuration.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct AgentWebhookConfig {+ /// URL to send webhook notifications to.+ pub url: String,+ /// Custom headers to include in webhook requests.+ pub headers: Option<HashMap<String, String>>,+ /// Custom metadata to include in webhook payloads.+ pub metadata: Option<HashMap<String, String>>,+ /// Event types to receive notifications for.+ pub events: Option<Vec<AgentWebhookEvent>>,+}++impl From<String> for AgentWebhookConfig {+ fn from(url: String) -> Self {+ Self {+ url,+ ..Default::default()+ }+ }+}++impl From<&str> for AgentWebhookConfig {+ fn from(url: &str) -> Self {+ Self {+ url: url.to_string(),+ ..Default::default()+ }+ }+}++/// Document metadata returned from scrape operations.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct DocumentMetadata {+ // Firecrawl specific+ #[serde(rename = "sourceURL")]+ pub source_url: Option<String>,+ pub status_code: Option<u16>,+ pub error: Option<String>,++ // Basic meta tags+ pub title: Option<String>,+ pub description: Option<String>,+ pub language: Option<String>,+ pub keywords: Option<String>,+ pub robots: Option<String>,++ // OpenGraph namespace+ pub og_title: Option<String>,+ pub og_description: Option<String>,+ pub og_url: Option<String>,+ pub og_image: Option<String>,+ pub og_audio: Option<String>,+ pub og_determiner: Option<String>,+ pub og_locale: Option<String>,+ pub og_locale_alternate: Option<Vec<String>>,+ pub og_site_name: Option<String>,+ pub og_video: Option<String>,++ // Article namespace+ pub article_section: Option<String>,+ pub article_tag: Option<String>,+ pub published_time: Option<String>,+ pub modified_time: Option<String>,++ // Dublin Core namespace+ pub dcterms_keywords: Option<String>,+ pub dc_description: Option<String>,+ pub dc_subject: Option<String>,+ pub dcterms_subject: Option<String>,+ pub dcterms_audience: Option<String>,+ pub dc_type: Option<String>,+ pub dcterms_type: Option<String>,+ pub dc_date: Option<String>,+ pub dc_date_created: Option<String>,+ pub dcterms_created: Option<String>,++ // Response metadata+ pub scrape_id: Option<String>,+ pub num_pages: Option<u32>,+ pub content_type: Option<String>,+ pub timezone: Option<String>,+ pub proxy_used: Option<String>,+ pub cache_state: Option<String>,+ pub cached_at: Option<String>,+ pub credits_used: Option<u32>,+ pub concurrency_limited: Option<bool>,+}++/// Extracted attribute result.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+pub struct AttributeResult {+ pub selector: String,+ pub attribute: String,+ pub values: Vec<String>,+}++/// Document returned from scrape operations.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct Document {+ /// Markdown content of the page.+ pub markdown: Option<String>,+ /// Filtered HTML content.+ pub html: Option<String>,+ /// Raw HTML content.+ pub raw_html: Option<String>,+ /// Structured JSON extraction result.+ pub json: Option<Value>,+ /// AI-generated summary.+ pub summary: Option<String>,+ /// Document metadata.+ pub metadata: Option<DocumentMetadata>,+ /// Links found on the page.+ pub links: Option<Vec<String>>,+ /// Images found on the page.+ pub images: Option<Vec<String>>,+ /// Screenshot URL or base64 data.+ pub screenshot: Option<String>,+ /// Extracted attributes.+ pub attributes: Option<Vec<AttributeResult>>,+ /// Action results.+ pub actions: Option<HashMap<String, Value>>,+ /// Warning message.+ pub warning: Option<String>,+ /// Change tracking data.+ pub change_tracking: Option<Value>,+ /// Branding analysis.+ pub branding: Option<Value>,+}++/// Job status types for crawl and batch operations.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "camelCase")]+pub enum JobStatus {+ Scraping,+ Completed,+ Failed,+ Cancelled,+}++/// Sitemap handling mode.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum SitemapMode {+ /// Skip sitemap entirely.+ Skip,+ /// Include sitemap links alongside discovered links.+ Include,+ /// Only use links from the sitemap.+ Only,+}++/// Agent model types.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "kebab-case")]+pub enum AgentModel {+ #[serde(rename = "spark-1-pro")]+ Spark1Pro,+ #[serde(rename = "spark-1-mini")]+ Spark1Mini,+}++/// Search source types.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum SearchSource {+ Web,+ News,+ Images,+}++/// Search category types.+#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum SearchCategory {+ Github,+ Research,+ Pdf,+}++/// Web search result.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchResultWeb {+ pub url: String,+ pub title: Option<String>,+ pub description: Option<String>,+ pub category: Option<String>,+}++/// News search result.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchResultNews {+ pub title: Option<String>,+ pub url: Option<String>,+ pub snippet: Option<String>,+ pub date: Option<String>,+ pub image_url: Option<String>,+ pub position: Option<u32>,+ pub category: Option<String>,+}++/// Image search result.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct SearchResultImage {+ pub title: Option<String>,+ pub image_url: Option<String>,+ pub image_width: Option<u32>,+ pub image_height: Option<u32>,+ pub url: Option<String>,+ pub position: Option<u32>,+}++/// Crawl error information.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlError {+ pub id: String,+ pub timestamp: Option<String>,+ pub url: String,+ pub code: Option<String>,+ pub error: String,+}++/// Crawl errors response.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CrawlErrorsResponse {+ pub errors: Vec<CrawlError>,+ #[serde(rename = "robotsBlocked")]+ pub robots_blocked: Vec<String>,+}diff --git a/apps/rust-sdk/tests/e2e_with_auth.rs b/apps/rust-sdk/tests/e2e_with_auth.rsindex 071537f50..05c6d0080 100644--- a/apps/rust-sdk/tests/e2e_with_auth.rs+++ b/apps/rust-sdk/tests/e2e_with_auth.rs@@ -1,5 +1,5 @@use dotenvy::dotenv;-use firecrawl::scrape::{ExtractOptions, ScrapeFormats, ScrapeOptions};+use firecrawl::scrape::{JsonOptions, ScrapeFormats, ScrapeOptions};use firecrawl::{FirecrawlApp, FirecrawlError};use serde_json::json;use std::env;@@ -123,8 +123,8 @@ async fn test_llm_extraction() {let api_key = env::var("TEST_API_KEY").ok();let app = FirecrawlApp::new_selfhosted(api_url, api_key).unwrap();let options = ScrapeOptions {- formats: vec! [ ScrapeFormats::Extract ].into(),- extract: ExtractOptions {+ formats: vec![ScrapeFormats::Json].into(),+ json_options: JsonOptions {prompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source".to_string().into(),schema: json!({"type": "object",diff --git a/apps/rust-sdk/tests/v2_e2e.rs b/apps/rust-sdk/tests/v2_e2e.rsnew file mode 100644index 000000000..f91398206--- /dev/null+++ b/apps/rust-sdk/tests/v2_e2e.rs@@ -0,0 +1,302 @@+//! End-to-end tests for Firecrawl v2 API.+//!+//! These tests require the following environment variables:+//! - API_URL: The Firecrawl API URL+//! - TEST_API_KEY: A valid API key (optional for self-hosted)+//!+//! Run with: cargo test --test v2_e2e -- --ignored++use dotenvy::dotenv;+use firecrawl::v2::{+ AgentOptions, BatchScrapeOptions, Client, CrawlOptions, Format, MapOptions, ScrapeOptions,+ SearchOptions, SitemapMode,+};+use serde_json::json;+use std::env;++fn get_client() -> Client {+ dotenv().ok();+ let api_url = env::var("API_URL").expect("API_URL environment variable is required");+ let api_key = env::var("TEST_API_KEY").ok();+ Client::new_selfhosted(api_url, api_key).expect("Failed to create client")+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_scrape() {+ let client = get_client();+ let doc = client+ .scrape("https://example.com", None)+ .await+ .expect("Scrape should succeed");++ assert!(doc.markdown.is_some(), "Response should contain markdown");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_scrape_with_options() {+ let client = get_client();+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),+ only_main_content: Some(true),+ ..Default::default()+ };++ let doc = client+ .scrape("https://example.com", options)+ .await+ .expect("Scrape with options should succeed");++ assert!(doc.markdown.is_some(), "Response should contain markdown");+ assert!(doc.html.is_some(), "Response should contain html");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_scrape_with_schema() {+ let client = get_client();++ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "description": { "type": "string" }+ }+ });++ let data = client+ .scrape_with_schema("https://example.com", schema, Some("Extract page info"))+ .await+ .expect("Schema scrape should succeed");++ assert!(data.is_object(), "Response should be a JSON object");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_search() {+ let client = get_client();+ let response = client+ .search("rust programming", None)+ .await+ .expect("Search should succeed");++ assert!(response.success, "Response should indicate success");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_search_with_options() {+ let client = get_client();+ let options = SearchOptions {+ limit: Some(5),+ ..Default::default()+ };++ let response = client+ .search("firecrawl web scraping", options)+ .await+ .expect("Search with options should succeed");++ assert!(response.success, "Response should indicate success");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_map() {+ let client = get_client();+ let response = client+ .map("https://example.com", None)+ .await+ .expect("Map should succeed");++ assert!(response.success, "Response should indicate success");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_map_with_options() {+ let client = get_client();+ let options = MapOptions {+ sitemap: Some(SitemapMode::Include),+ limit: Some(50),+ ..Default::default()+ };++ let response = client+ .map("https://example.com", options)+ .await+ .expect("Map with options should succeed");++ assert!(response.success, "Response should indicate success");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_crawl_async() {+ let client = get_client();+ let response = client+ .start_crawl("https://example.com", None)+ .await+ .expect("Start crawl should succeed");++ assert!(response.success, "Response should indicate success");+ assert!(!response.id.is_empty(), "Response should contain job ID");++ // Check status+ let status = client+ .get_crawl_status(&response.id)+ .await+ .expect("Get crawl status should succeed");+ assert!(+ matches!(+ status.status,+ firecrawl::v2::JobStatus::Scraping | firecrawl::v2::JobStatus::Completed+ ),+ "Status should be scraping or completed"+ );++ // Cancel the crawl to clean up+ let _ = client.cancel_crawl(&response.id).await;+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_crawl_sync() {+ let client = get_client();+ let options = CrawlOptions {+ limit: Some(2),+ poll_interval: Some(2000),+ ..Default::default()+ };++ let job = client+ .crawl("https://example.com", options)+ .await+ .expect("Crawl should succeed");++ assert!(+ job.status == firecrawl::v2::JobStatus::Completed,+ "Job should be completed"+ );+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_batch_scrape_async() {+ let client = get_client();+ let urls = vec![+ "https://example.com".to_string(),+ "https://example.org".to_string(),+ ];++ let response = client+ .start_batch_scrape(urls, None)+ .await+ .expect("Start batch scrape should succeed");++ assert!(response.success, "Response should indicate success");+ assert!(!response.id.is_empty(), "Response should contain job ID");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_batch_scrape_sync() {+ let client = get_client();+ let urls = vec!["https://example.com".to_string()];++ let options = BatchScrapeOptions {+ options: Some(ScrapeOptions {+ formats: Some(vec![Format::Markdown]),+ ..Default::default()+ }),+ poll_interval: Some(2000),+ ..Default::default()+ };++ let job = client+ .batch_scrape(urls, options)+ .await+ .expect("Batch scrape should succeed");++ assert!(+ job.status == firecrawl::v2::JobStatus::Completed,+ "Job should be completed"+ );+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_agent_async() {+ let client = get_client();+ let options = AgentOptions {+ urls: Some(vec!["https://example.com".to_string()]),+ prompt: "Describe what this website is about".to_string(),+ ..Default::default()+ };++ let response = client+ .start_agent(options)+ .await+ .expect("Start agent should succeed");++ assert!(response.success, "Response should indicate success");+ assert!(!response.id.is_empty(), "Response should contain task ID");+}++#[tokio::test]+#[ignore = "Requires API access"]+async fn test_v2_agent_with_schema() {+ let client = get_client();++ #[derive(Debug, serde::Deserialize)]+ #[allow(dead_code)]+ struct WebsiteInfo {+ title: Option<String>,+ description: Option<String>,+ }++ let schema = json!({+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "description": { "type": "string" }+ }+ });++ let result: Option<WebsiteInfo> = client+ .agent_with_schema(+ vec!["https://example.com".to_string()],+ "Extract the title and description",+ schema,+ )+ .await+ .expect("Agent with schema should succeed");++ // Agent may or may not return data depending on the page+ if let Some(info) = result {+ println!("Agent extracted: {:?}", info);+ }+}++// Test that the v2 client can be created with different configurations+// This test doesn't require API access+#[test]+fn test_v2_client_creation() {+ // Cloud client requires API key+ let cloud_result = Client::new("test-key");+ assert!(cloud_result.is_ok());++ // Cloud client without API key should fail+ let cloud_no_key = Client::new_selfhosted("https://api.firecrawl.dev", None::<&str>);+ assert!(cloud_no_key.is_err());++ // Self-hosted client without API key should work+ let selfhosted = Client::new_selfhosted("http://localhost:3000", None::<&str>);+ assert!(selfhosted.is_ok());++ // Self-hosted client with API key should work+ let selfhosted_with_key = Client::new_selfhosted("http://localhost:3000", Some("key"));+ assert!(selfhosted_with_key.is_ok());+}