Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

41.1 KB
Raw Blame
//! Scrape endpoint for Firecrawl API v2.

use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use std::collections::HashMap;

use crate::client::Client;
use crate::types::{
    fold_format_options, unfold_format_options, Action, AttributeSelector, ChangeTrackingOptions,
    Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
};
use crate::{AuditMetadata, FirecrawlError};

/// Options for scraping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(remote = "Self", rename_all = "camelCase")]
pub struct ScrapeOptions {
    /// Origin label for request attribution (e.g., "[email protected]").
    pub origin: Option<String>,

    /// Output formats to include in the response.
    pub formats: Option<Vec<Format>>,

    /// Additional HTTP headers to send with the request.
    pub headers: Option<HashMap<String, String>>,

    /// HTML tags to exclusively include in the output.
    pub include_tags: Option<Vec<String>>,

    /// HTML tags to exclude from the output.
    pub exclude_tags: Option<Vec<String>>,

    /// Only extract the main content of the page.
    pub only_main_content: Option<bool>,

    /// Timeout in milliseconds before returning an error.
    pub timeout: Option<u32>,

    /// Time to wait after page load before scraping (milliseconds).
    pub wait_for: Option<u32>,

    /// Emulate a mobile device.
    pub mobile: Option<bool>,

    /// Parser configurations (e.g., for PDFs).
    pub parsers: Option<Vec<ParserConfig>>,

    /// Browser automation actions to perform before scraping.
    pub actions: Option<Vec<Action>>,

    /// Location configuration for proxy routing.
    pub location: Option<LocationConfig>,

    /// Skip TLS certificate verification.
    pub skip_tls_verification: Option<bool>,

    /// Remove base64-encoded images from the output.
    pub remove_base64_images: Option<bool>,

    /// Enable fast mode for quicker scrapes with reduced accuracy.
    pub fast_mode: Option<bool>,

    /// Block advertisements on the page.
    pub block_ads: Option<bool>,

    /// Proxy type to use.
    pub proxy: Option<ProxyType>,

    /// Maximum age of cached content to accept (seconds).
    pub max_age: Option<u32>,

    /// Minimum age of cached content to accept (seconds).
    pub min_age: Option<u32>,

    /// Store the result in cache for future requests.
    pub store_in_cache: Option<bool>,

    /// Lockdown mode: serve only previously cached results, never make outbound requests.
    pub lockdown: Option<bool>,

    /// Scan the page content for prompt injection with any format except rawBase64,
    /// before LLM-backed formats run. A detection fails the scrape with
    /// SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check scans the whole page.
    pub check_prompt_injection: Option<bool>,

    /// Redact personally identifiable information from returned content.
    #[serde(rename = "redactPII")]
    pub redact_pii: Option<bool>,

    /// User attribution to include with SIEM logging events.
    pub audit_metadata: Option<AuditMetadata>,

    /// Persistent browser profile for maintaining state across scrapes.
    pub profile: Option<ProfileConfig>,

    /// Integration identifier for tracking.
    pub integration: Option<String>,

    /// JSON extraction options.
    pub json_options: Option<JsonOptions>,

    /// Screenshot options.
    pub screenshot_options: Option<ScreenshotOptions>,

    /// Change tracking options.
    pub change_tracking_options: Option<ChangeTrackingOptions>,

    /// Attribute selectors for extraction.
    pub attribute_selectors: Option<Vec<AttributeSelector>>,

    /// Enable Alexandria domain-tool discovery for this scrape.
    pub domain_tools: Option<bool>,

    /// Discovery response detail; defaults to summary.
    pub tool_detail: Option<crate::types::ToolDetail>,
}

impl Serialize for ScrapeOptions {
    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
        let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer)
            .map_err(ser::Error::custom)?;
        fold_format_options(&mut value);
        value.serialize(serializer)
    }
}

impl<'de> Deserialize<'de> for ScrapeOptions {
    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
        let mut value = Value::deserialize(deserializer)?;
        unfold_format_options(&mut value);
        ScrapeOptions::deserialize(value).map_err(de::Error::custom)
    }
}

/// Parser configuration for document parsing.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(untagged)]
pub enum ParserConfig {
    /// Simple parser type string.
    Simple(String),
    /// PDF parser with options.
    Pdf {
        #[serde(rename = "type")]
        parser_type: String,
        #[serde(skip_serializing_if = "Option::is_none")]
        mode: Option<String>,
        #[serde(rename = "maxPages", skip_serializing_if = "Option::is_none")]
        max_pages: Option<u32>,
        #[serde(skip_serializing_if = "Option::is_none")]
        pages: Option<bool>,
        #[serde(skip_serializing_if = "Option::is_none")]
        blocks: Option<bool>,
        /// Join PDF pages in document markdown with `\n\n---\n\n<!-- page N -->\n\n`.
        #[serde(rename = "pageMarkers", skip_serializing_if = "Option::is_none")]
        page_markers: Option<bool>,
    },
}

/// Request body for scrape endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct ScrapeRequest {
    url: String,
    #[serde(flatten)]
    options: ScrapeOptions,
}

/// Response from scrape endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ScrapeResponse {
    success: bool,
    data: Document,
    #[serde(skip_serializing_if = "Option::is_none")]
    warning: Option<String>,
}

#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaCall {
    pub provider: String,
    pub capability: String,
    pub options: Option<serde_json::Map<String, Value>>,
}

#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaOptions {
    #[serde(skip)]
    pub request_id: Option<String>,
    pub timeout: Option<u32>,
    pub integration: Option<String>,
    pub origin: Option<String>,
}

/// Matches the API's default and maximum execution timeout for Alexandria calls.
const ALEXANDRIA_MAX_TIMEOUT_MS: u32 = 120_000;
/// Extra time for the API to deliver a response after its execution deadline.
const ALEXANDRIA_RESPONSE_MARGIN_MS: u64 = 30_000;

/// Returns how long the client waits for an Alexandria response.
fn alexandria_transport_timeout(timeout: Option<u32>) -> std::time::Duration {
    let execution_ms = timeout
        .unwrap_or(ALEXANDRIA_MAX_TIMEOUT_MS)
        .min(ALEXANDRIA_MAX_TIMEOUT_MS);
    std::time::Duration::from_millis(u64::from(execution_ms) + ALEXANDRIA_RESPONSE_MARGIN_MS)
}

#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct AlexandriaRequest {
    alexandria: Vec<AlexandriaCall>,
    #[serde(flatten)]
    options: AlexandriaOptions,
}

#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeError {
    pub code: String,
    pub message: String,
    pub status: Option<u16>,
    /// Charge identifier, present when credits were captured before the failure.
    #[serde(default)]
    pub charge_id: Option<String>,
}

#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeResult {
    pub provider: Option<String>,
    pub capability: Option<String>,
    pub credits_cost: Option<u32>,
    pub data: Option<Value>,
    pub records: Option<u64>,
    pub upstream_status: Option<u16>,
    pub recorded_at: Option<String>,
    pub error: Option<AlexandriaScrapeError>,
}

impl AlexandriaScrapeResult {
    pub fn failed(&self) -> bool {
        self.error.is_some()
    }
}

#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeData {
    pub request_id: String,
    pub scrape_id: String,
    pub alexandria: Vec<AlexandriaScrapeResult>,
    pub credits_cost: u32,
}

#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct AlexandriaScrapePayload {
    alexandria: Vec<AlexandriaScrapeResult>,
    credits_cost: u32,
}

#[derive(Deserialize, Debug)]
struct AlexandriaScrapeResponse {
    scrape_id: String,
    data: AlexandriaScrapePayload,
}

/// Supported languages for scrape-bound browser execution.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ScrapeExecuteLanguage {
    Python,
    Node,
    Bash,
}

/// Options for executing code or a prompt in a scrape-bound browser session.
///
/// At least one of `code` or `prompt` must be provided.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteOptions {
    /// Code to execute (optional if `prompt` is provided).
    pub code: Option<String>,
    /// Natural-language instruction for the browser agent (optional if `code` is provided).
    pub prompt: Option<String>,
    /// Runtime language for the code.
    pub language: Option<ScrapeExecuteLanguage>,
    /// Execution timeout in seconds.
    pub timeout: Option<u32>,
    /// Optional origin tag for request attribution.
    pub origin: Option<String>,
}

/// Response from scrape-bound browser execution.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteResponse {
    /// Whether the request succeeded.
    pub success: bool,
    /// Live-view URL for the browser session.
    pub live_view_url: Option<String>,
    /// Interactive live-view URL for the browser session.
    pub interactive_live_view_url: Option<String>,
    /// Agent output when a prompt was used.
    pub output: Option<String>,
    /// Captured stdout from execution.
    pub stdout: Option<String>,
    /// Optional execution result payload.
    pub result: Option<String>,
    /// Captured stderr from execution.
    pub stderr: Option<String>,
    /// Process exit code.
    pub exit_code: Option<i32>,
    /// Whether execution was killed by timeout or system.
    pub killed: Option<bool>,
    /// Error message when execution fails.
    pub error: Option<String>,
}

/// Response from deleting a scrape-bound browser session.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeBrowserDeleteResponse {
    /// Whether the delete request succeeded.
    pub success: bool,
    /// Session duration in milliseconds when available.
    pub session_duration_ms: Option<u64>,
    /// Credits billed when available.
    pub credits_billed: Option<u32>,
    /// Error message when deletion fails.
    pub error: Option<String>,
}

impl Client {
    /// Scrapes a URL and returns the content in the requested formats.
    ///
    /// # Arguments
    ///
    /// * `url` - The URL to scrape.
    /// * `options` - Optional scrape configuration.
    ///
    /// # Returns
    ///
    /// A `Document` containing the scraped content.
    ///
    /// # Example
    ///
    /// ```no_run
    /// use firecrawl::{Client, ScrapeOptions, Format};
    ///
    /// #[tokio::main]
    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
    ///     let client = Client::new("your-api-key")?;
    ///
    ///     // Simple scrape
    ///     let document = client.scrape("https://example.com", None).await?;
    ///     println!("Markdown: {:?}", document.markdown);
    ///
    ///     // Scrape with options
    ///     let options = ScrapeOptions {
    ///         formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
    ///         only_main_content: Some(true),
    ///         ..Default::default()
    ///     };
    ///     let document = client.scrape("https://example.com", options).await?;
    ///     println!("Links: {:?}", document.links);
    ///
    ///     Ok(())
    /// }
    /// ```
    pub async fn scrape(
        &self,
        url: impl AsRef<str>,
        options: impl Into<Option<ScrapeOptions>>,
    ) -> Result<Document, FirecrawlError> {
        let mut options = options.into().unwrap_or_default();
        if options.origin.is_none() {
            options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
        }
        let body = ScrapeRequest {
            url: url.as_ref().to_string(),
            options,
        };

        let headers = self.prepare_headers(None);

        let response = self
            .client
            .post(self.url("/scrape"))
            .headers(headers)
            .json(&body)
            .send()
            .await
            .map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;

        let response: ScrapeResponse = self.handle_response(response, "scrape").await?;

        Ok(response.data)
    }

    pub async fn scrape_alexandria(
        &self,
        calls: Vec<AlexandriaCall>,
        options: impl Into<Option<AlexandriaOptions>>,
    ) -> Result<AlexandriaScrapeData, FirecrawlError> {
        if calls.is_empty() {
            return Err(FirecrawlError::Misuse(
                "at least one alexandria call is required".to_string(),
            ));
        }
        if calls.len() > 10 {
            return Err(FirecrawlError::Misuse(
                "at most 10 alexandria calls are allowed per request".to_string(),
            ));
        }
        for (index, call) in calls.iter().enumerate() {
            if call.provider.trim().is_empty() {
                return Err(FirecrawlError::Misuse(format!(
                    "alexandria call {index}: provider is required"
                )));
            }
            if call.capability.trim().is_empty() {
                return Err(FirecrawlError::Misuse(format!(
                    "alexandria call {index}: capability is required"
                )));
            }
        }
        let mut options = options.into().unwrap_or_default();
        if options.timeout == Some(0) {
            return Err(FirecrawlError::Misuse(
                "timeout must be positive".to_string(),
            ));
        }
        if options.origin.is_none() {
            options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
        }
        // Allow response delivery after the API's capped execution deadline.
        let request_timeout = alexandria_transport_timeout(options.timeout);
        let request_id = options
            .request_id
            .clone()
            .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
        if request_id.is_empty()
            || request_id.len() > 128
            || !request_id
                .bytes()
                .all(|c| c.is_ascii_alphanumeric() || b"._:-".contains(&c))
        {
            return Err(FirecrawlError::Misuse("Invalid request_id".into()));
        }
        let body = AlexandriaRequest {
            alexandria: calls,
            options,
        };

        let headers = self.prepare_headers(None);

        let response = self
            .client
            .post(self.url("/scrape"))
            .headers(headers)
            .header("x-request-id", &request_id)
            .json(&body)
            .timeout(request_timeout)
            .send()
            .await
            .map_err(|e| FirecrawlError::AlexandriaExecution {
                request_id: request_id.clone(),
                source: Box::new(FirecrawlError::HttpError(
                    "Executing alexandria calls".to_string(),
                    e,
                )),
            })?;

        let response: AlexandriaScrapeResponse = self
            .handle_response(response, "alexandria")
            .await
            .map_err(|e| FirecrawlError::AlexandriaExecution {
                request_id: request_id.clone(),
                source: Box::new(e),
            })?;

        Ok(AlexandriaScrapeData {
            request_id,
            scrape_id: response.scrape_id,
            alexandria: response.data.alexandria,
            credits_cost: response.data.credits_cost,
        })
    }

    /// Scrapes a URL with a JSON schema for structured extraction.
    ///
    /// This is a convenience method that combines scraping with JSON extraction.
    ///
    /// # Arguments
    ///
    /// * `url` - The URL to scrape.
    /// * `schema` - JSON schema for the extraction.
    /// * `prompt` - Optional extraction prompt.
    ///
    /// # Returns
    ///
    /// The extracted JSON value matching the schema.
    ///
    /// # Example
    ///
    /// ```no_run
    /// use firecrawl::Client;
    /// use serde_json::json;
    ///
    /// #[tokio::main]
    /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
    ///     let client = Client::new("your-api-key")?;
    ///
    ///     let schema = json!({
    ///         "type": "object",
    ///         "properties": {
    ///             "title": { "type": "string" },
    ///             "price": { "type": "number" }
    ///         }
    ///     });
    ///
    ///     let data = client.scrape_with_schema(
    ///         "https://example.com/product",
    ///         schema,
    ///         Some("Extract the product title and price")
    ///     ).await?;
    ///
    ///     println!("Extracted: {}", data);
    ///
    ///     Ok(())
    /// }
    /// ```
    pub async fn scrape_with_schema(
        &self,
        url: impl AsRef<str>,
        schema: Value,
        prompt: Option<impl AsRef<str>>,
    ) -> Result<Value, FirecrawlError> {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Json]),
            json_options: Some(JsonOptions {
                schema: Some(schema),
                prompt: prompt.map(|p| p.as_ref().to_string()),
                ..Default::default()
            }),
            ..Default::default()
        };

        let document = self.scrape(url, options).await?;
        Ok(document.json.unwrap_or(Value::Null))
    }

    /// Interacts with the browser session associated with a scrape job.
    ///
    /// # Arguments
    ///
    /// * `job_id` - The scrape job ID.
    /// * `options` - Execution options including code and runtime config.
    ///
    /// # Returns
    ///
    /// A `ScrapeExecuteResponse` containing execution output.
    pub async fn interact(
        &self,
        job_id: impl AsRef<str>,
        options: ScrapeExecuteOptions,
    ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
        let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
        let has_prompt = options
            .prompt
            .as_ref()
            .is_some_and(|p| !p.trim().is_empty());
        if !has_code && !has_prompt {
            return Err(FirecrawlError::Misuse(
                "Either 'code' or 'prompt' must be provided".into(),
            ));
        }

        let mut body = options;
        if body.language.is_none() {
            body.language = Some(ScrapeExecuteLanguage::Node);
        }
        if body.origin.is_none() {
            body.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
        }

        let response = self
            .client
            .post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
            .headers(self.prepare_headers(None))
            .json(&body)
            .send()
            .await
            .map_err(|e| {
                FirecrawlError::HttpError(
                    format!("Interacting with scrape browser for {}", job_id.as_ref()),
                    e,
                )
            })?;

        self.handle_response(response, "scrape interact").await
    }

    /// Stops the interaction session associated with a scrape job.
    ///
    /// # Arguments
    ///
    /// * `job_id` - The scrape job ID.
    ///
    /// # Returns
    ///
    /// A `ScrapeBrowserDeleteResponse` indicating stop status.
    pub async fn stop_interaction(
        &self,
        job_id: impl AsRef<str>,
    ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
        let response = self
            .client
            .delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
            .headers(self.prepare_headers(None))
            .send()
            .await
            .map_err(|e| {
                FirecrawlError::HttpError(
                    format!("Stopping interaction for {}", job_id.as_ref()),
                    e,
                )
            })?;

        self.handle_response(response, "stop interaction").await
    }

    /// Deprecated alias for [`Client::interact`].
    #[deprecated(note = "Use interact() instead")]
    pub async fn scrape_execute(
        &self,
        job_id: impl AsRef<str>,
        options: ScrapeExecuteOptions,
    ) -> Result<ScrapeExecuteResponse, FirecrawlError> {
        self.interact(job_id, options).await
    }

    /// Deprecated alias for [`Client::stop_interaction`].
    #[deprecated(note = "Use stop_interaction() instead")]
    pub async fn stop_interactive_browser(
        &self,
        job_id: impl AsRef<str>,
    ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
        self.stop_interaction(job_id).await
    }

    /// Deprecated alias for [`Client::stop_interaction`].
    #[deprecated(note = "Use stop_interaction() instead")]
    pub async fn delete_scrape_browser(
        &self,
        job_id: impl AsRef<str>,
    ) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
        self.stop_interaction(job_id).await
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::{HighlightsFormat, QueryFormat, QueryFormatMode, QuestionFormat};
    use serde_json::json;

    #[test]
    fn test_alexandria_transport_timeout() {
        for (timeout, expected_secs) in [
            (None, 150),
            (Some(1_000), 31),
            (Some(100_000), 130),
            (Some(120_000), 150),
            (Some(300_000), 150),
        ] {
            assert_eq!(
                alexandria_transport_timeout(timeout),
                std::time::Duration::from_secs(expected_secs),
                "timeout {timeout:?}"
            );
        }
    }

    #[test]
    fn test_query_format_serializes_mode() {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Query(QueryFormat {
                prompt: "What is Firecrawl?".to_string(),
                mode: Some(QueryFormatMode::DirectQuote),
            })]),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"][0],
            json!({
                "type": "query",
                "prompt": "What is Firecrawl?",
                "mode": "directQuote"
            })
        );
    }

    #[test]
    fn test_question_and_highlights_formats_serialize() {
        let options = ScrapeOptions {
            formats: Some(vec![
                Format::Question(QuestionFormat {
                    question: "What is Firecrawl?".to_string(),
                }),
                Format::Highlights(HighlightsFormat {
                    query: "What is Firecrawl?".to_string(),
                }),
            ]),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"][0],
            json!({
                "type": "question",
                "question": "What is Firecrawl?"
            })
        );
        assert_eq!(
            payload["formats"][1],
            json!({
                "type": "highlights",
                "query": "What is Firecrawl?"
            })
        );
    }

    #[test]
    fn test_json_options_serialize_as_json_format() {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Markdown, Format::Json]),
            json_options: Some(JsonOptions {
                prompt: Some("Extract the product name".to_string()),
                schema: Some(json!({ "type": "object" })),
                ..Default::default()
            }),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"],
            json!([
                "markdown",
                {
                    "type": "json",
                    "prompt": "Extract the product name",
                    "schema": { "type": "object" }
                }
            ])
        );
        assert!(payload.get("jsonOptions").is_none());
    }

    #[test]
    #[allow(deprecated)]
    fn test_json_options_add_json_format_and_drop_system_prompt() {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Markdown]),
            json_options: Some(JsonOptions {
                prompt: Some("Extract the title".to_string()),
                system_prompt: Some("You are an extractor".to_string()),
                ..Default::default()
            }),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"],
            json!(["markdown", { "type": "json", "prompt": "Extract the title" }])
        );
        assert!(payload.get("jsonOptions").is_none());
    }

    #[test]
    fn test_format_options_serialize_inside_format_objects() {
        let options = ScrapeOptions {
            formats: Some(vec![
                Format::Markdown,
                Format::Screenshot,
                Format::ChangeTracking,
                Format::Attributes,
            ]),
            screenshot_options: Some(ScreenshotOptions {
                full_page: Some(true),
                quality: Some(80),
                viewport: Some(crate::types::Viewport {
                    width: 1280,
                    height: 720,
                }),
            }),
            change_tracking_options: Some(ChangeTrackingOptions {
                modes: Some(vec![crate::types::ChangeTrackingMode::GitDiff]),
                tag: Some("daily".to_string()),
                ..Default::default()
            }),
            attribute_selectors: Some(vec![AttributeSelector {
                selector: "a".to_string(),
                attribute: "href".to_string(),
            }]),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"],
            json!([
                "markdown",
                {
                    "type": "screenshot",
                    "fullPage": true,
                    "quality": 80,
                    "viewport": { "width": 1280, "height": 720 }
                },
                { "type": "changeTracking", "modes": ["git-diff"], "tag": "daily" },
                { "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }
            ])
        );
        for key in [
            "screenshotOptions",
            "changeTrackingOptions",
            "attributeSelectors",
        ] {
            assert!(payload.get(key).is_none(), "{key} must not be top-level");
        }

        let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();
        assert!(matches!(
            restored.formats.as_deref(),
            Some([
                Format::Markdown,
                Format::Screenshot,
                Format::ChangeTracking,
                Format::Attributes
            ])
        ));
        assert_eq!(restored.screenshot_options.unwrap().quality, Some(80));
        assert_eq!(
            restored.change_tracking_options.unwrap().tag,
            Some("daily".to_string())
        );
        assert_eq!(restored.attribute_selectors.unwrap()[0].attribute, "href");
    }

    #[test]
    fn test_format_options_add_missing_format() {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Markdown]),
            screenshot_options: Some(ScreenshotOptions {
                full_page: Some(true),
                ..Default::default()
            }),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["formats"],
            json!(["markdown", { "type": "screenshot", "fullPage": true }])
        );
    }

    #[test]
    fn test_scrape_options_round_trip() {
        let options = ScrapeOptions {
            formats: Some(vec![Format::Markdown, Format::Json]),
            only_main_content: Some(true),
            json_options: Some(JsonOptions {
                prompt: Some("Extract the title".to_string()),
                schema: Some(json!({ "type": "object" })),
                ..Default::default()
            }),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();

        assert!(matches!(
            restored.formats.as_deref(),
            Some([Format::Markdown, Format::Json])
        ));
        assert_eq!(restored.only_main_content, Some(true));
        let json_options = restored.json_options.clone().unwrap();
        assert_eq!(json_options.prompt, Some("Extract the title".to_string()));
        assert_eq!(json_options.schema, Some(json!({ "type": "object" })));
        assert_eq!(serde_json::to_value(restored).unwrap(), payload);
    }

    #[test]
    fn test_scrape_options_serializes_domain_tools() {
        let options = ScrapeOptions {
            domain_tools: Some(true),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(payload["domainTools"], json!(true));
    }

    #[test]
    fn test_scrape_options_serializes_redact_pii() {
        let options = ScrapeOptions {
            redact_pii: Some(true),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(payload["redactPII"], json!(true));
        assert!(payload.get("formats").is_none());
    }

    #[test]
    fn test_scrape_options_serializes_check_prompt_injection() {
        let options = ScrapeOptions {
            check_prompt_injection: Some(true),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(payload["checkPromptInjection"], json!(true));

        let unset = serde_json::to_value(ScrapeOptions::default()).unwrap();
        assert!(unset.get("checkPromptInjection").is_none());
    }

    #[test]
    fn test_pdf_parser_serializes_blocks() {
        let options = ScrapeOptions {
            parsers: Some(vec![ParserConfig::Pdf {
                parser_type: "pdf".to_string(),
                mode: Some("auto".to_string()),
                max_pages: None,
                pages: Some(true),
                blocks: Some(true),
                page_markers: Some(true),
            }]),
            ..Default::default()
        };

        let payload = serde_json::to_value(options).unwrap();
        assert_eq!(
            payload["parsers"][0],
            json!({
                "type": "pdf",
                "mode": "auto",
                "pages": true,
                "blocks": true,
                "pageMarkers": true
            })
        );
    }

    #[tokio::test]
    async fn test_scrape_with_mock() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "data": {
                        "markdown": "# Example Domain\n\nThis is an example.",
                        "metadata": {
                            "sourceURL": "https://example.com",
                            "statusCode": 200,
                            "title": "Example Domain"
                        }
                    }
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let document = client.scrape("https://example.com", None).await.unwrap();

        assert!(document.markdown.is_some());
        assert!(document.markdown.unwrap().contains("Example Domain"));
        mock.assert();
    }

    #[tokio::test]
    async fn test_scrape_with_options() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "data": {
                        "markdown": "# Test",
                        "html": "<h1>Test</h1>",
                        "links": ["https://example.com/page1", "https://example.com/page2"],
                        "metadata": {
                            "sourceURL": "https://example.com",
                            "statusCode": 200
                        }
                    }
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let options = ScrapeOptions {
            formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
            only_main_content: Some(true),
            ..Default::default()
        };

        let document = client.scrape("https://example.com", options).await.unwrap();

        assert!(document.markdown.is_some());
        assert!(document.html.is_some());
        assert!(document.links.is_some());
        assert_eq!(document.links.unwrap().len(), 2);
        mock.assert();
    }

    #[tokio::test]
    async fn test_scrape_with_schema() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape")
            .match_body(mockito::Matcher::PartialJson(json!({
                "url": "https://example.com/product",
                "formats": [{
                    "type": "json",
                    "prompt": "Extract product info",
                    "schema": {
                        "type": "object",
                        "properties": {
                            "title": { "type": "string" },
                            "price": { "type": "number" }
                        }
                    }
                }]
            })))
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "data": {
                        "json": {
                            "title": "Product Name",
                            "price": 99.99
                        },
                        "metadata": {
                            "sourceURL": "https://example.com/product",
                            "statusCode": 200
                        }
                    }
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();

        let schema = json!({
            "type": "object",
            "properties": {
                "title": { "type": "string" },
                "price": { "type": "number" }
            }
        });

        let data = client
            .scrape_with_schema(
                "https://example.com/product",
                schema,
                Some("Extract product info"),
            )
            .await
            .unwrap();

        assert_eq!(data["title"], "Product Name");
        assert_eq!(data["price"], 99.99);
        mock.assert();
    }

    #[tokio::test]
    async fn test_scrape_error_response() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape")
            .with_status(400)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": false,
                    "error": "Invalid URL"
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let result = client.scrape("invalid-url", None).await;

        assert!(result.is_err());
        mock.assert();
    }

    #[tokio::test]
    async fn test_interact_with_mock() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape/job-123/interact")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "stdout": "ok",
                    "result": "done",
                    "stderr": "",
                    "exitCode": 0,
                    "killed": false
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let response = client
            .interact(
                "job-123",
                ScrapeExecuteOptions {
                    code: Some("console.log('ok')".to_string()),
                    timeout: Some(30),
                    ..Default::default()
                },
            )
            .await
            .unwrap();

        assert!(response.success);
        assert_eq!(response.exit_code, Some(0));
        assert_eq!(response.result, Some("done".to_string()));
        mock.assert();
    }

    #[tokio::test]
    async fn test_interact_with_prompt() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape/job-789/interact")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "output": "Clicked the login button",
                    "liveViewUrl": "https://live.example.com/view",
                    "interactiveLiveViewUrl": "https://live.example.com/interactive",
                    "stdout": "",
                    "exitCode": 0,
                    "killed": false
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let response = client
            .interact(
                "job-789",
                ScrapeExecuteOptions {
                    prompt: Some("Click the login button".to_string()),
                    ..Default::default()
                },
            )
            .await
            .unwrap();

        assert!(response.success);
        assert_eq!(
            response.output,
            Some("Clicked the login button".to_string())
        );
        assert_eq!(
            response.live_view_url,
            Some("https://live.example.com/view".to_string())
        );
        assert_eq!(
            response.interactive_live_view_url,
            Some("https://live.example.com/interactive".to_string())
        );
        mock.assert();
    }

    #[tokio::test]
    async fn test_stop_interaction_with_mock() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("DELETE", "/v2/scrape/job-123/interact")
            .with_status(200)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": true,
                    "sessionDurationMs": 1200,
                    "creditsBilled": 3
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let response = client.stop_interaction("job-123").await.unwrap();

        assert!(response.success);
        assert_eq!(response.session_duration_ms, Some(1200));
        assert_eq!(response.credits_billed, Some(3));
        mock.assert();
    }

    #[tokio::test]
    async fn test_interact_error_response() {
        let mut server = mockito::Server::new_async().await;

        let mock = server
            .mock("POST", "/v2/scrape/job-404/interact")
            .with_status(404)
            .with_header("content-type", "application/json")
            .with_body(
                json!({
                    "success": false,
                    "error": "Job not found."
                })
                .to_string(),
            )
            .create();

        let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
        let result = client
            .interact(
                "job-404",
                ScrapeExecuteOptions {
                    code: Some("console.log('ok')".to_string()),
                    ..Default::default()
                },
            )
            .await;

        assert!(result.is_err());
        mock.assert();
    }
}