Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
//! Scrape endpoint for Firecrawl API v2.
use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use std::collections::HashMap;
use crate::client::Client;
use crate::types::{
fold_format_options, unfold_format_options, Action, AttributeSelector, ChangeTrackingOptions,
Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
};
use crate::{AuditMetadata, FirecrawlError};
/// Options for scraping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(remote = "Self", rename_all = "camelCase")]
pub struct ScrapeOptions {
/// Origin label for request attribution (e.g., "[email protected]").
pub origin: Option<String>,
/// Output formats to include in the response.
pub formats: Option<Vec<Format>>,
/// Additional HTTP headers to send with the request.
pub headers: Option<HashMap<String, String>>,
/// HTML tags to exclusively include in the output.
pub include_tags: Option<Vec<String>>,
/// HTML tags to exclude from the output.
pub exclude_tags: Option<Vec<String>>,
/// Only extract the main content of the page.
pub only_main_content: Option<bool>,
/// Timeout in milliseconds before returning an error.
pub timeout: Option<u32>,
/// Time to wait after page load before scraping (milliseconds).
pub wait_for: Option<u32>,
/// Emulate a mobile device.
pub mobile: Option<bool>,
/// Parser configurations (e.g., for PDFs).
pub parsers: Option<Vec<ParserConfig>>,
/// Browser automation actions to perform before scraping.
pub actions: Option<Vec<Action>>,
/// Location configuration for proxy routing.
pub location: Option<LocationConfig>,
/// Skip TLS certificate verification.
pub skip_tls_verification: Option<bool>,
/// Remove base64-encoded images from the output.
pub remove_base64_images: Option<bool>,
/// Enable fast mode for quicker scrapes with reduced accuracy.
pub fast_mode: Option<bool>,
/// Block advertisements on the page.
pub block_ads: Option<bool>,
/// Proxy type to use.
pub proxy: Option<ProxyType>,
/// Maximum age of cached content to accept (seconds).
pub max_age: Option<u32>,
/// Minimum age of cached content to accept (seconds).
pub min_age: Option<u32>,
/// Store the result in cache for future requests.
pub store_in_cache: Option<bool>,
/// Lockdown mode: serve only previously cached results, never make outbound requests.
pub lockdown: Option<bool>,
/// Scan the page content for prompt injection with any format except rawBase64,
/// before LLM-backed formats run. A detection fails the scrape with
/// SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check scans the whole page.
pub check_prompt_injection: Option<bool>,
/// Redact personally identifiable information from returned content.
#[serde(rename = "redactPII")]
pub redact_pii: Option<bool>,
/// User attribution to include with SIEM logging events.
pub audit_metadata: Option<AuditMetadata>,
/// Persistent browser profile for maintaining state across scrapes.
pub profile: Option<ProfileConfig>,
/// Integration identifier for tracking.
pub integration: Option<String>,
/// JSON extraction options.
pub json_options: Option<JsonOptions>,
/// Screenshot options.
pub screenshot_options: Option<ScreenshotOptions>,
/// Change tracking options.
pub change_tracking_options: Option<ChangeTrackingOptions>,
/// Attribute selectors for extraction.
pub attribute_selectors: Option<Vec<AttributeSelector>>,
/// Enable Alexandria domain-tool discovery for this scrape.
pub domain_tools: Option<bool>,
/// Discovery response detail; defaults to summary.
pub tool_detail: Option<crate::types::ToolDetail>,
}
impl Serialize for ScrapeOptions {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer)
.map_err(ser::Error::custom)?;
fold_format_options(&mut value);
value.serialize(serializer)
}
}
impl<'de> Deserialize<'de> for ScrapeOptions {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let mut value = Value::deserialize(deserializer)?;
unfold_format_options(&mut value);
ScrapeOptions::deserialize(value).map_err(de::Error::custom)
}
}
/// Parser configuration for document parsing.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(untagged)]
pub enum ParserConfig {
/// Simple parser type string.
Simple(String),
/// PDF parser with options.
Pdf {
#[serde(rename = "type")]
parser_type: String,
#[serde(skip_serializing_if = "Option::is_none")]
mode: Option<String>,
#[serde(rename = "maxPages", skip_serializing_if = "Option::is_none")]
max_pages: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pages: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
blocks: Option<bool>,
/// Join PDF pages in document markdown with `\n\n---\n\n<!-- page N -->\n\n`.
#[serde(rename = "pageMarkers", skip_serializing_if = "Option::is_none")]
page_markers: Option<bool>,
},
}
/// Request body for scrape endpoint.
#[derive(Deserialize, Serialize, Debug, Default)]
#[serde(rename_all = "camelCase")]
struct ScrapeRequest {
url: String,
#[serde(flatten)]
options: ScrapeOptions,
}
/// Response from scrape endpoint.
#[derive(Deserialize, Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ScrapeResponse {
success: bool,
data: Document,
#[serde(skip_serializing_if = "Option::is_none")]
warning: Option<String>,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaCall {
pub provider: String,
pub capability: String,
pub options: Option<serde_json::Map<String, Value>>,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaOptions {
#[serde(skip)]
pub request_id: Option<String>,
pub timeout: Option<u32>,
pub integration: Option<String>,
pub origin: Option<String>,
}
/// Matches the API's default and maximum execution timeout for Alexandria calls.
const ALEXANDRIA_MAX_TIMEOUT_MS: u32 = 120_000;
/// Extra time for the API to deliver a response after its execution deadline.
const ALEXANDRIA_RESPONSE_MARGIN_MS: u64 = 30_000;
/// Returns how long the client waits for an Alexandria response.
fn alexandria_transport_timeout(timeout: Option<u32>) -> std::time::Duration {
let execution_ms = timeout
.unwrap_or(ALEXANDRIA_MAX_TIMEOUT_MS)
.min(ALEXANDRIA_MAX_TIMEOUT_MS);
std::time::Duration::from_millis(u64::from(execution_ms) + ALEXANDRIA_RESPONSE_MARGIN_MS)
}
#[derive(Serialize, Debug)]
#[serde(rename_all = "camelCase")]
struct AlexandriaRequest {
alexandria: Vec<AlexandriaCall>,
#[serde(flatten)]
options: AlexandriaOptions,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeError {
pub code: String,
pub message: String,
pub status: Option<u16>,
/// Charge identifier, present when credits were captured before the failure.
#[serde(default)]
pub charge_id: Option<String>,
}
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeResult {
pub provider: Option<String>,
pub capability: Option<String>,
pub credits_cost: Option<u32>,
pub data: Option<Value>,
pub records: Option<u64>,
pub upstream_status: Option<u16>,
pub recorded_at: Option<String>,
pub error: Option<AlexandriaScrapeError>,
}
impl AlexandriaScrapeResult {
pub fn failed(&self) -> bool {
self.error.is_some()
}
}
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct AlexandriaScrapeData {
pub request_id: String,
pub scrape_id: String,
pub alexandria: Vec<AlexandriaScrapeResult>,
pub credits_cost: u32,
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct AlexandriaScrapePayload {
alexandria: Vec<AlexandriaScrapeResult>,
credits_cost: u32,
}
#[derive(Deserialize, Debug)]
struct AlexandriaScrapeResponse {
scrape_id: String,
data: AlexandriaScrapePayload,
}
/// Supported languages for scrape-bound browser execution.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum ScrapeExecuteLanguage {
Python,
Node,
Bash,
}
/// Options for executing code or a prompt in a scrape-bound browser session.
///
/// At least one of `code` or `prompt` must be provided.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteOptions {
/// Code to execute (optional if `prompt` is provided).
pub code: Option<String>,
/// Natural-language instruction for the browser agent (optional if `code` is provided).
pub prompt: Option<String>,
/// Runtime language for the code.
pub language: Option<ScrapeExecuteLanguage>,
/// Execution timeout in seconds.
pub timeout: Option<u32>,
/// Optional origin tag for request attribution.
pub origin: Option<String>,
}
/// Response from scrape-bound browser execution.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeExecuteResponse {
/// Whether the request succeeded.
pub success: bool,
/// Live-view URL for the browser session.
pub live_view_url: Option<String>,
/// Interactive live-view URL for the browser session.
pub interactive_live_view_url: Option<String>,
/// Agent output when a prompt was used.
pub output: Option<String>,
/// Captured stdout from execution.
pub stdout: Option<String>,
/// Optional execution result payload.
pub result: Option<String>,
/// Captured stderr from execution.
pub stderr: Option<String>,
/// Process exit code.
pub exit_code: Option<i32>,
/// Whether execution was killed by timeout or system.
pub killed: Option<bool>,
/// Error message when execution fails.
pub error: Option<String>,
}
/// Response from deleting a scrape-bound browser session.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct ScrapeBrowserDeleteResponse {
/// Whether the delete request succeeded.
pub success: bool,
/// Session duration in milliseconds when available.
pub session_duration_ms: Option<u64>,
/// Credits billed when available.
pub credits_billed: Option<u32>,
/// Error message when deletion fails.
pub error: Option<String>,
}
impl Client {
/// Scrapes a URL and returns the content in the requested formats.
///
/// # Arguments
///
/// * `url` - The URL to scrape.
/// * `options` - Optional scrape configuration.
///
/// # Returns
///
/// A `Document` containing the scraped content.
///
/// # Example
///
/// ```no_run
/// use firecrawl::{Client, ScrapeOptions, Format};
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// // Simple scrape
/// let document = client.scrape("https://example.com", None).await?;
/// println!("Markdown: {:?}", document.markdown);
///
/// // Scrape with options
/// let options = ScrapeOptions {
/// formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
/// only_main_content: Some(true),
/// ..Default::default()
/// };
/// let document = client.scrape("https://example.com", options).await?;
/// println!("Links: {:?}", document.links);
///
/// Ok(())
/// }
/// ```
pub async fn scrape(
&self,
url: impl AsRef<str>,
options: impl Into<Option<ScrapeOptions>>,
) -> Result<Document, FirecrawlError> {
let mut options = options.into().unwrap_or_default();
if options.origin.is_none() {
options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
}
let body = ScrapeRequest {
url: url.as_ref().to_string(),
options,
};
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/scrape"))
.headers(headers)
.json(&body)
.send()
.await
.map_err(|e| FirecrawlError::HttpError(format!("Scraping {:?}", url.as_ref()), e))?;
let response: ScrapeResponse = self.handle_response(response, "scrape").await?;
Ok(response.data)
}
pub async fn scrape_alexandria(
&self,
calls: Vec<AlexandriaCall>,
options: impl Into<Option<AlexandriaOptions>>,
) -> Result<AlexandriaScrapeData, FirecrawlError> {
if calls.is_empty() {
return Err(FirecrawlError::Misuse(
"at least one alexandria call is required".to_string(),
));
}
if calls.len() > 10 {
return Err(FirecrawlError::Misuse(
"at most 10 alexandria calls are allowed per request".to_string(),
));
}
for (index, call) in calls.iter().enumerate() {
if call.provider.trim().is_empty() {
return Err(FirecrawlError::Misuse(format!(
"alexandria call {index}: provider is required"
)));
}
if call.capability.trim().is_empty() {
return Err(FirecrawlError::Misuse(format!(
"alexandria call {index}: capability is required"
)));
}
}
let mut options = options.into().unwrap_or_default();
if options.timeout == Some(0) {
return Err(FirecrawlError::Misuse(
"timeout must be positive".to_string(),
));
}
if options.origin.is_none() {
options.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
}
// Allow response delivery after the API's capped execution deadline.
let request_timeout = alexandria_transport_timeout(options.timeout);
let request_id = options
.request_id
.clone()
.unwrap_or_else(|| uuid::Uuid::new_v4().to_string());
if request_id.is_empty()
|| request_id.len() > 128
|| !request_id
.bytes()
.all(|c| c.is_ascii_alphanumeric() || b"._:-".contains(&c))
{
return Err(FirecrawlError::Misuse("Invalid request_id".into()));
}
let body = AlexandriaRequest {
alexandria: calls,
options,
};
let headers = self.prepare_headers(None);
let response = self
.client
.post(self.url("/scrape"))
.headers(headers)
.header("x-request-id", &request_id)
.json(&body)
.timeout(request_timeout)
.send()
.await
.map_err(|e| FirecrawlError::AlexandriaExecution {
request_id: request_id.clone(),
source: Box::new(FirecrawlError::HttpError(
"Executing alexandria calls".to_string(),
e,
)),
})?;
let response: AlexandriaScrapeResponse = self
.handle_response(response, "alexandria")
.await
.map_err(|e| FirecrawlError::AlexandriaExecution {
request_id: request_id.clone(),
source: Box::new(e),
})?;
Ok(AlexandriaScrapeData {
request_id,
scrape_id: response.scrape_id,
alexandria: response.data.alexandria,
credits_cost: response.data.credits_cost,
})
}
/// Scrapes a URL with a JSON schema for structured extraction.
///
/// This is a convenience method that combines scraping with JSON extraction.
///
/// # Arguments
///
/// * `url` - The URL to scrape.
/// * `schema` - JSON schema for the extraction.
/// * `prompt` - Optional extraction prompt.
///
/// # Returns
///
/// The extracted JSON value matching the schema.
///
/// # Example
///
/// ```no_run
/// use firecrawl::Client;
/// use serde_json::json;
///
/// #[tokio::main]
/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
/// let client = Client::new("your-api-key")?;
///
/// let schema = json!({
/// "type": "object",
/// "properties": {
/// "title": { "type": "string" },
/// "price": { "type": "number" }
/// }
/// });
///
/// let data = client.scrape_with_schema(
/// "https://example.com/product",
/// schema,
/// Some("Extract the product title and price")
/// ).await?;
///
/// println!("Extracted: {}", data);
///
/// Ok(())
/// }
/// ```
pub async fn scrape_with_schema(
&self,
url: impl AsRef<str>,
schema: Value,
prompt: Option<impl AsRef<str>>,
) -> Result<Value, FirecrawlError> {
let options = ScrapeOptions {
formats: Some(vec![Format::Json]),
json_options: Some(JsonOptions {
schema: Some(schema),
prompt: prompt.map(|p| p.as_ref().to_string()),
..Default::default()
}),
..Default::default()
};
let document = self.scrape(url, options).await?;
Ok(document.json.unwrap_or(Value::Null))
}
/// Interacts with the browser session associated with a scrape job.
///
/// # Arguments
///
/// * `job_id` - The scrape job ID.
/// * `options` - Execution options including code and runtime config.
///
/// # Returns
///
/// A `ScrapeExecuteResponse` containing execution output.
pub async fn interact(
&self,
job_id: impl AsRef<str>,
options: ScrapeExecuteOptions,
) -> Result<ScrapeExecuteResponse, FirecrawlError> {
let has_code = options.code.as_ref().is_some_and(|c| !c.trim().is_empty());
let has_prompt = options
.prompt
.as_ref()
.is_some_and(|p| !p.trim().is_empty());
if !has_code && !has_prompt {
return Err(FirecrawlError::Misuse(
"Either 'code' or 'prompt' must be provided".into(),
));
}
let mut body = options;
if body.language.is_none() {
body.language = Some(ScrapeExecuteLanguage::Node);
}
if body.origin.is_none() {
body.origin = Some(format!("rust-sdk@{}", env!("CARGO_PKG_VERSION")));
}
let response = self
.client
.post(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
.headers(self.prepare_headers(None))
.json(&body)
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Interacting with scrape browser for {}", job_id.as_ref()),
e,
)
})?;
self.handle_response(response, "scrape interact").await
}
/// Stops the interaction session associated with a scrape job.
///
/// # Arguments
///
/// * `job_id` - The scrape job ID.
///
/// # Returns
///
/// A `ScrapeBrowserDeleteResponse` indicating stop status.
pub async fn stop_interaction(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
let response = self
.client
.delete(self.url(&format!("/scrape/{}/interact", job_id.as_ref())))
.headers(self.prepare_headers(None))
.send()
.await
.map_err(|e| {
FirecrawlError::HttpError(
format!("Stopping interaction for {}", job_id.as_ref()),
e,
)
})?;
self.handle_response(response, "stop interaction").await
}
/// Deprecated alias for [`Client::interact`].
#[deprecated(note = "Use interact() instead")]
pub async fn scrape_execute(
&self,
job_id: impl AsRef<str>,
options: ScrapeExecuteOptions,
) -> Result<ScrapeExecuteResponse, FirecrawlError> {
self.interact(job_id, options).await
}
/// Deprecated alias for [`Client::stop_interaction`].
#[deprecated(note = "Use stop_interaction() instead")]
pub async fn stop_interactive_browser(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
self.stop_interaction(job_id).await
}
/// Deprecated alias for [`Client::stop_interaction`].
#[deprecated(note = "Use stop_interaction() instead")]
pub async fn delete_scrape_browser(
&self,
job_id: impl AsRef<str>,
) -> Result<ScrapeBrowserDeleteResponse, FirecrawlError> {
self.stop_interaction(job_id).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{HighlightsFormat, QueryFormat, QueryFormatMode, QuestionFormat};
use serde_json::json;
#[test]
fn test_alexandria_transport_timeout() {
for (timeout, expected_secs) in [
(None, 150),
(Some(1_000), 31),
(Some(100_000), 130),
(Some(120_000), 150),
(Some(300_000), 150),
] {
assert_eq!(
alexandria_transport_timeout(timeout),
std::time::Duration::from_secs(expected_secs),
"timeout {timeout:?}"
);
}
}
#[test]
fn test_query_format_serializes_mode() {
let options = ScrapeOptions {
formats: Some(vec![Format::Query(QueryFormat {
prompt: "What is Firecrawl?".to_string(),
mode: Some(QueryFormatMode::DirectQuote),
})]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"][0],
json!({
"type": "query",
"prompt": "What is Firecrawl?",
"mode": "directQuote"
})
);
}
#[test]
fn test_question_and_highlights_formats_serialize() {
let options = ScrapeOptions {
formats: Some(vec![
Format::Question(QuestionFormat {
question: "What is Firecrawl?".to_string(),
}),
Format::Highlights(HighlightsFormat {
query: "What is Firecrawl?".to_string(),
}),
]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"][0],
json!({
"type": "question",
"question": "What is Firecrawl?"
})
);
assert_eq!(
payload["formats"][1],
json!({
"type": "highlights",
"query": "What is Firecrawl?"
})
);
}
#[test]
fn test_json_options_serialize_as_json_format() {
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Json]),
json_options: Some(JsonOptions {
prompt: Some("Extract the product name".to_string()),
schema: Some(json!({ "type": "object" })),
..Default::default()
}),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"],
json!([
"markdown",
{
"type": "json",
"prompt": "Extract the product name",
"schema": { "type": "object" }
}
])
);
assert!(payload.get("jsonOptions").is_none());
}
#[test]
#[allow(deprecated)]
fn test_json_options_add_json_format_and_drop_system_prompt() {
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown]),
json_options: Some(JsonOptions {
prompt: Some("Extract the title".to_string()),
system_prompt: Some("You are an extractor".to_string()),
..Default::default()
}),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"],
json!(["markdown", { "type": "json", "prompt": "Extract the title" }])
);
assert!(payload.get("jsonOptions").is_none());
}
#[test]
fn test_format_options_serialize_inside_format_objects() {
let options = ScrapeOptions {
formats: Some(vec![
Format::Markdown,
Format::Screenshot,
Format::ChangeTracking,
Format::Attributes,
]),
screenshot_options: Some(ScreenshotOptions {
full_page: Some(true),
quality: Some(80),
viewport: Some(crate::types::Viewport {
width: 1280,
height: 720,
}),
}),
change_tracking_options: Some(ChangeTrackingOptions {
modes: Some(vec![crate::types::ChangeTrackingMode::GitDiff]),
tag: Some("daily".to_string()),
..Default::default()
}),
attribute_selectors: Some(vec![AttributeSelector {
selector: "a".to_string(),
attribute: "href".to_string(),
}]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"],
json!([
"markdown",
{
"type": "screenshot",
"fullPage": true,
"quality": 80,
"viewport": { "width": 1280, "height": 720 }
},
{ "type": "changeTracking", "modes": ["git-diff"], "tag": "daily" },
{ "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }
])
);
for key in [
"screenshotOptions",
"changeTrackingOptions",
"attributeSelectors",
] {
assert!(payload.get(key).is_none(), "{key} must not be top-level");
}
let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();
assert!(matches!(
restored.formats.as_deref(),
Some([
Format::Markdown,
Format::Screenshot,
Format::ChangeTracking,
Format::Attributes
])
));
assert_eq!(restored.screenshot_options.unwrap().quality, Some(80));
assert_eq!(
restored.change_tracking_options.unwrap().tag,
Some("daily".to_string())
);
assert_eq!(restored.attribute_selectors.unwrap()[0].attribute, "href");
}
#[test]
fn test_format_options_add_missing_format() {
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown]),
screenshot_options: Some(ScreenshotOptions {
full_page: Some(true),
..Default::default()
}),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["formats"],
json!(["markdown", { "type": "screenshot", "fullPage": true }])
);
}
#[test]
fn test_scrape_options_round_trip() {
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Json]),
only_main_content: Some(true),
json_options: Some(JsonOptions {
prompt: Some("Extract the title".to_string()),
schema: Some(json!({ "type": "object" })),
..Default::default()
}),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();
assert!(matches!(
restored.formats.as_deref(),
Some([Format::Markdown, Format::Json])
));
assert_eq!(restored.only_main_content, Some(true));
let json_options = restored.json_options.clone().unwrap();
assert_eq!(json_options.prompt, Some("Extract the title".to_string()));
assert_eq!(json_options.schema, Some(json!({ "type": "object" })));
assert_eq!(serde_json::to_value(restored).unwrap(), payload);
}
#[test]
fn test_scrape_options_serializes_domain_tools() {
let options = ScrapeOptions {
domain_tools: Some(true),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(payload["domainTools"], json!(true));
}
#[test]
fn test_scrape_options_serializes_redact_pii() {
let options = ScrapeOptions {
redact_pii: Some(true),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(payload["redactPII"], json!(true));
assert!(payload.get("formats").is_none());
}
#[test]
fn test_scrape_options_serializes_check_prompt_injection() {
let options = ScrapeOptions {
check_prompt_injection: Some(true),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(payload["checkPromptInjection"], json!(true));
let unset = serde_json::to_value(ScrapeOptions::default()).unwrap();
assert!(unset.get("checkPromptInjection").is_none());
}
#[test]
fn test_pdf_parser_serializes_blocks() {
let options = ScrapeOptions {
parsers: Some(vec![ParserConfig::Pdf {
parser_type: "pdf".to_string(),
mode: Some("auto".to_string()),
max_pages: None,
pages: Some(true),
blocks: Some(true),
page_markers: Some(true),
}]),
..Default::default()
};
let payload = serde_json::to_value(options).unwrap();
assert_eq!(
payload["parsers"][0],
json!({
"type": "pdf",
"mode": "auto",
"pages": true,
"blocks": true,
"pageMarkers": true
})
);
}
#[tokio::test]
async fn test_scrape_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"markdown": "# Example Domain\n\nThis is an example.",
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200,
"title": "Example Domain"
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let document = client.scrape("https://example.com", None).await.unwrap();
assert!(document.markdown.is_some());
assert!(document.markdown.unwrap().contains("Example Domain"));
mock.assert();
}
#[tokio::test]
async fn test_scrape_with_options() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"markdown": "# Test",
"html": "<h1>Test</h1>",
"links": ["https://example.com/page1", "https://example.com/page2"],
"metadata": {
"sourceURL": "https://example.com",
"statusCode": 200
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let options = ScrapeOptions {
formats: Some(vec![Format::Markdown, Format::Html, Format::Links]),
only_main_content: Some(true),
..Default::default()
};
let document = client.scrape("https://example.com", options).await.unwrap();
assert!(document.markdown.is_some());
assert!(document.html.is_some());
assert!(document.links.is_some());
assert_eq!(document.links.unwrap().len(), 2);
mock.assert();
}
#[tokio::test]
async fn test_scrape_with_schema() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.match_body(mockito::Matcher::PartialJson(json!({
"url": "https://example.com/product",
"formats": [{
"type": "json",
"prompt": "Extract product info",
"schema": {
"type": "object",
"properties": {
"title": { "type": "string" },
"price": { "type": "number" }
}
}
}]
})))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"data": {
"json": {
"title": "Product Name",
"price": 99.99
},
"metadata": {
"sourceURL": "https://example.com/product",
"statusCode": 200
}
}
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let schema = json!({
"type": "object",
"properties": {
"title": { "type": "string" },
"price": { "type": "number" }
}
});
let data = client
.scrape_with_schema(
"https://example.com/product",
schema,
Some("Extract product info"),
)
.await
.unwrap();
assert_eq!(data["title"], "Product Name");
assert_eq!(data["price"], 99.99);
mock.assert();
}
#[tokio::test]
async fn test_scrape_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape")
.with_status(400)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Invalid URL"
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client.scrape("invalid-url", None).await;
assert!(result.is_err());
mock.assert();
}
#[tokio::test]
async fn test_interact_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-123/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"stdout": "ok",
"result": "done",
"stderr": "",
"exitCode": 0,
"killed": false
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client
.interact(
"job-123",
ScrapeExecuteOptions {
code: Some("console.log('ok')".to_string()),
timeout: Some(30),
..Default::default()
},
)
.await
.unwrap();
assert!(response.success);
assert_eq!(response.exit_code, Some(0));
assert_eq!(response.result, Some("done".to_string()));
mock.assert();
}
#[tokio::test]
async fn test_interact_with_prompt() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-789/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"output": "Clicked the login button",
"liveViewUrl": "https://live.example.com/view",
"interactiveLiveViewUrl": "https://live.example.com/interactive",
"stdout": "",
"exitCode": 0,
"killed": false
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client
.interact(
"job-789",
ScrapeExecuteOptions {
prompt: Some("Click the login button".to_string()),
..Default::default()
},
)
.await
.unwrap();
assert!(response.success);
assert_eq!(
response.output,
Some("Clicked the login button".to_string())
);
assert_eq!(
response.live_view_url,
Some("https://live.example.com/view".to_string())
);
assert_eq!(
response.interactive_live_view_url,
Some("https://live.example.com/interactive".to_string())
);
mock.assert();
}
#[tokio::test]
async fn test_stop_interaction_with_mock() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("DELETE", "/v2/scrape/job-123/interact")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": true,
"sessionDurationMs": 1200,
"creditsBilled": 3
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let response = client.stop_interaction("job-123").await.unwrap();
assert!(response.success);
assert_eq!(response.session_duration_ms, Some(1200));
assert_eq!(response.credits_billed, Some(3));
mock.assert();
}
#[tokio::test]
async fn test_interact_error_response() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/v2/scrape/job-404/interact")
.with_status(404)
.with_header("content-type", "application/json")
.with_body(
json!({
"success": false,
"error": "Job not found."
})
.to_string(),
)
.create();
let client = Client::new_selfhosted(server.url(), Some("test_key")).unwrap();
let result = client
.interact(
"job-404",
ScrapeExecuteOptions {
code: Some("console.log('ok')".to_string()),
..Default::default()
},
)
.await;
assert!(result.is_err());
mock.assert();
}
}