Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(rust-sdk): add DocumentMetadata::provider for Exchange scrapes (#5001)
Gergő Móricz7599de41e7a3
diff --git a/apps/rust-sdk/CHANGELOG.md b/apps/rust-sdk/CHANGELOG.mdindex abf5e2a47..1b5661f4c 100644--- a/apps/rust-sdk/CHANGELOG.md+++ b/apps/rust-sdk/CHANGELOG.md@@ -1,5 +1,14 @@## CHANGELOG+## [2.21.5] - 2026-10-07++### Added++- `DocumentMetadata::provider` (`ScrapeProvider`) reports the third-party+ provider that served an Exchange scrape: its `id`, the access price in+ `credits_cost`, and every provider tried in `steps` (`ScrapeProviderStep`).+ It is `None` for any other scrape.+## [2.21.4] - 2026-10-07### Fixeddiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 29328766e..6c425b6db 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.21.4"+version = "2.21.5"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex ef8b27f3f..69908bece 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.21.4"+version = "2.21.5"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/serde_helpers.rs b/apps/rust-sdk/src/serde_helpers.rsindex 21fb59dc4..36550d1b7 100644--- a/apps/rust-sdk/src/serde_helpers.rs+++ b/apps/rust-sdk/src/serde_helpers.rs@@ -30,6 +30,17 @@ where}}+/// Reads a field as `T`, or as `None` when the value has another shape, such as+/// a page's own meta tag of the same name.+pub(crate) fn deserialize_or_none<'de, D, T>(deserializer: D) -> Result<Option<T>, D::Error>+where+ D: serde::Deserializer<'de>,+ T: serde::de::DeserializeOwned,+{+ Ok(Option::<Value>::deserialize(deserializer)?+ .and_then(|value| serde_json::from_value(value).ok()))+}+#[cfg(test)]mod tests {use serde::Deserialize;diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rsindex 93ade4cf7..791f74818 100644--- a/apps/rust-sdk/src/types.rs+++ b/apps/rust-sdk/src/types.rs@@ -4,7 +4,7 @@ use serde::{de, Deserialize, Deserializer, Serialize, Serializer};use serde_json::Value;use std::collections::HashMap;-use crate::serde_helpers::deserialize_string_or_array;+use crate::serde_helpers::{deserialize_or_none, deserialize_string_or_array};/// Available output formats for scraping operations.#[derive(Clone, Debug, PartialEq, Eq)]@@ -636,6 +636,33 @@ pub struct DocumentMetadata {pub cached_at: Option<String>,pub credits_used: Option<u32>,pub concurrency_limited: Option<bool>,+ /// The third-party provider that served an Exchange scrape. `None` for+ /// any other scrape.+ #[serde(default, deserialize_with = "deserialize_or_none")]+ pub provider: Option<ScrapeProvider>,+}++/// The third-party provider that served an Exchange scrape, what the access+/// cost in credits, and every provider tried for it in order.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeProvider {+ /// The provider that returned the data.+ pub id: String,+ pub credits_cost: u32,+ pub steps: Vec<ScrapeProviderStep>,+}++/// One provider tried for an Exchange scrape.+#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone, PartialEq)]+#[serde(rename_all = "camelCase")]+pub struct ScrapeProviderStep {+ pub provider: String,+ /// `matched`, `not_found` or `error`.+ pub status: String,+ pub credits_cost: Option<u32>,}/// Extracted attribute result.@@ -1265,6 +1292,63 @@ mod tests {assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));}+ #[test]+ fn test_document_metadata_provider() {+ let json = json!({+ "metadata": {+ "sourceURL": "https://profiles.example/in/example-person",+ "statusCode": 200,+ "provider": {+ "id": "globex",+ "creditsCost": 30,+ "steps": [+ { "provider": "acme", "status": "not_found", "creditsCost": 10 },+ { "provider": "globex", "status": "matched", "creditsCost": 20 },+ { "provider": "initech", "status": "error" }+ ]+ }+ }+ });+ let doc: Document = serde_json::from_value(json).unwrap();+ let provider = doc.metadata.unwrap().provider.unwrap();+ assert_eq!(provider.id, "globex");+ assert_eq!(provider.credits_cost, 30);+ assert_eq!(+ provider.steps,+ vec![+ ScrapeProviderStep {+ provider: "acme".to_string(),+ status: "not_found".to_string(),+ credits_cost: Some(10),+ },+ ScrapeProviderStep {+ provider: "globex".to_string(),+ status: "matched".to_string(),+ credits_cost: Some(20),+ },+ ScrapeProviderStep {+ provider: "initech".to_string(),+ status: "error".to_string(),+ credits_cost: None,+ },+ ]+ );+ }++ #[test]+ fn test_document_metadata_provider_absent_or_other_shape() {+ for metadata in [+ json!({ "statusCode": 200 }),+ json!({ "statusCode": 200, "provider": "Example Provider" }),+ json!({ "statusCode": 200, "provider": ["a", "b"] }),+ ] {+ let doc: Document = serde_json::from_value(json!({ "metadata": metadata })).unwrap();+ let meta = doc.metadata.unwrap();+ assert_eq!(meta.status_code, Some(200));+ assert_eq!(meta.provider, None);+ }+ }+#[test]fn test_format_menu_round_trip() {let format = Format::Menu;