Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(rust-sdk): handle metadata fields returned as arrays (#2948)
Mike Rossc8f2307f4061
diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rsindex 38c7fcc15..5dd2c1a47 100644--- a/apps/rust-sdk/src/document.rs+++ b/apps/rust-sdk/src/document.rs@@ -1,6 +1,8 @@use serde::{Deserialize, Serialize};use serde_json::Value;+use crate::serde_helpers::deserialize_string_or_array;+#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]#[serde(rename_all = "camelCase")]@@ -12,40 +14,68 @@ pub struct DocumentMetadata {pub error: Option<String>,// basic meta tags+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub title: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub language: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub keywords: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub robots: Option<String>,// og: namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_title: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_url: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_image: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_audio: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_determiner: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_locale: Option<String>,pub og_locale_alternate: Option<Vec<String>>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_site_name: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_video: Option<String>,// article: namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub article_section: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub article_tag: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub published_time: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub modified_time: Option<String>,// dc./dcterms. namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_keywords: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_subject: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_subject: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_audience: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_type: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_type: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_date: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_date_created: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_created: Option<String>,}@@ -83,3 +113,38 @@ pub struct Document {/// The warning message will contain any errors encountered during the extraction.pub warning: Option<String>,}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn test_full_document_with_array_metadata() {+ let json = json!({+ "markdown": "# Hello",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example Page",+ "description": ["A great page", "with multiple descriptions"],+ "robots": ["index", "follow"],+ "ogImage": ["https://img.jpg"],+ "language": "en",+ "keywords": ["rust", "sdk", "firecrawl"]+ }+ });+ let doc: Document = serde_json::from_value(json).unwrap();+ assert_eq!(doc.markdown, Some("# Hello".to_string()));+ let meta = doc.metadata;+ assert_eq!(meta.title, Some("Example Page".to_string()));+ assert_eq!(+ meta.description,+ Some("A great page, with multiple descriptions".to_string())+ );+ assert_eq!(meta.robots, Some("index, follow".to_string()));+ assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));+ assert_eq!(meta.language, Some("en".to_string()));+ assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));+ }+}diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex 9f69ac226..1b56c2e47 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -46,6 +46,7 @@ pub mod llmstxt;pub mod map;pub mod scrape;pub mod search;+pub(crate) mod serde_helpers;// v2 module (new API)pub mod v2;diff --git a/apps/rust-sdk/src/serde_helpers.rs b/apps/rust-sdk/src/serde_helpers.rsnew file mode 100644index 000000000..21fb59dc4--- /dev/null+++ b/apps/rust-sdk/src/serde_helpers.rs@@ -0,0 +1,85 @@+use serde::Deserialize;+use serde_json::Value;++/// Handles metadata fields that the API may return as either a string or an array of strings.+/// Arrays are joined with ", ".+pub(crate) fn deserialize_string_or_array<'de, D>(+ deserializer: D,+) -> Result<Option<String>, D::Error>+where+ D: serde::Deserializer<'de>,+{+ match Option::<Value>::deserialize(deserializer)? {+ None | Some(Value::Null) => Ok(None),+ Some(Value::String(s)) => Ok(Some(s)),+ Some(Value::Array(arr)) => {+ let strings: Vec<String> = arr+ .into_iter()+ .map(|v| match v {+ Value::String(s) => s,+ other => other.to_string(),+ })+ .collect();+ if strings.is_empty() {+ Ok(None)+ } else {+ Ok(Some(strings.join(", ")))+ }+ }+ Some(other) => Ok(Some(other.to_string())),+ }+}++#[cfg(test)]+mod tests {+ use serde::Deserialize;+ use serde_json::json;++ #[derive(Deserialize)]+ struct Helper {+ #[serde(default, deserialize_with = "super::deserialize_string_or_array")]+ field: Option<String>,+ }++ #[test]+ fn test_string_value() {+ let h: Helper = serde_json::from_value(json!({"field": "hello"})).unwrap();+ assert_eq!(h.field, Some("hello".to_string()));+ }++ #[test]+ fn test_array_value() {+ let h: Helper = serde_json::from_value(json!({"field": ["index", "follow"]})).unwrap();+ assert_eq!(h.field, Some("index, follow".to_string()));+ }++ #[test]+ fn test_null_value() {+ let h: Helper = serde_json::from_value(json!({"field": null})).unwrap();+ assert_eq!(h.field, None);+ }++ #[test]+ fn test_missing_field() {+ let h: Helper = serde_json::from_value(json!({})).unwrap();+ assert_eq!(h.field, None);+ }++ #[test]+ fn test_empty_array() {+ let h: Helper = serde_json::from_value(json!({"field": []})).unwrap();+ assert_eq!(h.field, None);+ }++ #[test]+ fn test_single_element_array() {+ let h: Helper = serde_json::from_value(json!({"field": ["noindex"]})).unwrap();+ assert_eq!(h.field, Some("noindex".to_string()));+ }++ #[test]+ fn test_serialization_roundtrip() {+ let h: Helper = serde_json::from_value(json!({"field": "index, follow"})).unwrap();+ assert_eq!(h.field, Some("index, follow".to_string()));+ }+}diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rsindex 89ffe68b8..5d119fd42 100644--- a/apps/rust-sdk/src/v2/types.rs+++ b/apps/rust-sdk/src/v2/types.rs@@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize};use serde_json::Value;use std::collections::HashMap;+use crate::serde_helpers::deserialize_string_or_array;+/// Available output formats for scraping operations.#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]#[serde(rename_all = "camelCase")]@@ -307,49 +309,83 @@ pub struct DocumentMetadata {pub error: Option<String>,// Basic meta tags+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub title: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub language: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub keywords: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub robots: Option<String>,// OpenGraph namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_title: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_url: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_image: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_audio: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_determiner: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_locale: Option<String>,pub og_locale_alternate: Option<Vec<String>>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_site_name: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub og_video: Option<String>,// Article namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub article_section: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub article_tag: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub published_time: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub modified_time: Option<String>,// Dublin Core namespace+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_keywords: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_description: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_subject: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_subject: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_audience: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_type: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_type: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_date: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dc_date_created: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub dcterms_created: Option<String>,// Response metadata+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub scrape_id: Option<String>,pub num_pages: Option<u32>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub content_type: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub timezone: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub proxy_used: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub cache_state: Option<String>,+ #[serde(default, deserialize_with = "deserialize_string_or_array")]pub cached_at: Option<String>,pub credits_used: Option<u32>,pub concurrency_limited: Option<bool>,@@ -508,3 +544,38 @@ pub struct CrawlErrorsResponse {#[serde(rename = "robotsBlocked")]pub robots_blocked: Vec<String>,}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn test_full_document_with_array_metadata() {+ let json = json!({+ "markdown": "# Hello",+ "metadata": {+ "sourceURL": "https://example.com",+ "statusCode": 200,+ "title": "Example Page",+ "description": ["A great page", "with multiple descriptions"],+ "robots": ["index", "follow"],+ "ogImage": ["https://img.jpg"],+ "language": "en",+ "keywords": ["rust", "sdk", "firecrawl"]+ }+ });+ let doc: Document = serde_json::from_value(json).unwrap();+ assert_eq!(doc.markdown, Some("# Hello".to_string()));+ let meta = doc.metadata.unwrap();+ assert_eq!(meta.title, Some("Example Page".to_string()));+ assert_eq!(+ meta.description,+ Some("A great page, with multiple descriptions".to_string())+ );+ assert_eq!(meta.robots, Some("index, follow".to_string()));+ assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));+ assert_eq!(meta.language, Some("en".to_string()));+ assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));+ }+}