Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(rust-sdk): handle metadata fields returned as arrays (#2948)

Mike Rossc8f2307f4061
diff --git a/apps/rust-sdk/src/document.rs b/apps/rust-sdk/src/document.rs
index 38c7fcc15..5dd2c1a47 100644
--- a/apps/rust-sdk/src/document.rs
+++ b/apps/rust-sdk/src/document.rs
@@ -1,6 +1,8 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use crate::serde_helpers::deserialize_string_or_array;
+
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
#[serde(rename_all = "camelCase")]
@@ -12,40 +14,68 @@ pub struct DocumentMetadata {
pub error: Option<String>,
// basic meta tags
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub title: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub language: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub keywords: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub robots: Option<String>,
// og: namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_title: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_url: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_image: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_audio: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_determiner: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_locale: Option<String>,
pub og_locale_alternate: Option<Vec<String>>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_site_name: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_video: Option<String>,
// article: namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_section: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_tag: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub published_time: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub modified_time: Option<String>,
// dc./dcterms. namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_keywords: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_subject: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_subject: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_audience: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_type: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_type: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date_created: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_created: Option<String>,
}
@@ -83,3 +113,38 @@ pub struct Document {
/// The warning message will contain any errors encountered during the extraction.
pub warning: Option<String>,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn test_full_document_with_array_metadata() {
+ let json = json!({
+ "markdown": "# Hello",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example Page",
+ "description": ["A great page", "with multiple descriptions"],
+ "robots": ["index", "follow"],
+ "ogImage": ["https://img.jpg"],
+ "language": "en",
+ "keywords": ["rust", "sdk", "firecrawl"]
+ }
+ });
+ let doc: Document = serde_json::from_value(json).unwrap();
+ assert_eq!(doc.markdown, Some("# Hello".to_string()));
+ let meta = doc.metadata;
+ assert_eq!(meta.title, Some("Example Page".to_string()));
+ assert_eq!(
+ meta.description,
+ Some("A great page, with multiple descriptions".to_string())
+ );
+ assert_eq!(meta.robots, Some("index, follow".to_string()));
+ assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));
+ assert_eq!(meta.language, Some("en".to_string()));
+ assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));
+ }
+}
diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs
index 9f69ac226..1b56c2e47 100644
--- a/apps/rust-sdk/src/lib.rs
+++ b/apps/rust-sdk/src/lib.rs
@@ -46,6 +46,7 @@ pub mod llmstxt;
pub mod map;
pub mod scrape;
pub mod search;
+pub(crate) mod serde_helpers;
// v2 module (new API)
pub mod v2;
diff --git a/apps/rust-sdk/src/serde_helpers.rs b/apps/rust-sdk/src/serde_helpers.rs
new file mode 100644
index 000000000..21fb59dc4
--- /dev/null
+++ b/apps/rust-sdk/src/serde_helpers.rs
@@ -0,0 +1,85 @@
+use serde::Deserialize;
+use serde_json::Value;
+
+/// Handles metadata fields that the API may return as either a string or an array of strings.
+/// Arrays are joined with ", ".
+pub(crate) fn deserialize_string_or_array<'de, D>(
+ deserializer: D,
+) -> Result<Option<String>, D::Error>
+where
+ D: serde::Deserializer<'de>,
+{
+ match Option::<Value>::deserialize(deserializer)? {
+ None | Some(Value::Null) => Ok(None),
+ Some(Value::String(s)) => Ok(Some(s)),
+ Some(Value::Array(arr)) => {
+ let strings: Vec<String> = arr
+ .into_iter()
+ .map(|v| match v {
+ Value::String(s) => s,
+ other => other.to_string(),
+ })
+ .collect();
+ if strings.is_empty() {
+ Ok(None)
+ } else {
+ Ok(Some(strings.join(", ")))
+ }
+ }
+ Some(other) => Ok(Some(other.to_string())),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use serde::Deserialize;
+ use serde_json::json;
+
+ #[derive(Deserialize)]
+ struct Helper {
+ #[serde(default, deserialize_with = "super::deserialize_string_or_array")]
+ field: Option<String>,
+ }
+
+ #[test]
+ fn test_string_value() {
+ let h: Helper = serde_json::from_value(json!({"field": "hello"})).unwrap();
+ assert_eq!(h.field, Some("hello".to_string()));
+ }
+
+ #[test]
+ fn test_array_value() {
+ let h: Helper = serde_json::from_value(json!({"field": ["index", "follow"]})).unwrap();
+ assert_eq!(h.field, Some("index, follow".to_string()));
+ }
+
+ #[test]
+ fn test_null_value() {
+ let h: Helper = serde_json::from_value(json!({"field": null})).unwrap();
+ assert_eq!(h.field, None);
+ }
+
+ #[test]
+ fn test_missing_field() {
+ let h: Helper = serde_json::from_value(json!({})).unwrap();
+ assert_eq!(h.field, None);
+ }
+
+ #[test]
+ fn test_empty_array() {
+ let h: Helper = serde_json::from_value(json!({"field": []})).unwrap();
+ assert_eq!(h.field, None);
+ }
+
+ #[test]
+ fn test_single_element_array() {
+ let h: Helper = serde_json::from_value(json!({"field": ["noindex"]})).unwrap();
+ assert_eq!(h.field, Some("noindex".to_string()));
+ }
+
+ #[test]
+ fn test_serialization_roundtrip() {
+ let h: Helper = serde_json::from_value(json!({"field": "index, follow"})).unwrap();
+ assert_eq!(h.field, Some("index, follow".to_string()));
+ }
+}
diff --git a/apps/rust-sdk/src/v2/types.rs b/apps/rust-sdk/src/v2/types.rs
index 89ffe68b8..5d119fd42 100644
--- a/apps/rust-sdk/src/v2/types.rs
+++ b/apps/rust-sdk/src/v2/types.rs
@@ -4,6 +4,8 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
+use crate::serde_helpers::deserialize_string_or_array;
+
/// Available output formats for scraping operations.
#[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
@@ -307,49 +309,83 @@ pub struct DocumentMetadata {
pub error: Option<String>,
// Basic meta tags
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub title: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub language: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub keywords: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub robots: Option<String>,
// OpenGraph namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_title: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_url: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_image: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_audio: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_determiner: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_locale: Option<String>,
pub og_locale_alternate: Option<Vec<String>>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_site_name: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub og_video: Option<String>,
// Article namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_section: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub article_tag: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub published_time: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub modified_time: Option<String>,
// Dublin Core namespace
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_keywords: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_description: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_subject: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_subject: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_audience: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_type: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_type: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dc_date_created: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub dcterms_created: Option<String>,
// Response metadata
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub scrape_id: Option<String>,
pub num_pages: Option<u32>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub content_type: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub timezone: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub proxy_used: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub cache_state: Option<String>,
+ #[serde(default, deserialize_with = "deserialize_string_or_array")]
pub cached_at: Option<String>,
pub credits_used: Option<u32>,
pub concurrency_limited: Option<bool>,
@@ -508,3 +544,38 @@ pub struct CrawlErrorsResponse {
#[serde(rename = "robotsBlocked")]
pub robots_blocked: Vec<String>,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn test_full_document_with_array_metadata() {
+ let json = json!({
+ "markdown": "# Hello",
+ "metadata": {
+ "sourceURL": "https://example.com",
+ "statusCode": 200,
+ "title": "Example Page",
+ "description": ["A great page", "with multiple descriptions"],
+ "robots": ["index", "follow"],
+ "ogImage": ["https://img.jpg"],
+ "language": "en",
+ "keywords": ["rust", "sdk", "firecrawl"]
+ }
+ });
+ let doc: Document = serde_json::from_value(json).unwrap();
+ assert_eq!(doc.markdown, Some("# Hello".to_string()));
+ let meta = doc.metadata.unwrap();
+ assert_eq!(meta.title, Some("Example Page".to_string()));
+ assert_eq!(
+ meta.description,
+ Some("A great page, with multiple descriptions".to_string())
+ );
+ assert_eq!(meta.robots, Some("index, follow".to_string()));
+ assert_eq!(meta.og_image, Some("https://img.jpg".to_string()));
+ assert_eq!(meta.language, Some("en".to_string()));
+ assert_eq!(meta.keywords, Some("rust, sdk, firecrawl".to_string()));
+ }
+}