Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(rust-sdk): send screenshot, changeTracking and attributes options inside their v2 format objects (#5232)
Erik Engervall64bc810adeeb
diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rsindex 4e81525eb..97d195cd0 100644--- a/apps/rust-sdk/src/parse.rs+++ b/apps/rust-sdk/src/parse.rs@@ -10,7 +10,7 @@ use std::collections::HashMap;use super::client::Client;use super::scrape::ParserConfig;use super::types::{- fold_json_options, unfold_json_options, AttributeSelector, Document, JsonOptions,+ fold_format_options, unfold_format_options, AttributeSelector, Document, JsonOptions,};use crate::{AuditMetadata, FirecrawlError};@@ -194,7 +194,7 @@ impl Serialize for ParseOptions {fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {let mut value = ParseOptions::serialize(self, serde_json::value::Serializer).map_err(ser::Error::custom)?;- fold_json_options(&mut value);+ fold_format_options(&mut value);value.serialize(serializer)}}@@ -202,7 +202,7 @@ impl Serialize for ParseOptions {impl<'de> Deserialize<'de> for ParseOptions {fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {let mut value = Value::deserialize(deserializer)?;- unfold_json_options(&mut value);+ unfold_format_options(&mut value);ParseOptions::deserialize(value).map_err(de::Error::custom)}}@@ -537,6 +537,28 @@ mod tests {);}+ #[test]+ fn test_parse_options_send_attribute_selectors_as_attributes_format() {+ let options = ParseOptions {+ formats: Some(vec![ParseFormat::Markdown, ParseFormat::Attributes]),+ attribute_selectors: Some(vec![AttributeSelector {+ selector: "a".to_string(),+ attribute: "href".to_string(),+ }]),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!([+ "markdown",+ { "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }+ ])+ );+ assert!(payload.get("attributeSelectors").is_none());+ }+#[test]fn test_parse_options_serializes_check_prompt_injection() {let options = ParseOptions {diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 31a5c1ff0..6e4c0fda7 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -6,7 +6,7 @@ use std::collections::HashMap;use crate::client::Client;use crate::types::{- fold_json_options, unfold_json_options, Action, AttributeSelector, ChangeTrackingOptions,+ fold_format_options, unfold_format_options, Action, AttributeSelector, ChangeTrackingOptions,Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,};use crate::{AuditMetadata, FirecrawlError};@@ -120,7 +120,7 @@ impl Serialize for ScrapeOptions {fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer).map_err(ser::Error::custom)?;- fold_json_options(&mut value);+ fold_format_options(&mut value);value.serialize(serializer)}}@@ -128,7 +128,7 @@ impl Serialize for ScrapeOptions {impl<'de> Deserialize<'de> for ScrapeOptions {fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {let mut value = Value::deserialize(deserializer)?;- unfold_json_options(&mut value);+ unfold_format_options(&mut value);ScrapeOptions::deserialize(value).map_err(de::Error::custom)}}@@ -793,6 +793,94 @@ mod tests {assert!(payload.get("jsonOptions").is_none());}+ #[test]+ fn test_format_options_serialize_inside_format_objects() {+ let options = ScrapeOptions {+ formats: Some(vec![+ Format::Markdown,+ Format::Screenshot,+ Format::ChangeTracking,+ Format::Attributes,+ ]),+ screenshot_options: Some(ScreenshotOptions {+ full_page: Some(true),+ quality: Some(80),+ viewport: Some(crate::types::Viewport {+ width: 1280,+ height: 720,+ }),+ }),+ change_tracking_options: Some(ChangeTrackingOptions {+ modes: Some(vec![crate::types::ChangeTrackingMode::GitDiff]),+ tag: Some("daily".to_string()),+ ..Default::default()+ }),+ attribute_selectors: Some(vec![AttributeSelector {+ selector: "a".to_string(),+ attribute: "href".to_string(),+ }]),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!([+ "markdown",+ {+ "type": "screenshot",+ "fullPage": true,+ "quality": 80,+ "viewport": { "width": 1280, "height": 720 }+ },+ { "type": "changeTracking", "modes": ["git-diff"], "tag": "daily" },+ { "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }+ ])+ );+ for key in [+ "screenshotOptions",+ "changeTrackingOptions",+ "attributeSelectors",+ ] {+ assert!(payload.get(key).is_none(), "{key} must not be top-level");+ }++ let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();+ assert!(matches!(+ restored.formats.as_deref(),+ Some([+ Format::Markdown,+ Format::Screenshot,+ Format::ChangeTracking,+ Format::Attributes+ ])+ ));+ assert_eq!(restored.screenshot_options.unwrap().quality, Some(80));+ assert_eq!(+ restored.change_tracking_options.unwrap().tag,+ Some("daily".to_string())+ );+ assert_eq!(restored.attribute_selectors.unwrap()[0].attribute, "href");+ }++ #[test]+ fn test_format_options_add_missing_format() {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown]),+ screenshot_options: Some(ScreenshotOptions {+ full_page: Some(true),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!(["markdown", { "type": "screenshot", "fullPage": true }])+ );+ }+#[test]fn test_scrape_options_round_trip() {let options = ScrapeOptions {diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rsindex f7aca42d6..829bd6f9e 100644--- a/apps/rust-sdk/src/search.rs+++ b/apps/rust-sdk/src/search.rs@@ -46,6 +46,7 @@ pub struct SearchOptions {pub country: Option<String>,/// Whether to ignore invalid URLs in results.+ #[serde(rename = "ignoreInvalidURLs")]pub ignore_invalid_urls: Option<bool>,/// Timeout in milliseconds.@@ -311,6 +312,19 @@ mod tests {assert!(default_value.get("country").is_none());}+ #[test]+ fn serializes_ignore_invalid_urls_option() {+ let options = SearchOptions {+ ignore_invalid_urls: Some(true),+ ..Default::default()+ };++ assert_eq!(+ serde_json::to_value(options).unwrap(),+ json!({ "ignoreInvalidURLs": true })+ );+ }+#[tokio::test]async fn test_search_with_mock() {let mut server = mockito::Server::new_async().await;diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rsindex ee9dbebc5..e6d1500bd 100644--- a/apps/rust-sdk/src/types.rs+++ b/apps/rust-sdk/src/types.rs@@ -341,56 +341,80 @@ pub struct JsonOptions {pub check_prompt_injection: Option<bool>,}-/// Moves a serialized `jsonOptions` into the `json` entry of `formats`, since /v2-/// rejects a top-level `jsonOptions` key. Adds the `json` format when it is not listed.-pub(crate) fn fold_json_options(value: &mut Value) {+/// Top-level option keys that /v2 only accepts inside their format object in `formats`.+const FORMAT_OPTIONS: [(&str, &str); 4] = [+ ("jsonOptions", "json"),+ ("screenshotOptions", "screenshot"),+ ("changeTrackingOptions", "changeTracking"),+ ("attributeSelectors", "attributes"),+];++/// Moves serialized `jsonOptions`, `screenshotOptions`, `changeTrackingOptions` and+/// `attributeSelectors` into their format object in `formats`, since /v2 rejects them as+/// top-level keys. Adds the format when it is not listed.+pub(crate) fn fold_format_options(value: &mut Value) {let Some(map) = value.as_object_mut() else {return;};- let Some(Value::Object(options)) = map.remove("jsonOptions") else {- return;- };-- let mut json_format = serde_json::Map::new();- json_format.insert("type".to_string(), Value::from("json"));- json_format.extend(options.into_iter().filter(|(key, _)| key != "systemPrompt"));- let json_format = Value::Object(json_format);-- let formats = map- .entry("formats")- .or_insert_with(|| Value::Array(Vec::new()));- let Some(formats) = formats.as_array_mut() else {- return;- };- match formats- .iter_mut()- .find(|format| format.as_str() == Some("json"))- {- Some(slot) => *slot = json_format,- None => formats.push(json_format),+ for (key, format_type) in FORMAT_OPTIONS {+ let mut format = serde_json::Map::new();+ format.insert("type".to_string(), Value::from(format_type));+ match map.remove(key) {+ Some(Value::Object(options)) => format.extend(+ options+ .into_iter()+ .filter(|(name, _)| name != "systemPrompt"),+ ),+ Some(selectors @ Value::Array(_)) => {+ format.insert("selectors".to_string(), selectors);+ }+ _ => continue,+ }+ let format = Value::Object(format);++ let formats = map+ .entry("formats")+ .or_insert_with(|| Value::Array(Vec::new()));+ let Some(formats) = formats.as_array_mut() else {+ return;+ };+ match formats+ .iter_mut()+ .find(|entry| entry.as_str() == Some(format_type))+ {+ Some(slot) => *slot = format,+ None => formats.push(format),+ }}}-/// Reverses `fold_json_options`: a `json` format object in `formats` becomes the-/// `"json"` format plus `jsonOptions`, so serialized options deserialize again.-pub(crate) fn unfold_json_options(value: &mut Value) {+/// Reverses `fold_format_options`: each format object in `formats` becomes the plain+/// format name plus its top-level options key, so serialized options deserialize again.+pub(crate) fn unfold_format_options(value: &mut Value) {let Some(map) = value.as_object_mut() else {return;};- let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {- return;- };- let Some(slot) = formats- .iter_mut()- .find(|format| format.get("type").and_then(Value::as_str) == Some("json"))- else {- return;- };- let Value::Object(mut options) = std::mem::replace(slot, Value::from("json")) else {- return;- };- options.remove("type");- map.insert("jsonOptions".to_string(), Value::Object(options));+ for (key, format_type) in FORMAT_OPTIONS {+ let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {+ return;+ };+ let Some(slot) = formats+ .iter_mut()+ .find(|entry| entry.get("type").and_then(Value::as_str) == Some(format_type))+ else {+ continue;+ };+ let Value::Object(mut options) = std::mem::replace(slot, Value::from(format_type)) else {+ continue;+ };+ options.remove("type");+ let options = if format_type == "attributes" {+ options.remove("selectors").unwrap_or_default()+ } else {+ Value::Object(options)+ };+ map.insert(key.to_string(), options);+ }}/// Location configuration for proxy routing.