Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(rust-sdk): send screenshot, changeTracking and attributes options inside their v2 format objects (#5232)

Erik Engervall64bc810adeeb
diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rs
index 4e81525eb..97d195cd0 100644
--- a/apps/rust-sdk/src/parse.rs
+++ b/apps/rust-sdk/src/parse.rs
@@ -10,7 +10,7 @@ use std::collections::HashMap;
use super::client::Client;
use super::scrape::ParserConfig;
use super::types::{
- fold_json_options, unfold_json_options, AttributeSelector, Document, JsonOptions,
+ fold_format_options, unfold_format_options, AttributeSelector, Document, JsonOptions,
};
use crate::{AuditMetadata, FirecrawlError};
@@ -194,7 +194,7 @@ impl Serialize for ParseOptions {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut value = ParseOptions::serialize(self, serde_json::value::Serializer)
.map_err(ser::Error::custom)?;
- fold_json_options(&mut value);
+ fold_format_options(&mut value);
value.serialize(serializer)
}
}
@@ -202,7 +202,7 @@ impl Serialize for ParseOptions {
impl<'de> Deserialize<'de> for ParseOptions {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let mut value = Value::deserialize(deserializer)?;
- unfold_json_options(&mut value);
+ unfold_format_options(&mut value);
ParseOptions::deserialize(value).map_err(de::Error::custom)
}
}
@@ -537,6 +537,28 @@ mod tests {
);
}
+ #[test]
+ fn test_parse_options_send_attribute_selectors_as_attributes_format() {
+ let options = ParseOptions {
+ formats: Some(vec![ParseFormat::Markdown, ParseFormat::Attributes]),
+ attribute_selectors: Some(vec![AttributeSelector {
+ selector: "a".to_string(),
+ attribute: "href".to_string(),
+ }]),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!([
+ "markdown",
+ { "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }
+ ])
+ );
+ assert!(payload.get("attributeSelectors").is_none());
+ }
+
#[test]
fn test_parse_options_serializes_check_prompt_injection() {
let options = ParseOptions {
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 31a5c1ff0..6e4c0fda7 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -6,7 +6,7 @@ use std::collections::HashMap;
use crate::client::Client;
use crate::types::{
- fold_json_options, unfold_json_options, Action, AttributeSelector, ChangeTrackingOptions,
+ fold_format_options, unfold_format_options, Action, AttributeSelector, ChangeTrackingOptions,
Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
};
use crate::{AuditMetadata, FirecrawlError};
@@ -120,7 +120,7 @@ impl Serialize for ScrapeOptions {
fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer)
.map_err(ser::Error::custom)?;
- fold_json_options(&mut value);
+ fold_format_options(&mut value);
value.serialize(serializer)
}
}
@@ -128,7 +128,7 @@ impl Serialize for ScrapeOptions {
impl<'de> Deserialize<'de> for ScrapeOptions {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
let mut value = Value::deserialize(deserializer)?;
- unfold_json_options(&mut value);
+ unfold_format_options(&mut value);
ScrapeOptions::deserialize(value).map_err(de::Error::custom)
}
}
@@ -793,6 +793,94 @@ mod tests {
assert!(payload.get("jsonOptions").is_none());
}
+ #[test]
+ fn test_format_options_serialize_inside_format_objects() {
+ let options = ScrapeOptions {
+ formats: Some(vec![
+ Format::Markdown,
+ Format::Screenshot,
+ Format::ChangeTracking,
+ Format::Attributes,
+ ]),
+ screenshot_options: Some(ScreenshotOptions {
+ full_page: Some(true),
+ quality: Some(80),
+ viewport: Some(crate::types::Viewport {
+ width: 1280,
+ height: 720,
+ }),
+ }),
+ change_tracking_options: Some(ChangeTrackingOptions {
+ modes: Some(vec![crate::types::ChangeTrackingMode::GitDiff]),
+ tag: Some("daily".to_string()),
+ ..Default::default()
+ }),
+ attribute_selectors: Some(vec![AttributeSelector {
+ selector: "a".to_string(),
+ attribute: "href".to_string(),
+ }]),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!([
+ "markdown",
+ {
+ "type": "screenshot",
+ "fullPage": true,
+ "quality": 80,
+ "viewport": { "width": 1280, "height": 720 }
+ },
+ { "type": "changeTracking", "modes": ["git-diff"], "tag": "daily" },
+ { "type": "attributes", "selectors": [{ "selector": "a", "attribute": "href" }] }
+ ])
+ );
+ for key in [
+ "screenshotOptions",
+ "changeTrackingOptions",
+ "attributeSelectors",
+ ] {
+ assert!(payload.get(key).is_none(), "{key} must not be top-level");
+ }
+
+ let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();
+ assert!(matches!(
+ restored.formats.as_deref(),
+ Some([
+ Format::Markdown,
+ Format::Screenshot,
+ Format::ChangeTracking,
+ Format::Attributes
+ ])
+ ));
+ assert_eq!(restored.screenshot_options.unwrap().quality, Some(80));
+ assert_eq!(
+ restored.change_tracking_options.unwrap().tag,
+ Some("daily".to_string())
+ );
+ assert_eq!(restored.attribute_selectors.unwrap()[0].attribute, "href");
+ }
+
+ #[test]
+ fn test_format_options_add_missing_format() {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown]),
+ screenshot_options: Some(ScreenshotOptions {
+ full_page: Some(true),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!(["markdown", { "type": "screenshot", "fullPage": true }])
+ );
+ }
+
#[test]
fn test_scrape_options_round_trip() {
let options = ScrapeOptions {
diff --git a/apps/rust-sdk/src/search.rs b/apps/rust-sdk/src/search.rs
index f7aca42d6..829bd6f9e 100644
--- a/apps/rust-sdk/src/search.rs
+++ b/apps/rust-sdk/src/search.rs
@@ -46,6 +46,7 @@ pub struct SearchOptions {
pub country: Option<String>,
/// Whether to ignore invalid URLs in results.
+ #[serde(rename = "ignoreInvalidURLs")]
pub ignore_invalid_urls: Option<bool>,
/// Timeout in milliseconds.
@@ -311,6 +312,19 @@ mod tests {
assert!(default_value.get("country").is_none());
}
+ #[test]
+ fn serializes_ignore_invalid_urls_option() {
+ let options = SearchOptions {
+ ignore_invalid_urls: Some(true),
+ ..Default::default()
+ };
+
+ assert_eq!(
+ serde_json::to_value(options).unwrap(),
+ json!({ "ignoreInvalidURLs": true })
+ );
+ }
+
#[tokio::test]
async fn test_search_with_mock() {
let mut server = mockito::Server::new_async().await;
diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rs
index ee9dbebc5..e6d1500bd 100644
--- a/apps/rust-sdk/src/types.rs
+++ b/apps/rust-sdk/src/types.rs
@@ -341,56 +341,80 @@ pub struct JsonOptions {
pub check_prompt_injection: Option<bool>,
}
-/// Moves a serialized `jsonOptions` into the `json` entry of `formats`, since /v2
-/// rejects a top-level `jsonOptions` key. Adds the `json` format when it is not listed.
-pub(crate) fn fold_json_options(value: &mut Value) {
+/// Top-level option keys that /v2 only accepts inside their format object in `formats`.
+const FORMAT_OPTIONS: [(&str, &str); 4] = [
+ ("jsonOptions", "json"),
+ ("screenshotOptions", "screenshot"),
+ ("changeTrackingOptions", "changeTracking"),
+ ("attributeSelectors", "attributes"),
+];
+
+/// Moves serialized `jsonOptions`, `screenshotOptions`, `changeTrackingOptions` and
+/// `attributeSelectors` into their format object in `formats`, since /v2 rejects them as
+/// top-level keys. Adds the format when it is not listed.
+pub(crate) fn fold_format_options(value: &mut Value) {
let Some(map) = value.as_object_mut() else {
return;
};
- let Some(Value::Object(options)) = map.remove("jsonOptions") else {
- return;
- };
-
- let mut json_format = serde_json::Map::new();
- json_format.insert("type".to_string(), Value::from("json"));
- json_format.extend(options.into_iter().filter(|(key, _)| key != "systemPrompt"));
- let json_format = Value::Object(json_format);
-
- let formats = map
- .entry("formats")
- .or_insert_with(|| Value::Array(Vec::new()));
- let Some(formats) = formats.as_array_mut() else {
- return;
- };
- match formats
- .iter_mut()
- .find(|format| format.as_str() == Some("json"))
- {
- Some(slot) => *slot = json_format,
- None => formats.push(json_format),
+ for (key, format_type) in FORMAT_OPTIONS {
+ let mut format = serde_json::Map::new();
+ format.insert("type".to_string(), Value::from(format_type));
+ match map.remove(key) {
+ Some(Value::Object(options)) => format.extend(
+ options
+ .into_iter()
+ .filter(|(name, _)| name != "systemPrompt"),
+ ),
+ Some(selectors @ Value::Array(_)) => {
+ format.insert("selectors".to_string(), selectors);
+ }
+ _ => continue,
+ }
+ let format = Value::Object(format);
+
+ let formats = map
+ .entry("formats")
+ .or_insert_with(|| Value::Array(Vec::new()));
+ let Some(formats) = formats.as_array_mut() else {
+ return;
+ };
+ match formats
+ .iter_mut()
+ .find(|entry| entry.as_str() == Some(format_type))
+ {
+ Some(slot) => *slot = format,
+ None => formats.push(format),
+ }
}
}
-/// Reverses `fold_json_options`: a `json` format object in `formats` becomes the
-/// `"json"` format plus `jsonOptions`, so serialized options deserialize again.
-pub(crate) fn unfold_json_options(value: &mut Value) {
+/// Reverses `fold_format_options`: each format object in `formats` becomes the plain
+/// format name plus its top-level options key, so serialized options deserialize again.
+pub(crate) fn unfold_format_options(value: &mut Value) {
let Some(map) = value.as_object_mut() else {
return;
};
- let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {
- return;
- };
- let Some(slot) = formats
- .iter_mut()
- .find(|format| format.get("type").and_then(Value::as_str) == Some("json"))
- else {
- return;
- };
- let Value::Object(mut options) = std::mem::replace(slot, Value::from("json")) else {
- return;
- };
- options.remove("type");
- map.insert("jsonOptions".to_string(), Value::Object(options));
+ for (key, format_type) in FORMAT_OPTIONS {
+ let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {
+ return;
+ };
+ let Some(slot) = formats
+ .iter_mut()
+ .find(|entry| entry.get("type").and_then(Value::as_str) == Some(format_type))
+ else {
+ continue;
+ };
+ let Value::Object(mut options) = std::mem::replace(slot, Value::from(format_type)) else {
+ continue;
+ };
+ options.remove("type");
+ let options = if format_type == "attributes" {
+ options.remove("selectors").unwrap_or_default()
+ } else {
+ Value::Object(options)
+ };
+ map.insert(key.to_string(), options);
+ }
}
/// Location configuration for proxy routing.