Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(rust-sdk): send json_options as the v2 json format object (#5207)

Erik Engervallaad6dcf27448
diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rs
index aa196b630..b3cc94227 100644
--- a/apps/rust-sdk/src/crawl.rs
+++ b/apps/rust-sdk/src/crawl.rs
@@ -491,6 +491,28 @@ mod tests {
mock.assert();
}
+ #[test]
+ fn test_crawl_scrape_options_send_json_options_as_json_format() {
+ let options = CrawlOptions {
+ scrape_options: Some(ScrapeOptions {
+ formats: Some(vec![crate::Format::Json]),
+ json_options: Some(crate::JsonOptions {
+ prompt: Some("Extract the title".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["scrapeOptions"]["formats"],
+ json!([{ "type": "json", "prompt": "Extract the title" }])
+ );
+ assert!(payload["scrapeOptions"].get("jsonOptions").is_none());
+ }
+
#[tokio::test]
async fn test_get_crawl_status_with_mock() {
let mut server = mockito::Server::new_async().await;
diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rs
index dedc0ba8c..4e81525eb 100644
--- a/apps/rust-sdk/src/parse.rs
+++ b/apps/rust-sdk/src/parse.rs
@@ -3,12 +3,15 @@
use std::path::Path;
use reqwest::multipart::{Form, Part};
-use serde::{Deserialize, Serialize};
+use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
+use serde_json::Value;
use std::collections::HashMap;
use super::client::Client;
use super::scrape::ParserConfig;
-use super::types::{AttributeSelector, Document, JsonOptions};
+use super::types::{
+ fold_json_options, unfold_json_options, AttributeSelector, Document, JsonOptions,
+};
use crate::{AuditMetadata, FirecrawlError};
/// Uploaded file payload for the `/v2/parse` endpoint.
@@ -137,7 +140,7 @@ struct ParseFormatsResponse {
/// (e.g. actions, waitFor, location, and screenshot/branding/changeTracking options).
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
+#[serde(remote = "Self", rename_all = "camelCase")]
pub struct ParseOptions {
/// Output formats to include in the response.
pub formats: Option<Vec<ParseFormat>>,
@@ -187,6 +190,23 @@ pub struct ParseOptions {
pub attribute_selectors: Option<Vec<AttributeSelector>>,
}
+impl Serialize for ParseOptions {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut value = ParseOptions::serialize(self, serde_json::value::Serializer)
+ .map_err(ser::Error::custom)?;
+ fold_json_options(&mut value);
+ value.serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for ParseOptions {
+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ let mut value = Value::deserialize(deserializer)?;
+ unfold_json_options(&mut value);
+ ParseOptions::deserialize(value).map_err(de::Error::custom)
+ }
+}
+
impl Client {
/// Parse an uploaded file and return the extracted document.
pub async fn parse(
@@ -488,6 +508,35 @@ mod tests {
Ok(())
}
+ #[test]
+ fn test_parse_options_send_json_options_as_json_format() {
+ let options = ParseOptions {
+ formats: Some(vec![ParseFormat::Json]),
+ json_options: Some(JsonOptions {
+ prompt: Some("Extract the title".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!([{ "type": "json", "prompt": "Extract the title" }])
+ );
+ assert!(payload.get("jsonOptions").is_none());
+
+ let restored: ParseOptions = serde_json::from_value(payload).unwrap();
+ assert!(matches!(
+ restored.formats.as_deref(),
+ Some([ParseFormat::Json])
+ ));
+ assert_eq!(
+ restored.json_options.and_then(|options| options.prompt),
+ Some("Extract the title".to_string())
+ );
+ }
+
#[test]
fn test_parse_options_serializes_check_prompt_injection() {
let options = ParseOptions {
diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rs
index 040c7f39b..31a5c1ff0 100644
--- a/apps/rust-sdk/src/scrape.rs
+++ b/apps/rust-sdk/src/scrape.rs
@@ -1,20 +1,20 @@
//! Scrape endpoint for Firecrawl API v2.
-use serde::{Deserialize, Serialize};
+use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};
use serde_json::Value;
use std::collections::HashMap;
use crate::client::Client;
use crate::types::{
- Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,
- LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
+ fold_json_options, unfold_json_options, Action, AttributeSelector, ChangeTrackingOptions,
+ Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,
};
use crate::{AuditMetadata, FirecrawlError};
/// Options for scraping a URL.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]
-#[serde(rename_all = "camelCase")]
+#[serde(remote = "Self", rename_all = "camelCase")]
pub struct ScrapeOptions {
/// Origin label for request attribution (e.g., "[email protected]").
pub origin: Option<String>,
@@ -116,6 +116,23 @@ pub struct ScrapeOptions {
pub tool_detail: Option<crate::types::ToolDetail>,
}
+impl Serialize for ScrapeOptions {
+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+ let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer)
+ .map_err(ser::Error::custom)?;
+ fold_json_options(&mut value);
+ value.serialize(serializer)
+ }
+}
+
+impl<'de> Deserialize<'de> for ScrapeOptions {
+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
+ let mut value = Value::deserialize(deserializer)?;
+ unfold_json_options(&mut value);
+ ScrapeOptions::deserialize(value).map_err(de::Error::custom)
+ }
+}
+
/// Parser configuration for document parsing.
#[derive(Deserialize, Serialize, Debug, Clone)]
#[serde(untagged)]
@@ -728,6 +745,81 @@ mod tests {
);
}
+ #[test]
+ fn test_json_options_serialize_as_json_format() {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Json]),
+ json_options: Some(JsonOptions {
+ prompt: Some("Extract the product name".to_string()),
+ schema: Some(json!({ "type": "object" })),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!([
+ "markdown",
+ {
+ "type": "json",
+ "prompt": "Extract the product name",
+ "schema": { "type": "object" }
+ }
+ ])
+ );
+ assert!(payload.get("jsonOptions").is_none());
+ }
+
+ #[test]
+ #[allow(deprecated)]
+ fn test_json_options_add_json_format_and_drop_system_prompt() {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown]),
+ json_options: Some(JsonOptions {
+ prompt: Some("Extract the title".to_string()),
+ system_prompt: Some("You are an extractor".to_string()),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ assert_eq!(
+ payload["formats"],
+ json!(["markdown", { "type": "json", "prompt": "Extract the title" }])
+ );
+ assert!(payload.get("jsonOptions").is_none());
+ }
+
+ #[test]
+ fn test_scrape_options_round_trip() {
+ let options = ScrapeOptions {
+ formats: Some(vec![Format::Markdown, Format::Json]),
+ only_main_content: Some(true),
+ json_options: Some(JsonOptions {
+ prompt: Some("Extract the title".to_string()),
+ schema: Some(json!({ "type": "object" })),
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+
+ let payload = serde_json::to_value(options).unwrap();
+ let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();
+
+ assert!(matches!(
+ restored.formats.as_deref(),
+ Some([Format::Markdown, Format::Json])
+ ));
+ assert_eq!(restored.only_main_content, Some(true));
+ let json_options = restored.json_options.clone().unwrap();
+ assert_eq!(json_options.prompt, Some("Extract the title".to_string()));
+ assert_eq!(json_options.schema, Some(json!({ "type": "object" })));
+ assert_eq!(serde_json::to_value(restored).unwrap(), payload);
+ }
+
#[test]
fn test_scrape_options_serializes_domain_tools() {
let options = ScrapeOptions {
@@ -871,6 +963,20 @@ mod tests {
let mock = server
.mock("POST", "/v2/scrape")
+ .match_body(mockito::Matcher::PartialJson(json!({
+ "url": "https://example.com/product",
+ "formats": [{
+ "type": "json",
+ "prompt": "Extract product info",
+ "schema": {
+ "type": "object",
+ "properties": {
+ "title": { "type": "string" },
+ "price": { "type": "number" }
+ }
+ }
+ }]
+ })))
.with_status(200)
.with_header("content-type", "application/json")
.with_body(
diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rs
index a27fa93b8..ee9dbebc5 100644
--- a/apps/rust-sdk/src/types.rs
+++ b/apps/rust-sdk/src/types.rs
@@ -330,6 +330,7 @@ pub struct JsonOptions {
/// JSON schema the output should adhere to.
pub schema: Option<Value>,
/// System prompt for the LLM agent.
+ #[deprecated(note = "the v2 json format has no system prompt; this field is not sent")]
pub system_prompt: Option<String>,
/// Extraction prompt for the LLM agent.
pub prompt: Option<String>,
@@ -340,6 +341,58 @@ pub struct JsonOptions {
pub check_prompt_injection: Option<bool>,
}
+/// Moves a serialized `jsonOptions` into the `json` entry of `formats`, since /v2
+/// rejects a top-level `jsonOptions` key. Adds the `json` format when it is not listed.
+pub(crate) fn fold_json_options(value: &mut Value) {
+ let Some(map) = value.as_object_mut() else {
+ return;
+ };
+ let Some(Value::Object(options)) = map.remove("jsonOptions") else {
+ return;
+ };
+
+ let mut json_format = serde_json::Map::new();
+ json_format.insert("type".to_string(), Value::from("json"));
+ json_format.extend(options.into_iter().filter(|(key, _)| key != "systemPrompt"));
+ let json_format = Value::Object(json_format);
+
+ let formats = map
+ .entry("formats")
+ .or_insert_with(|| Value::Array(Vec::new()));
+ let Some(formats) = formats.as_array_mut() else {
+ return;
+ };
+ match formats
+ .iter_mut()
+ .find(|format| format.as_str() == Some("json"))
+ {
+ Some(slot) => *slot = json_format,
+ None => formats.push(json_format),
+ }
+}
+
+/// Reverses `fold_json_options`: a `json` format object in `formats` becomes the
+/// `"json"` format plus `jsonOptions`, so serialized options deserialize again.
+pub(crate) fn unfold_json_options(value: &mut Value) {
+ let Some(map) = value.as_object_mut() else {
+ return;
+ };
+ let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {
+ return;
+ };
+ let Some(slot) = formats
+ .iter_mut()
+ .find(|format| format.get("type").and_then(Value::as_str) == Some("json"))
+ else {
+ return;
+ };
+ let Value::Object(mut options) = std::mem::replace(slot, Value::from("json")) else {
+ return;
+ };
+ options.remove("type");
+ map.insert("jsonOptions".to_string(), Value::Object(options));
+}
+
/// Location configuration for proxy routing.
#[serde_with::skip_serializing_none]
#[derive(Deserialize, Serialize, Debug, Default, Clone)]