Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(rust-sdk): send json_options as the v2 json format object (#5207)
Erik Engervallaad6dcf27448
diff --git a/apps/rust-sdk/src/crawl.rs b/apps/rust-sdk/src/crawl.rsindex aa196b630..b3cc94227 100644--- a/apps/rust-sdk/src/crawl.rs+++ b/apps/rust-sdk/src/crawl.rs@@ -491,6 +491,28 @@ mod tests {mock.assert();}+ #[test]+ fn test_crawl_scrape_options_send_json_options_as_json_format() {+ let options = CrawlOptions {+ scrape_options: Some(ScrapeOptions {+ formats: Some(vec![crate::Format::Json]),+ json_options: Some(crate::JsonOptions {+ prompt: Some("Extract the title".to_string()),+ ..Default::default()+ }),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["scrapeOptions"]["formats"],+ json!([{ "type": "json", "prompt": "Extract the title" }])+ );+ assert!(payload["scrapeOptions"].get("jsonOptions").is_none());+ }+#[tokio::test]async fn test_get_crawl_status_with_mock() {let mut server = mockito::Server::new_async().await;diff --git a/apps/rust-sdk/src/parse.rs b/apps/rust-sdk/src/parse.rsindex dedc0ba8c..4e81525eb 100644--- a/apps/rust-sdk/src/parse.rs+++ b/apps/rust-sdk/src/parse.rs@@ -3,12 +3,15 @@use std::path::Path;use reqwest::multipart::{Form, Part};-use serde::{Deserialize, Serialize};+use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};+use serde_json::Value;use std::collections::HashMap;use super::client::Client;use super::scrape::ParserConfig;-use super::types::{AttributeSelector, Document, JsonOptions};+use super::types::{+ fold_json_options, unfold_json_options, AttributeSelector, Document, JsonOptions,+};use crate::{AuditMetadata, FirecrawlError};/// Uploaded file payload for the `/v2/parse` endpoint.@@ -137,7 +140,7 @@ struct ParseFormatsResponse {/// (e.g. actions, waitFor, location, and screenshot/branding/changeTracking options).#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]+#[serde(remote = "Self", rename_all = "camelCase")]pub struct ParseOptions {/// Output formats to include in the response.pub formats: Option<Vec<ParseFormat>>,@@ -187,6 +190,23 @@ pub struct ParseOptions {pub attribute_selectors: Option<Vec<AttributeSelector>>,}+impl Serialize for ParseOptions {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut value = ParseOptions::serialize(self, serde_json::value::Serializer)+ .map_err(ser::Error::custom)?;+ fold_json_options(&mut value);+ value.serialize(serializer)+ }+}++impl<'de> Deserialize<'de> for ParseOptions {+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {+ let mut value = Value::deserialize(deserializer)?;+ unfold_json_options(&mut value);+ ParseOptions::deserialize(value).map_err(de::Error::custom)+ }+}+impl Client {/// Parse an uploaded file and return the extracted document.pub async fn parse(@@ -488,6 +508,35 @@ mod tests {Ok(())}+ #[test]+ fn test_parse_options_send_json_options_as_json_format() {+ let options = ParseOptions {+ formats: Some(vec![ParseFormat::Json]),+ json_options: Some(JsonOptions {+ prompt: Some("Extract the title".to_string()),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!([{ "type": "json", "prompt": "Extract the title" }])+ );+ assert!(payload.get("jsonOptions").is_none());++ let restored: ParseOptions = serde_json::from_value(payload).unwrap();+ assert!(matches!(+ restored.formats.as_deref(),+ Some([ParseFormat::Json])+ ));+ assert_eq!(+ restored.json_options.and_then(|options| options.prompt),+ Some("Extract the title".to_string())+ );+ }+#[test]fn test_parse_options_serializes_check_prompt_injection() {let options = ParseOptions {diff --git a/apps/rust-sdk/src/scrape.rs b/apps/rust-sdk/src/scrape.rsindex 040c7f39b..31a5c1ff0 100644--- a/apps/rust-sdk/src/scrape.rs+++ b/apps/rust-sdk/src/scrape.rs@@ -1,20 +1,20 @@//! Scrape endpoint for Firecrawl API v2.-use serde::{Deserialize, Serialize};+use serde::{de, ser, Deserialize, Deserializer, Serialize, Serializer};use serde_json::Value;use std::collections::HashMap;use crate::client::Client;use crate::types::{- Action, AttributeSelector, ChangeTrackingOptions, Document, Format, JsonOptions,- LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,+ fold_json_options, unfold_json_options, Action, AttributeSelector, ChangeTrackingOptions,+ Document, Format, JsonOptions, LocationConfig, ProfileConfig, ProxyType, ScreenshotOptions,};use crate::{AuditMetadata, FirecrawlError};/// Options for scraping a URL.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]-#[serde(rename_all = "camelCase")]+#[serde(remote = "Self", rename_all = "camelCase")]pub struct ScrapeOptions {/// Origin label for request attribution (e.g., "[email protected]").pub origin: Option<String>,@@ -116,6 +116,23 @@ pub struct ScrapeOptions {pub tool_detail: Option<crate::types::ToolDetail>,}+impl Serialize for ScrapeOptions {+ fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {+ let mut value = ScrapeOptions::serialize(self, serde_json::value::Serializer)+ .map_err(ser::Error::custom)?;+ fold_json_options(&mut value);+ value.serialize(serializer)+ }+}++impl<'de> Deserialize<'de> for ScrapeOptions {+ fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {+ let mut value = Value::deserialize(deserializer)?;+ unfold_json_options(&mut value);+ ScrapeOptions::deserialize(value).map_err(de::Error::custom)+ }+}+/// Parser configuration for document parsing.#[derive(Deserialize, Serialize, Debug, Clone)]#[serde(untagged)]@@ -728,6 +745,81 @@ mod tests {);}+ #[test]+ fn test_json_options_serialize_as_json_format() {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Json]),+ json_options: Some(JsonOptions {+ prompt: Some("Extract the product name".to_string()),+ schema: Some(json!({ "type": "object" })),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!([+ "markdown",+ {+ "type": "json",+ "prompt": "Extract the product name",+ "schema": { "type": "object" }+ }+ ])+ );+ assert!(payload.get("jsonOptions").is_none());+ }++ #[test]+ #[allow(deprecated)]+ fn test_json_options_add_json_format_and_drop_system_prompt() {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown]),+ json_options: Some(JsonOptions {+ prompt: Some("Extract the title".to_string()),+ system_prompt: Some("You are an extractor".to_string()),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ assert_eq!(+ payload["formats"],+ json!(["markdown", { "type": "json", "prompt": "Extract the title" }])+ );+ assert!(payload.get("jsonOptions").is_none());+ }++ #[test]+ fn test_scrape_options_round_trip() {+ let options = ScrapeOptions {+ formats: Some(vec![Format::Markdown, Format::Json]),+ only_main_content: Some(true),+ json_options: Some(JsonOptions {+ prompt: Some("Extract the title".to_string()),+ schema: Some(json!({ "type": "object" })),+ ..Default::default()+ }),+ ..Default::default()+ };++ let payload = serde_json::to_value(options).unwrap();+ let restored: ScrapeOptions = serde_json::from_value(payload.clone()).unwrap();++ assert!(matches!(+ restored.formats.as_deref(),+ Some([Format::Markdown, Format::Json])+ ));+ assert_eq!(restored.only_main_content, Some(true));+ let json_options = restored.json_options.clone().unwrap();+ assert_eq!(json_options.prompt, Some("Extract the title".to_string()));+ assert_eq!(json_options.schema, Some(json!({ "type": "object" })));+ assert_eq!(serde_json::to_value(restored).unwrap(), payload);+ }+#[test]fn test_scrape_options_serializes_domain_tools() {let options = ScrapeOptions {@@ -871,6 +963,20 @@ mod tests {let mock = server.mock("POST", "/v2/scrape")+ .match_body(mockito::Matcher::PartialJson(json!({+ "url": "https://example.com/product",+ "formats": [{+ "type": "json",+ "prompt": "Extract product info",+ "schema": {+ "type": "object",+ "properties": {+ "title": { "type": "string" },+ "price": { "type": "number" }+ }+ }+ }]+ }))).with_status(200).with_header("content-type", "application/json").with_body(diff --git a/apps/rust-sdk/src/types.rs b/apps/rust-sdk/src/types.rsindex a27fa93b8..ee9dbebc5 100644--- a/apps/rust-sdk/src/types.rs+++ b/apps/rust-sdk/src/types.rs@@ -330,6 +330,7 @@ pub struct JsonOptions {/// JSON schema the output should adhere to.pub schema: Option<Value>,/// System prompt for the LLM agent.+ #[deprecated(note = "the v2 json format has no system prompt; this field is not sent")]pub system_prompt: Option<String>,/// Extraction prompt for the LLM agent.pub prompt: Option<String>,@@ -340,6 +341,58 @@ pub struct JsonOptions {pub check_prompt_injection: Option<bool>,}+/// Moves a serialized `jsonOptions` into the `json` entry of `formats`, since /v2+/// rejects a top-level `jsonOptions` key. Adds the `json` format when it is not listed.+pub(crate) fn fold_json_options(value: &mut Value) {+ let Some(map) = value.as_object_mut() else {+ return;+ };+ let Some(Value::Object(options)) = map.remove("jsonOptions") else {+ return;+ };++ let mut json_format = serde_json::Map::new();+ json_format.insert("type".to_string(), Value::from("json"));+ json_format.extend(options.into_iter().filter(|(key, _)| key != "systemPrompt"));+ let json_format = Value::Object(json_format);++ let formats = map+ .entry("formats")+ .or_insert_with(|| Value::Array(Vec::new()));+ let Some(formats) = formats.as_array_mut() else {+ return;+ };+ match formats+ .iter_mut()+ .find(|format| format.as_str() == Some("json"))+ {+ Some(slot) => *slot = json_format,+ None => formats.push(json_format),+ }+}++/// Reverses `fold_json_options`: a `json` format object in `formats` becomes the+/// `"json"` format plus `jsonOptions`, so serialized options deserialize again.+pub(crate) fn unfold_json_options(value: &mut Value) {+ let Some(map) = value.as_object_mut() else {+ return;+ };+ let Some(formats) = map.get_mut("formats").and_then(Value::as_array_mut) else {+ return;+ };+ let Some(slot) = formats+ .iter_mut()+ .find(|format| format.get("type").and_then(Value::as_str) == Some("json"))+ else {+ return;+ };+ let Value::Object(mut options) = std::mem::replace(slot, Value::from("json")) else {+ return;+ };+ options.remove("type");+ map.insert("jsonOptions".to_string(), Value::Object(options));+}+/// Location configuration for proxy routing.#[serde_with::skip_serializing_none]#[derive(Deserialize, Serialize, Debug, Default, Clone)]