Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(go-sdk): send JsonOptions as the v2 json format object (#5206)

Erik Engervall71af3aae5461
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.go
index f73acfd37..145c26840 100644
--- a/apps/go-sdk/options.go
+++ b/apps/go-sdk/options.go
@@ -93,7 +93,7 @@ type ScrapeOptions struct {
RedactPII *bool `json:"redactPII,omitempty"`
AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`
Integration *string `json:"integration,omitempty"`
- JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
+ JsonOptions *JsonOptions `json:"-"`
DomainTools *bool `json:"domainTools,omitempty"`
// CheckPromptInjection scans the page content for prompt injection with any
// format except rawBase64, before LLM-backed formats run. A detection fails
@@ -112,11 +112,7 @@ func (o ScrapeOptions) MarshalJSON() ([]byte, error) {
scrapeOptions: scrapeOptions(o),
}
- if len(o.FormatOptions) > 0 {
- payload.Formats = o.FormatOptions
- } else if len(o.Formats) > 0 {
- payload.Formats = o.Formats
- }
+ payload.Formats = requestFormats(o.Formats, o.FormatOptions, o.JsonOptions)
return json.Marshal(payload)
}
@@ -271,6 +267,46 @@ type JsonOptions struct {
CheckPromptInjection *bool `json:"checkPromptInjection,omitempty"`
}
+// requestFormats builds the request's formats array. JsonOptions travels as the
+// json format object, since /v2 rejects a top-level jsonOptions key. It replaces
+// a listed json format, or is appended when none is listed.
+func requestFormats(formats []string, formatOptions []interface{}, jsonOptions *JsonOptions) interface{} {
+ if jsonOptions == nil {
+ if len(formatOptions) > 0 {
+ return formatOptions
+ }
+ if len(formats) > 0 {
+ return formats
+ }
+ return nil
+ }
+
+ items := make([]interface{}, 0, len(formats)+len(formatOptions)+1)
+ if len(formatOptions) > 0 {
+ items = append(items, formatOptions...)
+ } else {
+ for _, format := range formats {
+ items = append(items, format)
+ }
+ }
+
+ jsonFormat := struct {
+ Type string `json:"type"`
+ *JsonOptions
+ }{Type: "json", JsonOptions: jsonOptions}
+ for i, item := range items {
+ isJSON := item == "json"
+ if format, ok := item.(map[string]interface{}); ok {
+ isJSON = format["type"] == "json"
+ }
+ if isJSON {
+ items[i] = jsonFormat
+ return items
+ }
+ }
+ return append(items, jsonFormat)
+}
+
// Pointer helpers for optional fields.
// Bool returns a pointer to the given bool value.
diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.go
index 406aed31d..aa58489e6 100644
--- a/apps/go-sdk/options_test.go
+++ b/apps/go-sdk/options_test.go
@@ -175,6 +175,95 @@ func TestScrapeOptionsSerializesCheckPromptInjection(t *testing.T) {
}
}
+func TestScrapeOptionsSendsJsonOptionsAsJsonFormat(t *testing.T) {
+ payload, err := json.Marshal(ScrapeOptions{
+ Formats: []string{"markdown", "json"},
+ JsonOptions: &JsonOptions{
+ Prompt: "Extract the product name",
+ Schema: map[string]interface{}{"type": "object"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Marshal ScrapeOptions: %v", err)
+ }
+
+ jsonBody := string(payload)
+ want := `"formats":["markdown",{"type":"json","prompt":"Extract the product name","schema":{"type":"object"}}]`
+ if !strings.Contains(jsonBody, want) {
+ t.Fatalf("serialized formats = %s, want to contain %s", jsonBody, want)
+ }
+ if strings.Contains(jsonBody, "jsonOptions") {
+ t.Fatalf("serialized options = %s, want no top-level jsonOptions", jsonBody)
+ }
+}
+
+func TestScrapeOptionsAddsJsonFormatForJsonOptions(t *testing.T) {
+ payload, err := json.Marshal(ScrapeOptions{
+ FormatOptions: []interface{}{"markdown", QuestionFormat{Question: "What is it?"}},
+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},
+ })
+ if err != nil {
+ t.Fatalf("Marshal ScrapeOptions: %v", err)
+ }
+
+ want := `"formats":["markdown",{"type":"question","question":"What is it?"},{"type":"json","prompt":"Extract the title"}]`
+ if !strings.Contains(string(payload), want) {
+ t.Fatalf("serialized formats = %s, want to contain %s", payload, want)
+ }
+}
+
+func TestScrapeOptionsReplacesJsonFormatObjectWithJsonOptions(t *testing.T) {
+ payload, err := json.Marshal(ScrapeOptions{
+ FormatOptions: []interface{}{"markdown", map[string]interface{}{"type": "json"}},
+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},
+ })
+ if err != nil {
+ t.Fatalf("Marshal ScrapeOptions: %v", err)
+ }
+
+ want := `"formats":["markdown",{"type":"json","prompt":"Extract the title"}]`
+ if !strings.Contains(string(payload), want) {
+ t.Fatalf("serialized formats = %s, want to contain %s", payload, want)
+ }
+}
+
+func TestCrawlScrapeOptionsSendJsonOptionsAsJsonFormat(t *testing.T) {
+ payload, err := json.Marshal(CrawlOptions{
+ ScrapeOptions: &ScrapeOptions{
+ Formats: []string{"json"},
+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Marshal CrawlOptions: %v", err)
+ }
+
+ jsonBody := string(payload)
+ want := `"scrapeOptions":{"formats":[{"type":"json","prompt":"Extract the title"}]}`
+ if !strings.Contains(jsonBody, want) {
+ t.Fatalf("serialized crawl options = %s, want to contain %s", jsonBody, want)
+ }
+}
+
+func TestParseOptionsSendJsonOptionsAsJsonFormat(t *testing.T) {
+ payload, err := json.Marshal(ParseOptions{
+ Formats: []string{"json"},
+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},
+ })
+ if err != nil {
+ t.Fatalf("Marshal ParseOptions: %v", err)
+ }
+
+ jsonBody := string(payload)
+ want := `"formats":[{"type":"json","prompt":"Extract the title"}]`
+ if !strings.Contains(jsonBody, want) {
+ t.Fatalf("serialized parse options = %s, want to contain %s", jsonBody, want)
+ }
+ if strings.Contains(jsonBody, "jsonOptions") {
+ t.Fatalf("serialized parse options = %s, want no top-level jsonOptions", jsonBody)
+ }
+}
+
func TestScrapeOptionsSerializesDomainTools(t *testing.T) {
payload, err := json.Marshal(ScrapeOptions{
DomainTools: Bool(true),
diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.go
index d564fca24..dd55f51b3 100644
--- a/apps/go-sdk/parse.go
+++ b/apps/go-sdk/parse.go
@@ -91,7 +91,7 @@ type ParseOptions struct {
Integration *string `json:"integration,omitempty"`
RedactPII *bool `json:"redactPII,omitempty"`
AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`
- JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`
+ JsonOptions *JsonOptions `json:"-"`
// CheckPromptInjection scans the parsed content for prompt injection before
// LLM-backed formats run. A detection fails the parse with
// SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check scans the
@@ -109,11 +109,7 @@ func (o ParseOptions) MarshalJSON() ([]byte, error) {
parseOptions: parseOptions(o),
}
- if len(o.FormatOptions) > 0 {
- payload.Formats = o.FormatOptions
- } else if len(o.Formats) > 0 {
- payload.Formats = o.Formats
- }
+ payload.Formats = requestFormats(o.Formats, o.FormatOptions, o.JsonOptions)
return json.Marshal(payload)
}