Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(go-sdk): send JsonOptions as the v2 json format object (#5206)
Erik Engervall71af3aae5461
diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.goindex f73acfd37..145c26840 100644--- a/apps/go-sdk/options.go+++ b/apps/go-sdk/options.go@@ -93,7 +93,7 @@ type ScrapeOptions struct {RedactPII *bool `json:"redactPII,omitempty"`AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`Integration *string `json:"integration,omitempty"`- JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`+ JsonOptions *JsonOptions `json:"-"`DomainTools *bool `json:"domainTools,omitempty"`// CheckPromptInjection scans the page content for prompt injection with any// format except rawBase64, before LLM-backed formats run. A detection fails@@ -112,11 +112,7 @@ func (o ScrapeOptions) MarshalJSON() ([]byte, error) {scrapeOptions: scrapeOptions(o),}- if len(o.FormatOptions) > 0 {- payload.Formats = o.FormatOptions- } else if len(o.Formats) > 0 {- payload.Formats = o.Formats- }+ payload.Formats = requestFormats(o.Formats, o.FormatOptions, o.JsonOptions)return json.Marshal(payload)}@@ -271,6 +267,46 @@ type JsonOptions struct {CheckPromptInjection *bool `json:"checkPromptInjection,omitempty"`}+// requestFormats builds the request's formats array. JsonOptions travels as the+// json format object, since /v2 rejects a top-level jsonOptions key. It replaces+// a listed json format, or is appended when none is listed.+func requestFormats(formats []string, formatOptions []interface{}, jsonOptions *JsonOptions) interface{} {+ if jsonOptions == nil {+ if len(formatOptions) > 0 {+ return formatOptions+ }+ if len(formats) > 0 {+ return formats+ }+ return nil+ }++ items := make([]interface{}, 0, len(formats)+len(formatOptions)+1)+ if len(formatOptions) > 0 {+ items = append(items, formatOptions...)+ } else {+ for _, format := range formats {+ items = append(items, format)+ }+ }++ jsonFormat := struct {+ Type string `json:"type"`+ *JsonOptions+ }{Type: "json", JsonOptions: jsonOptions}+ for i, item := range items {+ isJSON := item == "json"+ if format, ok := item.(map[string]interface{}); ok {+ isJSON = format["type"] == "json"+ }+ if isJSON {+ items[i] = jsonFormat+ return items+ }+ }+ return append(items, jsonFormat)+}+// Pointer helpers for optional fields.// Bool returns a pointer to the given bool value.diff --git a/apps/go-sdk/options_test.go b/apps/go-sdk/options_test.goindex 406aed31d..aa58489e6 100644--- a/apps/go-sdk/options_test.go+++ b/apps/go-sdk/options_test.go@@ -175,6 +175,95 @@ func TestScrapeOptionsSerializesCheckPromptInjection(t *testing.T) {}}+func TestScrapeOptionsSendsJsonOptionsAsJsonFormat(t *testing.T) {+ payload, err := json.Marshal(ScrapeOptions{+ Formats: []string{"markdown", "json"},+ JsonOptions: &JsonOptions{+ Prompt: "Extract the product name",+ Schema: map[string]interface{}{"type": "object"},+ },+ })+ if err != nil {+ t.Fatalf("Marshal ScrapeOptions: %v", err)+ }++ jsonBody := string(payload)+ want := `"formats":["markdown",{"type":"json","prompt":"Extract the product name","schema":{"type":"object"}}]`+ if !strings.Contains(jsonBody, want) {+ t.Fatalf("serialized formats = %s, want to contain %s", jsonBody, want)+ }+ if strings.Contains(jsonBody, "jsonOptions") {+ t.Fatalf("serialized options = %s, want no top-level jsonOptions", jsonBody)+ }+}++func TestScrapeOptionsAddsJsonFormatForJsonOptions(t *testing.T) {+ payload, err := json.Marshal(ScrapeOptions{+ FormatOptions: []interface{}{"markdown", QuestionFormat{Question: "What is it?"}},+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},+ })+ if err != nil {+ t.Fatalf("Marshal ScrapeOptions: %v", err)+ }++ want := `"formats":["markdown",{"type":"question","question":"What is it?"},{"type":"json","prompt":"Extract the title"}]`+ if !strings.Contains(string(payload), want) {+ t.Fatalf("serialized formats = %s, want to contain %s", payload, want)+ }+}++func TestScrapeOptionsReplacesJsonFormatObjectWithJsonOptions(t *testing.T) {+ payload, err := json.Marshal(ScrapeOptions{+ FormatOptions: []interface{}{"markdown", map[string]interface{}{"type": "json"}},+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},+ })+ if err != nil {+ t.Fatalf("Marshal ScrapeOptions: %v", err)+ }++ want := `"formats":["markdown",{"type":"json","prompt":"Extract the title"}]`+ if !strings.Contains(string(payload), want) {+ t.Fatalf("serialized formats = %s, want to contain %s", payload, want)+ }+}++func TestCrawlScrapeOptionsSendJsonOptionsAsJsonFormat(t *testing.T) {+ payload, err := json.Marshal(CrawlOptions{+ ScrapeOptions: &ScrapeOptions{+ Formats: []string{"json"},+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},+ },+ })+ if err != nil {+ t.Fatalf("Marshal CrawlOptions: %v", err)+ }++ jsonBody := string(payload)+ want := `"scrapeOptions":{"formats":[{"type":"json","prompt":"Extract the title"}]}`+ if !strings.Contains(jsonBody, want) {+ t.Fatalf("serialized crawl options = %s, want to contain %s", jsonBody, want)+ }+}++func TestParseOptionsSendJsonOptionsAsJsonFormat(t *testing.T) {+ payload, err := json.Marshal(ParseOptions{+ Formats: []string{"json"},+ JsonOptions: &JsonOptions{Prompt: "Extract the title"},+ })+ if err != nil {+ t.Fatalf("Marshal ParseOptions: %v", err)+ }++ jsonBody := string(payload)+ want := `"formats":[{"type":"json","prompt":"Extract the title"}]`+ if !strings.Contains(jsonBody, want) {+ t.Fatalf("serialized parse options = %s, want to contain %s", jsonBody, want)+ }+ if strings.Contains(jsonBody, "jsonOptions") {+ t.Fatalf("serialized parse options = %s, want no top-level jsonOptions", jsonBody)+ }+}+func TestScrapeOptionsSerializesDomainTools(t *testing.T) {payload, err := json.Marshal(ScrapeOptions{DomainTools: Bool(true),diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.goindex d564fca24..dd55f51b3 100644--- a/apps/go-sdk/parse.go+++ b/apps/go-sdk/parse.go@@ -91,7 +91,7 @@ type ParseOptions struct {Integration *string `json:"integration,omitempty"`RedactPII *bool `json:"redactPII,omitempty"`AuditMetadata *AuditMetadata `json:"auditMetadata,omitempty"`- JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`+ JsonOptions *JsonOptions `json:"-"`// CheckPromptInjection scans the parsed content for prompt injection before// LLM-backed formats run. A detection fails the parse with// SCRAPE_PROMPT_INJECTION_DETECTED. Adds 4 credits when the check scans the@@ -109,11 +109,7 @@ func (o ParseOptions) MarshalJSON() ([]byte, error) {parseOptions: parseOptions(o),}- if len(o.FormatOptions) > 0 {- payload.Formats = o.FormatOptions- } else if len(o.Formats) > 0 {- payload.Formats = o.Formats- }+ payload.Formats = requestFormats(o.Formats, o.FormatOptions, o.JsonOptions)return json.Marshal(payload)}