Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(go-sdk): add GetParseFormats for GET /v2/parse/formats (#4908)
Gergő Móriczb2559740c04f
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdindex ff95fba48..d7524ca4b 100644--- a/apps/go-sdk/README.md+++ b/apps/go-sdk/README.md@@ -175,6 +175,16 @@ doc, err := client.Parse(ctx, file, &firecrawl.ParseOptions{fmt.Println(doc.Markdown)```+List the input formats the parse endpoint accepts. `Available` is `false` when+a format is supported but disabled on the deployment (for example, image OCR).++```go+formats, err := client.GetParseFormats(ctx)+for _, f := range formats {+ fmt.Println(f.Format, f.Kind, f.Extensions, f.MimeTypes, f.Available)+}+```+### CrawlCrawl a website and get content from multiple pages.diff --git a/apps/go-sdk/parse.go b/apps/go-sdk/parse.goindex 29a379e7e..4ee1376ed 100644--- a/apps/go-sdk/parse.go+++ b/apps/go-sdk/parse.go@@ -51,6 +51,25 @@ func NewParseFileFromBytes(filename string, content []byte) *ParseFile {}}+// ParseFormatKind classifies a parse input format. Unknown kinds returned by the+// API are preserved as-is.+type ParseFormatKind string++const (+ ParseFormatKindDocument ParseFormatKind = "document"+ ParseFormatKindImage ParseFormatKind = "image"+)++// ParseFormat describes one input file format accepted by the `/v2/parse` endpoint.+type ParseFormat struct {+ Format string `json:"format"`+ Kind ParseFormatKind `json:"kind"`+ Extensions []string `json:"extensions"`+ MimeTypes []string `json:"mimeTypes"`+ // Available is false when the format is supported but disabled on this deployment.+ Available bool `json:"available"`+}+// ParseOptions configures a parse request.//// Parse does not support browser-rendering features (actions, waitFor, location,@@ -127,3 +146,22 @@ func (c *Client) Parse(ctx context.Context, file *ParseFile, opts *ParseOptions)}return doc, nil}++// GetParseFormats lists the input file formats accepted by the `/v2/parse` endpoint.+func (c *Client) GetParseFormats(ctx context.Context) ([]ParseFormat, error) {+ raw, err := c.http.get(ctx, "/v2/parse/formats")+ if err != nil {+ return nil, err+ }++ data, err := extractDataAs[struct {+ Formats []ParseFormat `json:"formats"`+ }](raw)+ if err != nil {+ return nil, err+ }+ if data.Formats == nil {+ return nil, &FirecrawlError{Message: "parse formats response missing formats list"}+ }+ return data.Formats, nil+}diff --git a/apps/go-sdk/parse_test.go b/apps/go-sdk/parse_test.goindex 1bce28ad8..df2231951 100644--- a/apps/go-sdk/parse_test.go+++ b/apps/go-sdk/parse_test.go@@ -3,6 +3,7 @@ package firecrawlimport ("context""encoding/json"+ "errors""io""mime""mime/multipart"@@ -193,3 +194,122 @@ func TestDocumentUnmarshalsMenu(t *testing.T) {t.Errorf("item availability inStock = false, want true")}}++func TestGetParseFormats(t *testing.T) {+ authCh := make(chan string, 1)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet || r.URL.Path != "/v2/parse/formats" {+ t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)+ }+ authCh <- r.Header.Get("Authorization")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":{"formats":[+ {"format":"pdf","kind":"document","extensions":[".pdf"],"mimeTypes":["application/pdf"],"available":true},+ {"format":"png","kind":"image","extensions":[".png"],"mimeTypes":["image/png"],"available":false},+ {"format":"mp3","kind":"audio","extensions":[".mp3"],"mimeTypes":["audio/mpeg"],"available":true,"maxSizeBytes":1024}+ ]}}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-test"), option.WithAPIURL(server.URL))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ formats, err := client.GetParseFormats(context.Background())+ if err != nil {+ t.Fatalf("GetParseFormats: %v", err)+ }+ if gotAuth := <-authCh; gotAuth != "Bearer fc-test" {+ t.Errorf("Authorization = %q, want %q", gotAuth, "Bearer fc-test")+ }+ if len(formats) != 3 {+ t.Fatalf("formats = %d, want 3", len(formats))+ }++ pdf := formats[0]+ if pdf.Format != "pdf" || pdf.Kind != ParseFormatKindDocument || !pdf.Available {+ t.Errorf("pdf = %+v", pdf)+ }+ if len(pdf.Extensions) != 1 || pdf.Extensions[0] != ".pdf" {+ t.Errorf("pdf extensions = %v", pdf.Extensions)+ }+ if len(pdf.MimeTypes) != 1 || pdf.MimeTypes[0] != "application/pdf" {+ t.Errorf("pdf mimeTypes = %v", pdf.MimeTypes)+ }++ png := formats[1]+ if png.Kind != ParseFormatKindImage || png.Available {+ t.Errorf("png = %+v", png)+ }++ unknown := formats[2]+ if unknown.Kind != ParseFormatKind("audio") || unknown.Format != "mp3" || unknown.MimeTypes[0] != "audio/mpeg" {+ t.Errorf("unknown kind entry = %+v", unknown)+ }+}++func TestGetParseFormatsReturnsAuthenticationError(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusUnauthorized)+ _, _ = w.Write([]byte(`{"success":false,"error":"Unauthorized: Invalid token"}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-bad"), option.WithAPIURL(server.URL))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ formats, err := client.GetParseFormats(context.Background())+ if formats != nil {+ t.Errorf("formats = %v, want nil", formats)+ }+ var authErr *AuthenticationError+ if !errors.As(err, &authErr) {+ t.Fatalf("err = %T %v, want *AuthenticationError", err, err)+ }+ if authErr.StatusCode != http.StatusUnauthorized || authErr.Message != "Unauthorized: Invalid token" {+ t.Errorf("authErr = %+v", authErr.FirecrawlError)+ }+}++func TestGetParseFormatsReturnsServerError(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusInternalServerError)+ _, _ = w.Write([]byte(`{"success":false,"error":"boom"}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-test"), option.WithAPIURL(server.URL), option.WithMaxRetries(0))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ _, err = client.GetParseFormats(context.Background())+ var fcErr *FirecrawlError+ if !errors.As(err, &fcErr) || fcErr.StatusCode != http.StatusInternalServerError {+ t.Fatalf("err = %T %v, want *FirecrawlError with status 500", err, err)+ }+}++func TestGetParseFormatsRejectsMissingFormats(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":null}`))+ }))+ defer server.Close()++ client, err := NewClient(option.WithAPIKey("fc-test"), option.WithAPIURL(server.URL))+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ _, err = client.GetParseFormats(context.Background())+ var fcErr *FirecrawlError+ if !errors.As(err, &fcErr) {+ t.Fatalf("err = %T %v, want *FirecrawlError", err, err)+ }+}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 117f39006..e52a216c4 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.16.1"+const Version = "1.16.2"