Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(go-sdk): wrap the underlying transport error in FirecrawlError (#5311)
Gaurav Chadha8da8b946287d
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdindex a8b0f4b42..f8ef79a2e 100644--- a/apps/go-sdk/README.md+++ b/apps/go-sdk/README.md@@ -394,6 +394,56 @@ The SDK automatically retries transient failures:- **Not retried:** 401, 429, and other 4xx errors- **Backoff:** Exponential backoff with configurable factor+When every attempt fails at the transport layer, the returned `*FirecrawlError` wraps the+last underlying error, so errors raised by a custom `http.Client` (passed with+`option.WithHTTPClient`) can be matched with `errors.Is` or `errors.As`:++```go+var errResponseTooLarge = errors.New("response too large")++const maxResponseBytes = 20 << 20++type cappedBody struct {+ io.ReadCloser+ remaining int64+}++func (b *cappedBody) Read(p []byte) (int, error) {+ if b.remaining <= 0 {+ return 0, errResponseTooLarge+ }+ if int64(len(p)) > b.remaining {+ p = p[:b.remaining]+ }+ n, err := b.ReadCloser.Read(p)+ b.remaining -= int64(n)+ if b.remaining <= 0 {+ return n, errResponseTooLarge+ }+ return n, err+}++type cappingTransport struct{}++func (cappingTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ resp, err := http.DefaultTransport.RoundTrip(req)+ if err != nil {+ return nil, err+ }+ resp.Body = &cappedBody{ReadCloser: resp.Body, remaining: maxResponseBytes + 1}+ return resp, nil+}++client, _ := firecrawl.NewClient(+ option.WithHTTPClient(&http.Client{Transport: cappingTransport{}}),+ option.WithMaxRetries(0),+)+_, err := client.Scrape(ctx, "https://example.com", nil)+if errors.Is(err, errResponseTooLarge) {+ // The response exceeded maxResponseBytes.+}+```+## Context SupportAll methods accept a `context.Context` for cancellation and deadline control:diff --git a/apps/go-sdk/firecrawl_error.go b/apps/go-sdk/firecrawl_error.goindex c8649f8ad..b7008009e 100644--- a/apps/go-sdk/firecrawl_error.go+++ b/apps/go-sdk/firecrawl_error.go@@ -24,6 +24,8 @@ type FirecrawlError struct {Message string// RequiresAction is set when the API needs an out-of-band step first, such as accepting provider terms.RequiresAction *RequiresAction++ cause error}func (e *FirecrawlError) Error() string {@@ -36,6 +38,12 @@ func (e *FirecrawlError) Error() string {return fmt.Sprintf("firecrawl: %s", e.Message)}+// Unwrap returns the transport or body-read error that caused the request to fail, if any,+// so callers can match it with errors.Is and errors.As.+func (e *FirecrawlError) Unwrap() error {+ return e.cause+}+// AuthenticationError is returned when the API key is invalid (HTTP 401).type AuthenticationError struct {FirecrawlErrordiff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.goindex 4b2f4f32e..d5d3db2da 100644--- a/apps/go-sdk/http_client.go+++ b/apps/go-sdk/http_client.go@@ -226,7 +226,7 @@ func (h *httpClient) postMultipart(if fe, ok := lastErr.(*FirecrawlError); ok {return nil, fe}- return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}+ return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr), cause: lastErr}}return nil, &FirecrawlError{Message: "request failed"}}@@ -322,7 +322,7 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interfif fe, ok := lastErr.(*FirecrawlError); ok {return nil, fe}- return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}+ return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr), cause: lastErr}}return nil, &FirecrawlError{Message: "request failed"}}diff --git a/apps/go-sdk/transport_errors_test.go b/apps/go-sdk/transport_errors_test.gonew file mode 100644index 000000000..93da3eec0--- /dev/null+++ b/apps/go-sdk/transport_errors_test.go@@ -0,0 +1,107 @@+package firecrawl++import (+ "context"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/firecrawl/firecrawl/apps/go-sdk/option"+)++var errBodyTooLarge = errors.New("response body too large")++type limitedBody struct {+ body io.ReadCloser+ remaining int64+}++func (b *limitedBody) Read(p []byte) (int, error) {+ if b.remaining <= 0 {+ return 0, errBodyTooLarge+ }+ if int64(len(p)) > b.remaining {+ p = p[:b.remaining]+ }+ n, err := b.body.Read(p)+ b.remaining -= int64(n)+ return n, err+}++func (b *limitedBody) Close() error { return b.body.Close() }++type limitingTransport struct {+ limit int64+ calls int+}++func (t *limitingTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ t.calls+++ resp, err := http.DefaultTransport.RoundTrip(req)+ if err != nil {+ return nil, err+ }+ resp.Body = &limitedBody{body: resp.Body, remaining: t.limit}+ return resp, nil+}++func TestScrapeBodyReadErrorIsUnwrappable(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"success":true,"data":{"markdown":"` + strings.Repeat("a", 4096) + `"}}`))+ }))+ defer server.Close()++ transport := &limitingTransport{limit: 1024}+ client, err := NewClient(+ option.WithAPIKey("fc-test"),+ option.WithAPIURL(server.URL),+ option.WithHTTPClient(&http.Client{Transport: transport}),+ option.WithMaxRetries(1),+ option.WithBackoffFactor(0.001),+ )+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ _, err = client.Scrape(context.Background(), "https://example.com", nil)+ if !errors.Is(err, errBodyTooLarge) {+ t.Fatalf("expected errors.Is(err, errBodyTooLarge), got %T: %v", err, err)+ }+ var fe *FirecrawlError+ if !errors.As(err, &fe) {+ t.Fatalf("expected *FirecrawlError, got %T: %v", err, err)+ }+ if !strings.Contains(fe.Error(), "request failed after 1 retries") {+ t.Errorf("Error() = %q, want the existing retry message", fe.Error())+ }+ if transport.calls != 2 {+ t.Errorf("transport calls = %d, want 2 (retry behavior unchanged)", transport.calls)+ }+}++func TestScrapeRoundTripErrorIsUnwrappable(t *testing.T) {+ client, err := NewClient(+ option.WithAPIKey("fc-test"),+ option.WithAPIURL("http://firecrawl.invalid"),+ option.WithHTTPClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {+ return nil, errBodyTooLarge+ })}),+ option.WithMaxRetries(0),+ )+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ _, err = client.Scrape(context.Background(), "https://example.com", nil)+ if !errors.Is(err, errBodyTooLarge) {+ t.Fatalf("expected errors.Is(err, errBodyTooLarge), got %T: %v", err, err)+ }+}++type roundTripFunc func(*http.Request) (*http.Response, error)++func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 50344d60b..fabe945d8 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.16.7"+const Version = "1.16.8"