Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(go-sdk): wrap the underlying transport error in FirecrawlError (#5311)

Gaurav Chadha8da8b946287d
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.md
index a8b0f4b42..f8ef79a2e 100644
--- a/apps/go-sdk/README.md
+++ b/apps/go-sdk/README.md
@@ -394,6 +394,56 @@ The SDK automatically retries transient failures:
- **Not retried:** 401, 429, and other 4xx errors
- **Backoff:** Exponential backoff with configurable factor
+When every attempt fails at the transport layer, the returned `*FirecrawlError` wraps the
+last underlying error, so errors raised by a custom `http.Client` (passed with
+`option.WithHTTPClient`) can be matched with `errors.Is` or `errors.As`:
+
+```go
+var errResponseTooLarge = errors.New("response too large")
+
+const maxResponseBytes = 20 << 20
+
+type cappedBody struct {
+ io.ReadCloser
+ remaining int64
+}
+
+func (b *cappedBody) Read(p []byte) (int, error) {
+ if b.remaining <= 0 {
+ return 0, errResponseTooLarge
+ }
+ if int64(len(p)) > b.remaining {
+ p = p[:b.remaining]
+ }
+ n, err := b.ReadCloser.Read(p)
+ b.remaining -= int64(n)
+ if b.remaining <= 0 {
+ return n, errResponseTooLarge
+ }
+ return n, err
+}
+
+type cappingTransport struct{}
+
+func (cappingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ resp, err := http.DefaultTransport.RoundTrip(req)
+ if err != nil {
+ return nil, err
+ }
+ resp.Body = &cappedBody{ReadCloser: resp.Body, remaining: maxResponseBytes + 1}
+ return resp, nil
+}
+
+client, _ := firecrawl.NewClient(
+ option.WithHTTPClient(&http.Client{Transport: cappingTransport{}}),
+ option.WithMaxRetries(0),
+)
+_, err := client.Scrape(ctx, "https://example.com", nil)
+if errors.Is(err, errResponseTooLarge) {
+ // The response exceeded maxResponseBytes.
+}
+```
+
## Context Support
All methods accept a `context.Context` for cancellation and deadline control:
diff --git a/apps/go-sdk/firecrawl_error.go b/apps/go-sdk/firecrawl_error.go
index c8649f8ad..b7008009e 100644
--- a/apps/go-sdk/firecrawl_error.go
+++ b/apps/go-sdk/firecrawl_error.go
@@ -24,6 +24,8 @@ type FirecrawlError struct {
Message string
// RequiresAction is set when the API needs an out-of-band step first, such as accepting provider terms.
RequiresAction *RequiresAction
+
+ cause error
}
func (e *FirecrawlError) Error() string {
@@ -36,6 +38,12 @@ func (e *FirecrawlError) Error() string {
return fmt.Sprintf("firecrawl: %s", e.Message)
}
+// Unwrap returns the transport or body-read error that caused the request to fail, if any,
+// so callers can match it with errors.Is and errors.As.
+func (e *FirecrawlError) Unwrap() error {
+ return e.cause
+}
+
// AuthenticationError is returned when the API key is invalid (HTTP 401).
type AuthenticationError struct {
FirecrawlError
diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.go
index 4b2f4f32e..d5d3db2da 100644
--- a/apps/go-sdk/http_client.go
+++ b/apps/go-sdk/http_client.go
@@ -226,7 +226,7 @@ func (h *httpClient) postMultipart(
if fe, ok := lastErr.(*FirecrawlError); ok {
return nil, fe
}
- return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}
+ return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr), cause: lastErr}
}
return nil, &FirecrawlError{Message: "request failed"}
}
@@ -322,7 +322,7 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interf
if fe, ok := lastErr.(*FirecrawlError); ok {
return nil, fe
}
- return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}
+ return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr), cause: lastErr}
}
return nil, &FirecrawlError{Message: "request failed"}
}
diff --git a/apps/go-sdk/transport_errors_test.go b/apps/go-sdk/transport_errors_test.go
new file mode 100644
index 000000000..93da3eec0
--- /dev/null
+++ b/apps/go-sdk/transport_errors_test.go
@@ -0,0 +1,107 @@
+package firecrawl
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/firecrawl/firecrawl/apps/go-sdk/option"
+)
+
+var errBodyTooLarge = errors.New("response body too large")
+
+type limitedBody struct {
+ body io.ReadCloser
+ remaining int64
+}
+
+func (b *limitedBody) Read(p []byte) (int, error) {
+ if b.remaining <= 0 {
+ return 0, errBodyTooLarge
+ }
+ if int64(len(p)) > b.remaining {
+ p = p[:b.remaining]
+ }
+ n, err := b.body.Read(p)
+ b.remaining -= int64(n)
+ return n, err
+}
+
+func (b *limitedBody) Close() error { return b.body.Close() }
+
+type limitingTransport struct {
+ limit int64
+ calls int
+}
+
+func (t *limitingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ t.calls++
+ resp, err := http.DefaultTransport.RoundTrip(req)
+ if err != nil {
+ return nil, err
+ }
+ resp.Body = &limitedBody{body: resp.Body, remaining: t.limit}
+ return resp, nil
+}
+
+func TestScrapeBodyReadErrorIsUnwrappable(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"success":true,"data":{"markdown":"` + strings.Repeat("a", 4096) + `"}}`))
+ }))
+ defer server.Close()
+
+ transport := &limitingTransport{limit: 1024}
+ client, err := NewClient(
+ option.WithAPIKey("fc-test"),
+ option.WithAPIURL(server.URL),
+ option.WithHTTPClient(&http.Client{Transport: transport}),
+ option.WithMaxRetries(1),
+ option.WithBackoffFactor(0.001),
+ )
+ if err != nil {
+ t.Fatalf("NewClient: %v", err)
+ }
+
+ _, err = client.Scrape(context.Background(), "https://example.com", nil)
+ if !errors.Is(err, errBodyTooLarge) {
+ t.Fatalf("expected errors.Is(err, errBodyTooLarge), got %T: %v", err, err)
+ }
+ var fe *FirecrawlError
+ if !errors.As(err, &fe) {
+ t.Fatalf("expected *FirecrawlError, got %T: %v", err, err)
+ }
+ if !strings.Contains(fe.Error(), "request failed after 1 retries") {
+ t.Errorf("Error() = %q, want the existing retry message", fe.Error())
+ }
+ if transport.calls != 2 {
+ t.Errorf("transport calls = %d, want 2 (retry behavior unchanged)", transport.calls)
+ }
+}
+
+func TestScrapeRoundTripErrorIsUnwrappable(t *testing.T) {
+ client, err := NewClient(
+ option.WithAPIKey("fc-test"),
+ option.WithAPIURL("http://firecrawl.invalid"),
+ option.WithHTTPClient(&http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return nil, errBodyTooLarge
+ })}),
+ option.WithMaxRetries(0),
+ )
+ if err != nil {
+ t.Fatalf("NewClient: %v", err)
+ }
+
+ _, err = client.Scrape(context.Background(), "https://example.com", nil)
+ if !errors.Is(err, errBodyTooLarge) {
+ t.Fatalf("expected errors.Is(err, errBodyTooLarge), got %T: %v", err, err)
+ }
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index 50344d60b..fabe945d8 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.16.7"
+const Version = "1.16.8"