Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(go-sdk): expose batch scrape errors and job ID from BatchScrape (#5467)

Gaurav Chadha55ffb98760bf
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.md
index f8ef79a2e..6a63669a3 100644
--- a/apps/go-sdk/README.md
+++ b/apps/go-sdk/README.md
@@ -231,10 +231,35 @@ job, err := client.BatchScrape(ctx, urls, &firecrawl.BatchScrapeOptions{
Formats: []string{"markdown"},
},
})
+if err != nil {
+ log.Fatal(err)
+}
+
+fmt.Println("batch", job.ID, "scraped", len(job.Data), "pages")
+
+// Pages that responded with an HTTP error (404, 429, 5xx, ...) are still
+// returned in job.Data, with the original status in metadata.statusCode.
+for _, doc := range job.Data {
+ if status, _ := doc.Metadata["statusCode"].(float64); status >= 400 {
+ fmt.Printf("%s responded with HTTP %d\n", doc.Metadata["sourceURL"], int(status))
+ }
+}
+
+// URLs that could not be scraped at all (DNS errors, timeouts, ...) are
+// listed in job.Errors with a stable code, and robots.txt-blocked URLs in
+// job.RobotsBlocked.
+for _, e := range job.Errors {
+ fmt.Printf("%s failed: %s (%s)\n", e.URL, e.Code, e.Error)
+}
+for _, u := range job.RobotsBlocked {
+ fmt.Println(u, "blocked by robots.txt")
+}
// Or manage manually
resp, err := client.StartBatchScrape(ctx, urls, nil)
status, err := client.GetBatchScrapeStatus(ctx, resp.ID)
+errs, err := client.GetBatchScrapeErrors(ctx, resp.ID)
+fmt.Println(len(errs.Errors), "URLs failed,", len(errs.RobotsBlocked), "blocked by robots.txt")
_, err = client.CancelBatchScrape(ctx, resp.ID)
```
diff --git a/apps/go-sdk/batch_scrape_errors_test.go b/apps/go-sdk/batch_scrape_errors_test.go
new file mode 100644
index 000000000..e999a979e
--- /dev/null
+++ b/apps/go-sdk/batch_scrape_errors_test.go
@@ -0,0 +1,260 @@
+package firecrawl
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+
+ "github.com/firecrawl/firecrawl/apps/go-sdk/option"
+)
+
+const batchErrorsPayload = `{
+ "errors": [
+ {
+ "id": "job-1",
+ "timestamp": "2026-10-29T11:14:17.000Z",
+ "url": "https://does-not-resolve.invalid",
+ "code": "SCRAPE_DNS_RESOLUTION_ERROR",
+ "error": "DNS resolution error"
+ },
+ {
+ "id": "job-2",
+ "url": "https://slow.example.com",
+ "code": "SCRAPE_TIMEOUT",
+ "error": "Scrape timed out"
+ },
+ {
+ "id": "job-3",
+ "url": "https://terms.example.com",
+ "code": "SCRAPE_THIRD_PARTY_DATA_TERMS_REQUIRED",
+ "error": "An organization admin must accept the provider's terms first",
+ "requiresAction": {"type": "accept_terms", "terms": "acme", "version": "2", "url": "https://firecrawl.dev/terms/acme"}
+ }
+ ],
+ "robotsBlocked": ["https://example.com/private"]
+}`
+
+// batchServer fakes the batch scrape lifecycle: start -> status (completed) ->
+// errors, recording every request so tests can assert on what was called.
+type batchServer struct {
+ *httptest.Server
+ mu sync.Mutex
+ requests []*http.Request
+ // errorsStatus is the HTTP status the /errors endpoint responds with.
+ errorsStatus int
+ // errorsBody is the body the /errors endpoint responds with.
+ errorsBody string
+}
+
+func newBatchServer(t *testing.T) *batchServer {
+ t.Helper()
+ s := &batchServer{errorsStatus: http.StatusOK, errorsBody: batchErrorsPayload}
+ s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ s.mu.Lock()
+ s.requests = append(s.requests, r.Clone(r.Context()))
+ s.mu.Unlock()
+
+ w.Header().Set("Content-Type", "application/json")
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/v2/batch/scrape":
+ _, _ = w.Write([]byte(`{"success":true,"id":"batch-123","url":"` + s.URL + `/v2/batch/scrape/batch-123"}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v2/batch/scrape/batch-123":
+ _, _ = w.Write([]byte(`{"success":true,"status":"completed","total":1,"completed":1,"creditsUsed":1,"expiresAt":"2026-10-30T11:14:17.000Z","data":[{"markdown":"# ok","metadata":{"sourceURL":"https://example.com","statusCode":200}}]}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v2/batch/scrape/batch-123/errors":
+ w.WriteHeader(s.errorsStatus)
+ _, _ = w.Write([]byte(s.errorsBody))
+ case r.Method == http.MethodGet && r.URL.Path == "/v2/crawl/crawl-456":
+ _, _ = w.Write([]byte(`{"success":true,"status":"completed","total":1,"completed":1,"data":[]}`))
+ default:
+ w.WriteHeader(http.StatusNotFound)
+ _, _ = w.Write([]byte(`{"success":false,"error":"unexpected ` + r.Method + ` ` + r.URL.String() + `"}`))
+ }
+ }))
+ t.Cleanup(s.Close)
+ return s
+}
+
+func (s *batchServer) requestsTo(path string) []*http.Request {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ var out []*http.Request
+ for _, r := range s.requests {
+ if r.URL.Path == path {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+func newBatchClient(t *testing.T, s *batchServer) *Client {
+ t.Helper()
+ client, err := NewClient(
+ option.WithAPIKey("fc-test"),
+ option.WithAPIURL(s.URL),
+ option.WithMaxRetries(0),
+ )
+ if err != nil {
+ t.Fatalf("NewClient: %v", err)
+ }
+ return client
+}
+
+func assertBatchErrors(t *testing.T, got []BatchScrapeError, robotsBlocked []string) {
+ t.Helper()
+ if len(got) != 3 {
+ t.Fatalf("got %d errors, want 3: %+v", len(got), got)
+ }
+ dns := got[0]
+ if dns.ID != "job-1" || dns.URL != "https://does-not-resolve.invalid" || dns.Code != "SCRAPE_DNS_RESOLUTION_ERROR" || dns.Error != "DNS resolution error" || dns.Timestamp != "2026-10-29T11:14:17.000Z" {
+ t.Errorf("unexpected first error: %+v", dns)
+ }
+ if dns.RequiresAction != nil {
+ t.Errorf("first error should not require an action, got %+v", dns.RequiresAction)
+ }
+ if got[1].Code != "SCRAPE_TIMEOUT" || got[1].Timestamp != "" {
+ t.Errorf("unexpected second error: %+v", got[1])
+ }
+ terms := got[2]
+ if terms.RequiresAction == nil {
+ t.Fatalf("third error should carry requiresAction: %+v", terms)
+ }
+ if terms.RequiresAction.Type != "accept_terms" || terms.RequiresAction.Terms != "acme" || terms.RequiresAction.Version != "2" || terms.RequiresAction.URL != "https://firecrawl.dev/terms/acme" {
+ t.Errorf("unexpected requiresAction: %+v", terms.RequiresAction)
+ }
+ if len(robotsBlocked) != 1 || robotsBlocked[0] != "https://example.com/private" {
+ t.Errorf("unexpected robotsBlocked: %v", robotsBlocked)
+ }
+}
+
+func TestBatchScrapeReturnsIDAndErrors(t *testing.T) {
+ s := newBatchServer(t)
+ client := newBatchClient(t, s)
+
+ job, err := client.BatchScrapeWithPolling(context.Background(), []string{"https://example.com", "https://does-not-resolve.invalid"}, nil, 1, 30)
+ if err != nil {
+ t.Fatalf("BatchScrapeWithPolling: %v", err)
+ }
+
+ if job.ID != "batch-123" {
+ t.Errorf("job.ID = %q, want batch-123", job.ID)
+ }
+ if job.Status != "completed" {
+ t.Errorf("job.Status = %q, want completed", job.Status)
+ }
+ if len(job.Data) != 1 || job.Data[0].Markdown != "# ok" {
+ t.Fatalf("unexpected job.Data: %+v", job.Data)
+ }
+ if status, _ := job.Data[0].Metadata["statusCode"].(float64); status != 200 {
+ t.Errorf("document should keep its HTTP status code: %+v", job.Data[0].Metadata)
+ }
+ assertBatchErrors(t, job.Errors, job.RobotsBlocked)
+
+ reqs := s.requestsTo("/v2/batch/scrape/batch-123/errors")
+ if len(reqs) != 1 {
+ t.Fatalf("errors endpoint called %d times, want 1", len(reqs))
+ }
+ if got := reqs[0].Header.Get("Authorization"); got != "Bearer fc-test" {
+ t.Errorf("Authorization = %q, want Bearer fc-test", got)
+ }
+}
+
+func TestBatchScrapeFailsWhenErrorsEndpointFails(t *testing.T) {
+ s := newBatchServer(t)
+ s.errorsStatus = http.StatusInternalServerError
+ s.errorsBody = `{"success":false,"error":"Internal server error"}`
+ client := newBatchClient(t, s)
+
+ job, err := client.BatchScrapeWithPolling(context.Background(), []string{"https://example.com"}, nil, 1, 30)
+ if err == nil {
+ t.Fatalf("expected an error, got job %+v", job)
+ }
+ var fcErr *FirecrawlError
+ if !errors.As(err, &fcErr) {
+ t.Fatalf("error %T is not a *FirecrawlError: %v", err, err)
+ }
+ if fcErr.StatusCode != http.StatusInternalServerError {
+ t.Errorf("StatusCode = %d, want 500", fcErr.StatusCode)
+ }
+}
+
+func TestGetBatchScrapeErrors(t *testing.T) {
+ s := newBatchServer(t)
+ client := newBatchClient(t, s)
+
+ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")
+ if err != nil {
+ t.Fatalf("GetBatchScrapeErrors: %v", err)
+ }
+ assertBatchErrors(t, resp.Errors, resp.RobotsBlocked)
+}
+
+func TestGetBatchScrapeErrorsAcceptsDataEnvelope(t *testing.T) {
+ s := newBatchServer(t)
+ s.errorsBody = `{"success":true,"data":` + batchErrorsPayload + `}`
+ client := newBatchClient(t, s)
+
+ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")
+ if err != nil {
+ t.Fatalf("GetBatchScrapeErrors: %v", err)
+ }
+ assertBatchErrors(t, resp.Errors, resp.RobotsBlocked)
+}
+
+func TestGetBatchScrapeErrorsEmptyIsNotNil(t *testing.T) {
+ s := newBatchServer(t)
+ s.errorsBody = `{"errors":[],"robotsBlocked":[]}`
+ client := newBatchClient(t, s)
+
+ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")
+ if err != nil {
+ t.Fatalf("GetBatchScrapeErrors: %v", err)
+ }
+ if resp.Errors == nil || len(resp.Errors) != 0 {
+ t.Errorf("Errors = %#v, want empty non-nil slice", resp.Errors)
+ }
+ if resp.RobotsBlocked == nil || len(resp.RobotsBlocked) != 0 {
+ t.Errorf("RobotsBlocked = %#v, want empty non-nil slice", resp.RobotsBlocked)
+ }
+}
+
+func TestGetBatchScrapeErrorsRequiresJobID(t *testing.T) {
+ s := newBatchServer(t)
+ client := newBatchClient(t, s)
+
+ if _, err := client.GetBatchScrapeErrors(context.Background(), ""); err == nil {
+ t.Fatal("expected an error for an empty job ID")
+ }
+ if n := len(s.requests); n != 0 {
+ t.Errorf("made %d requests, want none", n)
+ }
+}
+
+func TestStatusResponsesCarryJobID(t *testing.T) {
+ s := newBatchServer(t)
+ client := newBatchClient(t, s)
+
+ batch, err := client.GetBatchScrapeStatus(context.Background(), "batch-123")
+ if err != nil {
+ t.Fatalf("GetBatchScrapeStatus: %v", err)
+ }
+ if batch.ID != "batch-123" {
+ t.Errorf("batch.ID = %q, want batch-123", batch.ID)
+ }
+ if batch.Errors != nil {
+ t.Errorf("GetBatchScrapeStatus should not fetch errors, got %+v", batch.Errors)
+ }
+ if n := len(s.requestsTo("/v2/batch/scrape/batch-123/errors")); n != 0 {
+ t.Errorf("GetBatchScrapeStatus called the errors endpoint %d times", n)
+ }
+
+ crawl, err := client.GetCrawlStatus(context.Background(), "crawl-456")
+ if err != nil {
+ t.Fatalf("GetCrawlStatus: %v", err)
+ }
+ if crawl.ID != "crawl-456" {
+ t.Errorf("crawl.ID = %q, want crawl-456", crawl.ID)
+ }
+}
diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.go
index 04c8605fc..894b5d765 100644
--- a/apps/go-sdk/firecrawl.go
+++ b/apps/go-sdk/firecrawl.go
@@ -221,6 +221,11 @@ func (c *Client) GetCrawlStatus(ctx context.Context, jobID string) (*CrawlJob, e
if err := json.Unmarshal(raw, &job); err != nil {
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}
}
+ // The status response does not echo the job ID; carry it over so callers
+ // can reach the per-job endpoints from the returned job.
+ if job.ID == "" {
+ job.ID = jobID
+ }
return &job, nil
}
@@ -340,21 +345,84 @@ func (c *Client) GetBatchScrapeStatus(ctx context.Context, jobID string) (*Batch
if err := json.Unmarshal(raw, &job); err != nil {
return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}
}
+ // The status response does not echo the job ID; carry it over so callers
+ // can reach the per-job endpoints from the returned job.
+ if job.ID == "" {
+ job.ID = jobID
+ }
return &job, nil
}
+// GetBatchScrapeErrors gets the URLs of a batch scrape job that failed to
+// scrape, along with the URLs blocked by robots.txt.
+//
+// Pages that responded with an HTTP error such as 404 or 429 are not reported
+// here; they are returned as documents with the original status in
+// Metadata["statusCode"].
+func (c *Client) GetBatchScrapeErrors(ctx context.Context, jobID string) (*BatchScrapeErrorsResponse, error) {
+ if jobID == "" {
+ return nil, &FirecrawlError{Message: "job ID is required"}
+ }
+
+ raw, err := c.http.get(ctx, "/v2/batch/scrape/"+jobID+"/errors")
+ if err != nil {
+ return nil, err
+ }
+
+ // The API returns the payload at the top level; accept a `data` envelope
+ // as well, like the other SDKs do.
+ var envelope struct {
+ BatchScrapeErrorsResponse
+ Data *BatchScrapeErrorsResponse `json:"data"`
+ }
+ if err := json.Unmarshal(raw, &envelope); err != nil {
+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}
+ }
+ resp := envelope.BatchScrapeErrorsResponse
+ if envelope.Data != nil {
+ resp = *envelope.Data
+ }
+ if resp.Errors == nil {
+ resp.Errors = []BatchScrapeError{}
+ }
+ if resp.RobotsBlocked == nil {
+ resp.RobotsBlocked = []string{}
+ }
+ return &resp, nil
+}
+
// BatchScrape batch-scrapes URLs and waits for completion with auto-polling.
+//
+// The returned job carries the batch ID, the scraped documents in Data, and
+// the URLs that failed to scrape in Errors (see BatchScrapeJob).
func (c *Client) BatchScrape(ctx context.Context, urls []string, opts *BatchScrapeOptions) (*BatchScrapeJob, error) {
return c.BatchScrapeWithPolling(ctx, urls, opts, defaultPollInterval, defaultJobTimeout)
}
// BatchScrapeWithPolling batch-scrapes URLs and waits for completion with custom polling settings.
+//
+// The returned job carries the batch ID, the scraped documents in Data, and
+// the URLs that failed to scrape in Errors (see BatchScrapeJob).
func (c *Client) BatchScrapeWithPolling(ctx context.Context, urls []string, opts *BatchScrapeOptions, pollIntervalSec, timeoutSec int) (*BatchScrapeJob, error) {
start, err := c.StartBatchScrape(ctx, urls, opts)
if err != nil {
return nil, err
}
- return c.pollBatchScrape(ctx, start.ID, pollIntervalSec, timeoutSec)
+ job, err := c.pollBatchScrape(ctx, start.ID, pollIntervalSec, timeoutSec)
+ if err != nil {
+ return nil, err
+ }
+
+ // The status endpoint only reports successful documents, and its counters
+ // exclude failed URLs, so the errors endpoint is the only place a caller can
+ // learn which URLs were dropped and why.
+ errs, err := c.GetBatchScrapeErrors(ctx, start.ID)
+ if err != nil {
+ return nil, err
+ }
+ job.Errors = errs.Errors
+ job.RobotsBlocked = errs.RobotsBlocked
+ return job, nil
}
// CancelBatchScrape cancels a running batch scrape job.
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index 273a80007..c66cb2d4d 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -407,6 +407,10 @@ type BatchScrapeResponse struct {
}
// BatchScrapeJob represents the status and results of a batch scrape job.
+//
+// ID is always set to the batch job ID, so callers of the synchronous
+// BatchScrape can still reach the per-job endpoints (GetBatchScrapeErrors,
+// CancelBatchScrape) afterwards.
type BatchScrapeJob struct {
ID string `json:"id,omitempty"`
Status string `json:"status"`
@@ -416,6 +420,18 @@ type BatchScrapeJob struct {
ExpiresAt string `json:"expiresAt,omitempty"`
Next string `json:"next,omitempty"`
Data []Document `json:"data,omitempty"`
+
+ // Errors lists the URLs that could not be scraped at all (for example DNS
+ // resolution errors or timeouts). Pages that responded with an HTTP error
+ // such as 404 or 429 are not listed here: they are returned in Data with the
+ // original status in Metadata["statusCode"].
+ //
+ // Populated by BatchScrape and BatchScrapeWithPolling once the job has
+ // finished. GetBatchScrapeStatus leaves it nil; use GetBatchScrapeErrors.
+ Errors []BatchScrapeError `json:"errors,omitempty"`
+ // RobotsBlocked lists the URLs skipped because robots.txt disallowed them.
+ // Populated under the same conditions as Errors.
+ RobotsBlocked []string `json:"robotsBlocked,omitempty"`
}
// IsDone returns true if the batch scrape job has finished.
@@ -423,6 +439,32 @@ func (b *BatchScrapeJob) IsDone() bool {
return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"
}
+// BatchScrapeError describes one URL in a batch scrape job that failed to scrape.
+type BatchScrapeError struct {
+ // ID is the ID of the individual scrape job for this URL.
+ ID string `json:"id"`
+ // Timestamp is when the scrape failed, in RFC 3339 format, if known.
+ Timestamp string `json:"timestamp,omitempty"`
+ // URL is the URL that failed to scrape.
+ URL string `json:"url"`
+ // Code is a stable error code, such as "SCRAPE_DNS_RESOLUTION_ERROR" or
+ // "SCRAPE_TIMEOUT". Empty when the failure has no classified code.
+ Code string `json:"code,omitempty"`
+ // Error is the human-readable error message.
+ Error string `json:"error"`
+ // RequiresAction is set when the page needs an out-of-band step first, such
+ // as accepting a data provider's terms.
+ RequiresAction *RequiresAction `json:"requiresAction,omitempty"`
+}
+
+// BatchScrapeErrorsResponse is returned by GetBatchScrapeErrors.
+type BatchScrapeErrorsResponse struct {
+ // Errors lists the URLs that failed to scrape. See BatchScrapeError.
+ Errors []BatchScrapeError `json:"errors"`
+ // RobotsBlocked lists the URLs skipped because robots.txt disallowed them.
+ RobotsBlocked []string `json:"robotsBlocked"`
+}
+
// LinkResult represents a discovered URL from a map request.
// The API may return links as plain strings or as objects with url/title/description.
type LinkResult struct {
diff --git a/apps/go-sdk/pagination_origin_test.go b/apps/go-sdk/pagination_origin_test.go
index b7c2fb753..4c16d9dbd 100644
--- a/apps/go-sdk/pagination_origin_test.go
+++ b/apps/go-sdk/pagination_origin_test.go
@@ -116,6 +116,8 @@ var paginationFollowers = []paginationFollower{
_, _ = w.Write([]byte(`{"success":true,"id":"abc"}`))
case r.URL.Path == "/v2/batch/scrape/abc" && r.URL.Query().Get("skip") == "":
_, _ = w.Write([]byte(`{"status":"completed","data":[{"markdown":"one"}],"next":"` + next + `"}`))
+ case r.URL.Path == "/v2/batch/scrape/abc/errors":
+ _, _ = w.Write([]byte(`{"errors":[],"robotsBlocked":[]}`))
default:
return false
}
diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.go
index fabe945d8..d69cf6db1 100644
--- a/apps/go-sdk/version.go
+++ b/apps/go-sdk/version.go
@@ -9,4 +9,4 @@ package firecrawl
// Bump this when preparing a new release. The publish-go-sdk GitHub workflow
// reads this value and creates the corresponding monorepo-prefixed tag on
// merge to main.
-const Version = "1.16.8"
+const Version = "1.16.9"