Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(go-sdk): expose batch scrape errors and job ID from BatchScrape (#5467)
Gaurav Chadha55ffb98760bf
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdindex f8ef79a2e..6a63669a3 100644--- a/apps/go-sdk/README.md+++ b/apps/go-sdk/README.md@@ -231,10 +231,35 @@ job, err := client.BatchScrape(ctx, urls, &firecrawl.BatchScrapeOptions{Formats: []string{"markdown"},},})+if err != nil {+ log.Fatal(err)+}++fmt.Println("batch", job.ID, "scraped", len(job.Data), "pages")++// Pages that responded with an HTTP error (404, 429, 5xx, ...) are still+// returned in job.Data, with the original status in metadata.statusCode.+for _, doc := range job.Data {+ if status, _ := doc.Metadata["statusCode"].(float64); status >= 400 {+ fmt.Printf("%s responded with HTTP %d\n", doc.Metadata["sourceURL"], int(status))+ }+}++// URLs that could not be scraped at all (DNS errors, timeouts, ...) are+// listed in job.Errors with a stable code, and robots.txt-blocked URLs in+// job.RobotsBlocked.+for _, e := range job.Errors {+ fmt.Printf("%s failed: %s (%s)\n", e.URL, e.Code, e.Error)+}+for _, u := range job.RobotsBlocked {+ fmt.Println(u, "blocked by robots.txt")+}// Or manage manuallyresp, err := client.StartBatchScrape(ctx, urls, nil)status, err := client.GetBatchScrapeStatus(ctx, resp.ID)+errs, err := client.GetBatchScrapeErrors(ctx, resp.ID)+fmt.Println(len(errs.Errors), "URLs failed,", len(errs.RobotsBlocked), "blocked by robots.txt")_, err = client.CancelBatchScrape(ctx, resp.ID)```diff --git a/apps/go-sdk/batch_scrape_errors_test.go b/apps/go-sdk/batch_scrape_errors_test.gonew file mode 100644index 000000000..e999a979e--- /dev/null+++ b/apps/go-sdk/batch_scrape_errors_test.go@@ -0,0 +1,260 @@+package firecrawl++import (+ "context"+ "errors"+ "net/http"+ "net/http/httptest"+ "sync"+ "testing"++ "github.com/firecrawl/firecrawl/apps/go-sdk/option"+)++const batchErrorsPayload = `{+ "errors": [+ {+ "id": "job-1",+ "timestamp": "2026-10-29T11:14:17.000Z",+ "url": "https://does-not-resolve.invalid",+ "code": "SCRAPE_DNS_RESOLUTION_ERROR",+ "error": "DNS resolution error"+ },+ {+ "id": "job-2",+ "url": "https://slow.example.com",+ "code": "SCRAPE_TIMEOUT",+ "error": "Scrape timed out"+ },+ {+ "id": "job-3",+ "url": "https://terms.example.com",+ "code": "SCRAPE_THIRD_PARTY_DATA_TERMS_REQUIRED",+ "error": "An organization admin must accept the provider's terms first",+ "requiresAction": {"type": "accept_terms", "terms": "acme", "version": "2", "url": "https://firecrawl.dev/terms/acme"}+ }+ ],+ "robotsBlocked": ["https://example.com/private"]+}`++// batchServer fakes the batch scrape lifecycle: start -> status (completed) ->+// errors, recording every request so tests can assert on what was called.+type batchServer struct {+ *httptest.Server+ mu sync.Mutex+ requests []*http.Request+ // errorsStatus is the HTTP status the /errors endpoint responds with.+ errorsStatus int+ // errorsBody is the body the /errors endpoint responds with.+ errorsBody string+}++func newBatchServer(t *testing.T) *batchServer {+ t.Helper()+ s := &batchServer{errorsStatus: http.StatusOK, errorsBody: batchErrorsPayload}+ s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ s.mu.Lock()+ s.requests = append(s.requests, r.Clone(r.Context()))+ s.mu.Unlock()++ w.Header().Set("Content-Type", "application/json")+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v2/batch/scrape":+ _, _ = w.Write([]byte(`{"success":true,"id":"batch-123","url":"` + s.URL + `/v2/batch/scrape/batch-123"}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v2/batch/scrape/batch-123":+ _, _ = w.Write([]byte(`{"success":true,"status":"completed","total":1,"completed":1,"creditsUsed":1,"expiresAt":"2026-10-30T11:14:17.000Z","data":[{"markdown":"# ok","metadata":{"sourceURL":"https://example.com","statusCode":200}}]}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v2/batch/scrape/batch-123/errors":+ w.WriteHeader(s.errorsStatus)+ _, _ = w.Write([]byte(s.errorsBody))+ case r.Method == http.MethodGet && r.URL.Path == "/v2/crawl/crawl-456":+ _, _ = w.Write([]byte(`{"success":true,"status":"completed","total":1,"completed":1,"data":[]}`))+ default:+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte(`{"success":false,"error":"unexpected ` + r.Method + ` ` + r.URL.String() + `"}`))+ }+ }))+ t.Cleanup(s.Close)+ return s+}++func (s *batchServer) requestsTo(path string) []*http.Request {+ s.mu.Lock()+ defer s.mu.Unlock()+ var out []*http.Request+ for _, r := range s.requests {+ if r.URL.Path == path {+ out = append(out, r)+ }+ }+ return out+}++func newBatchClient(t *testing.T, s *batchServer) *Client {+ t.Helper()+ client, err := NewClient(+ option.WithAPIKey("fc-test"),+ option.WithAPIURL(s.URL),+ option.WithMaxRetries(0),+ )+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }+ return client+}++func assertBatchErrors(t *testing.T, got []BatchScrapeError, robotsBlocked []string) {+ t.Helper()+ if len(got) != 3 {+ t.Fatalf("got %d errors, want 3: %+v", len(got), got)+ }+ dns := got[0]+ if dns.ID != "job-1" || dns.URL != "https://does-not-resolve.invalid" || dns.Code != "SCRAPE_DNS_RESOLUTION_ERROR" || dns.Error != "DNS resolution error" || dns.Timestamp != "2026-10-29T11:14:17.000Z" {+ t.Errorf("unexpected first error: %+v", dns)+ }+ if dns.RequiresAction != nil {+ t.Errorf("first error should not require an action, got %+v", dns.RequiresAction)+ }+ if got[1].Code != "SCRAPE_TIMEOUT" || got[1].Timestamp != "" {+ t.Errorf("unexpected second error: %+v", got[1])+ }+ terms := got[2]+ if terms.RequiresAction == nil {+ t.Fatalf("third error should carry requiresAction: %+v", terms)+ }+ if terms.RequiresAction.Type != "accept_terms" || terms.RequiresAction.Terms != "acme" || terms.RequiresAction.Version != "2" || terms.RequiresAction.URL != "https://firecrawl.dev/terms/acme" {+ t.Errorf("unexpected requiresAction: %+v", terms.RequiresAction)+ }+ if len(robotsBlocked) != 1 || robotsBlocked[0] != "https://example.com/private" {+ t.Errorf("unexpected robotsBlocked: %v", robotsBlocked)+ }+}++func TestBatchScrapeReturnsIDAndErrors(t *testing.T) {+ s := newBatchServer(t)+ client := newBatchClient(t, s)++ job, err := client.BatchScrapeWithPolling(context.Background(), []string{"https://example.com", "https://does-not-resolve.invalid"}, nil, 1, 30)+ if err != nil {+ t.Fatalf("BatchScrapeWithPolling: %v", err)+ }++ if job.ID != "batch-123" {+ t.Errorf("job.ID = %q, want batch-123", job.ID)+ }+ if job.Status != "completed" {+ t.Errorf("job.Status = %q, want completed", job.Status)+ }+ if len(job.Data) != 1 || job.Data[0].Markdown != "# ok" {+ t.Fatalf("unexpected job.Data: %+v", job.Data)+ }+ if status, _ := job.Data[0].Metadata["statusCode"].(float64); status != 200 {+ t.Errorf("document should keep its HTTP status code: %+v", job.Data[0].Metadata)+ }+ assertBatchErrors(t, job.Errors, job.RobotsBlocked)++ reqs := s.requestsTo("/v2/batch/scrape/batch-123/errors")+ if len(reqs) != 1 {+ t.Fatalf("errors endpoint called %d times, want 1", len(reqs))+ }+ if got := reqs[0].Header.Get("Authorization"); got != "Bearer fc-test" {+ t.Errorf("Authorization = %q, want Bearer fc-test", got)+ }+}++func TestBatchScrapeFailsWhenErrorsEndpointFails(t *testing.T) {+ s := newBatchServer(t)+ s.errorsStatus = http.StatusInternalServerError+ s.errorsBody = `{"success":false,"error":"Internal server error"}`+ client := newBatchClient(t, s)++ job, err := client.BatchScrapeWithPolling(context.Background(), []string{"https://example.com"}, nil, 1, 30)+ if err == nil {+ t.Fatalf("expected an error, got job %+v", job)+ }+ var fcErr *FirecrawlError+ if !errors.As(err, &fcErr) {+ t.Fatalf("error %T is not a *FirecrawlError: %v", err, err)+ }+ if fcErr.StatusCode != http.StatusInternalServerError {+ t.Errorf("StatusCode = %d, want 500", fcErr.StatusCode)+ }+}++func TestGetBatchScrapeErrors(t *testing.T) {+ s := newBatchServer(t)+ client := newBatchClient(t, s)++ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")+ if err != nil {+ t.Fatalf("GetBatchScrapeErrors: %v", err)+ }+ assertBatchErrors(t, resp.Errors, resp.RobotsBlocked)+}++func TestGetBatchScrapeErrorsAcceptsDataEnvelope(t *testing.T) {+ s := newBatchServer(t)+ s.errorsBody = `{"success":true,"data":` + batchErrorsPayload + `}`+ client := newBatchClient(t, s)++ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")+ if err != nil {+ t.Fatalf("GetBatchScrapeErrors: %v", err)+ }+ assertBatchErrors(t, resp.Errors, resp.RobotsBlocked)+}++func TestGetBatchScrapeErrorsEmptyIsNotNil(t *testing.T) {+ s := newBatchServer(t)+ s.errorsBody = `{"errors":[],"robotsBlocked":[]}`+ client := newBatchClient(t, s)++ resp, err := client.GetBatchScrapeErrors(context.Background(), "batch-123")+ if err != nil {+ t.Fatalf("GetBatchScrapeErrors: %v", err)+ }+ if resp.Errors == nil || len(resp.Errors) != 0 {+ t.Errorf("Errors = %#v, want empty non-nil slice", resp.Errors)+ }+ if resp.RobotsBlocked == nil || len(resp.RobotsBlocked) != 0 {+ t.Errorf("RobotsBlocked = %#v, want empty non-nil slice", resp.RobotsBlocked)+ }+}++func TestGetBatchScrapeErrorsRequiresJobID(t *testing.T) {+ s := newBatchServer(t)+ client := newBatchClient(t, s)++ if _, err := client.GetBatchScrapeErrors(context.Background(), ""); err == nil {+ t.Fatal("expected an error for an empty job ID")+ }+ if n := len(s.requests); n != 0 {+ t.Errorf("made %d requests, want none", n)+ }+}++func TestStatusResponsesCarryJobID(t *testing.T) {+ s := newBatchServer(t)+ client := newBatchClient(t, s)++ batch, err := client.GetBatchScrapeStatus(context.Background(), "batch-123")+ if err != nil {+ t.Fatalf("GetBatchScrapeStatus: %v", err)+ }+ if batch.ID != "batch-123" {+ t.Errorf("batch.ID = %q, want batch-123", batch.ID)+ }+ if batch.Errors != nil {+ t.Errorf("GetBatchScrapeStatus should not fetch errors, got %+v", batch.Errors)+ }+ if n := len(s.requestsTo("/v2/batch/scrape/batch-123/errors")); n != 0 {+ t.Errorf("GetBatchScrapeStatus called the errors endpoint %d times", n)+ }++ crawl, err := client.GetCrawlStatus(context.Background(), "crawl-456")+ if err != nil {+ t.Fatalf("GetCrawlStatus: %v", err)+ }+ if crawl.ID != "crawl-456" {+ t.Errorf("crawl.ID = %q, want crawl-456", crawl.ID)+ }+}diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.goindex 04c8605fc..894b5d765 100644--- a/apps/go-sdk/firecrawl.go+++ b/apps/go-sdk/firecrawl.go@@ -221,6 +221,11 @@ func (c *Client) GetCrawlStatus(ctx context.Context, jobID string) (*CrawlJob, eif err := json.Unmarshal(raw, &job); err != nil {return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}}+ // The status response does not echo the job ID; carry it over so callers+ // can reach the per-job endpoints from the returned job.+ if job.ID == "" {+ job.ID = jobID+ }return &job, nil}@@ -340,21 +345,84 @@ func (c *Client) GetBatchScrapeStatus(ctx context.Context, jobID string) (*Batchif err := json.Unmarshal(raw, &job); err != nil {return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}}+ // The status response does not echo the job ID; carry it over so callers+ // can reach the per-job endpoints from the returned job.+ if job.ID == "" {+ job.ID = jobID+ }return &job, nil}+// GetBatchScrapeErrors gets the URLs of a batch scrape job that failed to+// scrape, along with the URLs blocked by robots.txt.+//+// Pages that responded with an HTTP error such as 404 or 429 are not reported+// here; they are returned as documents with the original status in+// Metadata["statusCode"].+func (c *Client) GetBatchScrapeErrors(ctx context.Context, jobID string) (*BatchScrapeErrorsResponse, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.get(ctx, "/v2/batch/scrape/"+jobID+"/errors")+ if err != nil {+ return nil, err+ }++ // The API returns the payload at the top level; accept a `data` envelope+ // as well, like the other SDKs do.+ var envelope struct {+ BatchScrapeErrorsResponse+ Data *BatchScrapeErrorsResponse `json:"data"`+ }+ if err := json.Unmarshal(raw, &envelope); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ resp := envelope.BatchScrapeErrorsResponse+ if envelope.Data != nil {+ resp = *envelope.Data+ }+ if resp.Errors == nil {+ resp.Errors = []BatchScrapeError{}+ }+ if resp.RobotsBlocked == nil {+ resp.RobotsBlocked = []string{}+ }+ return &resp, nil+}+// BatchScrape batch-scrapes URLs and waits for completion with auto-polling.+//+// The returned job carries the batch ID, the scraped documents in Data, and+// the URLs that failed to scrape in Errors (see BatchScrapeJob).func (c *Client) BatchScrape(ctx context.Context, urls []string, opts *BatchScrapeOptions) (*BatchScrapeJob, error) {return c.BatchScrapeWithPolling(ctx, urls, opts, defaultPollInterval, defaultJobTimeout)}// BatchScrapeWithPolling batch-scrapes URLs and waits for completion with custom polling settings.+//+// The returned job carries the batch ID, the scraped documents in Data, and+// the URLs that failed to scrape in Errors (see BatchScrapeJob).func (c *Client) BatchScrapeWithPolling(ctx context.Context, urls []string, opts *BatchScrapeOptions, pollIntervalSec, timeoutSec int) (*BatchScrapeJob, error) {start, err := c.StartBatchScrape(ctx, urls, opts)if err != nil {return nil, err}- return c.pollBatchScrape(ctx, start.ID, pollIntervalSec, timeoutSec)+ job, err := c.pollBatchScrape(ctx, start.ID, pollIntervalSec, timeoutSec)+ if err != nil {+ return nil, err+ }++ // The status endpoint only reports successful documents, and its counters+ // exclude failed URLs, so the errors endpoint is the only place a caller can+ // learn which URLs were dropped and why.+ errs, err := c.GetBatchScrapeErrors(ctx, start.ID)+ if err != nil {+ return nil, err+ }+ job.Errors = errs.Errors+ job.RobotsBlocked = errs.RobotsBlocked+ return job, nil}// CancelBatchScrape cancels a running batch scrape job.diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex 273a80007..c66cb2d4d 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -407,6 +407,10 @@ type BatchScrapeResponse struct {}// BatchScrapeJob represents the status and results of a batch scrape job.+//+// ID is always set to the batch job ID, so callers of the synchronous+// BatchScrape can still reach the per-job endpoints (GetBatchScrapeErrors,+// CancelBatchScrape) afterwards.type BatchScrapeJob struct {ID string `json:"id,omitempty"`Status string `json:"status"`@@ -416,6 +420,18 @@ type BatchScrapeJob struct {ExpiresAt string `json:"expiresAt,omitempty"`Next string `json:"next,omitempty"`Data []Document `json:"data,omitempty"`++ // Errors lists the URLs that could not be scraped at all (for example DNS+ // resolution errors or timeouts). Pages that responded with an HTTP error+ // such as 404 or 429 are not listed here: they are returned in Data with the+ // original status in Metadata["statusCode"].+ //+ // Populated by BatchScrape and BatchScrapeWithPolling once the job has+ // finished. GetBatchScrapeStatus leaves it nil; use GetBatchScrapeErrors.+ Errors []BatchScrapeError `json:"errors,omitempty"`+ // RobotsBlocked lists the URLs skipped because robots.txt disallowed them.+ // Populated under the same conditions as Errors.+ RobotsBlocked []string `json:"robotsBlocked,omitempty"`}// IsDone returns true if the batch scrape job has finished.@@ -423,6 +439,32 @@ func (b *BatchScrapeJob) IsDone() bool {return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"}+// BatchScrapeError describes one URL in a batch scrape job that failed to scrape.+type BatchScrapeError struct {+ // ID is the ID of the individual scrape job for this URL.+ ID string `json:"id"`+ // Timestamp is when the scrape failed, in RFC 3339 format, if known.+ Timestamp string `json:"timestamp,omitempty"`+ // URL is the URL that failed to scrape.+ URL string `json:"url"`+ // Code is a stable error code, such as "SCRAPE_DNS_RESOLUTION_ERROR" or+ // "SCRAPE_TIMEOUT". Empty when the failure has no classified code.+ Code string `json:"code,omitempty"`+ // Error is the human-readable error message.+ Error string `json:"error"`+ // RequiresAction is set when the page needs an out-of-band step first, such+ // as accepting a data provider's terms.+ RequiresAction *RequiresAction `json:"requiresAction,omitempty"`+}++// BatchScrapeErrorsResponse is returned by GetBatchScrapeErrors.+type BatchScrapeErrorsResponse struct {+ // Errors lists the URLs that failed to scrape. See BatchScrapeError.+ Errors []BatchScrapeError `json:"errors"`+ // RobotsBlocked lists the URLs skipped because robots.txt disallowed them.+ RobotsBlocked []string `json:"robotsBlocked"`+}+// LinkResult represents a discovered URL from a map request.// The API may return links as plain strings or as objects with url/title/description.type LinkResult struct {diff --git a/apps/go-sdk/pagination_origin_test.go b/apps/go-sdk/pagination_origin_test.goindex b7c2fb753..4c16d9dbd 100644--- a/apps/go-sdk/pagination_origin_test.go+++ b/apps/go-sdk/pagination_origin_test.go@@ -116,6 +116,8 @@ var paginationFollowers = []paginationFollower{_, _ = w.Write([]byte(`{"success":true,"id":"abc"}`))case r.URL.Path == "/v2/batch/scrape/abc" && r.URL.Query().Get("skip") == "":_, _ = w.Write([]byte(`{"status":"completed","data":[{"markdown":"one"}],"next":"` + next + `"}`))+ case r.URL.Path == "/v2/batch/scrape/abc/errors":+ _, _ = w.Write([]byte(`{"errors":[],"robotsBlocked":[]}`))default:return false}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex fabe945d8..d69cf6db1 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.16.8"+const Version = "1.16.9"