Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(go-sdk): fix MapData.Links type and context-aware retry backoff
Devin AIcf359d4621bf
diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.goindex 1a4985b5f..39bb80feb 100644--- a/apps/go-sdk/http_client.go+++ b/apps/go-sdk/http_client.go@@ -77,7 +77,9 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interfvar lastErr errorfor attempt := 0; attempt <= h.maxRetries; attempt++ {if attempt > 0 {- h.sleepBackoff(attempt)+ if err := h.sleepBackoff(ctx, attempt); err != nil {+ return nil, err+ }// Reset the body reader for retries.if body != nil {@@ -104,6 +106,10 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interfresp, err := h.client.Do(req)if err != nil {+ // If context is cancelled, return immediately instead of retrying.+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }lastErr = errcontinue // Retry on transport errors.}@@ -151,9 +157,14 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interfreturn nil, &FirecrawlError{Message: "request failed"}}-func (h *httpClient) sleepBackoff(attempt int) {- delay := time.Duration(h.backoffFactor * 1000 * math.Pow(2, float64(attempt-1)))- time.Sleep(delay * time.Millisecond)+func (h *httpClient) sleepBackoff(ctx context.Context, attempt int) error {+ delay := time.Duration(h.backoffFactor*1000*math.Pow(2, float64(attempt-1))) * time.Millisecond+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(delay):+ return nil+ }}// extractError parses an API error response to get the message and error code.diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex fdba21ec2..d4335d74e 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -1,5 +1,7 @@package firecrawl+import "encoding/json"+// Document represents a scraped web page.type Document struct {Markdown string `json:"markdown,omitempty"`@@ -66,9 +68,36 @@ func (b *BatchScrapeJob) IsDone() bool {return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"}+// LinkResult represents a discovered URL from a map request.+// The API may return links as plain strings or as objects with url/title/description.+type LinkResult struct {+ URL string `json:"url,omitempty"`+ Title string `json:"title,omitempty"`+ Description string `json:"description,omitempty"`+}++// UnmarshalJSON handles both string and object link elements from the API.+func (l *LinkResult) UnmarshalJSON(data []byte) error {+ // Try as a plain string first.+ var s string+ if err := json.Unmarshal(data, &s); err == nil {+ l.URL = s+ return nil+ }++ // Otherwise unmarshal as an object.+ type linkAlias LinkResult+ var alias linkAlias+ if err := json.Unmarshal(data, &alias); err != nil {+ return err+ }+ *l = LinkResult(alias)+ return nil+}+// MapData represents the result of a map (URL discovery) request.type MapData struct {- Links []map[string]interface{} `json:"links,omitempty"`+ Links []LinkResult `json:"links,omitempty"`}// SearchData represents the result of a search request.