Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(go-sdk): fix MapData.Links type and context-aware retry backoff

Devin AIcf359d4621bf
diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.go
index 1a4985b5f..39bb80feb 100644
--- a/apps/go-sdk/http_client.go
+++ b/apps/go-sdk/http_client.go
@@ -77,7 +77,9 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interf
var lastErr error
for attempt := 0; attempt <= h.maxRetries; attempt++ {
if attempt > 0 {
- h.sleepBackoff(attempt)
+ if err := h.sleepBackoff(ctx, attempt); err != nil {
+ return nil, err
+ }
// Reset the body reader for retries.
if body != nil {
@@ -104,6 +106,10 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interf
resp, err := h.client.Do(req)
if err != nil {
+ // If context is cancelled, return immediately instead of retrying.
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
lastErr = err
continue // Retry on transport errors.
}
@@ -151,9 +157,14 @@ func (h *httpClient) doJSON(ctx context.Context, method, url string, body interf
return nil, &FirecrawlError{Message: "request failed"}
}
-func (h *httpClient) sleepBackoff(attempt int) {
- delay := time.Duration(h.backoffFactor * 1000 * math.Pow(2, float64(attempt-1)))
- time.Sleep(delay * time.Millisecond)
+func (h *httpClient) sleepBackoff(ctx context.Context, attempt int) error {
+ delay := time.Duration(h.backoffFactor*1000*math.Pow(2, float64(attempt-1))) * time.Millisecond
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(delay):
+ return nil
+ }
}
// extractError parses an API error response to get the message and error code.
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index fdba21ec2..d4335d74e 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -1,5 +1,7 @@
package firecrawl
+import "encoding/json"
+
// Document represents a scraped web page.
type Document struct {
Markdown string `json:"markdown,omitempty"`
@@ -66,9 +68,36 @@ func (b *BatchScrapeJob) IsDone() bool {
return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"
}
+// LinkResult represents a discovered URL from a map request.
+// The API may return links as plain strings or as objects with url/title/description.
+type LinkResult struct {
+ URL string `json:"url,omitempty"`
+ Title string `json:"title,omitempty"`
+ Description string `json:"description,omitempty"`
+}
+
+// UnmarshalJSON handles both string and object link elements from the API.
+func (l *LinkResult) UnmarshalJSON(data []byte) error {
+ // Try as a plain string first.
+ var s string
+ if err := json.Unmarshal(data, &s); err == nil {
+ l.URL = s
+ return nil
+ }
+
+ // Otherwise unmarshal as an object.
+ type linkAlias LinkResult
+ var alias linkAlias
+ if err := json.Unmarshal(data, &alias); err != nil {
+ return err
+ }
+ *l = LinkResult(alias)
+ return nil
+}
+
// MapData represents the result of a map (URL discovery) request.
type MapData struct {
- Links []map[string]interface{} `json:"links,omitempty"`
+ Links []LinkResult `json:"links,omitempty"`
}
// SearchData represents the result of a search request.