Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(go-sdk): add official Go SDK for Firecrawl v2 API
firecrawl-spring[bot]c654afd2a937
diff --git a/apps/go-sdk/README.md b/apps/go-sdk/README.mdnew file mode 100644index 000000000..237766e99--- /dev/null+++ b/apps/go-sdk/README.md@@ -0,0 +1,318 @@+# Firecrawl Go SDK++Go SDK for the [Firecrawl](https://firecrawl.dev) v2 web scraping API.++## Requirements++- **Go:** 1.23 or later++## Installation++```bash+go get github.com/mendableai/firecrawl-go+```++## API Key Setup++Get your API key from the [Firecrawl Dashboard](https://firecrawl.dev) and set it as an environment variable:++```bash+export FIRECRAWL_API_KEY="fc-your-api-key-here"+```++## Quick Start++```go+package main++import (+ "context"+ "fmt"+ "log"++ firecrawl "github.com/mendableai/firecrawl-go"+ "github.com/mendableai/firecrawl-go/option"+)++func main() {+ // Create a client (reads FIRECRAWL_API_KEY from environment)+ client, err := firecrawl.NewClient()+ if err != nil {+ log.Fatal(err)+ }++ // Or provide the API key directly+ client, err = firecrawl.NewClient(+ option.WithAPIKey("fc-your-api-key"),+ )+ if err != nil {+ log.Fatal(err)+ }++ ctx := context.Background()++ // Scrape a single page+ doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{+ Formats: []string{"markdown"},+ })+ if err != nil {+ log.Fatal(err)+ }+ fmt.Println(doc.Markdown)+}+```++## Configuration++```go+client, err := firecrawl.NewClient(+ option.WithAPIKey("fc-your-api-key"), // API key (or set FIRECRAWL_API_KEY env var)+ option.WithAPIURL("https://api.firecrawl.dev"), // Custom API URL+ option.WithMaxRetries(3), // Max retry attempts (default: 3)+ option.WithBackoffFactor(0.5), // Backoff factor in seconds (default: 0.5)+ option.WithTimeout(5 * time.Minute), // HTTP timeout (default: 5 minutes)+ option.WithHTTPClient(customHTTPClient), // Custom *http.Client+)+```++## API Reference++### Scrape++Scrape a single URL and get its content.++```go+// Basic scrape+doc, err := client.Scrape(ctx, "https://example.com", nil)++// With options+doc, err := client.Scrape(ctx, "https://example.com", &firecrawl.ScrapeOptions{+ Formats: []string{"markdown", "html"},+ OnlyMainContent: firecrawl.Bool(true),+ WaitFor: firecrawl.Int(5000),+ Location: &firecrawl.LocationConfig{Country: "US"},+})+```++#### Interactive Browser++Execute code in a scrape-bound browser session:++```go+resp, err := client.Interact(ctx, scrapeJobID, "document.title", &firecrawl.InteractParams{+ Language: "node",+ Timeout: firecrawl.Int(30),+})++// Stop the browser session+deleteResp, err := client.StopInteractiveBrowser(ctx, scrapeJobID)+```++### Crawl++Crawl a website and get content from multiple pages.++```go+// Auto-polling: starts the crawl and waits for completion+job, err := client.Crawl(ctx, "https://example.com", &firecrawl.CrawlOptions{+ Limit: firecrawl.Int(50),+ MaxDiscoveryDepth: firecrawl.Int(3),+ ScrapeOptions: &firecrawl.ScrapeOptions{+ Formats: []string{"markdown"},+ },+})++// Or manage polling manually+resp, err := client.StartCrawl(ctx, "https://example.com", &firecrawl.CrawlOptions{+ Limit: firecrawl.Int(50),+})++// Check status+status, err := client.GetCrawlStatus(ctx, resp.ID)++// Cancel+_, err = client.CancelCrawl(ctx, resp.ID)++// Get errors+errors, err := client.GetCrawlErrors(ctx, resp.ID)+```++### Batch Scrape++Scrape multiple URLs in a single batch job.++```go+urls := []string{+ "https://example.com/page1",+ "https://example.com/page2",+ "https://example.com/page3",+}++// Auto-polling+job, err := client.BatchScrape(ctx, urls, &firecrawl.BatchScrapeOptions{+ ScrapeOptions: &firecrawl.ScrapeOptions{+ Formats: []string{"markdown"},+ },+})++// Or manage manually+resp, err := client.StartBatchScrape(ctx, urls, nil)+status, err := client.GetBatchScrapeStatus(ctx, resp.ID)+_, err = client.CancelBatchScrape(ctx, resp.ID)+```++### Map++Discover URLs on a website.++```go+mapData, err := client.Map(ctx, "https://example.com", &firecrawl.MapOptions{+ Search: firecrawl.String("pricing"),+ IncludeSubdomains: firecrawl.Bool(true),+ Limit: firecrawl.Int(100),+})+```++### Search++Search the web and get scraped results.++```go+results, err := client.Search(ctx, "firecrawl web scraping", &firecrawl.SearchOptions{+ Limit: firecrawl.Int(5),+ ScrapeOptions: &firecrawl.ScrapeOptions{+ Formats: []string{"markdown"},+ },+})+```++### Agent++Run an AI-powered agent to extract structured data.++```go+// Auto-polling+status, err := client.Agent(ctx, &firecrawl.AgentOptions{+ Prompt: "Find all pricing plans and their features",+ URLs: []string{"https://example.com/pricing"},+ Schema: map[string]interface{}{+ "type": "object",+ "properties": map[string]interface{}{+ "plans": map[string]interface{}{+ "type": "array",+ "items": map[string]interface{}{+ "type": "object",+ "properties": map[string]interface{}{+ "name": map[string]interface{}{"type": "string"},+ "price": map[string]interface{}{"type": "string"},+ },+ },+ },+ },+ },+})++// Or manage manually+resp, err := client.StartAgent(ctx, &firecrawl.AgentOptions{+ Prompt: "Extract product information",+})+status, err := client.GetAgentStatus(ctx, resp.ID)+_, err = client.CancelAgent(ctx, resp.ID)+```++### Browser++Create and manage standalone browser sessions.++```go+// Create a browser session+session, err := client.Browser(ctx, &firecrawl.BrowserOptions{+ TTL: firecrawl.Int(300),+ StreamWebView: firecrawl.Bool(true),+})++// Execute code+result, err := client.BrowserExecute(ctx, session.ID, "echo 'hello'", &firecrawl.BrowserExecuteParams{+ Language: "bash",+ Timeout: firecrawl.Int(30),+})++// List sessions+list, err := client.ListBrowsers(ctx, "active")++// Delete session+_, err = client.DeleteBrowser(ctx, session.ID)+```++### Usage & Metrics++```go+// Check concurrency+concurrency, err := client.GetConcurrency(ctx)+fmt.Printf("Using %d of %d\n", concurrency.Concurrency, concurrency.MaxConcurrency)++// Check credit usage+credits, err := client.GetCreditUsage(ctx)+fmt.Printf("Remaining: %d of %d\n", credits.RemainingCredits, credits.PlanCredits)+```++## Error Handling++The SDK uses typed errors for different failure scenarios:++```go+doc, err := client.Scrape(ctx, "https://example.com", nil)+if err != nil {+ var authErr *firecrawl.AuthenticationError+ var rateErr *firecrawl.RateLimitError+ var timeoutErr *firecrawl.JobTimeoutError+ var fcErr *firecrawl.FirecrawlError++ switch {+ case errors.As(err, &authErr):+ fmt.Println("Invalid API key:", authErr.Message)+ case errors.As(err, &rateErr):+ fmt.Println("Rate limited:", rateErr.Message)+ case errors.As(err, &timeoutErr):+ fmt.Printf("Job %s timed out after %ds\n", timeoutErr.JobID, timeoutErr.TimeoutSeconds)+ case errors.As(err, &fcErr):+ fmt.Printf("API error (HTTP %d): %s\n", fcErr.StatusCode, fcErr.Message)+ default:+ fmt.Println("Unexpected error:", err)+ }+}+```++### Retry Logic++The SDK automatically retries transient failures:+- **Retried:** 408, 409, 5xx errors, and connection failures+- **Not retried:** 401, 429, and other 4xx errors+- **Backoff:** Exponential backoff with configurable factor++## Context Support++All methods accept a `context.Context` for cancellation and deadline control:++```go+ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+defer cancel()++doc, err := client.Scrape(ctx, "https://example.com", nil)+```++## Pointer Helpers++The SDK provides convenience functions for optional fields:++```go+firecrawl.Bool(true) // *bool+firecrawl.Int(50) // *int+firecrawl.Int64(1000) // *int64+firecrawl.String("test") // *string+firecrawl.Float64(0.5) // *float64+```++## License++MITdiff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.gonew file mode 100644index 000000000..d386d031b--- /dev/null+++ b/apps/go-sdk/firecrawl.go@@ -0,0 +1,796 @@+// Package firecrawl provides a Go client for the Firecrawl v2 web scraping API.+//+// Create a client using [NewClient] with functional options:+//+// client, err := firecrawl.NewClient(+// option.WithAPIKey("fc-your-api-key"),+// )+//+// Or let the client read the FIRECRAWL_API_KEY environment variable:+//+// client, err := firecrawl.NewClient()+package firecrawl++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "os"+ "time"++ "github.com/mendableai/firecrawl-go/option"+)++const (+ defaultPollInterval = 2 // seconds+ defaultJobTimeout = 300 // seconds+)++// Client is the Firecrawl v2 API client.+type Client struct {+ http *httpClient+}++// NewClient creates a new Firecrawl client.+//+// The API key is resolved in order from:+// 1. [option.WithAPIKey]+// 2. FIRECRAWL_API_KEY environment variable+//+// The API URL defaults to https://api.firecrawl.dev and can be overridden+// with [option.WithAPIURL] or the FIRECRAWL_API_URL environment variable.+func NewClient(opts ...option.RequestOption) (*Client, error) {+ cfg := &option.RequestConfig{+ MaxRetries: defaultMaxRetries,+ BackoffFactor: defaultBackoffFactor,+ }+ for _, opt := range opts {+ opt(cfg)+ }++ // Resolve API key.+ apiKey := cfg.APIKey+ if apiKey == "" {+ apiKey = os.Getenv("FIRECRAWL_API_KEY")+ }+ if apiKey == "" {+ return nil, &FirecrawlError{+ Message: "API key is required. Set it via option.WithAPIKey(), " ++ "or FIRECRAWL_API_KEY environment variable.",+ }+ }++ // Resolve API URL.+ apiURL := cfg.APIURL+ if apiURL == "" {+ apiURL = os.Getenv("FIRECRAWL_API_URL")+ }+ if apiURL == "" {+ apiURL = defaultAPIURL+ }++ // Resolve HTTP client.+ httpCl := cfg.HTTPClient+ if httpCl == nil {+ httpCl = &http.Client{Timeout: defaultTimeout}+ }++ hc := newHTTPClient(apiKey, apiURL, httpCl, cfg.MaxRetries, cfg.BackoffFactor)+ return &Client{http: hc}, nil+}++// ================================================================+// SCRAPE+// ================================================================++// Scrape scrapes a single URL and returns the document.+func (c *Client) Scrape(ctx context.Context, url string, opts *ScrapeOptions) (*Document, error) {+ if url == "" {+ return nil, &FirecrawlError{Message: "URL is required"}+ }++ body := map[string]interface{}{"url": url}+ mergeOptions(body, opts)++ raw, err := c.http.post("/v2/scrape", body, nil)+ if err != nil {+ return nil, err+ }++ doc, err := extractDataAs[Document](raw)+ if err != nil {+ return nil, err+ }+ return doc, nil+}++// Interact executes code in a scrape-bound browser session.+func (c *Client) Interact(ctx context.Context, jobID, code string, params *InteractParams) (*BrowserExecuteResponse, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }+ if code == "" {+ return nil, &FirecrawlError{Message: "code is required"}+ }++ body := map[string]interface{}{+ "code": code,+ "language": "node",+ }+ if params != nil {+ if params.Language != "" {+ body["language"] = params.Language+ }+ if params.Timeout != nil {+ body["timeout"] = *params.Timeout+ }+ if params.Origin != "" {+ body["origin"] = params.Origin+ }+ }++ raw, err := c.http.post("/v2/scrape/"+jobID+"/interact", body, nil)+ if err != nil {+ return nil, err+ }++ var resp BrowserExecuteResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// StopInteractiveBrowser stops the interactive browser session for a scrape job.+func (c *Client) StopInteractiveBrowser(ctx context.Context, jobID string) (*BrowserDeleteResponse, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.delete("/v2/scrape/" + jobID + "/interact")+ if err != nil {+ return nil, err+ }++ var resp BrowserDeleteResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// ================================================================+// CRAWL+// ================================================================++// StartCrawl starts an async crawl job and returns immediately.+func (c *Client) StartCrawl(ctx context.Context, url string, opts *CrawlOptions) (*CrawlResponse, error) {+ if url == "" {+ return nil, &FirecrawlError{Message: "URL is required"}+ }++ body := map[string]interface{}{"url": url}+ mergeOptions(body, opts)++ raw, err := c.http.post("/v2/crawl", body, nil)+ if err != nil {+ return nil, err+ }++ var resp CrawlResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// GetCrawlStatus gets the status and results of a crawl job.+func (c *Client) GetCrawlStatus(ctx context.Context, jobID string) (*CrawlJob, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.get("/v2/crawl/" + jobID)+ if err != nil {+ return nil, err+ }++ var job CrawlJob+ if err := json.Unmarshal(raw, &job); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &job, nil+}++// Crawl crawls a website and waits for completion with auto-polling.+func (c *Client) Crawl(ctx context.Context, url string, opts *CrawlOptions) (*CrawlJob, error) {+ return c.CrawlWithPolling(ctx, url, opts, defaultPollInterval, defaultJobTimeout)+}++// CrawlWithPolling crawls a website and waits for completion with custom polling settings.+func (c *Client) CrawlWithPolling(ctx context.Context, url string, opts *CrawlOptions, pollIntervalSec, timeoutSec int) (*CrawlJob, error) {+ start, err := c.StartCrawl(ctx, url, opts)+ if err != nil {+ return nil, err+ }+ return c.pollCrawl(ctx, start.ID, pollIntervalSec, timeoutSec)+}++// CancelCrawl cancels a running crawl job.+func (c *Client) CancelCrawl(ctx context.Context, jobID string) (map[string]interface{}, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.delete("/v2/crawl/" + jobID)+ if err != nil {+ return nil, err+ }++ var resp map[string]interface{}+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return resp, nil+}++// GetCrawlErrors gets errors from a crawl job.+func (c *Client) GetCrawlErrors(ctx context.Context, jobID string) (map[string]interface{}, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.get("/v2/crawl/" + jobID + "/errors")+ if err != nil {+ return nil, err+ }++ var resp map[string]interface{}+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return resp, nil+}++// ================================================================+// BATCH SCRAPE+// ================================================================++// StartBatchScrape starts an async batch scrape job.+func (c *Client) StartBatchScrape(ctx context.Context, urls []string, opts *BatchScrapeOptions) (*BatchScrapeResponse, error) {+ if len(urls) == 0 {+ return nil, &FirecrawlError{Message: "URLs list is required"}+ }++ body := map[string]interface{}{"urls": urls}+ var extraHeaders map[string]string++ if opts != nil {+ if opts.IdempotencyKey != nil && *opts.IdempotencyKey != "" {+ extraHeaders = map[string]string{"x-idempotency-key": *opts.IdempotencyKey}+ }+ mergeOptions(body, opts)+ // Flatten nested scrape options to top level as the API expects.+ if nested, ok := body["options"]; ok {+ delete(body, "options")+ if nestedMap, ok := nested.(map[string]interface{}); ok {+ batchFields := make(map[string]interface{})+ for k, v := range body {+ batchFields[k] = v+ }+ for k, v := range nestedMap {+ body[k] = v+ }+ for k, v := range batchFields {+ body[k] = v+ }+ }+ }+ }++ raw, err := c.http.post("/v2/batch/scrape", body, extraHeaders)+ if err != nil {+ return nil, err+ }++ var resp BatchScrapeResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// GetBatchScrapeStatus gets the status and results of a batch scrape job.+func (c *Client) GetBatchScrapeStatus(ctx context.Context, jobID string) (*BatchScrapeJob, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.get("/v2/batch/scrape/" + jobID)+ if err != nil {+ return nil, err+ }++ var job BatchScrapeJob+ if err := json.Unmarshal(raw, &job); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &job, nil+}++// BatchScrape batch-scrapes URLs and waits for completion with auto-polling.+func (c *Client) BatchScrape(ctx context.Context, urls []string, opts *BatchScrapeOptions) (*BatchScrapeJob, error) {+ return c.BatchScrapeWithPolling(ctx, urls, opts, defaultPollInterval, defaultJobTimeout)+}++// BatchScrapeWithPolling batch-scrapes URLs and waits for completion with custom polling settings.+func (c *Client) BatchScrapeWithPolling(ctx context.Context, urls []string, opts *BatchScrapeOptions, pollIntervalSec, timeoutSec int) (*BatchScrapeJob, error) {+ start, err := c.StartBatchScrape(ctx, urls, opts)+ if err != nil {+ return nil, err+ }+ return c.pollBatchScrape(ctx, start.ID, pollIntervalSec, timeoutSec)+}++// CancelBatchScrape cancels a running batch scrape job.+func (c *Client) CancelBatchScrape(ctx context.Context, jobID string) (map[string]interface{}, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.delete("/v2/batch/scrape/" + jobID)+ if err != nil {+ return nil, err+ }++ var resp map[string]interface{}+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return resp, nil+}++// ================================================================+// MAP+// ================================================================++// Map discovers URLs on a website.+func (c *Client) Map(ctx context.Context, url string, opts *MapOptions) (*MapData, error) {+ if url == "" {+ return nil, &FirecrawlError{Message: "URL is required"}+ }++ body := map[string]interface{}{"url": url}+ mergeOptions(body, opts)++ raw, err := c.http.post("/v2/map", body, nil)+ if err != nil {+ return nil, err+ }++ data, err := extractDataAs[MapData](raw)+ if err != nil {+ return nil, err+ }+ return data, nil+}++// ================================================================+// SEARCH+// ================================================================++// Search performs a web search.+func (c *Client) Search(ctx context.Context, query string, opts *SearchOptions) (*SearchData, error) {+ if query == "" {+ return nil, &FirecrawlError{Message: "query is required"}+ }++ body := map[string]interface{}{"query": query}+ mergeOptions(body, opts)++ raw, err := c.http.post("/v2/search", body, nil)+ if err != nil {+ return nil, err+ }++ data, err := extractDataAs[SearchData](raw)+ if err != nil {+ return nil, err+ }+ return data, nil+}++// ================================================================+// AGENT+// ================================================================++// StartAgent starts an async agent task.+func (c *Client) StartAgent(ctx context.Context, opts *AgentOptions) (*AgentResponse, error) {+ if opts == nil {+ return nil, &FirecrawlError{Message: "agent options are required"}+ }++ raw, err := c.http.post("/v2/agent", opts, nil)+ if err != nil {+ return nil, err+ }++ var resp AgentResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// GetAgentStatus gets the status of an agent task.+func (c *Client) GetAgentStatus(ctx context.Context, jobID string) (*AgentStatusResponse, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.get("/v2/agent/" + jobID)+ if err != nil {+ return nil, err+ }++ var resp AgentStatusResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// Agent runs an agent task and waits for completion with auto-polling.+func (c *Client) Agent(ctx context.Context, opts *AgentOptions) (*AgentStatusResponse, error) {+ return c.AgentWithPolling(ctx, opts, defaultPollInterval, defaultJobTimeout)+}++// AgentWithPolling runs an agent task and waits for completion with custom polling settings.+func (c *Client) AgentWithPolling(ctx context.Context, opts *AgentOptions, pollIntervalSec, timeoutSec int) (*AgentStatusResponse, error) {+ start, err := c.StartAgent(ctx, opts)+ if err != nil {+ return nil, err+ }+ if start.ID == "" {+ return nil, &FirecrawlError{Message: "agent start did not return a job ID"}+ }++ deadline := time.Now().Add(time.Duration(timeoutSec) * time.Second)+ for time.Now().Before(deadline) {+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ default:+ }++ status, err := c.GetAgentStatus(ctx, start.ID)+ if err != nil {+ return nil, err+ }+ if status.IsDone() {+ return status, nil+ }+ time.Sleep(time.Duration(pollIntervalSec) * time.Second)+ }++ return nil, &JobTimeoutError{+ FirecrawlError: FirecrawlError{Message: "agent job timed out"},+ JobID: start.ID,+ TimeoutSeconds: timeoutSec,+ }+}++// CancelAgent cancels a running agent task.+func (c *Client) CancelAgent(ctx context.Context, jobID string) (map[string]interface{}, error) {+ if jobID == "" {+ return nil, &FirecrawlError{Message: "job ID is required"}+ }++ raw, err := c.http.delete("/v2/agent/" + jobID)+ if err != nil {+ return nil, err+ }++ var resp map[string]interface{}+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return resp, nil+}++// ================================================================+// BROWSER+// ================================================================++// Browser creates a new browser session with default settings.+func (c *Client) Browser(ctx context.Context, opts *BrowserOptions) (*BrowserCreateResponse, error) {+ body := map[string]interface{}{}+ if opts != nil {+ if opts.TTL != nil {+ body["ttl"] = *opts.TTL+ }+ if opts.ActivityTTL != nil {+ body["activityTtl"] = *opts.ActivityTTL+ }+ if opts.StreamWebView != nil {+ body["streamWebView"] = *opts.StreamWebView+ }+ }++ raw, err := c.http.post("/v2/browser", body, nil)+ if err != nil {+ return nil, err+ }++ var resp BrowserCreateResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// BrowserExecute executes code in a browser session.+func (c *Client) BrowserExecute(ctx context.Context, sessionID, code string, params *BrowserExecuteParams) (*BrowserExecuteResponse, error) {+ if sessionID == "" {+ return nil, &FirecrawlError{Message: "session ID is required"}+ }+ if code == "" {+ return nil, &FirecrawlError{Message: "code is required"}+ }++ body := map[string]interface{}{+ "code": code,+ "language": "bash",+ }+ if params != nil {+ if params.Language != "" {+ body["language"] = params.Language+ }+ if params.Timeout != nil {+ body["timeout"] = *params.Timeout+ }+ }++ raw, err := c.http.post("/v2/browser/"+sessionID+"/execute", body, nil)+ if err != nil {+ return nil, err+ }++ var resp BrowserExecuteResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// DeleteBrowser deletes a browser session.+func (c *Client) DeleteBrowser(ctx context.Context, sessionID string) (*BrowserDeleteResponse, error) {+ if sessionID == "" {+ return nil, &FirecrawlError{Message: "session ID is required"}+ }++ raw, err := c.http.delete("/v2/browser/" + sessionID)+ if err != nil {+ return nil, err+ }++ var resp BrowserDeleteResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// ListBrowsers lists browser sessions with an optional status filter.+func (c *Client) ListBrowsers(ctx context.Context, status string) (*BrowserListResponse, error) {+ endpoint := "/v2/browser"+ if status != "" {+ endpoint += "?status=" + status+ }++ raw, err := c.http.get(endpoint)+ if err != nil {+ return nil, err+ }++ var resp BrowserListResponse+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// ================================================================+// USAGE & METRICS+// ================================================================++// GetConcurrency gets current concurrency usage.+func (c *Client) GetConcurrency(ctx context.Context) (*ConcurrencyCheck, error) {+ raw, err := c.http.get("/v2/concurrency-check")+ if err != nil {+ return nil, err+ }++ var resp ConcurrencyCheck+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// GetCreditUsage gets current credit usage.+func (c *Client) GetCreditUsage(ctx context.Context) (*CreditUsage, error) {+ raw, err := c.http.get("/v2/team/credit-usage")+ if err != nil {+ return nil, err+ }++ var resp CreditUsage+ if err := json.Unmarshal(raw, &resp); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &resp, nil+}++// ================================================================+// INTERNAL POLLING HELPERS+// ================================================================++func (c *Client) pollCrawl(ctx context.Context, jobID string, pollIntervalSec, timeoutSec int) (*CrawlJob, error) {+ deadline := time.Now().Add(time.Duration(timeoutSec) * time.Second)+ for time.Now().Before(deadline) {+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ default:+ }++ job, err := c.GetCrawlStatus(ctx, jobID)+ if err != nil {+ return nil, err+ }+ if job.IsDone() {+ return c.paginateCrawl(job)+ }+ time.Sleep(time.Duration(pollIntervalSec) * time.Second)+ }++ return nil, &JobTimeoutError{+ FirecrawlError: FirecrawlError{Message: "crawl job timed out"},+ JobID: jobID,+ TimeoutSeconds: timeoutSec,+ }+}++func (c *Client) pollBatchScrape(ctx context.Context, jobID string, pollIntervalSec, timeoutSec int) (*BatchScrapeJob, error) {+ deadline := time.Now().Add(time.Duration(timeoutSec) * time.Second)+ for time.Now().Before(deadline) {+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ default:+ }++ job, err := c.GetBatchScrapeStatus(ctx, jobID)+ if err != nil {+ return nil, err+ }+ if job.IsDone() {+ return c.paginateBatchScrape(job)+ }+ time.Sleep(time.Duration(pollIntervalSec) * time.Second)+ }++ return nil, &JobTimeoutError{+ FirecrawlError: FirecrawlError{Message: "batch scrape job timed out"},+ JobID: jobID,+ TimeoutSeconds: timeoutSec,+ }+}++func (c *Client) paginateCrawl(job *CrawlJob) (*CrawlJob, error) {+ if job.Data == nil {+ job.Data = []Document{}+ }+ current := job+ for current.Next != "" {+ raw, err := c.http.getAbsolute(current.Next)+ if err != nil {+ return nil, err+ }+ var nextPage CrawlJob+ if err := json.Unmarshal(raw, &nextPage); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode pagination response: %v", err)}+ }+ job.Data = append(job.Data, nextPage.Data...)+ current = &nextPage+ }+ return job, nil+}++func (c *Client) paginateBatchScrape(job *BatchScrapeJob) (*BatchScrapeJob, error) {+ if job.Data == nil {+ job.Data = []Document{}+ }+ current := job+ for current.Next != "" {+ raw, err := c.http.getAbsolute(current.Next)+ if err != nil {+ return nil, err+ }+ var nextPage BatchScrapeJob+ if err := json.Unmarshal(raw, &nextPage); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode pagination response: %v", err)}+ }+ job.Data = append(job.Data, nextPage.Data...)+ current = &nextPage+ }+ return job, nil+}++// ================================================================+// INTERNAL UTILITIES+// ================================================================++// InteractParams holds optional parameters for the Interact method.+type InteractParams struct {+ Language string+ Timeout *int+ Origin string+}++// BrowserOptions holds optional parameters for creating a browser session.+type BrowserOptions struct {+ TTL *int+ ActivityTTL *int+ StreamWebView *bool+}++// BrowserExecuteParams holds optional parameters for browser code execution.+type BrowserExecuteParams struct {+ Language string+ Timeout *int+}++// mergeOptions serializes the options struct and merges its fields into the body map.+func mergeOptions(body map[string]interface{}, opts interface{}) {+ if opts == nil {+ return+ }+ data, err := json.Marshal(opts)+ if err != nil {+ return+ }+ var optsMap map[string]interface{}+ if err := json.Unmarshal(data, &optsMap); err != nil {+ return+ }+ for k, v := range optsMap {+ body[k] = v+ }+}++// extractDataAs extracts the "data" field from a raw API response and deserializes it.+func extractDataAs[T any](raw json.RawMessage) (*T, error) {+ var envelope map[string]json.RawMessage+ if err := json.Unmarshal(raw, &envelope); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }++ dataRaw, ok := envelope["data"]+ if !ok {+ // Some endpoints return data at the top level.+ var result T+ if err := json.Unmarshal(raw, &result); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return &result, nil+ }++ var result T+ if err := json.Unmarshal(dataRaw, &result); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode data: %v", err)}+ }+ return &result, nil+}diff --git a/apps/go-sdk/firecrawl_error.go b/apps/go-sdk/firecrawl_error.gonew file mode 100644index 000000000..cf1139a1e--- /dev/null+++ b/apps/go-sdk/firecrawl_error.go@@ -0,0 +1,46 @@+package firecrawl++import "fmt"++// FirecrawlError represents an error returned by the Firecrawl API.+type FirecrawlError struct {+ // StatusCode is the HTTP status code (0 if not an HTTP error).+ StatusCode int+ // ErrorCode is the API error code, if any.+ ErrorCode string+ // Message is the human-readable error message.+ Message string+}++func (e *FirecrawlError) Error() string {+ if e.ErrorCode != "" {+ return fmt.Sprintf("firecrawl: HTTP %d [%s]: %s", e.StatusCode, e.ErrorCode, e.Message)+ }+ if e.StatusCode != 0 {+ return fmt.Sprintf("firecrawl: HTTP %d: %s", e.StatusCode, e.Message)+ }+ return fmt.Sprintf("firecrawl: %s", e.Message)+}++// AuthenticationError is returned when the API key is invalid (HTTP 401).+type AuthenticationError struct {+ FirecrawlError+}++// RateLimitError is returned when the rate limit is exceeded (HTTP 429).+type RateLimitError struct {+ FirecrawlError+}++// JobTimeoutError is returned when an async job exceeds its timeout.+type JobTimeoutError struct {+ FirecrawlError+ // JobID is the ID of the timed-out job.+ JobID string+ // TimeoutSeconds is the timeout that was exceeded.+ TimeoutSeconds int+}++func (e *JobTimeoutError) Error() string {+ return fmt.Sprintf("firecrawl: job %s timed out after %d seconds", e.JobID, e.TimeoutSeconds)+}diff --git a/apps/go-sdk/go.mod b/apps/go-sdk/go.modnew file mode 100644index 000000000..edf7631d3--- /dev/null+++ b/apps/go-sdk/go.mod@@ -0,0 +1,3 @@+module github.com/mendableai/firecrawl-go++go 1.23diff --git a/apps/go-sdk/go.sum b/apps/go-sdk/go.sumnew file mode 100644index 000000000..e69de29bbdiff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.gonew file mode 100644index 000000000..cea32c320--- /dev/null+++ b/apps/go-sdk/http_client.go@@ -0,0 +1,171 @@+package firecrawl++import (+ "bytes"+ "encoding/json"+ "fmt"+ "io"+ "math"+ "net/http"+ "strings"+ "time"+)++const (+ defaultAPIURL = "https://api.firecrawl.dev"+ defaultTimeout = 5 * time.Minute+ defaultMaxRetries = 3+ defaultBackoffFactor = 0.5+)++// httpClient is the internal HTTP client for the Firecrawl API.+type httpClient struct {+ client *http.Client+ apiKey string+ baseURL string+ maxRetries int+ backoffFactor float64+}++func newHTTPClient(apiKey, baseURL string, client *http.Client, maxRetries int, backoffFactor float64) *httpClient {+ baseURL = strings.TrimRight(baseURL, "/")+ return &httpClient{+ client: client,+ apiKey: apiKey,+ baseURL: baseURL,+ maxRetries: maxRetries,+ backoffFactor: backoffFactor,+ }+}++// post sends a POST request with a JSON body.+func (h *httpClient) post(path string, body interface{}, extraHeaders map[string]string) (json.RawMessage, error) {+ url := h.baseURL + path+ return h.doJSON("POST", url, body, extraHeaders)+}++// get sends a GET request.+func (h *httpClient) get(path string) (json.RawMessage, error) {+ url := h.baseURL + path+ return h.doJSON("GET", url, nil, nil)+}++// getAbsolute sends a GET request to an absolute URL (for pagination cursors).+func (h *httpClient) getAbsolute(absoluteURL string) (json.RawMessage, error) {+ return h.doJSON("GET", absoluteURL, nil, nil)+}++// delete sends a DELETE request.+func (h *httpClient) delete(path string) (json.RawMessage, error) {+ url := h.baseURL + path+ return h.doJSON("DELETE", url, nil, nil)+}++func (h *httpClient) doJSON(method, url string, body interface{}, extraHeaders map[string]string) (json.RawMessage, error) {+ var bodyReader io.Reader+ if body != nil {+ data, err := json.Marshal(body)+ if err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to serialize request body: %v", err)}+ }+ bodyReader = bytes.NewReader(data)+ }++ var lastErr error+ for attempt := 0; attempt <= h.maxRetries; attempt++ {+ if attempt > 0 {+ h.sleepBackoff(attempt)++ // Reset the body reader for retries.+ if body != nil {+ data, _ := json.Marshal(body)+ bodyReader = bytes.NewReader(data)+ }+ }++ req, err := http.NewRequest(method, url, bodyReader)+ if err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to create request: %v", err)}+ }++ req.Header.Set("Authorization", "Bearer "+h.apiKey)+ req.Header.Set("Content-Type", "application/json")+ for k, v := range extraHeaders {+ req.Header.Set(k, v)+ }++ resp, err := h.client.Do(req)+ if err != nil {+ lastErr = err+ continue // Retry on transport errors.+ }++ respBody, err := io.ReadAll(resp.Body)+ resp.Body.Close()+ if err != nil {+ lastErr = err+ continue+ }++ if resp.StatusCode >= 200 && resp.StatusCode < 300 {+ return json.RawMessage(respBody), nil+ }++ // Parse error details from the response.+ errMsg, errCode := extractError(respBody, resp.StatusCode)++ // Non-retryable client errors.+ switch resp.StatusCode {+ case 401:+ return nil, &AuthenticationError{+ FirecrawlError: FirecrawlError{StatusCode: 401, ErrorCode: errCode, Message: errMsg},+ }+ case 429:+ return nil, &RateLimitError{+ FirecrawlError: FirecrawlError{StatusCode: 429, ErrorCode: errCode, Message: errMsg},+ }+ }++ if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != 408 && resp.StatusCode != 409 {+ return nil, &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}+ }++ // Retryable: 408, 409, 5xx+ lastErr = &FirecrawlError{StatusCode: resp.StatusCode, ErrorCode: errCode, Message: errMsg}+ }++ if lastErr != nil {+ if fe, ok := lastErr.(*FirecrawlError); ok {+ return nil, fe+ }+ return nil, &FirecrawlError{Message: fmt.Sprintf("request failed after %d retries: %v", h.maxRetries, lastErr)}+ }+ return nil, &FirecrawlError{Message: "request failed"}+}++func (h *httpClient) sleepBackoff(attempt int) {+ delay := time.Duration(h.backoffFactor * 1000 * math.Pow(2, float64(attempt-1)))+ time.Sleep(delay * time.Millisecond)+}++// extractError parses an API error response to get the message and error code.+func extractError(body []byte, statusCode int) (string, string) {+ var parsed map[string]interface{}+ if err := json.Unmarshal(body, &parsed); err != nil {+ return fmt.Sprintf("HTTP %d error", statusCode), ""+ }++ msg := fmt.Sprintf("HTTP %d error", statusCode)+ if v, ok := parsed["error"]; ok {+ msg = fmt.Sprintf("%v", v)+ } else if v, ok := parsed["message"]; ok {+ msg = fmt.Sprintf("%v", v)+ }++ var errCode string+ if v, ok := parsed["code"]; ok && v != nil {+ errCode = fmt.Sprintf("%v", v)+ }++ return msg, errCode+}diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.gonew file mode 100644index 000000000..fdba21ec2--- /dev/null+++ b/apps/go-sdk/models.go@@ -0,0 +1,163 @@+package firecrawl++// Document represents a scraped web page.+type Document struct {+ Markdown string `json:"markdown,omitempty"`+ HTML string `json:"html,omitempty"`+ RawHTML string `json:"rawHtml,omitempty"`+ JSON interface{} `json:"json,omitempty"`+ Summary string `json:"summary,omitempty"`+ Metadata map[string]interface{} `json:"metadata,omitempty"`+ Links []string `json:"links,omitempty"`+ Images []string `json:"images,omitempty"`+ Screenshot string `json:"screenshot,omitempty"`+ Audio string `json:"audio,omitempty"`+ Attributes []map[string]interface{} `json:"attributes,omitempty"`+ Actions map[string]interface{} `json:"actions,omitempty"`+ Warning string `json:"warning,omitempty"`+ ChangeTracking map[string]interface{} `json:"changeTracking,omitempty"`+ Branding map[string]interface{} `json:"branding,omitempty"`+}++// CrawlResponse is returned when starting an async crawl.+type CrawlResponse struct {+ ID string `json:"id"`+ URL string `json:"url,omitempty"`+}++// CrawlJob represents the status and results of a crawl job.+type CrawlJob struct {+ ID string `json:"id,omitempty"`+ Status string `json:"status"`+ Total int `json:"total"`+ Completed int `json:"completed"`+ CreditsUsed *int `json:"creditsUsed,omitempty"`+ ExpiresAt string `json:"expiresAt,omitempty"`+ Next string `json:"next,omitempty"`+ Data []Document `json:"data,omitempty"`+}++// IsDone returns true if the crawl job has finished (completed, failed, or cancelled).+func (c *CrawlJob) IsDone() bool {+ return c.Status == "completed" || c.Status == "failed" || c.Status == "cancelled"+}++// BatchScrapeResponse is returned when starting an async batch scrape.+type BatchScrapeResponse struct {+ ID string `json:"id"`+ URL string `json:"url,omitempty"`+ InvalidURLs []string `json:"invalidURLs,omitempty"`+}++// BatchScrapeJob represents the status and results of a batch scrape job.+type BatchScrapeJob struct {+ ID string `json:"id,omitempty"`+ Status string `json:"status"`+ Total int `json:"total"`+ Completed int `json:"completed"`+ CreditsUsed *int `json:"creditsUsed,omitempty"`+ ExpiresAt string `json:"expiresAt,omitempty"`+ Next string `json:"next,omitempty"`+ Data []Document `json:"data,omitempty"`+}++// IsDone returns true if the batch scrape job has finished.+func (b *BatchScrapeJob) IsDone() bool {+ return b.Status == "completed" || b.Status == "failed" || b.Status == "cancelled"+}++// MapData represents the result of a map (URL discovery) request.+type MapData struct {+ Links []map[string]interface{} `json:"links,omitempty"`+}++// SearchData represents the result of a search request.+type SearchData struct {+ Web []map[string]interface{} `json:"web,omitempty"`+ News []map[string]interface{} `json:"news,omitempty"`+ Images []map[string]interface{} `json:"images,omitempty"`+}++// AgentResponse is returned when starting an async agent task.+type AgentResponse struct {+ Success bool `json:"success"`+ ID string `json:"id,omitempty"`+ Error string `json:"error,omitempty"`+}++// AgentStatusResponse represents the status and results of an agent task.+type AgentStatusResponse struct {+ Success bool `json:"success"`+ Status string `json:"status"`+ Error string `json:"error,omitempty"`+ Data interface{} `json:"data,omitempty"`+ Model string `json:"model,omitempty"`+ ExpiresAt string `json:"expiresAt,omitempty"`+ CreditsUsed *int `json:"creditsUsed,omitempty"`+}++// IsDone returns true if the agent task has finished.+func (a *AgentStatusResponse) IsDone() bool {+ return a.Status == "completed" || a.Status == "failed" || a.Status == "cancelled"+}++// BrowserCreateResponse is returned when creating a browser session.+type BrowserCreateResponse struct {+ Success bool `json:"success"`+ ID string `json:"id,omitempty"`+ CDPUrl string `json:"cdpUrl,omitempty"`+ LiveViewURL string `json:"liveViewUrl,omitempty"`+ ExpiresAt string `json:"expiresAt,omitempty"`+ Error string `json:"error,omitempty"`+}++// BrowserExecuteResponse is returned when executing code in a browser session.+type BrowserExecuteResponse struct {+ Success bool `json:"success"`+ Stdout string `json:"stdout,omitempty"`+ Result string `json:"result,omitempty"`+ Stderr string `json:"stderr,omitempty"`+ ExitCode *int `json:"exitCode,omitempty"`+ Killed *bool `json:"killed,omitempty"`+ Error string `json:"error,omitempty"`+}++// BrowserDeleteResponse is returned when deleting a browser session.+type BrowserDeleteResponse struct {+ Success bool `json:"success"`+ SessionDurationMs *int64 `json:"sessionDurationMs,omitempty"`+ CreditsBilled *int `json:"creditsBilled,omitempty"`+ Error string `json:"error,omitempty"`+}++// BrowserListResponse is returned when listing browser sessions.+type BrowserListResponse struct {+ Success bool `json:"success"`+ Sessions []BrowserSession `json:"sessions,omitempty"`+ Error string `json:"error,omitempty"`+}++// BrowserSession represents a browser session.+type BrowserSession struct {+ ID string `json:"id"`+ Status string `json:"status"`+ CDPUrl string `json:"cdpUrl,omitempty"`+ LiveViewURL string `json:"liveViewUrl,omitempty"`+ StreamWebView bool `json:"streamWebView,omitempty"`+ CreatedAt string `json:"createdAt,omitempty"`+ LastActivity string `json:"lastActivity,omitempty"`+}++// ConcurrencyCheck represents concurrency usage information.+type ConcurrencyCheck struct {+ Concurrency int `json:"concurrency"`+ MaxConcurrency int `json:"maxConcurrency"`+}++// CreditUsage represents credit usage information.+type CreditUsage struct {+ RemainingCredits int `json:"remainingCredits"`+ PlanCredits int `json:"planCredits"`+ BillingPeriodStart string `json:"billingPeriodStart,omitempty"`+ BillingPeriodEnd string `json:"billingPeriodEnd,omitempty"`+}diff --git a/apps/go-sdk/option/option.go b/apps/go-sdk/option/option.gonew file mode 100644index 000000000..29ac1284a--- /dev/null+++ b/apps/go-sdk/option/option.go@@ -0,0 +1,76 @@+// Package option provides functional options for configuring the Firecrawl client.+package option++import (+ "net/http"+ "time"+)++// RequestOption configures an individual request or the client.+type RequestOption func(*RequestConfig)++// RequestConfig holds the configuration for an HTTP request.+type RequestConfig struct {+ APIKey string+ APIURL string+ HTTPClient *http.Client+ MaxRetries int+ BackoffFactor float64+ ExtraHeaders map[string]string+}++// WithAPIKey sets the API key. Defaults to the FIRECRAWL_API_KEY environment variable.+func WithAPIKey(key string) RequestOption {+ return func(c *RequestConfig) {+ c.APIKey = key+ }+}++// WithAPIURL sets the API base URL. Defaults to https://api.firecrawl.dev.+func WithAPIURL(url string) RequestOption {+ return func(c *RequestConfig) {+ c.APIURL = url+ }+}++// WithHTTPClient sets a custom *http.Client for all requests.+func WithHTTPClient(client *http.Client) RequestOption {+ return func(c *RequestConfig) {+ c.HTTPClient = client+ }+}++// WithMaxRetries sets the maximum number of automatic retries for transient failures.+// Default: 3.+func WithMaxRetries(n int) RequestOption {+ return func(c *RequestConfig) {+ c.MaxRetries = n+ }+}++// WithBackoffFactor sets the exponential backoff factor in seconds. Default: 0.5.+func WithBackoffFactor(f float64) RequestOption {+ return func(c *RequestConfig) {+ c.BackoffFactor = f+ }+}++// WithTimeout sets the HTTP client timeout. Default: 5 minutes.+func WithTimeout(d time.Duration) RequestOption {+ return func(c *RequestConfig) {+ if c.HTTPClient == nil {+ c.HTTPClient = &http.Client{}+ }+ c.HTTPClient.Timeout = d+ }+}++// WithHeader adds an extra header to all requests.+func WithHeader(key, value string) RequestOption {+ return func(c *RequestConfig) {+ if c.ExtraHeaders == nil {+ c.ExtraHeaders = make(map[string]string)+ }+ c.ExtraHeaders[key] = value+ }+}diff --git a/apps/go-sdk/options.go b/apps/go-sdk/options.gonew file mode 100644index 000000000..93559aafe--- /dev/null+++ b/apps/go-sdk/options.go@@ -0,0 +1,132 @@+package firecrawl++// ScrapeOptions configures a single-page scrape request.+type ScrapeOptions struct {+ Formats []string `json:"formats,omitempty"`+ Headers map[string]string `json:"headers,omitempty"`+ IncludeTags []string `json:"includeTags,omitempty"`+ ExcludeTags []string `json:"excludeTags,omitempty"`+ OnlyMainContent *bool `json:"onlyMainContent,omitempty"`+ Timeout *int `json:"timeout,omitempty"`+ WaitFor *int `json:"waitFor,omitempty"`+ Mobile *bool `json:"mobile,omitempty"`+ Parsers []interface{} `json:"parsers,omitempty"`+ Actions []map[string]interface{} `json:"actions,omitempty"`+ Location *LocationConfig `json:"location,omitempty"`+ SkipTLSVerification *bool `json:"skipTlsVerification,omitempty"`+ RemoveBase64Images *bool `json:"removeBase64Images,omitempty"`+ BlockAds *bool `json:"blockAds,omitempty"`+ Proxy *string `json:"proxy,omitempty"`+ MaxAge *int64 `json:"maxAge,omitempty"`+ StoreInCache *bool `json:"storeInCache,omitempty"`+ Integration *string `json:"integration,omitempty"`+ JsonOptions *JsonOptions `json:"jsonOptions,omitempty"`+}++// CrawlOptions configures a crawl request.+type CrawlOptions struct {+ Prompt *string `json:"prompt,omitempty"`+ ExcludePaths []string `json:"excludePaths,omitempty"`+ IncludePaths []string `json:"includePaths,omitempty"`+ MaxDiscoveryDepth *int `json:"maxDiscoveryDepth,omitempty"`+ Sitemap *string `json:"sitemap,omitempty"`+ IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`+ DeduplicateSimilarURLs *bool `json:"deduplicateSimilarURLs,omitempty"`+ Limit *int `json:"limit,omitempty"`+ CrawlEntireDomain *bool `json:"crawlEntireDomain,omitempty"`+ AllowExternalLinks *bool `json:"allowExternalLinks,omitempty"`+ AllowSubdomains *bool `json:"allowSubdomains,omitempty"`+ Delay *int `json:"delay,omitempty"`+ MaxConcurrency *int `json:"maxConcurrency,omitempty"`+ Webhook interface{} `json:"webhook,omitempty"`+ ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"`+ RegexOnFullURL *bool `json:"regexOnFullURL,omitempty"`+ ZeroDataRetention *bool `json:"zeroDataRetention,omitempty"`+ Integration *string `json:"integration,omitempty"`+}++// BatchScrapeOptions configures a batch scrape request.+type BatchScrapeOptions struct {+ ScrapeOptions *ScrapeOptions `json:"options,omitempty"`+ Webhook interface{} `json:"webhook,omitempty"`+ AppendToID *string `json:"appendToId,omitempty"`+ IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"`+ MaxConcurrency *int `json:"maxConcurrency,omitempty"`+ ZeroDataRetention *bool `json:"zeroDataRetention,omitempty"`+ IdempotencyKey *string `json:"-"` // Sent as HTTP header, not in body+ Integration *string `json:"integration,omitempty"`+}++// MapOptions configures a map (URL discovery) request.+type MapOptions struct {+ Search *string `json:"search,omitempty"`+ Sitemap *string `json:"sitemap,omitempty"`+ IncludeSubdomains *bool `json:"includeSubdomains,omitempty"`+ IgnoreQueryParameters *bool `json:"ignoreQueryParameters,omitempty"`+ Limit *int `json:"limit,omitempty"`+ Timeout *int `json:"timeout,omitempty"`+ Integration *string `json:"integration,omitempty"`+ Location *LocationConfig `json:"location,omitempty"`+}++// SearchOptions configures a search request.+type SearchOptions struct {+ Sources []interface{} `json:"sources,omitempty"`+ Categories []interface{} `json:"categories,omitempty"`+ Limit *int `json:"limit,omitempty"`+ TBS *string `json:"tbs,omitempty"`+ Location *string `json:"location,omitempty"`+ IgnoreInvalidURLs *bool `json:"ignoreInvalidURLs,omitempty"`+ Timeout *int `json:"timeout,omitempty"`+ ScrapeOptions *ScrapeOptions `json:"scrapeOptions,omitempty"`+ Integration *string `json:"integration,omitempty"`+}++// AgentOptions configures an agent request.+type AgentOptions struct {+ URLs []string `json:"urls,omitempty"`+ Prompt string `json:"prompt"`+ Schema map[string]interface{} `json:"schema,omitempty"`+ Integration *string `json:"integration,omitempty"`+ MaxCredits *int `json:"maxCredits,omitempty"`+ StrictConstrainToURLs *bool `json:"strictConstrainToURLs,omitempty"`+ Model *string `json:"model,omitempty"`+ Webhook *WebhookConfig `json:"webhook,omitempty"`+}++// LocationConfig specifies geolocation for requests.+type LocationConfig struct {+ Country string `json:"country,omitempty"`+ Languages []string `json:"languages,omitempty"`+}++// WebhookConfig configures webhook notifications.+type WebhookConfig struct {+ URL string `json:"url"`+ Headers map[string]string `json:"headers,omitempty"`+ Metadata map[string]string `json:"metadata,omitempty"`+ Events []string `json:"events,omitempty"`+}++// JsonOptions configures JSON extraction within formats.+type JsonOptions struct {+ Prompt string `json:"prompt,omitempty"`+ Schema map[string]interface{} `json:"schema,omitempty"`+}++// Pointer helpers for optional fields.++// Bool returns a pointer to the given bool value.+func Bool(v bool) *bool { return &v }++// Int returns a pointer to the given int value.+func Int(v int) *int { return &v }++// Int64 returns a pointer to the given int64 value.+func Int64(v int64) *int64 { return &v }++// String returns a pointer to the given string value.+func String(v string) *string { return &v }++// Float64 returns a pointer to the given float64 value.+func Float64(v float64) *float64 { return &v }