Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(go-sdk): pin pagination next URLs to the api_url origin (#4867)
Gergő Móriczd46d816aa87d
diff --git a/.github/workflows/test-go-sdk.yml b/.github/workflows/test-go-sdk.ymlindex bbeafa166..dfeaad2b5 100644--- a/.github/workflows/test-go-sdk.yml+++ b/.github/workflows/test-go-sdk.yml@@ -14,6 +14,9 @@ on:- apps/go-sdk/**workflow_dispatch:+permissions:+ contents: read+jobs:build-and-vet:name: Build and Vet@@ -23,10 +26,12 @@ jobs:github.event_name == 'pull_request' ||(github.event_name == 'push' && github.ref == 'refs/heads/main')steps:- - uses: actions/checkout@v4+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false- name: Set up Go- uses: actions/setup-go@v5+ uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0with:go-version: "1.23"cache-dependency-path: apps/go-sdk/go.sum@@ -38,3 +43,7 @@ jobs:- name: Vetworking-directory: ./apps/go-sdkrun: go vet ./...++ - name: Test+ working-directory: ./apps/go-sdk+ run: go test -race ./...diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.goindex d676d97fd..4b2f4f32e 100644--- a/apps/go-sdk/http_client.go+++ b/apps/go-sdk/http_client.go@@ -10,6 +10,7 @@ import ("mime/multipart""net/http""net/textproto"+ "net/url""strings""time")@@ -61,9 +62,45 @@ func (h *httpClient) get(ctx context.Context, path string) (json.RawMessage, errreturn h.doJSON(ctx, "GET", url, nil, nil)}-// getAbsolute sends a GET request to an absolute URL (for pagination cursors).+// getAbsolute sends a GET request to a pagination cursor URL, pinned to the+// API origin by pinToAPIOrigin.func (h *httpClient) getAbsolute(ctx context.Context, absoluteURL string) (json.RawMessage, error) {- return h.doJSON(ctx, "GET", absoluteURL, nil, nil)+ pinned, err := pinToAPIOrigin(h.baseURL, absoluteURL)+ if err != nil {+ return nil, &FirecrawlError{Message: err.Error()}+ }+ return h.doJSON(ctx, "GET", pinned, nil, nil)+}++// pinToAPIOrigin rewrites an absolute or protocol-relative URL onto apiURL's+// scheme and host, keeping its path and query, so credentials never leave the+// API origin. Relative URLs are resolved against apiURL. It returns an error if+// apiURL is not an absolute URL.+func pinToAPIOrigin(apiURL, rawURL string) (string, error) {+ base, err := url.Parse(apiURL)+ if err != nil || base.Scheme == "" || base.Host == "" {+ return "", fmt.Errorf("api_url must be an absolute URL, got %q", apiURL)+ }+ ref, err := url.Parse(rawURL)+ if err != nil {+ return "", fmt.Errorf("invalid pagination URL %q: %v", rawURL, err)+ }+ ref.Fragment = ""+ ref.RawFragment = ""+ if ref.Scheme == "" && ref.Host == "" {+ if !strings.HasSuffix(base.Path, "/") {+ base.Path += "/"+ if base.RawPath != "" {+ base.RawPath += "/"+ }+ }+ return base.ResolveReference(ref).String(), nil+ }+ pinned := url.URL{Scheme: base.Scheme, Host: base.Host, Path: ref.Path, RawPath: ref.RawPath, RawQuery: ref.RawQuery}+ if pinned.Path == "" {+ pinned.Path = "/"+ }+ return pinned.String(), nil}// delete sends a DELETE request.diff --git a/apps/go-sdk/pagination_origin_test.go b/apps/go-sdk/pagination_origin_test.gonew file mode 100644index 000000000..b7c2fb753--- /dev/null+++ b/apps/go-sdk/pagination_origin_test.go@@ -0,0 +1,227 @@+package firecrawl++import (+ "context"+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"++ "github.com/firecrawl/firecrawl/apps/go-sdk/option"+)++func TestPinToAPIOrigin(t *testing.T) {+ cases := []struct {+ name string+ apiURL string+ next string+ want string+ }{+ {"same origin", "https://api.firecrawl.dev", "https://api.firecrawl.dev/v2/crawl/abc?skip=10", "https://api.firecrawl.dev/v2/crawl/abc?skip=10"},+ {"cross host", "https://api.firecrawl.dev", "https://evil.example/v2/crawl/abc?skip=10", "https://api.firecrawl.dev/v2/crawl/abc?skip=10"},+ {"protocol relative", "https://api.firecrawl.dev", "//evil.example/v2/crawl/abc?skip=10", "https://api.firecrawl.dev/v2/crawl/abc?skip=10"},+ {"different port", "https://api.firecrawl.dev", "https://api.firecrawl.dev:8443/v2/crawl/abc", "https://api.firecrawl.dev/v2/crawl/abc"},+ {"different scheme", "https://api.firecrawl.dev", "http://api.firecrawl.dev/v2/crawl/abc", "https://api.firecrawl.dev/v2/crawl/abc"},+ {"self-hosted http api url", "http://localhost:3002", "https://evil.example/v2/crawl/abc?skip=10", "http://localhost:3002/v2/crawl/abc?skip=10"},+ {"userinfo and fragment dropped", "https://api.firecrawl.dev", "https://user:[email protected]/v2/crawl/abc?skip=10#frag", "https://api.firecrawl.dev/v2/crawl/abc?skip=10"},+ {"empty path", "https://api.firecrawl.dev", "https://evil.example", "https://api.firecrawl.dev/"},+ {"relative path", "https://api.firecrawl.dev", "/v2/crawl/abc?skip=10", "https://api.firecrawl.dev/v2/crawl/abc?skip=10"},+ {"relative path with api url prefix", "https://proxy.example/firecrawl", "v2/crawl/abc?skip=10", "https://proxy.example/firecrawl/v2/crawl/abc?skip=10"},+ {"relative path with escaped api url prefix", "https://proxy.example/a%2Fb", "v2/crawl/abc", "https://proxy.example/a%2Fb/v2/crawl/abc"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got, err := pinToAPIOrigin(tc.apiURL, tc.next)+ if err != nil {+ t.Fatalf("pinToAPIOrigin: %v", err)+ }+ if got != tc.want {+ t.Errorf("pinToAPIOrigin(%q, %q) = %q, want %q", tc.apiURL, tc.next, got, tc.want)+ }+ })+ }+}++func TestPinToAPIOriginRejectsRelativeAPIURL(t *testing.T) {+ for _, apiURL := range []string{"localhost:3002", "/v2", ""} {+ if _, err := pinToAPIOrigin(apiURL, "https://evil.example/v2/crawl/abc"); err == nil {+ t.Errorf("pinToAPIOrigin(%q, ...) returned no error", apiURL)+ }+ }+}++// recordingTransport records every outgoing request and only forwards those+// addressed to the test server, so tests can assert that no request was aimed+// at a foreign host without ever reaching the real network.+type recordingTransport struct {+ apiHost string+ mu sync.Mutex+ requests []*http.Request+}++func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) {+ rt.mu.Lock()+ rt.requests = append(rt.requests, r.Clone(r.Context()))+ rt.mu.Unlock()+ if r.URL.Scheme != "http" || r.URL.Host != rt.apiHost {+ return nil, fmt.Errorf("refusing request outside the test server origin: %s", r.URL)+ }+ return http.DefaultTransport.RoundTrip(r)+}++func (rt *recordingTransport) snapshot() []*http.Request {+ rt.mu.Lock()+ defer rt.mu.Unlock()+ return append([]*http.Request(nil), rt.requests...)+}++type paginationFollower struct {+ name string+ pagePath string+ route func(w http.ResponseWriter, r *http.Request, next string) bool+ run func(ctx context.Context, c *Client) (int, error)+}++var paginationFollowers = []paginationFollower{+ {+ name: "crawl",+ pagePath: "/v2/crawl/abc",+ route: func(w http.ResponseWriter, r *http.Request, next string) bool {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v2/crawl":+ _, _ = w.Write([]byte(`{"success":true,"id":"abc"}`))+ case r.URL.Path == "/v2/crawl/abc" && r.URL.Query().Get("skip") == "":+ _, _ = w.Write([]byte(`{"status":"completed","data":[{"markdown":"one"}],"next":"` + next + `"}`))+ default:+ return false+ }+ return true+ },+ run: func(ctx context.Context, c *Client) (int, error) {+ job, err := c.CrawlWithPolling(ctx, "https://example.com", nil, 1, 30)+ if err != nil {+ return 0, err+ }+ return len(job.Data), nil+ },+ },+ {+ name: "batch scrape",+ pagePath: "/v2/batch/scrape/abc",+ route: func(w http.ResponseWriter, r *http.Request, next string) bool {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/v2/batch/scrape":+ _, _ = w.Write([]byte(`{"success":true,"id":"abc"}`))+ case r.URL.Path == "/v2/batch/scrape/abc" && r.URL.Query().Get("skip") == "":+ _, _ = w.Write([]byte(`{"status":"completed","data":[{"markdown":"one"}],"next":"` + next + `"}`))+ default:+ return false+ }+ return true+ },+ run: func(ctx context.Context, c *Client) (int, error) {+ job, err := c.BatchScrapeWithPolling(ctx, []string{"https://example.com"}, nil, 1, 30)+ if err != nil {+ return 0, err+ }+ return len(job.Data), nil+ },+ },+ {+ name: "monitor check",+ pagePath: "/v2/monitor/m1/checks/c1",+ route: func(w http.ResponseWriter, r *http.Request, next string) bool {+ if r.URL.Path != "/v2/monitor/m1/checks/c1" || r.URL.Query().Get("skip") != "" {+ return false+ }+ _, _ = w.Write([]byte(`{"success":true,"data":{"id":"c1","pages":[{"url":"https://example.com/one"}]},"next":"` + next + `"}`))+ return true+ },+ run: func(ctx context.Context, c *Client) (int, error) {+ detail, err := c.GetMonitorCheck(ctx, "m1", "c1", nil)+ if err != nil {+ return 0, err+ }+ return len(detail.Pages), nil+ },+ },+}++func TestPaginationNextURLPinnedToAPIOrigin(t *testing.T) {+ nextForms := []struct {+ name string+ next func(apiHost string) string+ }{+ {"same origin", func(apiHost string) string { return "http://" + apiHost }},+ {"cross host", func(string) string { return "https://evil.example" }},+ {"protocol relative", func(string) string { return "//evil.example" }},+ {"different port", func(apiHost string) string { return "http://" + strings.Split(apiHost, ":")[0] + ":1" }},+ {"different scheme", func(apiHost string) string { return "https://" + apiHost }},+ {"relative", func(string) string { return "" }},+ }++ for _, f := range paginationFollowers {+ for _, form := range nextForms {+ t.Run(f.name+"/"+form.name, func(t *testing.T) {+ var next string+ server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if f.route(w, r, next) {+ return+ }+ if r.URL.Path == f.pagePath && r.URL.Query().Get("skip") == "10" {+ if f.name == "monitor check" {+ _, _ = w.Write([]byte(`{"success":true,"data":{"id":"c1","pages":[{"url":"https://example.com/two"}]}}`))+ } else {+ _, _ = w.Write([]byte(`{"status":"completed","data":[{"markdown":"two"}]}`))+ }+ return+ }+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte(`{"error":"unexpected ` + r.Method + ` ` + r.URL.String() + `"}`))+ }))+ apiHost := server.Listener.Addr().String()+ next = form.next(apiHost) + f.pagePath + "?skip=10"+ server.Start()+ defer server.Close()++ transport := &recordingTransport{apiHost: apiHost}+ client, err := NewClient(+ option.WithAPIKey("fc-test"),+ option.WithAPIURL(server.URL),+ option.WithHTTPClient(&http.Client{Transport: transport}),+ option.WithMaxRetries(0),+ )+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }++ count, err := f.run(context.Background(), client)+ if err != nil {+ t.Fatalf("%s: %v", f.name, err)+ }+ if count != 2 {+ t.Errorf("got %d items, want 2 (both pages)", count)+ }++ var followed *http.Request+ for _, r := range transport.snapshot() {+ if r.URL.Scheme != "http" || r.URL.Host != apiHost {+ t.Errorf("request sent outside the API origin: %s", r.URL)+ }+ if r.URL.Path == f.pagePath && r.URL.Query().Get("skip") == "10" {+ followed = r+ }+ }+ if followed == nil {+ t.Fatalf("next page %q was not requested from the API origin", next)+ }+ if got := followed.Header.Get("Authorization"); got != "Bearer fc-test" {+ t.Errorf("Authorization = %q, want Bearer fc-test", got)+ }+ })+ }+ }+}diff --git a/apps/go-sdk/version.go b/apps/go-sdk/version.goindex 7746ffc25..117f39006 100644--- a/apps/go-sdk/version.go+++ b/apps/go-sdk/version.go@@ -9,4 +9,4 @@ package firecrawl// Bump this when preparing a new release. The publish-go-sdk GitHub workflow// reads this value and creates the corresponding monorepo-prefixed tag on// merge to main.-const Version = "1.16.0"+const Version = "1.16.1"