Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat(download): gzip fallback for binary downloads

Data37637e0ac421
diff --git a/internal/handlers/download.go b/internal/handlers/download.go
index 58341083..37696f3b 100644
--- a/internal/handlers/download.go
+++ b/internal/handlers/download.go
@@ -1,10 +1,13 @@
package handlers
import (
+ "compress/gzip"
"context"
"encoding/base64"
"errors"
"fmt"
+ "io"
+ "log/slog"
"net/http"
"net/url"
"os"
@@ -415,6 +418,23 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {
if writeDownloadGuardError(w, requestGuard.BlockedError(), maxDownloadBytes) {
return
}
+ // Chrome aborts navigation for binary downloads (.gz, etc.).
+ // Fall back to a direct Go HTTP fetch using the browser's cookies.
+ if isNavigationAborted(err) {
+ slog.Info("download: Chrome navigation aborted, falling back to direct fetch", "url", dlURL)
+ body, mime, status, fetchErr := h.fetchDirectWithCookies(tCtx, browserCtx, dlURL, maxDownloadBytes)
+ if fetchErr != nil {
+ httpx.Error(w, 502, fmt.Errorf("download fallback: %w", fetchErr))
+ return
+ }
+ if status >= 400 {
+ httpx.Error(w, 502, fmt.Errorf("remote server returned HTTP %d", status))
+ return
+ }
+ responseMIME = mime
+ h.writeDownloadResponse(w, body, responseMIME, dlURL, output, filePath, raw, maxDownloadBytes)
+ return
+ }
httpx.Error(w, 502, fmt.Errorf("navigate to download URL: %w", err))
return
}
@@ -458,19 +478,22 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {
httpx.Error(w, 500, fmt.Errorf("get response body: %w", err))
return
}
- if len(body) > maxDownloadBytes {
+ h.writeDownloadResponse(w, body, responseMIME, dlURL, output, filePath, raw, maxDownloadBytes)
+}
+
+func (h *Handlers) writeDownloadResponse(w http.ResponseWriter, body []byte, mime, dlURL, output, filePath string, raw bool, maxBytes int) {
+ if len(body) > maxBytes {
httpx.ErrorCode(w, http.StatusRequestEntityTooLarge, "download_too_large",
- downloadTooLargeError(int64(len(body)), maxDownloadBytes).Error(), false, map[string]any{
- "maxBytes": maxDownloadBytes,
+ downloadTooLargeError(int64(len(body)), maxBytes).Error(), false, map[string]any{
+ "maxBytes": maxBytes,
})
return
}
- if responseMIME == "" {
- responseMIME = "application/octet-stream"
+ if mime == "" {
+ mime = "application/octet-stream"
}
- // Write to file.
if output == "file" {
if filePath == "" {
httpx.Error(w, 400, fmt.Errorf("path required when output=file"))
@@ -500,29 +523,150 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {
"status": "saved",
"path": filePath,
"size": len(body),
- "contentType": responseMIME,
+ "contentType": mime,
})
return
}
- // Raw bytes.
if raw {
- w.Header().Set("Content-Type", responseMIME)
+ w.Header().Set("Content-Type", mime)
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body)))
w.WriteHeader(200)
_, _ = w.Write(body)
return
}
- // Default: base64 JSON response.
httpx.JSON(w, 200, map[string]any{
"data": base64.StdEncoding.EncodeToString(body),
- "contentType": responseMIME,
+ "contentType": mime,
"size": len(body),
"url": dlURL,
})
}
+// fetchDirectWithCookies performs a Go HTTP fetch using cookies extracted from
+// the browser session. Used as a fallback when Chrome's navigation aborts
+// (e.g. for .gz files or other binary downloads).
+func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx context.Context, dlURL string, maxBytes int) (body []byte, contentType string, statusCode int, err error) {
+ parsed, err := url.Parse(dlURL)
+ if err != nil {
+ return nil, "", 0, fmt.Errorf("parse URL: %w", err)
+ }
+
+ // Pull cookies from the browser for this URL.
+ var browserCookies []*network.Cookie
+ if fetchErr := chromedp.Run(browserCtx,
+ chromedp.ActionFunc(func(ctx context.Context) error {
+ var err error
+ browserCookies, err = network.GetCookies().WithURLs([]string{dlURL}).Do(ctx)
+ return err
+ }),
+ ); fetchErr != nil {
+ slog.Debug("download fallback: failed to get browser cookies", "err", fetchErr)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, "GET", dlURL, nil)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ if h.Config.UserAgent != "" {
+ req.Header.Set("User-Agent", h.Config.UserAgent)
+ }
+ req.Header.Set("Accept", "*/*")
+ for _, c := range browserCookies {
+ if domainMatchesCookie(c.Domain, parsed.Host) {
+ req.AddCookie(&http.Cookie{Name: c.Name, Value: c.Value})
+ }
+ }
+
+ client := &http.Client{
+ Timeout: 30 * time.Second,
+ CheckRedirect: func(req *http.Request, via []*http.Request) error {
+ if len(via) >= 10 {
+ return fmt.Errorf("too many redirects")
+ }
+ host := req.URL.Hostname()
+ if netguard.IsLocalHost(host) {
+ return fmt.Errorf("redirect to local network blocked: %s", host)
+ }
+ if _, err := netguard.ResolveAndValidatePublicIPs(req.Context(), host); err != nil {
+ return fmt.Errorf("redirect to private network blocked: %s", host)
+ }
+ return nil
+ },
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ reader := io.Reader(resp.Body)
+
+ // Decompress gzip if the content itself is gzip-encoded (e.g. .gz files)
+ // but NOT if the transport encoding is gzip (Go handles that automatically).
+ if isGzipContent(resp.Header.Get("Content-Type"), dlURL) &&
+ !strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
+ gz, gzErr := gzip.NewReader(resp.Body)
+ if gzErr != nil {
+ return nil, "", 0, fmt.Errorf("gzip decompress: %w", gzErr)
+ }
+ defer func() { _ = gz.Close() }()
+ reader = gz
+ }
+
+ data, err := io.ReadAll(io.LimitReader(reader, int64(maxBytes)+1))
+ if err != nil {
+ return nil, "", 0, err
+ }
+ if len(data) > maxBytes {
+ return nil, "", 0, errDownloadTooLarge
+ }
+
+ ct := resp.Header.Get("Content-Type")
+ if ct == "" {
+ ct = "application/octet-stream"
+ }
+ // If we decompressed gzip, report the inner content type
+ if isGzipContent(ct, dlURL) {
+ ct = inferDecompressedContentType(dlURL, ct)
+ }
+
+ return data, ct, resp.StatusCode, nil
+}
+
+func domainMatchesCookie(cookieDomain, host string) bool {
+ cookieDomain = strings.TrimPrefix(cookieDomain, ".")
+ return strings.EqualFold(host, cookieDomain) ||
+ strings.HasSuffix(host, "."+cookieDomain)
+}
+
+func isGzipContent(contentType, rawURL string) bool {
+ if strings.Contains(strings.ToLower(contentType), "gzip") {
+ return true
+ }
+ return strings.HasSuffix(strings.ToLower(rawURL), ".gz")
+}
+
+func inferDecompressedContentType(rawURL, fallback string) string {
+ lower := strings.ToLower(rawURL)
+ // Strip .gz to infer inner type
+ if strings.HasSuffix(lower, ".xml.gz") {
+ return "application/xml"
+ }
+ if strings.HasSuffix(lower, ".json.gz") {
+ return "application/json"
+ }
+ if strings.HasSuffix(lower, ".txt.gz") || strings.HasSuffix(lower, ".csv.gz") {
+ return "text/plain"
+ }
+ return "application/octet-stream"
+}
+
+func isNavigationAborted(err error) bool {
+ return err != nil && strings.Contains(err.Error(), "net::ERR_ABORTED")
+}
+
// HandleTabDownload fetches a URL using the browser session for a tab identified by path ID.
//
// @Endpoint GET /tabs/{id}/download
diff --git a/internal/handlers/download_fallback_test.go b/internal/handlers/download_fallback_test.go
new file mode 100644
index 00000000..b6d7dd05
--- /dev/null
+++ b/internal/handlers/download_fallback_test.go
@@ -0,0 +1,141 @@
+package handlers
+
+import (
+ "compress/gzip"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestIsGzipContent(t *testing.T) {
+ tests := []struct {
+ contentType string
+ url string
+ want bool
+ }{
+ {"application/gzip", "https://example.com/file", true},
+ {"application/x-gzip", "https://example.com/file", true},
+ {"application/xml", "https://example.com/sitemap.xml.gz", true},
+ {"application/xml", "https://example.com/sitemap.xml", false},
+ {"text/html", "https://example.com/page", false},
+ {"", "https://example.com/data.gz", true},
+ }
+ for _, tt := range tests {
+ if got := isGzipContent(tt.contentType, tt.url); got != tt.want {
+ t.Errorf("isGzipContent(%q, %q) = %v, want %v", tt.contentType, tt.url, got, tt.want)
+ }
+ }
+}
+
+func TestInferDecompressedContentType(t *testing.T) {
+ tests := []struct {
+ url string
+ fallback string
+ want string
+ }{
+ {"https://example.com/sitemap.xml.gz", "application/gzip", "application/xml"},
+ {"https://example.com/data.json.gz", "application/gzip", "application/json"},
+ {"https://example.com/log.txt.gz", "application/gzip", "text/plain"},
+ {"https://example.com/data.csv.gz", "application/gzip", "text/plain"},
+ {"https://example.com/archive.gz", "application/gzip", "application/octet-stream"},
+ }
+ for _, tt := range tests {
+ if got := inferDecompressedContentType(tt.url, tt.fallback); got != tt.want {
+ t.Errorf("inferDecompressedContentType(%q) = %q, want %q", tt.url, got, tt.want)
+ }
+ }
+}
+
+func TestDomainMatchesCookie(t *testing.T) {
+ tests := []struct {
+ cookieDomain string
+ host string
+ want bool
+ }{
+ {".example.com", "www.example.com", true},
+ {".example.com", "example.com", true},
+ {"example.com", "example.com", true},
+ {".example.com", "evil.com", false},
+ {".other.com", "example.com", false},
+ }
+ for _, tt := range tests {
+ if got := domainMatchesCookie(tt.cookieDomain, tt.host); got != tt.want {
+ t.Errorf("domainMatchesCookie(%q, %q) = %v, want %v", tt.cookieDomain, tt.host, got, tt.want)
+ }
+ }
+}
+
+func TestIsNavigationAborted(t *testing.T) {
+ tests := []struct {
+ err error
+ want bool
+ }{
+ {nil, false},
+ {io.EOF, false},
+ {&navError{"page load error net::ERR_ABORTED"}, true},
+ {&navError{"page load error net::ERR_CONNECTION_REFUSED"}, false},
+ }
+ for _, tt := range tests {
+ if got := isNavigationAborted(tt.err); got != tt.want {
+ t.Errorf("isNavigationAborted(%v) = %v, want %v", tt.err, got, tt.want)
+ }
+ }
+}
+
+type navError struct{ msg string }
+
+func (e *navError) Error() string { return e.msg }
+
+func TestFetchDirectWithCookies_GzipDecompression(t *testing.T) {
+ // Create a test server that serves gzip content
+ xmlContent := `<?xml version="1.0"?><urlset><url><loc>https://example.com</loc></url></urlset>`
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/sitemap.xml.gz" {
+ w.Header().Set("Content-Type", "application/gzip")
+ gz := gzip.NewWriter(w)
+ _, _ = gz.Write([]byte(xmlContent))
+ _ = gz.Close()
+ return
+ }
+ if r.URL.Path == "/plain.txt" {
+ w.Header().Set("Content-Type", "text/plain")
+ _, _ = w.Write([]byte("hello world"))
+ return
+ }
+ w.WriteHeader(404)
+ }))
+ defer srv.Close()
+
+ client := srv.Client()
+ resp, err := client.Get(srv.URL + "/sitemap.xml.gz")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ // Verify the server is sending gzip content
+ if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "gzip") {
+ t.Fatalf("expected gzip content-type, got %q", ct)
+ }
+
+ // Manually decompress to verify our detection works
+ if isGzipContent(resp.Header.Get("Content-Type"), srv.URL+"/sitemap.xml.gz") {
+ gz, err := gzip.NewReader(resp.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = gz.Close() }()
+ body, err := io.ReadAll(gz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(body) != xmlContent {
+ t.Errorf("decompressed content = %q, want %q", body, xmlContent)
+ }
+ } else {
+ t.Error("expected isGzipContent to return true for .gz URL with gzip content-type")
+ }
+}
diff --git a/tests/e2e/fixtures/sitemap.xml.gz b/tests/e2e/fixtures/sitemap.xml.gz
new file mode 100644
index 00000000..6d2abf69
Binary files /dev/null and b/tests/e2e/fixtures/sitemap.xml.gz differ
diff --git a/tests/e2e/scenarios-cli/files-full.sh b/tests/e2e/scenarios-cli/files-full.sh
index ee728cc0..72772dbf 100644
--- a/tests/e2e/scenarios-cli/files-full.sh
+++ b/tests/e2e/scenarios-cli/files-full.sh
@@ -61,3 +61,57 @@ pt_ok pdf --scale 0.5 -o /tmp/e2e-scaled.pdf
rm -f /tmp/e2e-scaled.pdf
end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "pinchtab download .gz file (gzip fallback)"
+
+# Serve the fixture .gz file from a temporary public-facing server.
+# In CI the FIXTURES_URL is blocked by SSRF guards, so we use a
+# Python one-liner to serve the file on an ephemeral port.
+GZ_PORT=0
+GZ_PID=""
+GZ_FIXTURE="${GROUP_DIR}/../fixtures/sitemap.xml.gz"
+
+if [ -f "$GZ_FIXTURE" ]; then
+ # Start a minimal HTTP server for the .gz file
+ GZ_DIR=$(mktemp -d)
+ cp "$GZ_FIXTURE" "$GZ_DIR/sitemap.xml.gz"
+
+ python3 -m http.server 0 --directory "$GZ_DIR" --bind 0.0.0.0 &>/tmp/e2e-gz-server.log &
+ GZ_PID=$!
+ sleep 1
+
+ # Extract the port from the log
+ GZ_PORT=$(grep -oE 'port [0-9]+' /tmp/e2e-gz-server.log | head -1 | grep -oE '[0-9]+')
+
+ if [ -n "$GZ_PORT" ] && [ "$GZ_PORT" != "0" ]; then
+ # The server binds 0.0.0.0, access via the host the pinchtab instance can reach.
+ # In Docker, this is typically the host gateway or the container hostname.
+ GZ_HOST="${E2E_GZ_HOST:-host.docker.internal}"
+ GZ_URL="http://${GZ_HOST}:${GZ_PORT}/sitemap.xml.gz"
+
+ pt_ok download "$GZ_URL" -o /tmp/e2e-download-gz.xml
+ if [ -f /tmp/e2e-download-gz.xml ]; then
+ if grep -q "example.com" /tmp/e2e-download-gz.xml; then
+ echo -e " ${GREEN}✓${NC} .gz file downloaded and decompressed"
+ ((ASSERTIONS_PASSED++)) || true
+ else
+ echo -e " ${RED}✗${NC} file downloaded but content not decompressed"
+ ((ASSERTIONS_FAILED++)) || true
+ fi
+ rm -f /tmp/e2e-download-gz.xml
+ else
+ echo -e " ${RED}✗${NC} .gz download file not created"
+ ((ASSERTIONS_FAILED++)) || true
+ fi
+ else
+ echo -e " ${YELLOW}⚠${NC} could not start gz fixture server, skipping"
+ fi
+
+ [ -n "$GZ_PID" ] && kill "$GZ_PID" 2>/dev/null
+ rm -rf "$GZ_DIR" /tmp/e2e-gz-server.log
+else
+ echo -e " ${YELLOW}⚠${NC} gz fixture not found, skipping"
+fi
+
+end_test