High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat(download): gzip fallback for binary downloads
Data37637e0ac421
diff --git a/internal/handlers/download.go b/internal/handlers/download.goindex 58341083..37696f3b 100644--- a/internal/handlers/download.go+++ b/internal/handlers/download.go@@ -1,10 +1,13 @@package handlersimport (+ "compress/gzip""context""encoding/base64""errors""fmt"+ "io"+ "log/slog""net/http""net/url""os"@@ -415,6 +418,23 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {if writeDownloadGuardError(w, requestGuard.BlockedError(), maxDownloadBytes) {return}+ // Chrome aborts navigation for binary downloads (.gz, etc.).+ // Fall back to a direct Go HTTP fetch using the browser's cookies.+ if isNavigationAborted(err) {+ slog.Info("download: Chrome navigation aborted, falling back to direct fetch", "url", dlURL)+ body, mime, status, fetchErr := h.fetchDirectWithCookies(tCtx, browserCtx, dlURL, maxDownloadBytes)+ if fetchErr != nil {+ httpx.Error(w, 502, fmt.Errorf("download fallback: %w", fetchErr))+ return+ }+ if status >= 400 {+ httpx.Error(w, 502, fmt.Errorf("remote server returned HTTP %d", status))+ return+ }+ responseMIME = mime+ h.writeDownloadResponse(w, body, responseMIME, dlURL, output, filePath, raw, maxDownloadBytes)+ return+ }httpx.Error(w, 502, fmt.Errorf("navigate to download URL: %w", err))return}@@ -458,19 +478,22 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {httpx.Error(w, 500, fmt.Errorf("get response body: %w", err))return}- if len(body) > maxDownloadBytes {+ h.writeDownloadResponse(w, body, responseMIME, dlURL, output, filePath, raw, maxDownloadBytes)+}++func (h *Handlers) writeDownloadResponse(w http.ResponseWriter, body []byte, mime, dlURL, output, filePath string, raw bool, maxBytes int) {+ if len(body) > maxBytes {httpx.ErrorCode(w, http.StatusRequestEntityTooLarge, "download_too_large",- downloadTooLargeError(int64(len(body)), maxDownloadBytes).Error(), false, map[string]any{- "maxBytes": maxDownloadBytes,+ downloadTooLargeError(int64(len(body)), maxBytes).Error(), false, map[string]any{+ "maxBytes": maxBytes,})return}- if responseMIME == "" {- responseMIME = "application/octet-stream"+ if mime == "" {+ mime = "application/octet-stream"}- // Write to file.if output == "file" {if filePath == "" {httpx.Error(w, 400, fmt.Errorf("path required when output=file"))@@ -500,29 +523,150 @@ func (h *Handlers) HandleDownload(w http.ResponseWriter, r *http.Request) {"status": "saved","path": filePath,"size": len(body),- "contentType": responseMIME,+ "contentType": mime,})return}- // Raw bytes.if raw {- w.Header().Set("Content-Type", responseMIME)+ w.Header().Set("Content-Type", mime)w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body)))w.WriteHeader(200)_, _ = w.Write(body)return}- // Default: base64 JSON response.httpx.JSON(w, 200, map[string]any{"data": base64.StdEncoding.EncodeToString(body),- "contentType": responseMIME,+ "contentType": mime,"size": len(body),"url": dlURL,})}+// fetchDirectWithCookies performs a Go HTTP fetch using cookies extracted from+// the browser session. Used as a fallback when Chrome's navigation aborts+// (e.g. for .gz files or other binary downloads).+func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx context.Context, dlURL string, maxBytes int) (body []byte, contentType string, statusCode int, err error) {+ parsed, err := url.Parse(dlURL)+ if err != nil {+ return nil, "", 0, fmt.Errorf("parse URL: %w", err)+ }++ // Pull cookies from the browser for this URL.+ var browserCookies []*network.Cookie+ if fetchErr := chromedp.Run(browserCtx,+ chromedp.ActionFunc(func(ctx context.Context) error {+ var err error+ browserCookies, err = network.GetCookies().WithURLs([]string{dlURL}).Do(ctx)+ return err+ }),+ ); fetchErr != nil {+ slog.Debug("download fallback: failed to get browser cookies", "err", fetchErr)+ }++ req, err := http.NewRequestWithContext(ctx, "GET", dlURL, nil)+ if err != nil {+ return nil, "", 0, err+ }+ if h.Config.UserAgent != "" {+ req.Header.Set("User-Agent", h.Config.UserAgent)+ }+ req.Header.Set("Accept", "*/*")+ for _, c := range browserCookies {+ if domainMatchesCookie(c.Domain, parsed.Host) {+ req.AddCookie(&http.Cookie{Name: c.Name, Value: c.Value})+ }+ }++ client := &http.Client{+ Timeout: 30 * time.Second,+ CheckRedirect: func(req *http.Request, via []*http.Request) error {+ if len(via) >= 10 {+ return fmt.Errorf("too many redirects")+ }+ host := req.URL.Hostname()+ if netguard.IsLocalHost(host) {+ return fmt.Errorf("redirect to local network blocked: %s", host)+ }+ if _, err := netguard.ResolveAndValidatePublicIPs(req.Context(), host); err != nil {+ return fmt.Errorf("redirect to private network blocked: %s", host)+ }+ return nil+ },+ }+ resp, err := client.Do(req)+ if err != nil {+ return nil, "", 0, err+ }+ defer func() { _ = resp.Body.Close() }()++ reader := io.Reader(resp.Body)++ // Decompress gzip if the content itself is gzip-encoded (e.g. .gz files)+ // but NOT if the transport encoding is gzip (Go handles that automatically).+ if isGzipContent(resp.Header.Get("Content-Type"), dlURL) &&+ !strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {+ gz, gzErr := gzip.NewReader(resp.Body)+ if gzErr != nil {+ return nil, "", 0, fmt.Errorf("gzip decompress: %w", gzErr)+ }+ defer func() { _ = gz.Close() }()+ reader = gz+ }++ data, err := io.ReadAll(io.LimitReader(reader, int64(maxBytes)+1))+ if err != nil {+ return nil, "", 0, err+ }+ if len(data) > maxBytes {+ return nil, "", 0, errDownloadTooLarge+ }++ ct := resp.Header.Get("Content-Type")+ if ct == "" {+ ct = "application/octet-stream"+ }+ // If we decompressed gzip, report the inner content type+ if isGzipContent(ct, dlURL) {+ ct = inferDecompressedContentType(dlURL, ct)+ }++ return data, ct, resp.StatusCode, nil+}++func domainMatchesCookie(cookieDomain, host string) bool {+ cookieDomain = strings.TrimPrefix(cookieDomain, ".")+ return strings.EqualFold(host, cookieDomain) ||+ strings.HasSuffix(host, "."+cookieDomain)+}++func isGzipContent(contentType, rawURL string) bool {+ if strings.Contains(strings.ToLower(contentType), "gzip") {+ return true+ }+ return strings.HasSuffix(strings.ToLower(rawURL), ".gz")+}++func inferDecompressedContentType(rawURL, fallback string) string {+ lower := strings.ToLower(rawURL)+ // Strip .gz to infer inner type+ if strings.HasSuffix(lower, ".xml.gz") {+ return "application/xml"+ }+ if strings.HasSuffix(lower, ".json.gz") {+ return "application/json"+ }+ if strings.HasSuffix(lower, ".txt.gz") || strings.HasSuffix(lower, ".csv.gz") {+ return "text/plain"+ }+ return "application/octet-stream"+}++func isNavigationAborted(err error) bool {+ return err != nil && strings.Contains(err.Error(), "net::ERR_ABORTED")+}+// HandleTabDownload fetches a URL using the browser session for a tab identified by path ID.//// @Endpoint GET /tabs/{id}/downloaddiff --git a/internal/handlers/download_fallback_test.go b/internal/handlers/download_fallback_test.gonew file mode 100644index 00000000..b6d7dd05--- /dev/null+++ b/internal/handlers/download_fallback_test.go@@ -0,0 +1,141 @@+package handlers++import (+ "compress/gzip"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+)++func TestIsGzipContent(t *testing.T) {+ tests := []struct {+ contentType string+ url string+ want bool+ }{+ {"application/gzip", "https://example.com/file", true},+ {"application/x-gzip", "https://example.com/file", true},+ {"application/xml", "https://example.com/sitemap.xml.gz", true},+ {"application/xml", "https://example.com/sitemap.xml", false},+ {"text/html", "https://example.com/page", false},+ {"", "https://example.com/data.gz", true},+ }+ for _, tt := range tests {+ if got := isGzipContent(tt.contentType, tt.url); got != tt.want {+ t.Errorf("isGzipContent(%q, %q) = %v, want %v", tt.contentType, tt.url, got, tt.want)+ }+ }+}++func TestInferDecompressedContentType(t *testing.T) {+ tests := []struct {+ url string+ fallback string+ want string+ }{+ {"https://example.com/sitemap.xml.gz", "application/gzip", "application/xml"},+ {"https://example.com/data.json.gz", "application/gzip", "application/json"},+ {"https://example.com/log.txt.gz", "application/gzip", "text/plain"},+ {"https://example.com/data.csv.gz", "application/gzip", "text/plain"},+ {"https://example.com/archive.gz", "application/gzip", "application/octet-stream"},+ }+ for _, tt := range tests {+ if got := inferDecompressedContentType(tt.url, tt.fallback); got != tt.want {+ t.Errorf("inferDecompressedContentType(%q) = %q, want %q", tt.url, got, tt.want)+ }+ }+}++func TestDomainMatchesCookie(t *testing.T) {+ tests := []struct {+ cookieDomain string+ host string+ want bool+ }{+ {".example.com", "www.example.com", true},+ {".example.com", "example.com", true},+ {"example.com", "example.com", true},+ {".example.com", "evil.com", false},+ {".other.com", "example.com", false},+ }+ for _, tt := range tests {+ if got := domainMatchesCookie(tt.cookieDomain, tt.host); got != tt.want {+ t.Errorf("domainMatchesCookie(%q, %q) = %v, want %v", tt.cookieDomain, tt.host, got, tt.want)+ }+ }+}++func TestIsNavigationAborted(t *testing.T) {+ tests := []struct {+ err error+ want bool+ }{+ {nil, false},+ {io.EOF, false},+ {&navError{"page load error net::ERR_ABORTED"}, true},+ {&navError{"page load error net::ERR_CONNECTION_REFUSED"}, false},+ }+ for _, tt := range tests {+ if got := isNavigationAborted(tt.err); got != tt.want {+ t.Errorf("isNavigationAborted(%v) = %v, want %v", tt.err, got, tt.want)+ }+ }+}++type navError struct{ msg string }++func (e *navError) Error() string { return e.msg }++func TestFetchDirectWithCookies_GzipDecompression(t *testing.T) {+ // Create a test server that serves gzip content+ xmlContent := `<?xml version="1.0"?><urlset><url><loc>https://example.com</loc></url></urlset>`++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/sitemap.xml.gz" {+ w.Header().Set("Content-Type", "application/gzip")+ gz := gzip.NewWriter(w)+ _, _ = gz.Write([]byte(xmlContent))+ _ = gz.Close()+ return+ }+ if r.URL.Path == "/plain.txt" {+ w.Header().Set("Content-Type", "text/plain")+ _, _ = w.Write([]byte("hello world"))+ return+ }+ w.WriteHeader(404)+ }))+ defer srv.Close()++ client := srv.Client()+ resp, err := client.Get(srv.URL + "/sitemap.xml.gz")+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = resp.Body.Close() }()++ // Verify the server is sending gzip content+ if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "gzip") {+ t.Fatalf("expected gzip content-type, got %q", ct)+ }++ // Manually decompress to verify our detection works+ if isGzipContent(resp.Header.Get("Content-Type"), srv.URL+"/sitemap.xml.gz") {+ gz, err := gzip.NewReader(resp.Body)+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = gz.Close() }()+ body, err := io.ReadAll(gz)+ if err != nil {+ t.Fatal(err)+ }+ if string(body) != xmlContent {+ t.Errorf("decompressed content = %q, want %q", body, xmlContent)+ }+ } else {+ t.Error("expected isGzipContent to return true for .gz URL with gzip content-type")+ }+}diff --git a/tests/e2e/fixtures/sitemap.xml.gz b/tests/e2e/fixtures/sitemap.xml.gznew file mode 100644index 00000000..6d2abf69Binary files /dev/null and b/tests/e2e/fixtures/sitemap.xml.gz differdiff --git a/tests/e2e/scenarios-cli/files-full.sh b/tests/e2e/scenarios-cli/files-full.shindex ee728cc0..72772dbf 100644--- a/tests/e2e/scenarios-cli/files-full.sh+++ b/tests/e2e/scenarios-cli/files-full.sh@@ -61,3 +61,57 @@ pt_ok pdf --scale 0.5 -o /tmp/e2e-scaled.pdfrm -f /tmp/e2e-scaled.pdfend_test++# ─────────────────────────────────────────────────────────────────+start_test "pinchtab download .gz file (gzip fallback)"++# Serve the fixture .gz file from a temporary public-facing server.+# In CI the FIXTURES_URL is blocked by SSRF guards, so we use a+# Python one-liner to serve the file on an ephemeral port.+GZ_PORT=0+GZ_PID=""+GZ_FIXTURE="${GROUP_DIR}/../fixtures/sitemap.xml.gz"++if [ -f "$GZ_FIXTURE" ]; then+ # Start a minimal HTTP server for the .gz file+ GZ_DIR=$(mktemp -d)+ cp "$GZ_FIXTURE" "$GZ_DIR/sitemap.xml.gz"++ python3 -m http.server 0 --directory "$GZ_DIR" --bind 0.0.0.0 &>/tmp/e2e-gz-server.log &+ GZ_PID=$!+ sleep 1++ # Extract the port from the log+ GZ_PORT=$(grep -oE 'port [0-9]+' /tmp/e2e-gz-server.log | head -1 | grep -oE '[0-9]+')++ if [ -n "$GZ_PORT" ] && [ "$GZ_PORT" != "0" ]; then+ # The server binds 0.0.0.0, access via the host the pinchtab instance can reach.+ # In Docker, this is typically the host gateway or the container hostname.+ GZ_HOST="${E2E_GZ_HOST:-host.docker.internal}"+ GZ_URL="http://${GZ_HOST}:${GZ_PORT}/sitemap.xml.gz"++ pt_ok download "$GZ_URL" -o /tmp/e2e-download-gz.xml+ if [ -f /tmp/e2e-download-gz.xml ]; then+ if grep -q "example.com" /tmp/e2e-download-gz.xml; then+ echo -e " ${GREEN}✓${NC} .gz file downloaded and decompressed"+ ((ASSERTIONS_PASSED++)) || true+ else+ echo -e " ${RED}✗${NC} file downloaded but content not decompressed"+ ((ASSERTIONS_FAILED++)) || true+ fi+ rm -f /tmp/e2e-download-gz.xml+ else+ echo -e " ${RED}✗${NC} .gz download file not created"+ ((ASSERTIONS_FAILED++)) || true+ fi+ else+ echo -e " ${YELLOW}⚠${NC} could not start gz fixture server, skipping"+ fi++ [ -n "$GZ_PID" ] && kill "$GZ_PID" 2>/dev/null+ rm -rf "$GZ_DIR" /tmp/e2e-gz-server.log+else+ echo -e " ${YELLOW}⚠${NC} gz fixture not found, skipping"+fi++end_test