High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
docs: add gzip and allowlist notes, clean up redundant comments
Data69671e228666
diff --git a/docs/endpoints.md b/docs/endpoints.mdindex 7c2d0b5a..3331bca6 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -222,6 +222,8 @@ GET /cache/statusNotes:- download and upload endpoints are gated by `security.allowDownload` and `security.allowUpload`+- download automatically decompresses `.gz` files and returns the decompressed content+- `security.downloadAllowedDomains` can whitelist specific domains (bypasses SSRF checks for those domains)- clipboard endpoints are gated by `security.allowClipboard`- upload uses a JSON body with `selector` and `files`diff --git a/internal/handlers/download.go b/internal/handlers/download.goindex 648fca6c..acd15713 100644--- a/internal/handlers/download.go+++ b/internal/handlers/download.go@@ -57,8 +57,7 @@ func (g *downloadURLGuard) Validate(rawURL string) error {return fmt.Errorf("internal or blocked host")}- // Check domain allowlist first. If a domain is explicitly allowed,- // skip the IP validation (allows internal/docker network hosts).+ // Allowlisted domains bypass IP validation (e.g. internal docker hosts).if len(g.allowedDomains) > 0 {result := idpi.CheckDomain(rawURL, config.IDPIConfig{Enabled: true,@@ -68,11 +67,9 @@ func (g *downloadURLGuard) Validate(rawURL string) error {if result.Blocked {return fmt.Errorf("domain not allowed by security.downloadAllowedDomains")}- // Domain is explicitly allowed, skip IP validationreturn nil}- // No allowlist configured — enforce public IP requirementctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)defer cancel()@@ -559,15 +556,9 @@ func (h *Handlers) writeDownloadResponse(w http.ResponseWriter, body []byte, mim})}-// fetchDirectWithCookies performs a Go HTTP fetch using cookies extracted from-// the browser session. Used as a fallback when Chrome's navigation aborts-// (e.g. for .gz files or other binary downloads).+// fetchDirectWithCookies performs a Go HTTP fetch with browser cookies.+// Fallback for when Chrome navigation aborts (e.g. .gz files).func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx context.Context, dlURL string, maxBytes int) (body []byte, contentType string, statusCode int, err error) {- // Pull cookies from the browser for this URL via CDP.- // Note: CDP's GetCookies with a URL already returns domain-scoped cookies.- // We forward all cookies including HttpOnly ones - this is intentional for- // download functionality (browsers wouldn't expose HttpOnly to JS, but we're- // acting as the browser itself, not as JS running in the page).var browserCookies []*network.Cookieif fetchErr := chromedp.Run(browserCtx,chromedp.ActionFunc(func(ctx context.Context) error {@@ -588,8 +579,6 @@ func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx contex}req.Header.Set("Accept", "*/*")- // Forward browser cookies. CDP already scopes cookies by URL/domain,- // so we just add them directly without additional filtering.for _, c := range browserCookies {req.AddCookie(&http.Cookie{Name: c.Name, Value: c.Value})}@@ -616,8 +605,6 @@ func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx contex}defer func() { _ = resp.Body.Close() }()- // Check Content-Length early to avoid reading oversized responses.- // This provides a fast-fail for obviously too-large downloads.if cl := resp.Header.Get("Content-Length"); cl != "" {if contentLength, parseErr := strconv.ParseInt(cl, 10, 64); parseErr == nil {if contentLength > int64(maxBytes) {@@ -630,8 +617,6 @@ func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx contexisGzip := isGzipContent(resp.Header.Get("Content-Type"), dlURL) &&!strings.EqualFold(resp.Header.Get("Content-Encoding"), "gzip")- // Decompress gzip if the content itself is gzip-encoded (e.g. .gz files)- // but NOT if the transport encoding is gzip (Go handles that automatically).if isGzip {gz, gzErr := gzip.NewReader(resp.Body)if gzErr != nil {@@ -641,10 +626,7 @@ func (h *Handlers) fetchDirectWithCookies(ctx context.Context, browserCtx contexreader = gz}- // LimitReader protects against gzip bombs: a small compressed payload could- // decompress to gigabytes. By limiting the *decompressed* stream (not the- // compressed input), we bound memory usage regardless of compression ratio.- // Default maxBytes is 20MB, capped at 100MB via config.+ // LimitReader on decompressed stream protects against gzip bombs.data, err := io.ReadAll(io.LimitReader(reader, int64(maxBytes)+1))if err != nil {return nil, "", 0, errdiff --git a/tests/e2e/scenarios-cli/files-full.sh b/tests/e2e/scenarios-cli/files-full.shindex 2955316f..12142b49 100644--- a/tests/e2e/scenarios-cli/files-full.sh+++ b/tests/e2e/scenarios-cli/files-full.sh@@ -65,9 +65,7 @@ end_test# ─────────────────────────────────────────────────────────────────start_test "pinchtab download .gz file (gzip fallback)"-# The fixtures domain is allowlisted in security.downloadAllowedDomains for e2e tests.-# This tests the full fallback path: Chrome navigation aborts on .gz, we fall back-# to Go HTTP fetch with cookie forwarding, decompress gzip, return plain XML.+# fixtures domain allowlisted in e2e config for this testGZ_URL="${FIXTURES_URL:-http://fixtures:80}/sitemap.xml.gz"pt_ok download "$GZ_URL" -o /tmp/e2e-download-gz.xml