High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: wait for retained network bodies
Mariofe885de04355
diff --git a/docs/endpoints.md b/docs/endpoints.mdindex 066783fd..2944818b 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -475,11 +475,14 @@ Network query parameters:- `limit`- `bufferSize`- `body=true` on detail requests+- `waitRetained=true` on detail requests to wait briefly for asynchronous retained-body capture instead of failing over immediately+- `timeoutMs` on detail requests to bound the retained-body wait window (default 2000, max 30000)Response body behavior for network detail/export:- by default, response bodies are fetched on demand from live CDP state and may no longer be available for older requests- when `server.retainNetworkBodies=true`, PinchTab opportunistically retains bounded response bodies in the in-memory network buffer and returns the retained body first+- retained-body detail responses may expose `bodyPending=true` while capture is still in flight, or `bodySkipped=true` with `bodySkipReason` when retention was intentionally not completed- retained bodies are capped by `server.retainNetworkBodyMaxBytes`; oversized retained bodies are truncated and marked with `bodyTruncated=true`- retained responses may include `bodyRetained=true`diff --git a/internal/bridge/observe/network.go b/internal/bridge/observe/network.goindex 34a4ec5e..cff2cccd 100644--- a/internal/bridge/observe/network.go+++ b/internal/bridge/observe/network.go@@ -56,6 +56,9 @@ type NetworkEntry struct {ResponseBody string `json:"responseBody,omitempty"`Base64Encoded bool `json:"base64Encoded,omitempty"`BodyRetained bool `json:"bodyRetained,omitempty"`+ BodyPending bool `json:"bodyPending,omitempty"`+ BodySkipped bool `json:"bodySkipped,omitempty"`+ BodySkipReason string `json:"bodySkipReason,omitempty"`BodyTruncated bool `json:"bodyTruncated,omitempty"`BodyError string `json:"bodyError,omitempty"`}@@ -487,6 +490,12 @@ func (nm *NetworkMonitor) StartCaptureWithSize(tabCtx context.Context, tabID strif e.EncodedDataLength > 0 {entry.Size = int64(e.EncodedDataLength)}+ if nm.bodyRetentionEnabled() {+ entry.BodyPending = true+ entry.BodySkipped = false+ entry.BodySkipReason = ""+ entry.BodyError = ""+ }})buf.MarkRequestEnd(string(e.RequestID))go nm.maybeRetainBody(tabCtx, buf, string(e.RequestID))@@ -539,16 +548,30 @@ func (nm *NetworkMonitor) ClearAll() {}}+func (nm *NetworkMonitor) bodyRetentionEnabled() bool {+ nm.mu.RLock()+ defer nm.mu.RUnlock()+ return nm.retainBodies+}+func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBuffer, requestID string) {nm.mu.RLock()enabled := nm.retainBodiesmaxBytes := nm.retainBodyMaxBytesnm.mu.RUnlock()if !enabled {+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyPending = false+ entry.BodySkipped = true+ entry.BodySkipReason = "retention disabled"+ })return}if buf.RetainedBytes() >= nm.retainBodyMaxPerTab {buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyPending = false+ entry.BodySkipped = true+ entry.BodySkipReason = "retention budget exceeded"entry.BodyError = "retention budget exceeded"})return@@ -558,6 +581,9 @@ func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBudefer func() { <-nm.retainBodySemaphore }()default:buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyPending = false+ entry.BodySkipped = true+ entry.BodySkipReason = "retention concurrency limit reached"entry.BodyError = "retention concurrency limit reached"})return@@ -565,6 +591,7 @@ func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBubody, base64Encoded, err := GetResponseBodyDirect(tabCtx, requestID)if err != nil {buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyPending = falseentry.BodyError = err.Error()})return@@ -577,6 +604,9 @@ func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBuremainingBudget := int(nm.retainBodyMaxPerTab - buf.RetainedBytes())if remainingBudget <= 0 {buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyPending = false+ entry.BodySkipped = true+ entry.BodySkipReason = "retention budget exceeded"entry.BodyError = "retention budget exceeded"})return@@ -589,6 +619,9 @@ func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBuentry.ResponseBody = bodyentry.Base64Encoded = base64Encodedentry.BodyRetained = true+ entry.BodyPending = false+ entry.BodySkipped = false+ entry.BodySkipReason = ""entry.BodyTruncated = truncatedentry.BodyError = ""})diff --git a/internal/handlers/network.go b/internal/handlers/network.goindex 9d997256..534badb2 100644--- a/internal/handlers/network.go+++ b/internal/handlers/network.go@@ -5,12 +5,18 @@ import ("fmt""net/http""strconv"+ "strings""time""github.com/pinchtab/pinchtab/internal/bridge""github.com/pinchtab/pinchtab/internal/httpx")+const (+ defaultWaitRetainedTimeout = 2000 * time.Millisecond+ maxWaitRetainedTimeout = 30 * time.Second+)+// parseBufferSize extracts an optional bufferSize query param. Returns 0 if absent.func parseBufferSize(r *http.Request) int {if v := r.URL.Query().Get("bufferSize"); v != "" {@@ -21,6 +27,82 @@ func parseBufferSize(r *http.Request) int {return 0}+func parseBoolQuery(value string) bool {+ switch strings.ToLower(strings.TrimSpace(value)) {+ case "1", "true", "yes", "on":+ return true+ default:+ return false+ }+}++func parseWaitRetainedTimeout(r *http.Request) time.Duration {+ if v := r.URL.Query().Get("timeoutMs"); v != "" {+ if n, err := strconv.Atoi(v); err == nil {+ switch {+ case n <= 0:+ return 0+ case n > int(maxWaitRetainedTimeout/time.Millisecond):+ return maxWaitRetainedTimeout+ default:+ return time.Duration(n) * time.Millisecond+ }+ }+ }+ return defaultWaitRetainedTimeout+}++func waitForRetainedBody(buf *bridge.NetworkBuffer, requestID string, timeout time.Duration) (bridge.NetworkEntry, bool) {+ if timeout <= 0 {+ return buf.Get(requestID)+ }+ deadline := time.Now().Add(timeout)+ for {+ entry, ok := buf.Get(requestID)+ if !ok {+ return bridge.NetworkEntry{}, false+ }+ if entry.BodyRetained || !entry.BodyPending || entry.BodyError != "" {+ return entry, true+ }+ if time.Now().After(deadline) {+ return entry, true+ }+ remaining := time.Until(deadline)+ if remaining > 25*time.Millisecond {+ remaining = 25 * time.Millisecond+ }+ time.Sleep(remaining)+ }+}++func populateRetainedBodyResult(result map[string]any, entry bridge.NetworkEntry) {+ if entry.ResponseBody != "" || entry.BodyRetained {+ result["responseBody"] = entry.ResponseBody+ }+ if entry.Base64Encoded {+ result["base64Encoded"] = entry.Base64Encoded+ }+ if entry.BodyRetained {+ result["bodyRetained"] = true+ }+ if entry.BodyPending {+ result["bodyPending"] = true+ }+ if entry.BodySkipped {+ result["bodySkipped"] = true+ }+ if entry.BodySkipReason != "" {+ result["bodySkipReason"] = entry.BodySkipReason+ }+ if entry.BodyTruncated {+ result["bodyTruncated"] = true+ }+ if entry.BodyError != "" {+ result["bodyError"] = entry.BodyError+ }+}+// HandleNetwork lists recent network entries for a tab.//// @Endpoint GET /network@@ -160,16 +242,19 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {// Optionally include response bodyif r.URL.Query().Get("body") == "true" && entry.Finished && !entry.Failed {- if entry.BodyRetained {- result["responseBody"] = entry.ResponseBody- result["base64Encoded"] = entry.Base64Encoded- result["bodyRetained"] = true- if entry.BodyTruncated {- result["bodyTruncated"] = true- }- if entry.BodyError != "" {- result["bodyError"] = entry.BodyError+ waitRetained := parseBoolQuery(r.URL.Query().Get("waitRetained"))+ if waitRetained && entry.BodyPending {+ entry, ok = waitForRetainedBody(buf, requestID, parseWaitRetainedTimeout(r))+ if !ok {+ httpx.Error(w, 404, fmt.Errorf("request %s not found", requestID))+ return}+ result["entry"] = entry+ }+ if entry.BodyRetained {+ populateRetainedBodyResult(result, entry)+ } else if entry.BodyPending || (entry.BodyError != "" && !entry.BodySkipped) {+ populateRetainedBodyResult(result, entry)} else {body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)if err != nil {@@ -186,6 +271,7 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {result["bodyRetained"] = true}}+ populateRetainedBodyResult(result, entry)}httpx.JSON(w, 200, result)diff --git a/internal/handlers/network_test.go b/internal/handlers/network_test.goindex 34ebafa7..179422d1 100644--- a/internal/handlers/network_test.go+++ b/internal/handlers/network_test.go@@ -257,6 +257,101 @@ func TestHandleNetworkByIDUsesRetainedBody(t *testing.T) {}}+func TestHandleNetworkByIDWaitRetainedReturnsBodyAfterPendingResolves(t *testing.T) {+ nm := bridge.NewNetworkMonitor(100)+ buf := nm.GetOrCreateBufferForTest("tab1")+ buf.Add(bridge.NetworkEntry{+ RequestID: "pending-1",+ URL: "https://api.example.com/data",+ Method: "GET",+ ResourceType: "XHR",+ Finished: true,+ BodyPending: true,+ })+ h := newNetworkTestHandler(nm)++ go func() {+ time.Sleep(40 * time.Millisecond)+ buf.Update("pending-1", func(entry *bridge.NetworkEntry) {+ entry.ResponseBody = "{\"ok\":true}"+ entry.BodyRetained = true+ entry.BodyPending = false+ })+ }()++ req := httptest.NewRequest("GET", "/network/pending-1?body=true&waitRetained=true&timeoutMs=250", nil)+ req.SetPathValue("requestId", "pending-1")+ w := httptest.NewRecorder()+ h.HandleNetworkByID(w, req)++ if w.Code != 200 {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }+ var got map[string]any+ if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {+ t.Fatalf("decode: %v", err)+ }+ if got["responseBody"] != "{\"ok\":true}" {+ t.Fatalf("expected retained response body, got %v", got["responseBody"])+ }+ if got["bodyRetained"] != true {+ t.Fatalf("expected bodyRetained=true, got %v", got["bodyRetained"])+ }+ if _, ok := got["bodyPending"]; ok {+ t.Fatalf("expected bodyPending to clear after wait, got %v", got["bodyPending"])+ }+ entryMap, ok := got["entry"].(map[string]any)+ if !ok {+ t.Fatalf("expected entry map, got %T", got["entry"])+ }+ if entryMap["bodyRetained"] != true {+ t.Fatalf("expected nested entry bodyRetained=true, got %v", entryMap["bodyRetained"])+ }+}++func TestHandleNetworkByIDWaitRetainedTimeoutReturnsPendingState(t *testing.T) {+ nm := bridge.NewNetworkMonitor(100)+ buf := nm.GetOrCreateBufferForTest("tab1")+ buf.Add(bridge.NetworkEntry{+ RequestID: "pending-timeout",+ URL: "https://api.example.com/data",+ Method: "GET",+ ResourceType: "XHR",+ Finished: true,+ BodyPending: true,+ })+ h := newNetworkTestHandler(nm)++ req := httptest.NewRequest("GET", "/network/pending-timeout?body=true&waitRetained=true&timeoutMs=20", nil)+ req.SetPathValue("requestId", "pending-timeout")+ w := httptest.NewRecorder()+ h.HandleNetworkByID(w, req)++ if w.Code != 200 {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }+ var got map[string]any+ if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {+ t.Fatalf("decode: %v", err)+ }+ if got["bodyPending"] != true {+ t.Fatalf("expected bodyPending=true on timeout, got %v", got["bodyPending"])+ }+ if _, ok := got["responseBody"]; ok {+ t.Fatalf("expected no responseBody on timeout, got %v", got["responseBody"])+ }+ if _, ok := got["bodyError"]; ok {+ t.Fatalf("expected no bodyError on timeout, got %v", got["bodyError"])+ }+ entryMap, ok := got["entry"].(map[string]any)+ if !ok {+ t.Fatalf("expected entry map, got %T", got["entry"])+ }+ if entryMap["bodyPending"] != true {+ t.Fatalf("expected nested entry bodyPending=true, got %v", entryMap["bodyPending"])+ }+}+func TestHandleNetworkByID_Found(t *testing.T) {nm := bridge.NewNetworkMonitor(100)seedBuffer(nm, "tab1")diff --git a/tests/e2e/config/pinchtab-network-retain-bodies.json b/tests/e2e/config/pinchtab-network-retain-bodies.jsonindex 5a7aa47a..6539feb6 100644--- a/tests/e2e/config/pinchtab-network-retain-bodies.json+++ b/tests/e2e/config/pinchtab-network-retain-bodies.json@@ -1,7 +1,81 @@{"server": {+ "bind": "0.0.0.0",+ "port": "9999",+ "token": "e2e-token","networkBufferSize": 200,"retainNetworkBodies": true,"retainNetworkBodyMaxBytes": 262144+ },+ "browser": {+ "extensionPaths": [+ "/extensions/test-extension"+ ]+ },+ "instanceDefaults": {+ "maxTabs": 10,+ "stealthLevel": "light"+ },+ "multiInstance": {+ "strategy": "always-on",+ "allocationPolicy": "fcfs",+ "restart": {+ "maxRestarts": 20,+ "initBackoffSec": 2,+ "maxBackoffSec": 60,+ "stableAfterSec": 300+ }+ },+ "timeouts": {+ "waitNavMs": 100+ },+ "scheduler": {+ "enabled": true,+ "strategy": "fair-fifo",+ "maxQueueSize": 5,+ "maxPerAgent": 3,+ "maxInflight": 2,+ "workerCount": 2,+ "maxBatchSize": 5,+ "resultTTLSec": 120+ },+ "security": {+ "allowEvaluate": true,+ "allowDownload": true,+ "allowScreencast": true,+ "allowCookies": true,+ "allowUpload": true,+ "allowClipboard": true,+ "allowStateExport": true,+ "allowedDomains": [+ "localhost",+ "127.0.0.1",+ "::1",+ "fixtures",+ "httpbin.org"+ ],+ "downloadAllowedDomains": ["fixtures", "httpbin.org"],+ "attach": {+ "enabled": true,+ "allowHosts": [+ "127.0.0.1",+ "localhost",+ "::1",+ "pinchtab-bridge"+ ],+ "allowSchemes": [+ "ws",+ "wss",+ "http",+ "https"+ ]+ },+ "idpi": {+ "enabled": true,+ "strictMode": false,+ "scanContent": true,+ "wrapContent": false,+ "shieldThreshold": 30+ }}}diff --git a/tests/e2e/docker-compose-multi.yml b/tests/e2e/docker-compose-multi.ymlindex 20667ff7..2b7e919e 100644--- a/tests/e2e/docker-compose-multi.yml+++ b/tests/e2e/docker-compose-multi.yml@@ -75,6 +75,18 @@ services:- "127.0.0.1:9995:9999"healthcheck: *pinchtab-healthcheck+ pinchtab-retain:+ <<: *pinchtab-base+ environment:+ PINCHTAB_CONFIG: /data/e2e-config/pinchtab-network-retain-bodies.json+ volumes:+ - ./config/pinchtab-network-retain-bodies.json:/fixture-config/pinchtab-network-retain-bodies.json:ro+ - ./fixtures/test-extension:/extensions/test-extension+ command: ["/bin/sh", "-lc", "mkdir -p /data/e2e-config && cp /fixture-config/pinchtab-network-retain-bodies.json /data/e2e-config/pinchtab-network-retain-bodies.json && exec /usr/local/bin/docker-entrypoint.sh pinchtab server"]+ ports:+ - "127.0.0.1:9993:9999"+ healthcheck: *pinchtab-healthcheck+pinchtab-lite:<<: *pinchtab-baseenvironment:@@ -148,6 +160,8 @@ services:condition: service_healthypinchtab-full:condition: service_healthy+ pinchtab-retain:+ condition: service_healthypinchtab-lite:condition: service_healthypinchtab-bridge:@@ -160,6 +174,7 @@ services:- E2E_AUTOCLOSE_SERVER=http://pinchtab-autoclose:9999- E2E_MEDIUM_SERVER=http://pinchtab-medium:9999- E2E_FULL_SERVER=http://pinchtab-full:9999+ - E2E_RETAIN_SERVER=http://pinchtab-retain:9999- E2E_LITE_SERVER=http://pinchtab-lite:9999- E2E_SERVER_TOKEN=e2e-token- E2E_BRIDGE_URL=http://pinchtab-bridge:9999diff --git a/tests/e2e/scenarios/api/network-retain-body.sh b/tests/e2e/scenarios/api/network-retain-body.shindex c4c0b511..cc23930d 100644--- a/tests/e2e/scenarios/api/network-retain-body.sh+++ b/tests/e2e/scenarios/api/network-retain-body.sh@@ -6,12 +6,19 @@ source "${GROUP_DIR}/../../helpers/api.sh"start_test "network detail: retained response body"+if [ -z "${E2E_RETAIN_SERVER:-}" ]; then+ echo " E2E_RETAIN_SERVER not set, skipping retained-body smoke"+ end_test+ exit 0+fi++NETWORK_RETAIN_OLD_SERVER="$E2E_SERVER"+E2E_SERVER="$E2E_RETAIN_SERVER"+trap 'E2E_SERVER="$NETWORK_RETAIN_OLD_SERVER"' EXIT+pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/network-retain-body.html\"}"assert_ok "navigate to retention fixture"-# Give the fixture fetch a moment to complete and land in the network buffer.-sleep 1-NETWORK_JSON=$(e2e_curl -s "${E2E_SERVER}/network?type=XHR&limit=20")REQ_ID=$(echo "$NETWORK_JSON" | jq -r '.items[] | select(.url | contains("network-retain-body.json")) | .requestId' | head -n1)if [ -z "$REQ_ID" ] || [ "$REQ_ID" = "null" ]; then@@ -24,7 +31,7 @@ fiecho -e " ${GREEN}✓${NC} found request id: $REQ_ID"((ASSERTIONS_PASSED++)) || true-DETAIL=$(e2e_curl -s "${E2E_SERVER}/network/${REQ_ID}?body=true")+DETAIL=$(e2e_curl -s "${E2E_SERVER}/network/${REQ_ID}?body=true&waitRetained=true&timeoutMs=2000")echo "$DETAIL" | jq -e '.bodyRetained == true' >/dev/null 2>&1if [ $? -eq 0 ]; then@@ -46,4 +53,14 @@ else((ASSERTIONS_FAILED++)) || truefi+echo "$DETAIL" | jq -e '(.bodyPending // false) == false' >/dev/null 2>&1+if [ $? -eq 0 ]; then+ echo -e " ${GREEN}✓${NC} bounded wait completed without leaving bodyPending=true"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} expected bodyPending=false after bounded wait"+ echo "$DETAIL" | jq .+ ((ASSERTIONS_FAILED++)) || true+fi+end_testdiff --git a/tests/e2e/scenarios/manifest.json b/tests/e2e/scenarios/manifest.jsonindex f624a9a8..fe89cca1 100644--- a/tests/e2e/scenarios/manifest.json+++ b/tests/e2e/scenarios/manifest.json@@ -35,6 +35,11 @@"ready": ["E2E_SERVER", "E2E_FULL_SERVER"],"tags": ["network", "route", "interception", "smoke"]},+ "api/network-retain-body.sh": {+ "services": ["pinchtab", "pinchtab-retain", "fixtures"],+ "ready": ["E2E_SERVER", "E2E_RETAIN_SERVER"],+ "tags": ["network", "retain", "body", "smoke"]+ },"api/secrets-extended.sh": {"tags": ["actions", "secrets", "credentials"]},diff --git a/tests/tools/runner/internal/e2e/e2e_test.go b/tests/tools/runner/internal/e2e/e2e_test.goindex e0e9de25..724515b5 100644--- a/tests/tools/runner/internal/e2e/e2e_test.go+++ b/tests/tools/runner/internal/e2e/e2e_test.go@@ -53,12 +53,13 @@ func TestDryRunExtendedPlan(t *testing.T) {out := stdout.String()for _, want := range []string{"suite: extended",- "docker compose -f tests/e2e/docker-compose-multi.yml up -d pinchtab pinchtab-secure pinchtab-medium pinchtab-full pinchtab-lite pinchtab-bridge fixtures",+ "docker compose -f tests/e2e/docker-compose-multi.yml up -d pinchtab pinchtab-secure pinchtab-medium pinchtab-full pinchtab-retain pinchtab-lite pinchtab-bridge fixtures","run --rm --no-deps","E2E_READY_TARGETS=E2E_SERVER E2E_SECURE_SERVER","E2E_SUMMARY_TITLE=PinchTab E2E API Extended Suite","runner-api /bin/bash /e2e/run.sh scenario=actions-basic.sh","scenario=actions-extended.sh",+ "scenario=network-retain-body.sh","E2E_SUMMARY_TITLE=PinchTab E2E CLI Extended Suite","runner-cli /bin/bash /e2e/run.sh scenario=actions-basic.sh","scenario=actions-extended.sh",@@ -172,6 +173,20 @@ func TestScenarioMetadataDefaultsAndManifestOverrides(t *testing.T) {if !hasString(orchestrator.Tags, "multiinstance") || !hasString(orchestrator.Tags, "bridge") {t.Fatalf("expected orchestrator tags, got: %#v", orchestrator.Tags)}++ retain, ok := catalog.find("api", "network-retain-body.sh")+ if !ok {+ t.Fatal("api/network-retain-body.sh missing from scenario catalog")+ }+ if got := strings.Join(retain.Services, " "); got != "pinchtab pinchtab-retain fixtures" {+ t.Fatalf("unexpected retained-body services: %s", got)+ }+ if got := strings.Join(retain.Ready, " "); got != "E2E_SERVER E2E_RETAIN_SERVER" {+ t.Fatalf("unexpected retained-body ready targets: %s", got)+ }+ if !hasString(retain.Tags, "retain") || !hasString(retain.Tags, "smoke") {+ t.Fatalf("expected retained-body tags, got: %#v", retain.Tags)+ }}func hasString(values []string, want string) bool {diff --git a/tests/tools/runner/internal/e2e/manifest.go b/tests/tools/runner/internal/e2e/manifest.goindex bf63a0c5..98f44e33 100644--- a/tests/tools/runner/internal/e2e/manifest.go+++ b/tests/tools/runner/internal/e2e/manifest.go@@ -24,6 +24,7 @@ var composeServiceOrder = []string{"pinchtab-autoclose","pinchtab-medium","pinchtab-full",+ "pinchtab-retain","pinchtab-lite","pinchtab-bridge","fixtures",@@ -35,6 +36,7 @@ var readyTargetOrder = []string{"E2E_AUTOCLOSE_SERVER","E2E_MEDIUM_SERVER","E2E_FULL_SERVER",+ "E2E_RETAIN_SERVER","E2E_LITE_SERVER","E2E_BRIDGE_URL|60|E2E_BRIDGE_TOKEN",}