High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add optional retained network response bodies
Marioe4c3801dd87f
diff --git a/docs/endpoints.md b/docs/endpoints.mdindex ceac56cb..99023b36 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -457,12 +457,19 @@ Network query parameters:- `bufferSize`- `body=true` on detail requests+Response body behavior for network detail/export:++- by default, response bodies are fetched on demand from live CDP state and may no longer be available for older requests+- when `server.retainNetworkBodies=true`, PinchTab opportunistically retains bounded response bodies in the in-memory network buffer and returns the retained body first+- retained bodies are capped by `server.retainNetworkBodyMaxBytes`; oversized retained bodies are truncated and marked with `bodyTruncated=true`+- retained responses may include `bodyRetained=true`+Network export query parameters:- `format` — `har` (default) or `ndjson`. Pluggable: new formats register at startup.- `output=file` — save to disk instead of streaming to response- `path` — filename when `output=file` (auto-generated if omitted, required for `/export/stream`)-- `body=true` — include response bodies (fetched on demand, 10 MB cap per entry)+- `body=true` — include response bodies (fetched on demand by default; retained-body mode can make this durable for bounded entries)- `redact` — `true` (default) redacts Cookie/Authorization/Set-Cookie. `false` exports raw headers.- all standard network filters (`filter`, `method`, `status`, `type`, `limit`)diff --git a/docs/reference/config.md b/docs/reference/config.mdindex e6c5d5c9..1f055971 100644--- a/docs/reference/config.md+++ b/docs/reference/config.md@@ -174,6 +174,8 @@ Current nested file-config shape:"stateDir": "/path/to/state","engine": "chrome","networkBufferSize": 100,+ "retainNetworkBodies": false,+ "retainNetworkBodyMaxBytes": 262144,"trustProxyHeaders": false,"cookieSecure": null},diff --git a/internal/bridge/bridge.go b/internal/bridge/bridge.goindex f977f7b5..3584bb39 100644--- a/internal/bridge/bridge.go+++ b/internal/bridge/bridge.go@@ -74,6 +74,9 @@ func New(allocCtx, browserCtx context.Context, cfg *config.RuntimeConfig) *BridgLogStore: logStore,stealthLaunchMode: stealth.LaunchModeUninitialized,}+ if cfg != nil {+ b.netMonitor.ConfigureBodyRetention(cfg.RetainNetworkBodies, cfg.RetainNetworkBodyMaxBytes)+ }b.routeMgr = NewRouteManager(func() []string {if b.Config == nil {return nildiff --git a/internal/bridge/bridge_lifecycle.go b/internal/bridge/bridge_lifecycle.goindex c4894984..7e980a1a 100644--- a/internal/bridge/bridge_lifecycle.go+++ b/internal/bridge/bridge_lifecycle.go@@ -261,6 +261,7 @@ func (b *Bridge) RestartBrowser(cfg *config.RuntimeConfig) error {if cfg.NetworkBufferSize > 0 {b.netMonitor = NewNetworkMonitor(cfg.NetworkBufferSize)}+ b.netMonitor.ConfigureBodyRetention(cfg.RetainNetworkBodies, cfg.RetainNetworkBodyMaxBytes)b.fingerprintMu.Lock()b.fingerprintOverlays = make(map[string]bool)b.fingerprintMu.Unlock()diff --git a/internal/bridge/observe/network.go b/internal/bridge/observe/network.goindex b07a632f..de2bae7b 100644--- a/internal/bridge/observe/network.go+++ b/internal/bridge/observe/network.go@@ -18,6 +18,8 @@ import (// DefaultNetworkBufferSize is the default number of entries kept per tab.const DefaultNetworkBufferSize = 100+const defaultRetainBodyMaxBytesPerTab = 4 << 20+const defaultRetainBodyConcurrency = 4const (maxNetworkURLBytes = 8 * 1024@@ -51,14 +53,20 @@ type NetworkEntry struct {Error string `json:"error,omitempty"`Finished bool `json:"finished"`Failed bool `json:"failed"`+ ResponseBody string `json:"responseBody,omitempty"`+ Base64Encoded bool `json:"base64Encoded,omitempty"`+ BodyRetained bool `json:"bodyRetained,omitempty"`+ BodyTruncated bool `json:"bodyTruncated,omitempty"`+ BodyError string `json:"bodyError,omitempty"`}// NetworkBuffer is a thread-safe ring buffer of network entries for a single tab.type NetworkBuffer struct {- mu sync.RWMutex- entries []NetworkEntry- index map[string]int- maxSize int+ mu sync.RWMutex+ entries []NetworkEntry+ index map[string]int+ maxSize int+ retainedBytes int64// Inflight tracking is independent of the ring buffer so eviction// doesn't corrupt the count. inflightIDs holds request IDs currently@@ -138,6 +146,12 @@ func (nb *NetworkBuffer) Add(entry NetworkEntry) {isNew = trueif len(nb.entries) >= nb.maxSize {oldest := nb.entries[0]+ if oldest.BodyRetained {+ nb.retainedBytes -= int64(len(oldest.ResponseBody))+ if nb.retainedBytes < 0 {+ nb.retainedBytes = 0+ }+ }delete(nb.index, oldest.RequestID)nb.entries = nb.entries[1:]for i, e := range nb.entries {@@ -201,8 +215,25 @@ func (nb *NetworkBuffer) Update(requestID string, fn func(*NetworkEntry)) {if !ok {return}+ before := nb.entries[idx]fn(&nb.entries[idx])nb.entries[idx] = normalizeNetworkEntry(nb.entries[idx])+ after := nb.entries[idx]+ if before.BodyRetained {+ nb.retainedBytes -= int64(len(before.ResponseBody))+ }+ if after.BodyRetained {+ nb.retainedBytes += int64(len(after.ResponseBody))+ }+ if nb.retainedBytes < 0 {+ nb.retainedBytes = 0+ }+}++func (nb *NetworkBuffer) RetainedBytes() int64 {+ nb.mu.RLock()+ defer nb.mu.RUnlock()+ return nb.retainedBytes}// List returns all entries, optionally filtered.@@ -288,10 +319,14 @@ func MatchStatusRange(status int, pattern string) bool {// NetworkMonitor manages network capture for all tabs.type NetworkMonitor struct {- mu sync.RWMutex- buffers map[string]*NetworkBuffer- listeners map[string]context.CancelFunc- bufSize int+ mu sync.RWMutex+ buffers map[string]*NetworkBuffer+ listeners map[string]context.CancelFunc+ bufSize int+ retainBodies bool+ retainBodyMaxBytes int+ retainBodyMaxPerTab int64+ retainBodySemaphore chan struct{}}// NewNetworkMonitor creates a new monitor with the given per-tab buffer size.@@ -301,10 +336,25 @@ func NewNetworkMonitor(bufferSize int) *NetworkMonitor {bufferSize = DefaultNetworkBufferSize}return &NetworkMonitor{- buffers: make(map[string]*NetworkBuffer),- listeners: make(map[string]context.CancelFunc),- bufSize: bufferSize,+ buffers: make(map[string]*NetworkBuffer),+ listeners: make(map[string]context.CancelFunc),+ bufSize: bufferSize,+ retainBodies: false,+ retainBodyMaxBytes: 0,+ retainBodyMaxPerTab: defaultRetainBodyMaxBytesPerTab,+ retainBodySemaphore: make(chan struct{}, defaultRetainBodyConcurrency),+ }+}++// ConfigureBodyRetention enables optional bounded response-body retention.+func (nm *NetworkMonitor) ConfigureBodyRetention(enabled bool, maxBytes int) {+ nm.mu.Lock()+ defer nm.mu.Unlock()+ nm.retainBodies = enabled+ if maxBytes < 0 {+ maxBytes = 0}+ nm.retainBodyMaxBytes = maxBytes}func (nm *NetworkMonitor) getOrCreateBuffer(tabID string) *NetworkBuffer {@@ -424,6 +474,7 @@ func (nm *NetworkMonitor) StartCaptureWithSize(tabCtx context.Context, tabID str}})buf.MarkRequestEnd(string(e.RequestID))+ go nm.maybeRetainBody(tabCtx, buf, string(e.RequestID))case *network.EventLoadingFailed:buf.Update(string(e.RequestID), func(entry *NetworkEntry) {@@ -473,6 +524,61 @@ func (nm *NetworkMonitor) ClearAll() {}}+func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBuffer, requestID string) {+ nm.mu.RLock()+ enabled := nm.retainBodies+ maxBytes := nm.retainBodyMaxBytes+ nm.mu.RUnlock()+ if !enabled {+ return+ }+ if buf.RetainedBytes() >= nm.retainBodyMaxPerTab {+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyError = "retention budget exceeded"+ })+ return+ }+ select {+ case nm.retainBodySemaphore <- struct{}{}:+ defer func() { <-nm.retainBodySemaphore }()+ default:+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyError = "retention concurrency limit reached"+ })+ return+ }+ body, base64Encoded, err := GetResponseBodyDirect(tabCtx, requestID)+ if err != nil {+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyError = err.Error()+ })+ return+ }+ truncated := false+ if maxBytes > 0 && len(body) > maxBytes {+ body = body[:maxBytes]+ truncated = true+ }+ remainingBudget := int(nm.retainBodyMaxPerTab - buf.RetainedBytes())+ if remainingBudget <= 0 {+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.BodyError = "retention budget exceeded"+ })+ return+ }+ if len(body) > remainingBudget {+ body = body[:remainingBudget]+ truncated = true+ }+ buf.Update(requestID, func(entry *NetworkEntry) {+ entry.ResponseBody = body+ entry.Base64Encoded = base64Encoded+ entry.BodyRetained = true+ entry.BodyTruncated = truncated+ entry.BodyError = ""+ })+}+// GetResponseBody fetches the response body for a specific request via CDP.func (nm *NetworkMonitor) GetResponseBody(tabCtx context.Context, requestID string) (string, bool, error) {var body stringdiff --git a/internal/config/config_load.go b/internal/config/config_load.goindex 5efa8913..dc3d312b 100644--- a/internal/config/config_load.go+++ b/internal/config/config_load.go@@ -47,25 +47,27 @@ func Load() *RuntimeConfig {CookieSecure: nil,// Security defaults- AllowEvaluate: false,- AllowMacro: false,- AllowScreencast: false,- AllowDownload: false,- AllowCookies: false,- AllowNetworkIntercept: false,- AllowedDomains: append([]string(nil), defaultLocalAllowedDomains...),- DownloadAllowedDomains: nil,- DownloadMaxBytes: DefaultDownloadMaxBytes,- AllowUpload: false,- AllowClipboard: false,- AllowStateExport: false,- StateEncryptionKey: "",- EnableActionGuards: true,- UploadMaxRequestBytes: DefaultUploadMaxRequestBytes,- UploadMaxFiles: DefaultUploadMaxFiles,- UploadMaxFileBytes: DefaultUploadMaxFileBytes,- UploadMaxTotalBytes: DefaultUploadMaxTotalBytes,- MaxRedirects: -1, // Unlimited by default; set to N to limit redirect hops+ AllowEvaluate: false,+ AllowMacro: false,+ AllowScreencast: false,+ AllowDownload: false,+ AllowCookies: false,+ AllowNetworkIntercept: false,+ RetainNetworkBodies: false,+ RetainNetworkBodyMaxBytes: 256 * 1024,+ AllowedDomains: append([]string(nil), defaultLocalAllowedDomains...),+ DownloadAllowedDomains: nil,+ DownloadMaxBytes: DefaultDownloadMaxBytes,+ AllowUpload: false,+ AllowClipboard: false,+ AllowStateExport: false,+ StateEncryptionKey: "",+ EnableActionGuards: true,+ UploadMaxRequestBytes: DefaultUploadMaxRequestBytes,+ UploadMaxFiles: DefaultUploadMaxFiles,+ UploadMaxFileBytes: DefaultUploadMaxFileBytes,+ UploadMaxTotalBytes: DefaultUploadMaxTotalBytes,+ MaxRedirects: -1, // Unlimited by default; set to N to limit redirect hops// Browser / instance defaultsHeadless: true,@@ -263,6 +265,12 @@ func applyFileConfig(cfg *RuntimeConfig, fc *FileConfig) {if fc.Server.NetworkBufferSize != nil && *fc.Server.NetworkBufferSize > 0 {cfg.NetworkBufferSize = ClampNetworkBufferSize(*fc.Server.NetworkBufferSize)}+ if fc.Server.RetainNetworkBodies != nil {+ cfg.RetainNetworkBodies = *fc.Server.RetainNetworkBodies+ }+ if fc.Server.RetainNetworkBodyMaxBytes != nil && *fc.Server.RetainNetworkBodyMaxBytes >= 0 {+ cfg.RetainNetworkBodyMaxBytes = *fc.Server.RetainNetworkBodyMaxBytes+ }if fc.Server.TrustProxyHeaders != nil {cfg.TrustProxyHeaders = *fc.Server.TrustProxyHeaders}diff --git a/internal/config/config_types.go b/internal/config/config_types.goindex f1dd475b..5ee27a67 100644--- a/internal/config/config_types.go+++ b/internal/config/config_types.go@@ -110,7 +110,9 @@ type RuntimeConfig struct {Engine string// Network monitoring- NetworkBufferSize int // Per-tab network buffer size (default 100)+ NetworkBufferSize int // Per-tab network buffer size (default 100)+ RetainNetworkBodies bool // When true, opportunistically retain response bodies in the per-tab network buffer+ RetainNetworkBodyMaxBytes int // Max retained response-body bytes per entry when RetainNetworkBodies is enabled// Scheduler settings (dashboard mode only)Scheduler SchedulerConfig@@ -257,14 +259,16 @@ type FileConfig struct {}type ServerConfig struct {- Port string `json:"port,omitempty"`- Bind string `json:"bind,omitempty"`- Token string `json:"token,omitempty"`- StateDir string `json:"stateDir,omitempty"`- Engine string `json:"engine,omitempty"`- NetworkBufferSize *int `json:"networkBufferSize,omitempty"`- TrustProxyHeaders *bool `json:"trustProxyHeaders,omitempty"`- CookieSecure *bool `json:"cookieSecure,omitempty"`+ Port string `json:"port,omitempty"`+ Bind string `json:"bind,omitempty"`+ Token string `json:"token,omitempty"`+ StateDir string `json:"stateDir,omitempty"`+ Engine string `json:"engine,omitempty"`+ NetworkBufferSize *int `json:"networkBufferSize,omitempty"`+ RetainNetworkBodies *bool `json:"retainNetworkBodies,omitempty"`+ RetainNetworkBodyMaxBytes *int `json:"retainNetworkBodyMaxBytes,omitempty"`+ TrustProxyHeaders *bool `json:"trustProxyHeaders,omitempty"`+ CookieSecure *bool `json:"cookieSecure,omitempty"`}type SessionsFileConfig struct {diff --git a/internal/config/transfer_limits.go b/internal/config/transfer_limits.goindex 1eb879cb..8f6ef1f4 100644--- a/internal/config/transfer_limits.go+++ b/internal/config/transfer_limits.go@@ -3,6 +3,7 @@ package configconst (DefaultDownloadMaxBytes = 20 << 20MaxDownloadMaxBytes = 100 << 20+ MaxRetainNetworkBodyMaxBytes = 10 << 20DefaultUploadMaxRequestBytes = 10 << 20MaxUploadMaxRequestBytes = 100 << 20DefaultUploadMaxFiles = 8diff --git a/internal/config/validate.go b/internal/config/validate.goindex 07e00587..406456be 100644--- a/internal/config/validate.go+++ b/internal/config/validate.go@@ -40,6 +40,14 @@ func ValidateFileConfig(fc *FileConfig) []error {})}}+ if fc.Server.RetainNetworkBodyMaxBytes != nil {+ if *fc.Server.RetainNetworkBodyMaxBytes < 0 || *fc.Server.RetainNetworkBodyMaxBytes > MaxRetainNetworkBodyMaxBytes {+ errs = append(errs, ValidationError{+ Field: "server.retainNetworkBodyMaxBytes",+ Message: fmt.Sprintf("must be between 0 and %d (got %d)", MaxRetainNetworkBodyMaxBytes, *fc.Server.RetainNetworkBodyMaxBytes),+ })+ }+ }if fc.MultiInstance.InstancePortStart != nil && fc.MultiInstance.InstancePortEnd != nil {if *fc.MultiInstance.InstancePortStart > *fc.MultiInstance.InstancePortEnd {errs = append(errs, ValidationError{diff --git a/internal/handlers/network.go b/internal/handlers/network.goindex 565745bb..3b328231 100644--- a/internal/handlers/network.go+++ b/internal/handlers/network.go@@ -159,12 +159,24 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {// Optionally include response bodyif r.URL.Query().Get("body") == "true" && entry.Finished && !entry.Failed {- body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)- if err != nil {- result["bodyError"] = err.Error()+ if entry.BodyRetained {+ result["responseBody"] = entry.ResponseBody+ result["base64Encoded"] = entry.Base64Encoded+ result["bodyRetained"] = true+ if entry.BodyTruncated {+ result["bodyTruncated"] = true+ }+ if entry.BodyError != "" {+ result["bodyError"] = entry.BodyError+ }} else {- result["responseBody"] = body- result["base64Encoded"] = base64Encoded+ body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)+ if err != nil {+ result["bodyError"] = err.Error()+ } else {+ result["responseBody"] = body+ result["base64Encoded"] = base64Encoded+ }}}diff --git a/internal/handlers/network_test.go b/internal/handlers/network_test.goindex 0f3c69a3..34ebafa7 100644--- a/internal/handlers/network_test.go+++ b/internal/handlers/network_test.go@@ -221,6 +221,42 @@ func TestHandleNetwork_NilMonitor(t *testing.T) {}}+func TestHandleNetworkByIDUsesRetainedBody(t *testing.T) {+ nm := bridge.NewNetworkMonitor(100)+ buf := nm.GetOrCreateBufferForTest("tab1")+ const retainedBody = "{\"ok\":true}"+ buf.Add(bridge.NetworkEntry{+ RequestID: "retained-1",+ URL: "https://api.example.com/data",+ Method: "GET",+ ResourceType: "XHR",+ Finished: true,+ ResponseBody: retainedBody,+ Base64Encoded: false,+ BodyRetained: true,+ })+ h := newNetworkTestHandler(nm)++ req := httptest.NewRequest("GET", "/network/retained-1?body=true", nil)+ req.SetPathValue("requestId", "retained-1")+ w := httptest.NewRecorder()+ h.HandleNetworkByID(w, req)++ if w.Code != 200 {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }+ var got map[string]any+ if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil {+ t.Fatalf("decode: %v", err)+ }+ if got["responseBody"] != retainedBody {+ t.Fatalf("expected retained response body, got %v", got["responseBody"])+ }+ if got["bodyRetained"] != true {+ t.Fatalf("expected bodyRetained=true, got %v", got["bodyRetained"])+ }+}+func TestHandleNetworkByID_Found(t *testing.T) {nm := bridge.NewNetworkMonitor(100)seedBuffer(nm, "tab1")diff --git a/internal/schema/config.json b/internal/schema/config.jsonindex 333851af..ed203501 100644--- a/internal/schema/config.json+++ b/internal/schema/config.json@@ -177,6 +177,21 @@}]},+ "retainNetworkBodies": {+ "$ref": "#/definitions/nullableBoolean"+ },+ "retainNetworkBodyMaxBytes": {+ "anyOf": [+ {+ "type": "integer",+ "minimum": 0,+ "maximum": 10485760+ },+ {+ "type": "null"+ }+ ]+ },"trustProxyHeaders": {"$ref": "#/definitions/nullableBoolean"},diff --git a/tests/e2e/config/pinchtab-network-retain-bodies.json b/tests/e2e/config/pinchtab-network-retain-bodies.jsonnew file mode 100644index 00000000..5a7aa47a--- /dev/null+++ b/tests/e2e/config/pinchtab-network-retain-bodies.json@@ -0,0 +1,7 @@+{+ "server": {+ "networkBufferSize": 200,+ "retainNetworkBodies": true,+ "retainNetworkBodyMaxBytes": 262144+ }+}diff --git a/tests/e2e/scenarios/api/network-retain-body.sh b/tests/e2e/scenarios/api/network-retain-body.shnew file mode 100644index 00000000..c4c0b511--- /dev/null+++ b/tests/e2e/scenarios/api/network-retain-body.sh@@ -0,0 +1,49 @@+#!/bin/bash+# network-retain-body.sh — retained network response body smoke.++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/api.sh"++start_test "network detail: retained response body"++pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/network-retain-body.html\"}"+assert_ok "navigate to retention fixture"++# Give the fixture fetch a moment to complete and land in the network buffer.+sleep 1++NETWORK_JSON=$(e2e_curl -s "${E2E_SERVER}/network?type=XHR&limit=20")+REQ_ID=$(echo "$NETWORK_JSON" | jq -r '.items[] | select(.url | contains("network-retain-body.json")) | .requestId' | head -n1)+if [ -z "$REQ_ID" ] || [ "$REQ_ID" = "null" ]; then+ echo -e " ${RED}✗${NC} could not find retained-body request in network buffer"+ ((ASSERTIONS_FAILED++)) || true+ end_test+ exit 0+fi++echo -e " ${GREEN}✓${NC} found request id: $REQ_ID"+((ASSERTIONS_PASSED++)) || true++DETAIL=$(e2e_curl -s "${E2E_SERVER}/network/${REQ_ID}?body=true")++echo "$DETAIL" | jq -e '.bodyRetained == true' >/dev/null 2>&1+if [ $? -eq 0 ]; then+ echo -e " ${GREEN}✓${NC} bodyRetained=true"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} expected bodyRetained=true"+ echo "$DETAIL" | jq .+ ((ASSERTIONS_FAILED++)) || true+fi++echo "$DETAIL" | jq -e '.responseBody | contains("retained-body-ok")' >/dev/null 2>&1+if [ $? -eq 0 ]; then+ echo -e " ${GREEN}✓${NC} retained response body contains expected payload"+ ((ASSERTIONS_PASSED++)) || true+else+ echo -e " ${RED}✗${NC} retained response body missing expected payload"+ echo "$DETAIL" | jq .+ ((ASSERTIONS_FAILED++)) || true+fi++end_testdiff --git a/tests/tools/fixtures/network-retain-body.html b/tests/tools/fixtures/network-retain-body.htmlnew file mode 100644index 00000000..7e680a5e--- /dev/null+++ b/tests/tools/fixtures/network-retain-body.html@@ -0,0 +1,27 @@+<!doctype html>+<html>+<head>+ <meta charset="utf-8">+ <title>Network Retain Body</title>+</head>+<body>+ <h1>Network Retain Body</h1>+ <script>+ window.__retainFetchDone = false;+ window.__retainFetchOk = false;+ window.__retainFetchText = "";+ fetch('/network-retain-body.json')+ .then(r => r.text())+ .then(t => {+ window.__retainFetchDone = true;+ window.__retainFetchOk = true;+ window.__retainFetchText = t;+ })+ .catch(e => {+ window.__retainFetchDone = true;+ window.__retainFetchOk = false;+ window.__retainFetchText = String(e);+ });+ </script>+</body>+</html>diff --git a/tests/tools/fixtures/network-retain-body.json b/tests/tools/fixtures/network-retain-body.jsonnew file mode 100644index 00000000..ba5c6a5c--- /dev/null+++ b/tests/tools/fixtures/network-retain-body.json@@ -0,0 +1 @@+{"message":"retained-body-ok","items":[1,2,3],"kind":"fixture"}