Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: retained binary bodies are corrupted because base64Encoded is never set (PIN-056)

Luigi Agosti415f4e59bf05
diff --git a/internal/bridge/observe/network.go b/internal/bridge/observe/network.go
index be498f29..f3f21a4f 100644
--- a/internal/bridge/observe/network.go
+++ b/internal/bridge/observe/network.go
@@ -9,7 +9,6 @@ import (
"sync"
"time"
- "github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/network"
"github.com/chromedp/chromedp"
"github.com/pinchtab/pinchtab/internal/config"
@@ -280,7 +279,7 @@ func (nm *NetworkMonitor) bodyRetentionEnabled() bool {
// fetchResponseBody reads the body from the browser. Indirected so retention
// policy can be exercised without a live CDP target.
-var fetchResponseBody = GetResponseBodyDirect
+var fetchResponseBody = GetResponseBody
func skipRetainedBody(buf *NetworkBuffer, requestID, reason string) {
buf.Update(requestID, func(entry *NetworkEntry) {
@@ -394,13 +393,28 @@ func (nm *NetworkMonitor) IsTabIdle(tabID string) (bool, bool) {
return count == 0, true
}
-func (nm *NetworkMonitor) GetResponseBody(tabCtx context.Context, requestID string) (string, bool, error) {
+// GetResponseBody is the only response-body fetcher. It reads Network.getResponseBody
+// as raw JSON so the body and the base64Encoded flag describing it come from the same
+// call and travel together.
+//
+// It deliberately does NOT use cdproto's typed constructor plus Do: that helper
+// base64-decodes the payload inside the dependency and returns []byte, so by the time
+// it returns the flag is gone and the string holds raw bytes. A second fetcher built
+// that way reported base64Encoded=false for every response, which meant a binary body
+// was retained as a string of U+FFFD once JSON-encoded — and made clampRetainedBody's
+// drop-and-mark branch unreachable in production, since nothing ever set the flag it
+// tests.
+func GetResponseBody(ctx context.Context, requestID string) (string, bool, error) {
var body string
var base64Encoded bool
- err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error {
+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
+ executor := chromedp.FromContext(ctx).Target
+ if executor == nil {
+ return fmt.Errorf("no CDP executor available")
+ }
var result json.RawMessage
- if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Network.getResponseBody", map[string]any{
+ if err := executor.Execute(ctx, "Network.getResponseBody", map[string]any{
"requestId": requestID,
}, &result); err != nil {
return err
@@ -420,28 +434,6 @@ func (nm *NetworkMonitor) GetResponseBody(tabCtx context.Context, requestID stri
return body, base64Encoded, err
}
-func GetResponseBodyDirect(ctx context.Context, requestID string) (string, bool, error) {
- var body string
- var base64Encoded bool
-
- err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
- executor := chromedp.FromContext(ctx).Target
- if executor == nil {
- return fmt.Errorf("no CDP executor available")
- }
- params := network.GetResponseBody(network.RequestID(requestID))
- resp, err := params.Do(cdp.WithExecutor(ctx, executor))
- if err != nil {
- return err
- }
- body = string(resp)
- base64Encoded = false
- return nil
- }))
-
- return body, base64Encoded, err
-}
-
func normalizeNetworkEntry(entry NetworkEntry) NetworkEntry {
entry.URL = sanitize.TruncateUTF8Bytes(entry.URL, maxNetworkURLBytes)
entry.Method = sanitize.TruncateUTF8Bytes(entry.Method, maxNetworkMethodBytes)
diff --git a/internal/bridge/observe/network_body_binary_test.go b/internal/bridge/observe/network_body_binary_test.go
new file mode 100644
index 00000000..9dad9ff9
--- /dev/null
+++ b/internal/bridge/observe/network_body_binary_test.go
@@ -0,0 +1,215 @@
+package observe
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "fmt"
+ "image"
+ "image/color"
+ "image/png"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+ "unicode/utf8"
+
+ "github.com/chromedp/chromedp"
+ "github.com/pinchtab/pinchtab/internal/testbrowser"
+)
+
+// The retention path drove a fetcher that hardcoded base64Encoded to false, so a binary
+// body was retained as raw bytes and JSON-encoded into U+FFFD soup, and the drop-and-mark
+// branch that exists to prevent exactly that could never fire. Driven through the REAL
+// fetcher against real image responses — the seam-based clamp test stayed green through
+// all of it, because the seam supplied the flag production never set.
+func TestBinaryResponseBodiesCarryTheCDPFlagOnEveryPath(t *testing.T) {
+ small := pngBytes(t, 8)
+ large := pngBytes(t, 96)
+ // Multibyte on purpose: the limit can land mid-rune, which is what the rune-safe cut exists for.
+ textBody := strings.Repeat("héllo wörld ", 100)
+
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/small.png":
+ w.Header().Set("Content-Type", "image/png")
+ _, _ = w.Write(small)
+ case "/large.png":
+ w.Header().Set("Content-Type", "image/png")
+ _, _ = w.Write(large)
+ case "/body.txt":
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ _, _ = w.Write([]byte(textBody))
+ default:
+ w.Header().Set("Content-Type", "text/html")
+ _, _ = fmt.Fprintf(w, `<img src="/small.png"><img src="/large.png">
+ <script>fetch("/body.txt");</script>`)
+ }
+ }))
+ defer srv.Close()
+
+ // Between the two images: the small one stays under it and is retained, the large one
+ // is over and must be dropped rather than cut, since a base64 fragment is undecodable
+ // in whole rather than only at the tail.
+ limit := (base64Len(len(small)) + base64Len(len(large))) / 2
+ if base64Len(len(small)) >= limit || base64Len(len(large)) <= limit {
+ t.Fatalf("fixture does not straddle the limit: small=%d large=%d limit=%d",
+ base64Len(len(small)), base64Len(len(large)), limit)
+ }
+ if len(textBody) <= limit {
+ t.Fatalf("text fixture (%d) must also exceed the limit %d, or truncation is untested", len(textBody), limit)
+ }
+
+ ctx, cancel := browserContext(t)
+ defer cancel()
+
+ nm := NewNetworkMonitor(0)
+ nm.ConfigureBodyRetention(true, limit)
+ if err := nm.StartCapture(ctx, "tab1"); err != nil {
+ t.Fatal(err)
+ }
+ if err := chromedp.Run(ctx, chromedp.Navigate(srv.URL+"/index.html")); err != nil {
+ t.Fatal(err)
+ }
+
+ buf := nm.GetBuffer("tab1")
+ entries := waitForSettledEntries(t, buf, "/small.png", "/large.png", "/body.txt")
+
+ smallEntry := entries["/small.png"]
+ if !smallEntry.BodyRetained {
+ t.Fatalf("the under-budget image was not retained: %+v", skipSummary(smallEntry))
+ }
+ if !smallEntry.Base64Encoded {
+ t.Error("the retained image reports base64Encoded=false, so a caller decodes U+FFFD instead of a PNG")
+ }
+ if decoded, err := base64.StdEncoding.DecodeString(smallEntry.ResponseBody); err != nil {
+ t.Errorf("the retained image body is not decodable base64: %v", err)
+ } else if !bytes.Equal(decoded, small) {
+ t.Errorf("the retained image decodes to %d bytes, want the %d served", len(decoded), len(small))
+ }
+ if strings.ContainsRune(smallEntry.ResponseBody, '�') {
+ t.Error("the retained image body carries U+FFFD; bytes were destroyed on the way in")
+ }
+
+ // The export/on-demand path and the retention path must agree about the same request:
+ // they are one fetcher now, and this is what proves it at runtime rather than by grep.
+ liveBody, liveBase64, err := GetResponseBody(ctx, smallEntry.RequestID)
+ if err != nil {
+ t.Fatalf("live fetch of the retained request failed: %v", err)
+ }
+ if liveBase64 != smallEntry.Base64Encoded {
+ t.Errorf("live fetch reports base64Encoded=%v but retention recorded %v for the same request",
+ liveBase64, smallEntry.Base64Encoded)
+ }
+ if !liveBase64 {
+ t.Error("the on-demand path reports base64Encoded=false for an image/png response")
+ }
+ if liveBody != smallEntry.ResponseBody {
+ t.Error("live fetch and retention returned different bodies for the same request")
+ }
+
+ largeEntry := entries["/large.png"]
+ if largeEntry.BodyRetained || largeEntry.ResponseBody != "" {
+ t.Errorf("the over-budget image was retained (%d bytes) instead of dropped; a base64 fragment is undecodable",
+ len(largeEntry.ResponseBody))
+ }
+ if !largeEntry.BodySkipped {
+ t.Error("the over-budget image is neither retained nor marked skipped, so a caller cannot tell it is absent")
+ }
+ if !strings.Contains(largeEntry.BodySkipReason, "base64 body exceeds") {
+ t.Errorf("skip reason %q does not say a base64 body exceeded a budget", largeEntry.BodySkipReason)
+ }
+
+ // A text body is unaffected: still decoded, still flagged false, still cut rather than
+ // dropped, and still cut on a rune boundary.
+ textEntry := entries["/body.txt"]
+ if !textEntry.BodyRetained {
+ t.Fatalf("the text body was not retained: %+v", skipSummary(textEntry))
+ }
+ if textEntry.Base64Encoded {
+ t.Error("the text body reports base64Encoded=true")
+ }
+ if !textEntry.BodyTruncated {
+ t.Error("the over-limit text body was not marked truncated")
+ }
+ if !strings.HasPrefix(textBody, textEntry.ResponseBody) {
+ t.Errorf("the retained text %q is not a prefix of the response", textEntry.ResponseBody)
+ }
+ if len(textEntry.ResponseBody) > limit {
+ t.Errorf("the retained text is %d bytes, over the %d limit", len(textEntry.ResponseBody), limit)
+ }
+ if !utf8.ValidString(textEntry.ResponseBody) {
+ t.Error("the retained text is not valid UTF-8; the cut split a rune")
+ }
+}
+
+func browserContext(t *testing.T) (context.Context, context.CancelFunc) {
+ t.Helper()
+ chromePath := testbrowser.Path(t)
+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(
+ chromedp.DefaultExecAllocatorOptions[:],
+ chromedp.ExecPath(chromePath),
+ chromedp.Flag("headless", true),
+ chromedp.Flag("no-sandbox", true),
+ )...)
+ ctx, cancelCtx := chromedp.NewContext(alloc)
+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)
+ return ctx, func() {
+ cancelTimeout()
+ cancelCtx()
+ cancelAlloc()
+ }
+}
+
+// waitForSettledEntries waits until every named path has an entry whose body work is
+// finished — retained, skipped or errored — so the assertions never read a pending row.
+func waitForSettledEntries(t *testing.T, buf *NetworkBuffer, paths ...string) map[string]NetworkEntry {
+ t.Helper()
+ deadline := time.Now().Add(25 * time.Second)
+ for {
+ found := map[string]NetworkEntry{}
+ for _, entry := range buf.List(NetworkFilter{}) {
+ for _, path := range paths {
+ if strings.HasSuffix(entry.URL, path) && !entry.BodyPending && entry.Finished {
+ found[path] = entry
+ }
+ }
+ }
+ if len(found) == len(paths) {
+ return found
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("only %d/%d responses settled: %v", len(found), len(paths), found)
+ }
+ time.Sleep(50 * time.Millisecond)
+ }
+}
+
+func skipSummary(entry NetworkEntry) string {
+ return fmt.Sprintf("skipped=%v reason=%q error=%q base64=%v bytes=%d",
+ entry.BodySkipped, entry.BodySkipReason, entry.BodyError, entry.Base64Encoded, len(entry.ResponseBody))
+}
+
+// base64Len is the encoded size CDP reports a body at, which is what the retention
+// budget is measured against for a binary response.
+func base64Len(raw int) int {
+ return base64.StdEncoding.EncodedLen(raw)
+}
+
+// pngBytes builds a real PNG rather than arbitrary bytes, so Chrome classifies the
+// response as binary and CDP sets base64Encoded — the whole subject of this test.
+func pngBytes(t *testing.T, size int) []byte {
+ t.Helper()
+ img := image.NewRGBA(image.Rect(0, 0, size, size))
+ for y := 0; y < size; y++ {
+ for x := 0; x < size; x++ {
+ img.Set(x, y, color.RGBA{R: uint8(x * 7), G: uint8(y * 11), B: uint8(x ^ y), A: 255})
+ }
+ }
+ var out bytes.Buffer
+ if err := png.Encode(&out, img); err != nil {
+ t.Fatal(err)
+ }
+ return out.Bytes()
+}
diff --git a/internal/bridge/observe/network_body_fetcher_test.go b/internal/bridge/observe/network_body_fetcher_test.go
new file mode 100644
index 00000000..c3d30b9e
--- /dev/null
+++ b/internal/bridge/observe/network_body_fetcher_test.go
@@ -0,0 +1,98 @@
+package observe
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// The behavioural proof of this rule is browser-backed, and the lightweight CI run
+// installs no browser — so the wiring it depends on gets a source-level guard that runs
+// everywhere. Two fetchers existed; the one the retention path used called cdproto's
+// typed GetResponseBodyParams.Do, which base64-decodes inside the dependency and returns
+// []byte, so it had no flag left to report and hardcoded false.
+func TestOneResponseBodyFetcherThatReportsTheCDPFlag(t *testing.T) {
+ const fetcher = "func GetResponseBody(ctx context.Context, requestID string) (string, bool, error)"
+
+ body := readRepoCode(t, "network.go")
+
+ if !strings.Contains(body, fetcher) {
+ t.Fatalf("network.go no longer declares %q; this census would guard nothing", fetcher)
+ }
+ if !strings.Contains(body, "base64Encoded = resp.Base64Encoded") {
+ t.Error("the fetcher no longer reads base64Encoded off the CDP result, so the flag is not the browser's answer")
+ }
+ if strings.Contains(body, "base64Encoded = false") {
+ t.Error("network.go assigns base64Encoded a constant; a binary body then retains as raw bytes and JSON-encodes to U+FFFD")
+ }
+ if strings.Contains(body, "network.GetResponseBody(") {
+ t.Error("network.go fetches a body through cdproto's typed GetResponseBody, which decodes inside the dependency and destroys the flag")
+ }
+ if !strings.Contains(body, "var fetchResponseBody = GetResponseBody") {
+ t.Error("the retention seam no longer defaults to GetResponseBody, so retention can drive a different fetcher than export")
+ }
+
+ // The on-demand and export call sites must reach the same fetcher. A second
+ // implementation reachable from handlers is how the two paths came to disagree about
+ // the same request in the first place.
+ var handlerCalls int
+ for _, name := range []string{"network.go", "network_export.go"} {
+ handlerBody := readRepoCode(t, filepath.Join("..", "..", "handlers", name))
+ if strings.Contains(handlerBody, "GetResponseBodyDirect") {
+ t.Errorf("handlers/%s reaches GetResponseBodyDirect instead of the single fetcher", name)
+ }
+ for _, line := range strings.Split(handlerBody, "\n") {
+ if !strings.Contains(line, "GetResponseBody(") {
+ continue
+ }
+ handlerCalls++
+ if !strings.Contains(line, "bridge.GetResponseBody(") {
+ t.Errorf("handlers/%s fetches a body outside the single fetcher: %s", name, strings.TrimSpace(line))
+ }
+ }
+ assertFlagNotDiscarded(t, "handlers/"+name, handlerBody)
+ }
+ if handlerCalls == 0 {
+ t.Error("no body-fetch call site found in handlers; the call-site half of this census guards nothing")
+ }
+
+ // The same rule inside this package: one owner is not enough if a wrapper re-creates
+ // the old behaviour by throwing the flag away.
+ assertFlagNotDiscarded(t, "network.go", body)
+}
+
+// assertFlagNotDiscarded bans a caller that takes the body and drops the flag. Routing
+// every site through one fetcher does not fix anything on its own — a thin wrapper that
+// reads the body and returns a constant false is byte-for-byte the defect this card is
+// about, wearing a different name, and a ban keyed on the fetcher's NAME cannot see it.
+func assertFlagNotDiscarded(t *testing.T, label, code string) {
+ t.Helper()
+ for _, line := range strings.Split(code, "\n") {
+ if !strings.Contains(line, "GetResponseBody(") {
+ continue
+ }
+ if strings.Contains(line, ", _, err") || strings.Contains(line, ", _, _") {
+ t.Errorf("%s discards the base64Encoded flag: %s", label, strings.TrimSpace(line))
+ }
+ }
+}
+
+// readRepoCode returns the file with comment lines dropped, so a doc comment naming the
+// banned shape does not trip a ban on it — the prose explaining why a form is wrong is
+// exactly where that form gets written down.
+func readRepoCode(t *testing.T, path string) string {
+ t.Helper()
+ raw, err := os.ReadFile(path) // #nosec G304 -- fixed paths inside this repo.
+ if err != nil {
+ t.Fatalf("cannot read %s, so this census guards nothing: %v", path, err)
+ }
+ var code []string
+ for _, line := range strings.Split(string(raw), "\n") {
+ if strings.HasPrefix(strings.TrimSpace(line), "//") {
+ continue
+ }
+ code = append(code, line)
+ }
+ return strings.Join(code, "\n")
+}
diff --git a/internal/bridge/observe_facade.go b/internal/bridge/observe_facade.go
index 40a494de..72e68cbe 100644
--- a/internal/bridge/observe_facade.go
+++ b/internal/bridge/observe_facade.go
@@ -120,8 +120,8 @@ func matchStatusRange(status int, pattern string) bool {
return bridgeobserve.MatchStatusRange(status, pattern)
}
-func GetResponseBodyDirect(ctx context.Context, requestID string) (string, bool, error) {
- return bridgeobserve.GetResponseBodyDirect(ctx, requestID)
+func GetResponseBody(ctx context.Context, requestID string) (string, bool, error) {
+ return bridgeobserve.GetResponseBody(ctx, requestID)
}
func (b *Bridge) GetMemoryMetrics(tabID string) (*MemoryMetrics, error) {
diff --git a/internal/handlers/network.go b/internal/handlers/network.go
index 2eb1aa47..ec38831e 100644
--- a/internal/handlers/network.go
+++ b/internal/handlers/network.go
@@ -335,7 +335,7 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {
}
switch {
case bodyMode == networkBodyModeLiveOnly:
- body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)
+ body, base64Encoded, err := bridge.GetResponseBody(tabCtx, requestID)
if err != nil {
result["bodyError"] = err.Error()
} else {
@@ -348,7 +348,7 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {
case entry.BodyPending || entry.BodyError != "":
populateRetainedBodyResult(result, entry)
default:
- body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)
+ body, base64Encoded, err := bridge.GetResponseBody(tabCtx, requestID)
if err != nil {
result["bodyError"] = err.Error()
} else {
diff --git a/internal/handlers/network_export.go b/internal/handlers/network_export.go
index 7d7e9b30..b922e23a 100644
--- a/internal/handlers/network_export.go
+++ b/internal/handlers/network_export.go
@@ -193,7 +193,7 @@ func resolveExportBody(ctx context.Context, nm *bridge.NetworkMonitor, entry bri
if entry.BodyRetained {
return clampExportBody(entry.ResponseBody, entry.Base64Encoded)
}
- body, b64, _ := nm.GetResponseBody(ctx, entry.RequestID)
+ body, b64, _ := bridge.GetResponseBody(ctx, entry.RequestID)
return clampExportBody(body, b64)
}