High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: retained binary bodies are corrupted because base64Encoded is never set (PIN-056)
Luigi Agosti415f4e59bf05
diff --git a/internal/bridge/observe/network.go b/internal/bridge/observe/network.goindex be498f29..f3f21a4f 100644--- a/internal/bridge/observe/network.go+++ b/internal/bridge/observe/network.go@@ -9,7 +9,6 @@ import ("sync""time"- "github.com/chromedp/cdproto/cdp""github.com/chromedp/cdproto/network""github.com/chromedp/chromedp""github.com/pinchtab/pinchtab/internal/config"@@ -280,7 +279,7 @@ func (nm *NetworkMonitor) bodyRetentionEnabled() bool {// fetchResponseBody reads the body from the browser. Indirected so retention// policy can be exercised without a live CDP target.-var fetchResponseBody = GetResponseBodyDirect+var fetchResponseBody = GetResponseBodyfunc skipRetainedBody(buf *NetworkBuffer, requestID, reason string) {buf.Update(requestID, func(entry *NetworkEntry) {@@ -394,13 +393,28 @@ func (nm *NetworkMonitor) IsTabIdle(tabID string) (bool, bool) {return count == 0, true}-func (nm *NetworkMonitor) GetResponseBody(tabCtx context.Context, requestID string) (string, bool, error) {+// GetResponseBody is the only response-body fetcher. It reads Network.getResponseBody+// as raw JSON so the body and the base64Encoded flag describing it come from the same+// call and travel together.+//+// It deliberately does NOT use cdproto's typed constructor plus Do: that helper+// base64-decodes the payload inside the dependency and returns []byte, so by the time+// it returns the flag is gone and the string holds raw bytes. A second fetcher built+// that way reported base64Encoded=false for every response, which meant a binary body+// was retained as a string of U+FFFD once JSON-encoded — and made clampRetainedBody's+// drop-and-mark branch unreachable in production, since nothing ever set the flag it+// tests.+func GetResponseBody(ctx context.Context, requestID string) (string, bool, error) {var body stringvar base64Encoded bool- err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error {+ err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {+ executor := chromedp.FromContext(ctx).Target+ if executor == nil {+ return fmt.Errorf("no CDP executor available")+ }var result json.RawMessage- if err := chromedp.FromContext(ctx).Target.Execute(ctx, "Network.getResponseBody", map[string]any{+ if err := executor.Execute(ctx, "Network.getResponseBody", map[string]any{"requestId": requestID,}, &result); err != nil {return err@@ -420,28 +434,6 @@ func (nm *NetworkMonitor) GetResponseBody(tabCtx context.Context, requestID strireturn body, base64Encoded, err}-func GetResponseBodyDirect(ctx context.Context, requestID string) (string, bool, error) {- var body string- var base64Encoded bool-- err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {- executor := chromedp.FromContext(ctx).Target- if executor == nil {- return fmt.Errorf("no CDP executor available")- }- params := network.GetResponseBody(network.RequestID(requestID))- resp, err := params.Do(cdp.WithExecutor(ctx, executor))- if err != nil {- return err- }- body = string(resp)- base64Encoded = false- return nil- }))-- return body, base64Encoded, err-}-func normalizeNetworkEntry(entry NetworkEntry) NetworkEntry {entry.URL = sanitize.TruncateUTF8Bytes(entry.URL, maxNetworkURLBytes)entry.Method = sanitize.TruncateUTF8Bytes(entry.Method, maxNetworkMethodBytes)diff --git a/internal/bridge/observe/network_body_binary_test.go b/internal/bridge/observe/network_body_binary_test.gonew file mode 100644index 00000000..9dad9ff9--- /dev/null+++ b/internal/bridge/observe/network_body_binary_test.go@@ -0,0 +1,215 @@+package observe++import (+ "bytes"+ "context"+ "encoding/base64"+ "fmt"+ "image"+ "image/color"+ "image/png"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"+ "unicode/utf8"++ "github.com/chromedp/chromedp"+ "github.com/pinchtab/pinchtab/internal/testbrowser"+)++// The retention path drove a fetcher that hardcoded base64Encoded to false, so a binary+// body was retained as raw bytes and JSON-encoded into U+FFFD soup, and the drop-and-mark+// branch that exists to prevent exactly that could never fire. Driven through the REAL+// fetcher against real image responses — the seam-based clamp test stayed green through+// all of it, because the seam supplied the flag production never set.+func TestBinaryResponseBodiesCarryTheCDPFlagOnEveryPath(t *testing.T) {+ small := pngBytes(t, 8)+ large := pngBytes(t, 96)+ // Multibyte on purpose: the limit can land mid-rune, which is what the rune-safe cut exists for.+ textBody := strings.Repeat("héllo wörld ", 100)++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/small.png":+ w.Header().Set("Content-Type", "image/png")+ _, _ = w.Write(small)+ case "/large.png":+ w.Header().Set("Content-Type", "image/png")+ _, _ = w.Write(large)+ case "/body.txt":+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")+ _, _ = w.Write([]byte(textBody))+ default:+ w.Header().Set("Content-Type", "text/html")+ _, _ = fmt.Fprintf(w, `<img src="/small.png"><img src="/large.png">+ <script>fetch("/body.txt");</script>`)+ }+ }))+ defer srv.Close()++ // Between the two images: the small one stays under it and is retained, the large one+ // is over and must be dropped rather than cut, since a base64 fragment is undecodable+ // in whole rather than only at the tail.+ limit := (base64Len(len(small)) + base64Len(len(large))) / 2+ if base64Len(len(small)) >= limit || base64Len(len(large)) <= limit {+ t.Fatalf("fixture does not straddle the limit: small=%d large=%d limit=%d",+ base64Len(len(small)), base64Len(len(large)), limit)+ }+ if len(textBody) <= limit {+ t.Fatalf("text fixture (%d) must also exceed the limit %d, or truncation is untested", len(textBody), limit)+ }++ ctx, cancel := browserContext(t)+ defer cancel()++ nm := NewNetworkMonitor(0)+ nm.ConfigureBodyRetention(true, limit)+ if err := nm.StartCapture(ctx, "tab1"); err != nil {+ t.Fatal(err)+ }+ if err := chromedp.Run(ctx, chromedp.Navigate(srv.URL+"/index.html")); err != nil {+ t.Fatal(err)+ }++ buf := nm.GetBuffer("tab1")+ entries := waitForSettledEntries(t, buf, "/small.png", "/large.png", "/body.txt")++ smallEntry := entries["/small.png"]+ if !smallEntry.BodyRetained {+ t.Fatalf("the under-budget image was not retained: %+v", skipSummary(smallEntry))+ }+ if !smallEntry.Base64Encoded {+ t.Error("the retained image reports base64Encoded=false, so a caller decodes U+FFFD instead of a PNG")+ }+ if decoded, err := base64.StdEncoding.DecodeString(smallEntry.ResponseBody); err != nil {+ t.Errorf("the retained image body is not decodable base64: %v", err)+ } else if !bytes.Equal(decoded, small) {+ t.Errorf("the retained image decodes to %d bytes, want the %d served", len(decoded), len(small))+ }+ if strings.ContainsRune(smallEntry.ResponseBody, '�') {+ t.Error("the retained image body carries U+FFFD; bytes were destroyed on the way in")+ }++ // The export/on-demand path and the retention path must agree about the same request:+ // they are one fetcher now, and this is what proves it at runtime rather than by grep.+ liveBody, liveBase64, err := GetResponseBody(ctx, smallEntry.RequestID)+ if err != nil {+ t.Fatalf("live fetch of the retained request failed: %v", err)+ }+ if liveBase64 != smallEntry.Base64Encoded {+ t.Errorf("live fetch reports base64Encoded=%v but retention recorded %v for the same request",+ liveBase64, smallEntry.Base64Encoded)+ }+ if !liveBase64 {+ t.Error("the on-demand path reports base64Encoded=false for an image/png response")+ }+ if liveBody != smallEntry.ResponseBody {+ t.Error("live fetch and retention returned different bodies for the same request")+ }++ largeEntry := entries["/large.png"]+ if largeEntry.BodyRetained || largeEntry.ResponseBody != "" {+ t.Errorf("the over-budget image was retained (%d bytes) instead of dropped; a base64 fragment is undecodable",+ len(largeEntry.ResponseBody))+ }+ if !largeEntry.BodySkipped {+ t.Error("the over-budget image is neither retained nor marked skipped, so a caller cannot tell it is absent")+ }+ if !strings.Contains(largeEntry.BodySkipReason, "base64 body exceeds") {+ t.Errorf("skip reason %q does not say a base64 body exceeded a budget", largeEntry.BodySkipReason)+ }++ // A text body is unaffected: still decoded, still flagged false, still cut rather than+ // dropped, and still cut on a rune boundary.+ textEntry := entries["/body.txt"]+ if !textEntry.BodyRetained {+ t.Fatalf("the text body was not retained: %+v", skipSummary(textEntry))+ }+ if textEntry.Base64Encoded {+ t.Error("the text body reports base64Encoded=true")+ }+ if !textEntry.BodyTruncated {+ t.Error("the over-limit text body was not marked truncated")+ }+ if !strings.HasPrefix(textBody, textEntry.ResponseBody) {+ t.Errorf("the retained text %q is not a prefix of the response", textEntry.ResponseBody)+ }+ if len(textEntry.ResponseBody) > limit {+ t.Errorf("the retained text is %d bytes, over the %d limit", len(textEntry.ResponseBody), limit)+ }+ if !utf8.ValidString(textEntry.ResponseBody) {+ t.Error("the retained text is not valid UTF-8; the cut split a rune")+ }+}++func browserContext(t *testing.T) (context.Context, context.CancelFunc) {+ t.Helper()+ chromePath := testbrowser.Path(t)+ alloc, cancelAlloc := chromedp.NewExecAllocator(context.Background(), append(+ chromedp.DefaultExecAllocatorOptions[:],+ chromedp.ExecPath(chromePath),+ chromedp.Flag("headless", true),+ chromedp.Flag("no-sandbox", true),+ )...)+ ctx, cancelCtx := chromedp.NewContext(alloc)+ ctx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second)+ return ctx, func() {+ cancelTimeout()+ cancelCtx()+ cancelAlloc()+ }+}++// waitForSettledEntries waits until every named path has an entry whose body work is+// finished — retained, skipped or errored — so the assertions never read a pending row.+func waitForSettledEntries(t *testing.T, buf *NetworkBuffer, paths ...string) map[string]NetworkEntry {+ t.Helper()+ deadline := time.Now().Add(25 * time.Second)+ for {+ found := map[string]NetworkEntry{}+ for _, entry := range buf.List(NetworkFilter{}) {+ for _, path := range paths {+ if strings.HasSuffix(entry.URL, path) && !entry.BodyPending && entry.Finished {+ found[path] = entry+ }+ }+ }+ if len(found) == len(paths) {+ return found+ }+ if time.Now().After(deadline) {+ t.Fatalf("only %d/%d responses settled: %v", len(found), len(paths), found)+ }+ time.Sleep(50 * time.Millisecond)+ }+}++func skipSummary(entry NetworkEntry) string {+ return fmt.Sprintf("skipped=%v reason=%q error=%q base64=%v bytes=%d",+ entry.BodySkipped, entry.BodySkipReason, entry.BodyError, entry.Base64Encoded, len(entry.ResponseBody))+}++// base64Len is the encoded size CDP reports a body at, which is what the retention+// budget is measured against for a binary response.+func base64Len(raw int) int {+ return base64.StdEncoding.EncodedLen(raw)+}++// pngBytes builds a real PNG rather than arbitrary bytes, so Chrome classifies the+// response as binary and CDP sets base64Encoded — the whole subject of this test.+func pngBytes(t *testing.T, size int) []byte {+ t.Helper()+ img := image.NewRGBA(image.Rect(0, 0, size, size))+ for y := 0; y < size; y++ {+ for x := 0; x < size; x++ {+ img.Set(x, y, color.RGBA{R: uint8(x * 7), G: uint8(y * 11), B: uint8(x ^ y), A: 255})+ }+ }+ var out bytes.Buffer+ if err := png.Encode(&out, img); err != nil {+ t.Fatal(err)+ }+ return out.Bytes()+}diff --git a/internal/bridge/observe/network_body_fetcher_test.go b/internal/bridge/observe/network_body_fetcher_test.gonew file mode 100644index 00000000..c3d30b9e--- /dev/null+++ b/internal/bridge/observe/network_body_fetcher_test.go@@ -0,0 +1,98 @@+package observe++import (+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// The behavioural proof of this rule is browser-backed, and the lightweight CI run+// installs no browser — so the wiring it depends on gets a source-level guard that runs+// everywhere. Two fetchers existed; the one the retention path used called cdproto's+// typed GetResponseBodyParams.Do, which base64-decodes inside the dependency and returns+// []byte, so it had no flag left to report and hardcoded false.+func TestOneResponseBodyFetcherThatReportsTheCDPFlag(t *testing.T) {+ const fetcher = "func GetResponseBody(ctx context.Context, requestID string) (string, bool, error)"++ body := readRepoCode(t, "network.go")++ if !strings.Contains(body, fetcher) {+ t.Fatalf("network.go no longer declares %q; this census would guard nothing", fetcher)+ }+ if !strings.Contains(body, "base64Encoded = resp.Base64Encoded") {+ t.Error("the fetcher no longer reads base64Encoded off the CDP result, so the flag is not the browser's answer")+ }+ if strings.Contains(body, "base64Encoded = false") {+ t.Error("network.go assigns base64Encoded a constant; a binary body then retains as raw bytes and JSON-encodes to U+FFFD")+ }+ if strings.Contains(body, "network.GetResponseBody(") {+ t.Error("network.go fetches a body through cdproto's typed GetResponseBody, which decodes inside the dependency and destroys the flag")+ }+ if !strings.Contains(body, "var fetchResponseBody = GetResponseBody") {+ t.Error("the retention seam no longer defaults to GetResponseBody, so retention can drive a different fetcher than export")+ }++ // The on-demand and export call sites must reach the same fetcher. A second+ // implementation reachable from handlers is how the two paths came to disagree about+ // the same request in the first place.+ var handlerCalls int+ for _, name := range []string{"network.go", "network_export.go"} {+ handlerBody := readRepoCode(t, filepath.Join("..", "..", "handlers", name))+ if strings.Contains(handlerBody, "GetResponseBodyDirect") {+ t.Errorf("handlers/%s reaches GetResponseBodyDirect instead of the single fetcher", name)+ }+ for _, line := range strings.Split(handlerBody, "\n") {+ if !strings.Contains(line, "GetResponseBody(") {+ continue+ }+ handlerCalls+++ if !strings.Contains(line, "bridge.GetResponseBody(") {+ t.Errorf("handlers/%s fetches a body outside the single fetcher: %s", name, strings.TrimSpace(line))+ }+ }+ assertFlagNotDiscarded(t, "handlers/"+name, handlerBody)+ }+ if handlerCalls == 0 {+ t.Error("no body-fetch call site found in handlers; the call-site half of this census guards nothing")+ }++ // The same rule inside this package: one owner is not enough if a wrapper re-creates+ // the old behaviour by throwing the flag away.+ assertFlagNotDiscarded(t, "network.go", body)+}++// assertFlagNotDiscarded bans a caller that takes the body and drops the flag. Routing+// every site through one fetcher does not fix anything on its own — a thin wrapper that+// reads the body and returns a constant false is byte-for-byte the defect this card is+// about, wearing a different name, and a ban keyed on the fetcher's NAME cannot see it.+func assertFlagNotDiscarded(t *testing.T, label, code string) {+ t.Helper()+ for _, line := range strings.Split(code, "\n") {+ if !strings.Contains(line, "GetResponseBody(") {+ continue+ }+ if strings.Contains(line, ", _, err") || strings.Contains(line, ", _, _") {+ t.Errorf("%s discards the base64Encoded flag: %s", label, strings.TrimSpace(line))+ }+ }+}++// readRepoCode returns the file with comment lines dropped, so a doc comment naming the+// banned shape does not trip a ban on it — the prose explaining why a form is wrong is+// exactly where that form gets written down.+func readRepoCode(t *testing.T, path string) string {+ t.Helper()+ raw, err := os.ReadFile(path) // #nosec G304 -- fixed paths inside this repo.+ if err != nil {+ t.Fatalf("cannot read %s, so this census guards nothing: %v", path, err)+ }+ var code []string+ for _, line := range strings.Split(string(raw), "\n") {+ if strings.HasPrefix(strings.TrimSpace(line), "//") {+ continue+ }+ code = append(code, line)+ }+ return strings.Join(code, "\n")+}diff --git a/internal/bridge/observe_facade.go b/internal/bridge/observe_facade.goindex 40a494de..72e68cbe 100644--- a/internal/bridge/observe_facade.go+++ b/internal/bridge/observe_facade.go@@ -120,8 +120,8 @@ func matchStatusRange(status int, pattern string) bool {return bridgeobserve.MatchStatusRange(status, pattern)}-func GetResponseBodyDirect(ctx context.Context, requestID string) (string, bool, error) {- return bridgeobserve.GetResponseBodyDirect(ctx, requestID)+func GetResponseBody(ctx context.Context, requestID string) (string, bool, error) {+ return bridgeobserve.GetResponseBody(ctx, requestID)}func (b *Bridge) GetMemoryMetrics(tabID string) (*MemoryMetrics, error) {diff --git a/internal/handlers/network.go b/internal/handlers/network.goindex 2eb1aa47..ec38831e 100644--- a/internal/handlers/network.go+++ b/internal/handlers/network.go@@ -335,7 +335,7 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {}switch {case bodyMode == networkBodyModeLiveOnly:- body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)+ body, base64Encoded, err := bridge.GetResponseBody(tabCtx, requestID)if err != nil {result["bodyError"] = err.Error()} else {@@ -348,7 +348,7 @@ func (h *Handlers) HandleNetworkByID(w http.ResponseWriter, r *http.Request) {case entry.BodyPending || entry.BodyError != "":populateRetainedBodyResult(result, entry)default:- body, base64Encoded, err := bridge.GetResponseBodyDirect(tabCtx, requestID)+ body, base64Encoded, err := bridge.GetResponseBody(tabCtx, requestID)if err != nil {result["bodyError"] = err.Error()} else {diff --git a/internal/handlers/network_export.go b/internal/handlers/network_export.goindex 7d7e9b30..b922e23a 100644--- a/internal/handlers/network_export.go+++ b/internal/handlers/network_export.go@@ -193,7 +193,7 @@ func resolveExportBody(ctx context.Context, nm *bridge.NetworkMonitor, entry briif entry.BodyRetained {return clampExportBody(entry.ResponseBody, entry.Base64Encoded)}- body, b64, _ := nm.GetResponseBody(ctx, entry.RequestID)+ body, b64, _ := bridge.GetResponseBody(ctx, entry.RequestID)return clampExportBody(body, b64)}