High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: proxied responses carry two X-Request-Id values and only one is… (PIN-138)
Luigi Agosti7c489ec2d5a7
diff --git a/internal/handlers/middleware_owned_headers_test.go b/internal/handlers/middleware_owned_headers_test.gonew file mode 100644index 00000000..3df9c257--- /dev/null+++ b/internal/handlers/middleware_owned_headers_test.go@@ -0,0 +1,97 @@+package handlers++import (+ "crypto/tls"+ "net/http"+ "net/http/httptest"+ "sort"+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/config"+ "github.com/pinchtab/pinchtab/internal/httpx"+)++func headersSetByMiddleware(t *testing.T, mw func(http.Handler) http.Handler, r *http.Request) []string {+ t.Helper()++ rec := httptest.NewRecorder()+ mw(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})).ServeHTTP(rec, r)++ var names []string+ for name := range rec.Header() {+ names = append(names, name)+ }+ return names+}++// The owned set is what every proxy hop filters out of an upstream response. It is pinned+// against the middlewares themselves rather than a hand-kept list, so the next header the+// outer chain starts setting cannot be the one that keeps arriving twice.+func TestTheOwnedResponseHeaderSetIsExactlyWhatTheOuterChainSets(t *testing.T) {+ plain := httptest.NewRequest("GET", "/tabs", nil)+ secure := httptest.NewRequest("GET", "/tabs", nil)+ secure.TLS = &tls.ConnectionState{}++ cfg := &config.RuntimeConfig{}+ set := map[string]bool{}+ for _, name := range headersSetByMiddleware(t, RequestIDMiddleware, plain) {+ set[name] = true+ }+ for _, r := range []*http.Request{plain, secure} {+ for _, name := range headersSetByMiddleware(t, func(next http.Handler) http.Handler {+ return SecurityHeadersMiddleware(cfg, next)+ }, r) {+ set[name] = true+ }+ }++ if len(set) < 4 {+ t.Fatalf("the two middlewares set %v; that is fewer headers than this guard was written over, so it would pass vacuously", set)+ }++ var got []string+ for name := range set {+ got = append(got, name)+ }+ want := httpx.OuterChainResponseHeaders()+ sort.Strings(got)+ sort.Strings(want)++ if strings.Join(got, ",") != strings.Join(want, ",") {+ t.Errorf("the outer chain sets %v but httpx.OuterChainResponseHeaders() declares %v.\nA header the chain sets and the set omits arrives twice on every proxied response, and the second value is one no log holds; a header the set names and the chain no longer sets is filtered out of upstream responses that legitimately own it.", got, want)+ }+}++func TestTheOuterChainSetsItsHeadersSingleValued(t *testing.T) {+ rec := httptest.NewRecorder()+ req := httptest.NewRequest("GET", "/tabs", nil)+ req.TLS = &tls.ConnectionState{}++ handler := RequestIDMiddleware(SecurityHeadersMiddleware(&config.RuntimeConfig{}, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})))+ handler.ServeHTTP(rec, req)++ for _, name := range httpx.OuterChainResponseHeaders() {+ if got := rec.Header().Values(name); len(got) > 1 {+ t.Errorf("%s = %v: the outer chain itself is the second writer, so no proxy filter can make it single-valued", name, got)+ }+ }+}++func TestRequestIDMiddlewareStampsTheSameIDOnRequestAndResponse(t *testing.T) {+ rec := httptest.NewRecorder()+ req := httptest.NewRequest("GET", "/tabs", nil)++ var seen string+ RequestIDMiddleware(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) {+ seen = r.Header.Get(httpx.HeaderRequestID)+ })).ServeHTTP(rec, req)++ response := rec.Header().Get(httpx.HeaderRequestID)+ if response == "" {+ t.Fatal("no request id on the response, so a caller has nothing to quote")+ }+ if seen != response {+ t.Errorf("downstream request carries %q while the response carries %q; the id the outer process logs must be the one a downstream hop can propagate", seen, response)+ }+}diff --git a/internal/httpx/proxyheaders.go b/internal/httpx/proxyheaders.gonew file mode 100644index 00000000..db3869ab--- /dev/null+++ b/internal/httpx/proxyheaders.go@@ -0,0 +1,70 @@+package httpx++import (+ "net/http"+ "strings"+)++const (+ HeaderRequestID = "X-Request-Id"+ HeaderContentSecurityPolicy = "Content-Security-Policy"+ HeaderFrameOptions = "X-Frame-Options"+ HeaderContentTypeOptions = "X-Content-Type-Options"+ HeaderStrictTransportSecurity = "Strict-Transport-Security"+)++var outerChainResponseHeaders = []string{+ HeaderRequestID,+ HeaderContentSecurityPolicy,+ HeaderFrameOptions,+ HeaderContentTypeOptions,+ HeaderStrictTransportSecurity,+}++var hopByHopHeaders = []string{+ "Connection",+ "Keep-Alive",+ "Proxy-Authenticate",+ "Proxy-Authorization",+ "Te",+ "Trailers",+ "Transfer-Encoding",+ "Upgrade",+ "Host",+}++func OuterChainResponseHeaders() []string {+ return append([]string(nil), outerChainResponseHeaders...)+}++func OuterChainOwnsResponseHeader(key string) bool {+ return containsFold(outerChainResponseHeaders, key)+}++func IsHopByHopHeader(key string) bool {+ return containsFold(hopByHopHeaders, key)+}++func containsFold(names []string, key string) bool {+ for _, name := range names {+ if strings.EqualFold(name, key) {+ return true+ }+ }+ return false+}++// CopyProxiedResponseHeaders is the one response-header copy every proxy hop uses. The+// outer middleware chain has already set its own X-Request-Id and security headers on this+// response, so appending the upstream copies makes them multi-valued: a caller then sees+// two request ids, one of which the outer process never minted and therefore never logged.+func CopyProxiedResponseHeaders(dst, src http.Header) {+ for key, values := range src {+ if IsHopByHopHeader(key) || OuterChainOwnsResponseHeader(key) {+ continue+ }+ for _, v := range values {+ dst.Add(key, v)+ }+ }+}diff --git a/internal/httpx/proxyheaders_test.go b/internal/httpx/proxyheaders_test.gonew file mode 100644index 00000000..989d2160--- /dev/null+++ b/internal/httpx/proxyheaders_test.go@@ -0,0 +1,155 @@+package httpx++import (+ "go/ast"+ "go/parser"+ "go/token"+ "net/http"+ "path/filepath"+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/srccensus"+)++func TestCopyProxiedResponseHeadersLeavesTheOuterChainsHeadersSingleValued(t *testing.T) {+ owned := OuterChainResponseHeaders()+ if len(owned) < 4 {+ t.Fatalf("owned header set = %v, want at least the request id and the three security headers", owned)+ }++ dst := http.Header{}+ for _, name := range owned {+ dst.Set(name, "outer-"+name)+ }+ src := http.Header{"Content-Type": {"application/json"}}+ for _, name := range owned {+ src.Set(name, "upstream-"+name)+ }++ CopyProxiedResponseHeaders(dst, src)++ for _, name := range owned {+ if got := dst.Values(name); len(got) != 1 {+ t.Errorf("%s = %v, want exactly one value: a caller cannot tell which of two request ids the outer process logged", name, got)+ } else if got[0] != "outer-"+name {+ t.Errorf("%s = %q, want the outer chain's own value", name, got[0])+ }+ }+ if got := dst.Get("Content-Type"); got != "application/json" {+ t.Errorf("Content-Type = %q, want the upstream header copied through", got)+ }+}++func TestCopyProxiedResponseHeadersMatchesOwnedHeadersCaseInsensitively(t *testing.T) {+ dst := http.Header{}+ dst.Set(HeaderRequestID, "outer")++ src := http.Header{}+ src["x-request-id"] = []string{"upstream"}++ CopyProxiedResponseHeaders(dst, src)++ if got := dst.Values(HeaderRequestID); len(got) != 1 || got[0] != "outer" {+ t.Errorf("%s = %v, want the outer value alone — an upstream spelling the canonicaliser did not touch must still be filtered", HeaderRequestID, got)+ }+}++func TestCopyProxiedResponseHeadersDropsHopByHopHeaders(t *testing.T) {+ dst := http.Header{}+ src := http.Header{+ "Transfer-Encoding": {"chunked"},+ "Connection": {"keep-alive"},+ "X-Custom": {"kept"},+ }++ CopyProxiedResponseHeaders(dst, src)++ for _, name := range []string{"Transfer-Encoding", "Connection"} {+ if got := dst.Get(name); got != "" {+ t.Errorf("%s = %q, want it dropped: an upstream framing header is not the outer response's to republish", name, got)+ }+ }+ if got := dst.Get("X-Custom"); got != "kept" {+ t.Errorf("X-Custom = %q, want the upstream header copied through", got)+ }+}++func TestOwnedResponseHeadersAreDistinctAndCanonical(t *testing.T) {+ seen := map[string]bool{}+ for _, name := range OuterChainResponseHeaders() {+ lower := strings.ToLower(name)+ if seen[lower] {+ t.Errorf("%q is listed twice in the owned header set", name)+ }+ seen[lower] = true+ if name != http.CanonicalHeaderKey(name) {+ t.Errorf("%q is not the canonical spelling %q, so a census comparing it to a source literal misses", name, http.CanonicalHeaderKey(name))+ }+ }+}++func TestOuterChainResponseHeadersCannotBeMutatedByCallers(t *testing.T) {+ got := OuterChainResponseHeaders()+ got[0] = "X-Not-Owned"++ if !OuterChainOwnsResponseHeader(HeaderRequestID) {+ t.Fatalf("a caller editing the returned slice changed the owned set, so one proxy hop could stop filtering %s", HeaderRequestID)+ }+}++// This reached three copy sites at once because each grew its own copy loop. The rule is+// therefore about the SHAPE: a response header added under a key the code does not name is+// by construction republishing somebody else's header set, which is what makes the outer+// chain's headers multi-valued. A literal key — Vary: Origin — is a deliberate append and+// stays legal.+func TestNoProxyHopAddsResponseHeadersUnderAKeyItDoesNotName(t *testing.T) {+ files := srccensus.Tree(t, filepath.Join("..", ".."), 100)++ helperCalls := 0+ for _, file := range files {+ parsed, err := parser.ParseFile(token.NewFileSet(), file.Name, file.Text, 0)+ if err != nil {+ t.Errorf("%s: cannot parse, so this census can neither clear nor flag it: %v", file.Name, err)+ continue+ }+ ast.Inspect(parsed, func(node ast.Node) bool {+ call, ok := node.(*ast.CallExpr)+ if !ok {+ return true+ }+ sel, ok := call.Fun.(*ast.SelectorExpr)+ if !ok {+ return true+ }+ if sel.Sel.Name == "CopyProxiedResponseHeaders" {+ helperCalls+++ return true+ }+ if sel.Sel.Name != "Add" || len(call.Args) == 0 || !isHeaderAccessorCall(sel.X) {+ return true+ }+ if _, literal := call.Args[0].(*ast.BasicLit); literal {+ return true+ }+ t.Errorf("%s adds a response header under a variable key instead of calling httpx.CopyProxiedResponseHeaders; the outer chain has already set its request id and security headers on this response, so appending an upstream copy makes them multi-valued and the second request id appears in no log", file.Name)+ return true+ })+ }++ if helperCalls < 3 {+ t.Fatalf("found %d call(s) to CopyProxiedResponseHeaders in the whole module, want at least the known proxy hops; the helper has been inlined again, or this census has stopped seeing the module", helperCalls)+ }+}++// isHeaderAccessorCall reports whether expr is a Header() call — w.Header().Add(...) — as+// opposed to a plain Header field, which is how the REQUEST direction is built and is not+// this rule's subject.+func isHeaderAccessorCall(expr ast.Expr) bool {+ call, ok := expr.(*ast.CallExpr)+ if !ok {+ return false+ }+ sel, ok := call.Fun.(*ast.SelectorExpr)+ return ok && sel.Sel.Name == "Header"+}diff --git a/internal/instance/bridge_client.go b/internal/instance/bridge_client.goindex f39ee08e..3e0c37f1 100644--- a/internal/instance/bridge_client.go+++ b/internal/instance/bridge_client.go@@ -177,11 +177,7 @@ func (bc *BridgeClient) ProxyWithTabID(w http.ResponseWriter, r *http.Request, p}defer func() { _ = resp.Body.Close() }()- for key, values := range resp.Header {- for _, v := range values {- w.Header().Add(key, v)- }- }+ httpx.CopyProxiedResponseHeaders(w.Header(), resp.Header)w.WriteHeader(resp.StatusCode)_, _ = io.Copy(w, resp.Body)}@@ -202,13 +198,11 @@ func (bc *BridgeClient) ProxyToTab(w http.ResponseWriter, r *http.Request, port,}for key, values := range r.Header {- switch key {- case "Host", "Connection", "Keep-Alive", "Proxy-Authenticate",- "Proxy-Authorization", "Te", "Trailers", "Transfer-Encoding", "Upgrade":- default:- for _, v := range values {- proxyReq.Header.Add(key, v)- }+ if httpx.IsHopByHopHeader(key) {+ continue+ }+ for _, v := range values {+ proxyReq.Header.Add(key, v)}}@@ -219,11 +213,7 @@ func (bc *BridgeClient) ProxyToTab(w http.ResponseWriter, r *http.Request, port,}defer func() { _ = resp.Body.Close() }()- for key, values := range resp.Header {- for _, v := range values {- w.Header().Add(key, v)- }- }+ httpx.CopyProxiedResponseHeaders(w.Header(), resp.Header)w.WriteHeader(resp.StatusCode)_, _ = io.Copy(w, resp.Body)}diff --git a/internal/instance/bridge_client_proxy_headers_test.go b/internal/instance/bridge_client_proxy_headers_test.gonew file mode 100644index 00000000..76eda693--- /dev/null+++ b/internal/instance/bridge_client_proxy_headers_test.go@@ -0,0 +1,112 @@+package instance++import (+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/httpx"+)++func bridgeStub(t *testing.T) (*BridgeClient, string) {+ t.Helper()++ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ for _, name := range httpx.OuterChainResponseHeaders() {+ w.Header().Set(name, "instance-"+name)+ }+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(200)+ _, _ = w.Write([]byte(`{"ok":true}`))+ }))+ t.Cleanup(srv.Close)++ parsed, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatal(err)+ }+ return NewBridgeClient(), parsed.Port()+}++func assertOuterChainHeadersSurviveAlone(t *testing.T, header http.Header) {+ t.Helper()++ for _, name := range httpx.OuterChainResponseHeaders() {+ got := header.Values(name)+ if len(got) != 1 {+ t.Errorf("%s = %v, want exactly one value — the instance's own copy is a value the outer process never minted or logged", name, got)+ continue+ }+ if got[0] != "outer-"+name {+ t.Errorf("%s = %q, want the outer chain's value", name, got[0])+ }+ }+ if got := header.Get("Content-Type"); got != "application/json" {+ t.Errorf("Content-Type = %q, want the instance's own response header copied through", got)+ }+}++func recorderWithOuterChainHeaders() *httptest.ResponseRecorder {+ rec := httptest.NewRecorder()+ for _, name := range httpx.OuterChainResponseHeaders() {+ rec.Header().Set(name, "outer-"+name)+ }+ return rec+}++func TestProxyWithTabIDDoesNotDoubleTheOuterChainsResponseHeaders(t *testing.T) {+ client, port := bridgeStub(t)++ rec := recorderWithOuterChainHeaders()+ req := httptest.NewRequest("POST", "/find", strings.NewReader(`{"text":"Buy"}`))++ client.ProxyWithTabID(rec, req, port, "tab-1", "/find")++ if rec.Code != 200 {+ t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())+ }+ assertOuterChainHeadersSurviveAlone(t, rec.Header())+}++func TestProxyToTabDoesNotDoubleTheOuterChainsResponseHeaders(t *testing.T) {+ client, port := bridgeStub(t)++ rec := recorderWithOuterChainHeaders()+ req := httptest.NewRequest("GET", "/tabs/tab-1/text", nil)++ client.ProxyToTab(rec, req, port, "tab-1", "/text")++ if rec.Code != 200 {+ t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())+ }+ assertOuterChainHeadersSurviveAlone(t, rec.Header())+}++func TestProxyToTabForwardsRequestHeadersButNotHopByHopOnes(t *testing.T) {+ var seen http.Header+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ seen = r.Header.Clone()+ w.WriteHeader(200)+ }))+ defer srv.Close()++ parsed, err := url.Parse(srv.URL)+ if err != nil {+ t.Fatal(err)+ }++ req := httptest.NewRequest("GET", "/tabs/tab-1/text", nil)+ req.Header.Set("Authorization", "Bearer token")+ req.Header.Set("Te", "trailers")++ NewBridgeClient().ProxyToTab(httptest.NewRecorder(), req, parsed.Port(), "tab-1", "/text")++ if got := seen.Get("Authorization"); got != "Bearer token" {+ t.Errorf("Authorization = %q, want it forwarded to the instance", got)+ }+ if got := seen.Get("Te"); got != "" {+ t.Errorf("Te = %q, want the hop-by-hop header dropped on the proxy hop", got)+ }+}diff --git a/internal/orchestrator/handlers_tabs.go b/internal/orchestrator/handlers_tabs.goindex 1f4b8574..4aab5155 100644--- a/internal/orchestrator/handlers_tabs.go+++ b/internal/orchestrator/handlers_tabs.go@@ -126,11 +126,7 @@ func (o *Orchestrator) doInstanceRequest(ctx context.Context, method string, hea}func copyProxyResponse(w http.ResponseWriter, resp *http.Response, body []byte) {- for key, values := range resp.Header {- for _, v := range values {- w.Header().Add(key, v)- }- }+ httpx.CopyProxiedResponseHeaders(w.Header(), resp.Header)w.WriteHeader(resp.StatusCode)_, _ = w.Write(body)}diff --git a/internal/orchestrator/handlers_tabs_proxy_headers_test.go b/internal/orchestrator/handlers_tabs_proxy_headers_test.gonew file mode 100644index 00000000..4bd1f7ee--- /dev/null+++ b/internal/orchestrator/handlers_tabs_proxy_headers_test.go@@ -0,0 +1,40 @@+package orchestrator++import (+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/pinchtab/pinchtab/internal/httpx"+)++func TestCopyProxyResponseDoesNotDoubleTheOuterChainsResponseHeaders(t *testing.T) {+ owned := httpx.OuterChainResponseHeaders()++ rec := httptest.NewRecorder()+ for _, name := range owned {+ rec.Header().Set(name, "outer-"+name)+ }++ upstream := &http.Response{StatusCode: 200, Header: http.Header{}}+ for _, name := range owned {+ upstream.Header.Set(name, "instance-"+name)+ }+ upstream.Header.Set("Content-Type", "application/json")++ copyProxyResponse(rec, upstream, []byte(`{"ok":true}`))++ for _, name := range owned {+ got := rec.Header().Values(name)+ if len(got) != 1 {+ t.Errorf("%s = %v, want exactly one value — the instance's copy is untraceable in the outer process's log", name, got)+ continue+ }+ if got[0] != "outer-"+name {+ t.Errorf("%s = %q, want the outer chain's value", name, got[0])+ }+ }+ if got := rec.Header().Get("Content-Type"); got != "application/json" {+ t.Errorf("Content-Type = %q, want the instance's own response header copied through", got)+ }+}diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.goindex 7ad8c956..498dd223 100644--- a/internal/proxy/proxy.go+++ b/internal/proxy/proxy.go@@ -32,18 +32,6 @@ type Options struct {OnResponse func(origReq *http.Request, body []byte)}-var hopByHopHeaders = map[string]struct{}{- "connection": {},- "keep-alive": {},- "proxy-authenticate": {},- "proxy-authorization": {},- "te": {},- "trailers": {},- "transfer-encoding": {},- "upgrade": {},- "host": {},-}-var strippedProxyRequestHeaders = map[string]struct{}{"cookie": {},"forwarded": {},@@ -97,7 +85,7 @@ func Forward(w http.ResponseWriter, r *http.Request, targetURL *url.URL, opts Op}defer func() { _ = resp.Body.Close() }()- copyHeaders(w.Header(), resp.Header)+ httpx.CopyProxiedResponseHeaders(w.Header(), resp.Header)// Enrich activity from response headers (always available, regardless of body size).enrichActivityFromHeaders(r, resp.Header)@@ -175,24 +163,12 @@ func isWebSocketUpgrade(r *http.Request) bool {return false}-func copyHeaders(dst, src http.Header) {- for k, vv := range src {- if _, skip := hopByHopHeaders[strings.ToLower(k)]; skip {- continue- }- for _, v := range vv {- dst.Add(k, v)- }- }-}-func copyRequestHeaders(dst, src http.Header) {for k, vv := range src {- lower := strings.ToLower(k)- if _, skip := hopByHopHeaders[lower]; skip {+ if httpx.IsHopByHopHeader(k) {continue}- if _, skip := strippedProxyRequestHeaders[lower]; skip {+ if _, skip := strippedProxyRequestHeaders[strings.ToLower(k)]; skip {continue}for _, v := range vv {diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.goindex af225d92..311fe84f 100644--- a/internal/proxy/proxy_test.go+++ b/internal/proxy/proxy_test.go@@ -4,6 +4,8 @@ import ("encoding/json""net/http""net/http/httptest"++ "github.com/pinchtab/pinchtab/internal/httpx""testing")@@ -162,3 +164,36 @@ func TestIsWebSocketUpgrade(t *testing.T) {})}}++func TestHTTP_DoesNotDoubleTheOuterChainsResponseHeaders(t *testing.T) {+ owned := httpx.OuterChainResponseHeaders()+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ for _, name := range owned {+ w.Header().Set(name, "instance-"+name)+ }+ w.WriteHeader(200)+ }))+ defer srv.Close()++ req := httptest.NewRequest("GET", "/tabs", nil)+ rec := httptest.NewRecorder()+ for _, name := range owned {+ rec.Header().Set(name, "outer-"+name)+ }++ HTTP(rec, req, srv.URL+"/tabs")++ if rec.Code != 200 {+ t.Fatalf("expected 200, got %d", rec.Code)+ }+ for _, name := range owned {+ got := rec.Header().Values(name)+ if len(got) != 1 {+ t.Errorf("%s = %v, want exactly one value — the second is minted by the instance and appears in no log the outer process writes", name, got)+ continue+ }+ if got[0] != "outer-"+name {+ t.Errorf("%s = %q, want the outer chain's value, which is the one it logged", name, got[0])+ }+ }+}