High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(observability): carry a proxied failure reason across the hop whatever the status, so the front door counts a failed batch (PIN-274)
Luigi Agosti7626456d65d4
diff --git a/docs/reference/metrics.md b/docs/reference/metrics.mdindex 1713b33b..40163c19 100644--- a/docs/reference/metrics.md+++ b/docs/reference/metrics.md@@ -40,6 +40,12 @@ status: it counts in `requestsFailed`, appears in `failures.recent` with the runlogs at `WARN`. Watching `requestsFailed` therefore sees an agent whose batches are allfailing, which is the shape of traffic these endpoints exist to serve.+This holds at **both** layers. The reason travels the proxy hop on its own response+headers, so a batch served by an instance and proxied by the front door moves the front+door's `requestsFailed` too — which is the number an operator running `pinchtab server`+curls, since `GET /metrics` there reports the front door's own counters. A proxied+response carrying no reason records nothing, whatever its status.+**The two layers are never added together.** A single `requestsTotal` covering bothwould mean two things at once, and an operator cannot act on that number: a spikewould not say whether clients are being turned away at the door or the browser isdiff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.goindex 2f4b0579..207d9794 100644--- a/internal/handlers/middleware.go+++ b/internal/handlers/middleware.go@@ -56,8 +56,13 @@ func requestLogLevel(status int, reasonRecorded bool) slog.Level {// channels: the status, or a reason the handler published beside a success status.// The producer that wrote the response is the only thing that knows the second// case, which is why it is a recorded fact rather than something re-derived here.+//+// "A reason was recorded" means EITHER field, matching what RecordFailureReason+// accepts: it publishes when a code or a message is present, so keying this on the+// message alone left a producer that carried only a code recorded on the wire and+// uncounted here.func requestFailed(sw *httpx.StatusWriter) bool {- return sw.Code >= 400 || sw.FailureMessage != ""+ return sw.Code >= 400 || sw.FailureCode != "" || sw.FailureMessage != ""}func LoggingMiddleware(next http.Handler) http.Handler {@@ -91,7 +96,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {if sw.FailureMessage != "" {attrs = append(attrs, "code", sw.FailureCode, "error", sw.FailureMessage)}- slog.Log(r.Context(), requestLogLevel(sw.Code, sw.FailureMessage != ""), "request", attrs...)+ slog.Log(r.Context(), requestLogLevel(sw.Code, requestFailed(sw)), "request", attrs...)})}diff --git a/internal/handlers/multistep_failure_test.go b/internal/handlers/multistep_failure_test.goindex a793ca5e..79c8a547 100644--- a/internal/handlers/multistep_failure_test.go+++ b/internal/handlers/multistep_failure_test.go@@ -7,12 +7,15 @@ import ("log/slog""net/http""net/http/httptest"+ "net/url""strings""testing""github.com/pinchtab/pinchtab/internal/activity""github.com/pinchtab/pinchtab/internal/browserops""github.com/pinchtab/pinchtab/internal/config"+ "github.com/pinchtab/pinchtab/internal/httpx"+ "github.com/pinchtab/pinchtab/internal/proxy")// recordedRun is what the three durable channels held after one multi-step run.@@ -213,3 +216,95 @@ func assertStepCounts(t *testing.T, event map[string]any, total, successful, fai}}}++// proxiedRun drives the FRONT DOOR's shape: an upstream instance answering exactly+// what the shared writer now produces, through the real proxy, under the real+// logging middleware. The differential above is the bridge shape — one process, no+// hop — which is why it could not see that the reason stopped at the boundary.+func proxiedRun(t *testing.T, status int, code, message string) recordedRun {+ t.Helper()++ logs := &bytes.Buffer{}+ previous := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))+ t.Cleanup(func() { slog.SetDefault(previous) })+ resetObservabilityForTests()++ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ if code != "" {+ // The producer's own publisher, so the headers under test are the ones a+ // real instance stamps rather than a literal this test invented.+ httpx.RecordFailureReason(w, code, message)+ }+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)+ _, _ = w.Write([]byte(`{"status":"ok"}`))+ }))+ t.Cleanup(upstream.Close)++ target, err := url.Parse(upstream.URL + "/tabs/tab1/actions")+ if err != nil {+ t.Fatal(err)+ }+ chain := LoggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ proxy.Forward(w, r, target, proxy.Options{})+ }))++ req := httptest.NewRequest(http.MethodPost, "/tabs/tab1/actions", strings.NewReader(`{}`))+ w := httptest.NewRecorder()+ chain.ServeHTTP(w, req)++ failed, _ := SnapshotMetrics()["requestsFailed"].(uint64)+ failures := allRecentFailureEvents(t)+ encoded, err := json.Marshal(failures)+ if err != nil {+ t.Fatal(err)+ }+ return recordedRun{+ channels: map[string]string{"the server log": logs.String(), "failures.recent": string(encoded)},+ failures: failures,+ failedDelta: failed,+ }+}++// The hop the card's fix stopped at. GET /metrics on the front door serves the+// front door's OWN counters by design, so an operator running `pinchtab server`+// watches these numbers — and for a batch in which every step failed they did not+// move, because the reason travelled beside a 200 and the proxy dropped it on the+// status.+//+// The healthy row is the load-bearing half: removing the status gate is exactly the+// change that could start counting ordinary proxied traffic as failures, and the+// header's presence is what must be doing the work instead.+func TestTheFrontDoorRecordsAProxiedFailureReasonBesideA200(t *testing.T) {+ for _, tc := range []struct {+ name string+ status int+ code string+ message string+ wantFailed uint64+ wantLevel string+ }{+ {"a batch: 200 carrying a reason", http.StatusOK, "ref_not_found", "3 of 3 steps failed: " + firstBatchFailure, 1, "level=WARN"},+ {"a single action: 4xx carrying a reason", http.StatusNotFound, "ref_not_found", firstBatchFailure, 1, "level=WARN"},+ {"ordinary traffic: 200 with no reason", http.StatusOK, "", "", 0, "level=INFO"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ run := proxiedRun(t, tc.status, tc.code, tc.message)++ if run.failedDelta != tc.wantFailed {+ t.Errorf("the front door's requestsFailed moved by %d, want %d — this is the counter an operator curls at /metrics",+ run.failedDelta, tc.wantFailed)+ }+ if len(run.failures) != int(tc.wantFailed) {+ t.Errorf("failures.recent holds %d entries, want %d:\n%s", len(run.failures), tc.wantFailed, run.channels["failures.recent"])+ }+ if !strings.Contains(run.channels["the server log"], tc.wantLevel) {+ t.Errorf("the front door logged at the wrong level, want %s:\n%s", tc.wantLevel, run.channels["the server log"])+ }+ if tc.wantFailed > 0 && !strings.Contains(run.channels["failures.recent"], tc.message) {+ t.Errorf("failures.recent does not carry the reason the instance published:\n%s", run.channels["failures.recent"])+ }+ })+ }+}diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.goindex 5cdae984..413c9e9e 100644--- a/internal/proxy/proxy.go+++ b/internal/proxy/proxy.go@@ -187,16 +187,17 @@ func HTTP(w http.ResponseWriter, r *http.Request, targetURL string) {Forward(w, r, parsed, Options{})}-// enrichActivityFromHeaders extracts tab ID from upstream response headers-// and enriches the activity event. This works for all response sizes,-// unlike body-based enrichment which is limited to small JSON responses.// recordProxiedFailureReason carries the reason across the hop: the instance's error// producer stamped these headers on the response it serialised, so reading them here// keeps the reason coming from the producer — never from re-parsing the body.+// The status is deliberately NOT consulted. A multi-step run answers 200 with its+// failures in the body and publishes the reason beside it, so a status gate here+// dropped exactly that case and left the front door's counter, failures.recent, log+// level and activity record unmoved for a batch in which every step failed. The+// header being present is the whole condition, and it is a stronger one: only a+// producer that called RecordFailureReason stamps it, so ordinary 200 traffic+// records nothing and cannot be counted as a failure.func recordProxiedFailureReason(w http.ResponseWriter, resp *http.Response) {- if resp.StatusCode < 400 {- return- }code := strings.TrimSpace(resp.Header.Get(httpx.FailureCodeHeader))if code == "" {return@@ -204,6 +205,9 @@ func recordProxiedFailureReason(w http.ResponseWriter, resp *http.Response) {httpx.RecordFailureReason(w, code, resp.Header.Get(httpx.FailureMessageHeader))}+// enrichActivityFromHeaders extracts the tab id from upstream response headers and+// enriches the activity event. It works for all response sizes, unlike body-based+// enrichment, which is limited to small JSON responses.func enrichActivityFromHeaders(origReq *http.Request, respHeaders http.Header) {tabID := strings.TrimSpace(respHeaders.Get(activity.HeaderPTTabID))if tabID != "" {