Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(observability): carry a proxied failure reason across the hop whatever the status, so the front door counts a failed batch (PIN-274)

Luigi Agosti7626456d65d4
diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md
index 1713b33b..40163c19 100644
--- a/docs/reference/metrics.md
+++ b/docs/reference/metrics.md
@@ -40,6 +40,12 @@ status: it counts in `requestsFailed`, appears in `failures.recent` with the run
logs at `WARN`. Watching `requestsFailed` therefore sees an agent whose batches are all
failing, which is the shape of traffic these endpoints exist to serve.
+This holds at **both** layers. The reason travels the proxy hop on its own response
+headers, so a batch served by an instance and proxied by the front door moves the front
+door's `requestsFailed` too — which is the number an operator running `pinchtab server`
+curls, since `GET /metrics` there reports the front door's own counters. A proxied
+response carrying no reason records nothing, whatever its status.
+
**The two layers are never added together.** A single `requestsTotal` covering both
would mean two things at once, and an operator cannot act on that number: a spike
would not say whether clients are being turned away at the door or the browser is
diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.go
index 2f4b0579..207d9794 100644
--- a/internal/handlers/middleware.go
+++ b/internal/handlers/middleware.go
@@ -56,8 +56,13 @@ func requestLogLevel(status int, reasonRecorded bool) slog.Level {
// channels: the status, or a reason the handler published beside a success status.
// The producer that wrote the response is the only thing that knows the second
// case, which is why it is a recorded fact rather than something re-derived here.
+//
+// "A reason was recorded" means EITHER field, matching what RecordFailureReason
+// accepts: it publishes when a code or a message is present, so keying this on the
+// message alone left a producer that carried only a code recorded on the wire and
+// uncounted here.
func requestFailed(sw *httpx.StatusWriter) bool {
- return sw.Code >= 400 || sw.FailureMessage != ""
+ return sw.Code >= 400 || sw.FailureCode != "" || sw.FailureMessage != ""
}
func LoggingMiddleware(next http.Handler) http.Handler {
@@ -91,7 +96,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {
if sw.FailureMessage != "" {
attrs = append(attrs, "code", sw.FailureCode, "error", sw.FailureMessage)
}
- slog.Log(r.Context(), requestLogLevel(sw.Code, sw.FailureMessage != ""), "request", attrs...)
+ slog.Log(r.Context(), requestLogLevel(sw.Code, requestFailed(sw)), "request", attrs...)
})
}
diff --git a/internal/handlers/multistep_failure_test.go b/internal/handlers/multistep_failure_test.go
index a793ca5e..79c8a547 100644
--- a/internal/handlers/multistep_failure_test.go
+++ b/internal/handlers/multistep_failure_test.go
@@ -7,12 +7,15 @@ import (
"log/slog"
"net/http"
"net/http/httptest"
+ "net/url"
"strings"
"testing"
"github.com/pinchtab/pinchtab/internal/activity"
"github.com/pinchtab/pinchtab/internal/browserops"
"github.com/pinchtab/pinchtab/internal/config"
+ "github.com/pinchtab/pinchtab/internal/httpx"
+ "github.com/pinchtab/pinchtab/internal/proxy"
)
// recordedRun is what the three durable channels held after one multi-step run.
@@ -213,3 +216,95 @@ func assertStepCounts(t *testing.T, event map[string]any, total, successful, fai
}
}
}
+
+// proxiedRun drives the FRONT DOOR's shape: an upstream instance answering exactly
+// what the shared writer now produces, through the real proxy, under the real
+// logging middleware. The differential above is the bridge shape — one process, no
+// hop — which is why it could not see that the reason stopped at the boundary.
+func proxiedRun(t *testing.T, status int, code, message string) recordedRun {
+ t.Helper()
+
+ logs := &bytes.Buffer{}
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+ resetObservabilityForTests()
+
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ if code != "" {
+ // The producer's own publisher, so the headers under test are the ones a
+ // real instance stamps rather than a literal this test invented.
+ httpx.RecordFailureReason(w, code, message)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ _, _ = w.Write([]byte(`{"status":"ok"}`))
+ }))
+ t.Cleanup(upstream.Close)
+
+ target, err := url.Parse(upstream.URL + "/tabs/tab1/actions")
+ if err != nil {
+ t.Fatal(err)
+ }
+ chain := LoggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ proxy.Forward(w, r, target, proxy.Options{})
+ }))
+
+ req := httptest.NewRequest(http.MethodPost, "/tabs/tab1/actions", strings.NewReader(`{}`))
+ w := httptest.NewRecorder()
+ chain.ServeHTTP(w, req)
+
+ failed, _ := SnapshotMetrics()["requestsFailed"].(uint64)
+ failures := allRecentFailureEvents(t)
+ encoded, err := json.Marshal(failures)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return recordedRun{
+ channels: map[string]string{"the server log": logs.String(), "failures.recent": string(encoded)},
+ failures: failures,
+ failedDelta: failed,
+ }
+}
+
+// The hop the card's fix stopped at. GET /metrics on the front door serves the
+// front door's OWN counters by design, so an operator running `pinchtab server`
+// watches these numbers — and for a batch in which every step failed they did not
+// move, because the reason travelled beside a 200 and the proxy dropped it on the
+// status.
+//
+// The healthy row is the load-bearing half: removing the status gate is exactly the
+// change that could start counting ordinary proxied traffic as failures, and the
+// header's presence is what must be doing the work instead.
+func TestTheFrontDoorRecordsAProxiedFailureReasonBesideA200(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ status int
+ code string
+ message string
+ wantFailed uint64
+ wantLevel string
+ }{
+ {"a batch: 200 carrying a reason", http.StatusOK, "ref_not_found", "3 of 3 steps failed: " + firstBatchFailure, 1, "level=WARN"},
+ {"a single action: 4xx carrying a reason", http.StatusNotFound, "ref_not_found", firstBatchFailure, 1, "level=WARN"},
+ {"ordinary traffic: 200 with no reason", http.StatusOK, "", "", 0, "level=INFO"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ run := proxiedRun(t, tc.status, tc.code, tc.message)
+
+ if run.failedDelta != tc.wantFailed {
+ t.Errorf("the front door's requestsFailed moved by %d, want %d — this is the counter an operator curls at /metrics",
+ run.failedDelta, tc.wantFailed)
+ }
+ if len(run.failures) != int(tc.wantFailed) {
+ t.Errorf("failures.recent holds %d entries, want %d:\n%s", len(run.failures), tc.wantFailed, run.channels["failures.recent"])
+ }
+ if !strings.Contains(run.channels["the server log"], tc.wantLevel) {
+ t.Errorf("the front door logged at the wrong level, want %s:\n%s", tc.wantLevel, run.channels["the server log"])
+ }
+ if tc.wantFailed > 0 && !strings.Contains(run.channels["failures.recent"], tc.message) {
+ t.Errorf("failures.recent does not carry the reason the instance published:\n%s", run.channels["failures.recent"])
+ }
+ })
+ }
+}
diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go
index 5cdae984..413c9e9e 100644
--- a/internal/proxy/proxy.go
+++ b/internal/proxy/proxy.go
@@ -187,16 +187,17 @@ func HTTP(w http.ResponseWriter, r *http.Request, targetURL string) {
Forward(w, r, parsed, Options{})
}
-// enrichActivityFromHeaders extracts tab ID from upstream response headers
-// and enriches the activity event. This works for all response sizes,
-// unlike body-based enrichment which is limited to small JSON responses.
// recordProxiedFailureReason carries the reason across the hop: the instance's error
// producer stamped these headers on the response it serialised, so reading them here
// keeps the reason coming from the producer — never from re-parsing the body.
+// The status is deliberately NOT consulted. A multi-step run answers 200 with its
+// failures in the body and publishes the reason beside it, so a status gate here
+// dropped exactly that case and left the front door's counter, failures.recent, log
+// level and activity record unmoved for a batch in which every step failed. The
+// header being present is the whole condition, and it is a stronger one: only a
+// producer that called RecordFailureReason stamps it, so ordinary 200 traffic
+// records nothing and cannot be counted as a failure.
func recordProxiedFailureReason(w http.ResponseWriter, resp *http.Response) {
- if resp.StatusCode < 400 {
- return
- }
code := strings.TrimSpace(resp.Header.Get(httpx.FailureCodeHeader))
if code == "" {
return
@@ -204,6 +205,9 @@ func recordProxiedFailureReason(w http.ResponseWriter, resp *http.Response) {
httpx.RecordFailureReason(w, code, resp.Header.Get(httpx.FailureMessageHeader))
}
+// enrichActivityFromHeaders extracts the tab id from upstream response headers and
+// enriches the activity event. It works for all response sizes, unlike body-based
+// enrichment, which is limited to small JSON responses.
func enrichActivityFromHeaders(origReq *http.Request, respHeaders http.Header) {
tabID := strings.TrimSpace(respHeaders.Get(activity.HeaderPTTabID))
if tabID != "" {