Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: record why a request failed in all three failure sinks and log 4xx/5xx at a severity an operator can route on (PIN-243)

Luigi Agostif3bcf4cb4846
diff --git a/dashboard/src/generated/types.ts b/dashboard/src/generated/types.ts
index febb96b3..94b9da4b 100644
--- a/dashboard/src/generated/types.ts
+++ b/dashboard/src/generated/types.ts
@@ -116,6 +116,8 @@ export interface ActivityLogEvent {
action?: string;
route?: RouteMetadata;
ref?: string;
+ code?: string;
+ error?: string;
}
export interface ActivityLogResponse {
events: ActivityLogEvent[];
diff --git a/internal/activity/activity.go b/internal/activity/activity.go
index 0ef47028..93507cd6 100644
--- a/internal/activity/activity.go
+++ b/internal/activity/activity.go
@@ -56,6 +56,8 @@ type Event struct {
Action string `json:"action,omitempty"`
Route *browserops.RouteMetadata `json:"route,omitempty"`
Ref string `json:"ref,omitempty"`
+ Code string `json:"code,omitempty"`
+ Error string `json:"error,omitempty"`
}
type Filter struct {
diff --git a/internal/activity/api.go b/internal/activity/api.go
index aed8456a..6a755dde 100644
--- a/internal/activity/api.go
+++ b/internal/activity/api.go
@@ -53,6 +53,8 @@ func RegisterHandlers(mux *http.ServeMux, rec Recorder) {
Action: event.Action,
Route: convertRoute(event.Route),
Ref: event.Ref,
+ Code: event.Code,
+ Error: event.Error,
})
}
diff --git a/internal/activity/context.go b/internal/activity/context.go
index 171db7bb..5754b390 100644
--- a/internal/activity/context.go
+++ b/internal/activity/context.go
@@ -81,6 +81,7 @@ func Middleware(rec Recorder, source string, next http.Handler) http.Handler {
evt := state.snapshot()
evt.Status = sw.Code
evt.DurationMs = time.Since(start).Milliseconds()
+ evt.Code, evt.Error = sw.FailureCode, sw.FailureMessage
if evt.RequestID == "" {
evt.RequestID = requestIDFor(r, sw)
}
diff --git a/internal/api/types/types.go b/internal/api/types/types.go
index 14ac8155..295d14ee 100644
--- a/internal/api/types/types.go
+++ b/internal/api/types/types.go
@@ -119,6 +119,8 @@ type ActivityLogEvent struct {
Action string `json:"action,omitempty"`
Route *RouteMetadata `json:"route,omitempty"`
Ref string `json:"ref,omitempty"`
+ Code string `json:"code,omitempty"`
+ Error string `json:"error,omitempty"`
}
type ActivityLogResponse struct {
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index f1ba2624..4e6e7a4e 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -647,9 +647,8 @@ func (h *Handlers) HandleAction(w http.ResponseWriter, r *http.Request) {
if actionErr != nil {
if strings.HasPrefix(actionErr.Error(), "unknown action") {
kinds := h.Bridge.AvailableActions()
- httpx.JSON(w, 400, map[string]string{
- "error": fmt.Sprintf("%s - valid values: %s", actionErr.Error(), strings.Join(kinds, ", ")),
- })
+ message := fmt.Sprintf("%s - valid values: %s", actionErr.Error(), strings.Join(kinds, ", "))
+ httpx.JSONError(w, 400, "unknown_action_kind", message, map[string]string{"error": message})
return
}
if errors.Is(actionErr, bridge.ErrInvalidActionRequest) {
diff --git a/internal/handlers/failure_reason_test.go b/internal/handlers/failure_reason_test.go
new file mode 100644
index 00000000..df1fafe1
--- /dev/null
+++ b/internal/handlers/failure_reason_test.go
@@ -0,0 +1,213 @@
+package handlers
+
+import (
+ "bytes"
+ "encoding/json"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/activity"
+ "github.com/pinchtab/pinchtab/internal/httpx"
+)
+
+// failingRequest drives the production middleware chain — activity outside, logging
+// inside, exactly as internal/server stacks them — around a handler that fails the way a
+// real one does, and returns what each of the three sinks recorded for that one request:
+// the server log, the activity JSONL on disk, and failures.recent.
+func failingRequest(t *testing.T, handler http.HandlerFunc) (logOutput string, activityLine map[string]any, failures []map[string]any) {
+ t.Helper()
+
+ logs := &bytes.Buffer{}
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ logDir := t.TempDir()
+ store, err := activity.NewStore(logDir, 1)
+ if err != nil {
+ t.Fatalf("activity store: %v", err)
+ }
+ resetObservabilityForTests()
+
+ chain := activity.Middleware(store, "client", LoggingMiddleware(handler))
+ req := httptest.NewRequest(http.MethodPost, "/tabs/tab1/action", strings.NewReader(`{"kind":"click","ref":"e99"}`))
+ req.Header.Set("X-PinchTab-Source", "client")
+ chain.ServeHTTP(httptest.NewRecorder(), req)
+
+ return logs.String(), lastActivityEvent(t, logDir), recentFailureEvents(t)
+}
+
+func lastActivityEvent(t *testing.T, logDir string) map[string]any {
+ t.Helper()
+
+ paths, err := filepath.Glob(filepath.Join(logDir, "events-*.jsonl"))
+ if err != nil || len(paths) == 0 {
+ t.Fatalf("no activity log written in %s (glob err %v); the sink under test recorded nothing", logDir, err)
+ }
+ body, err := os.ReadFile(paths[len(paths)-1])
+ if err != nil {
+ t.Fatal(err)
+ }
+ lines := strings.Split(strings.TrimSpace(string(body)), "\n")
+ var event map[string]any
+ if err := json.Unmarshal([]byte(lines[len(lines)-1]), &event); err != nil {
+ t.Fatalf("decode activity line %q: %v", lines[len(lines)-1], err)
+ }
+ return event
+}
+
+func recentFailureEvents(t *testing.T) []map[string]any {
+ t.Helper()
+
+ snapshot := FailureSnapshot(LayerInstance)
+ events, _ := snapshot["recent"].([]map[string]any)
+ if len(events) == 0 {
+ t.Fatal("failures.recent is empty; the request was not recorded as a failure at all")
+ }
+ return events
+}
+
+const staleRefReason = "action click: ref e99 not found - take a /snapshot first"
+
+func failWithReason(w http.ResponseWriter, _ *http.Request) {
+ httpx.ErrorCode(w, http.StatusInternalServerError, "action_failed", staleRefReason, true, nil)
+}
+
+// The defect: all three sinks recorded THAT a request failed and none recorded WHY. They
+// are asserted together from one request, because the point is that they were
+// independently reasonless — fixing one and calling it done is the failure mode.
+func TestAllThreeSinksCarryTheReasonForOneFailedRequest(t *testing.T) {
+ logs, event, failures := failingRequest(t, failWithReason)
+
+ if !strings.Contains(logs, staleRefReason) {
+ t.Errorf("the server log does not carry the reason the caller was given:\n%s", logs)
+ }
+ if !strings.Contains(logs, "action_failed") {
+ t.Errorf("the server log does not carry the error code:\n%s", logs)
+ }
+ if got, _ := event["error"].(string); got != staleRefReason {
+ t.Errorf("activity error = %q, want %q — the record carries tabId, action and ref, and was one field short of a complete one", got, staleRefReason)
+ }
+ if got, _ := event["code"].(string); got != "action_failed" {
+ t.Errorf("activity code = %q, want action_failed", got)
+ }
+ last := failures[len(failures)-1]
+ if got, _ := last["message"].(string); got != staleRefReason {
+ t.Errorf("failures.recent message = %q, want %q", got, staleRefReason)
+ }
+ if got, _ := last["code"].(string); got != "action_failed" {
+ t.Errorf("failures.recent code = %q, want action_failed", got)
+ }
+ if got, _ := last["type"].(string); got != "http_error" {
+ t.Errorf("type = %q; it names the kind of record and must not be repurposed as the reason", got)
+ }
+}
+
+// `grep level=ERROR` is what an operator actually runs, so that is what this asserts.
+func TestRequestsLogAtASeverityAnOperatorCanRouteOn(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ status int
+ wantLevel string
+ wantReason bool
+ }{
+ {name: "a server fault", status: http.StatusInternalServerError, wantLevel: "level=ERROR", wantReason: true},
+ {name: "a caller error", status: http.StatusNotFound, wantLevel: "level=WARN", wantReason: true},
+ {name: "a success", status: http.StatusOK, wantLevel: "level=INFO"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ logs := &bytes.Buffer{}
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ handler := LoggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if tc.status >= 400 {
+ httpx.ErrorCode(w, tc.status, "action_failed", staleRefReason, false, nil)
+ return
+ }
+ httpx.JSON(w, tc.status, map[string]any{"success": true})
+ }))
+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/tabs/tab1/action", nil))
+
+ line := logs.String()
+ if !strings.Contains(line, tc.wantLevel) {
+ t.Errorf("a %d logged as %q, want %s — a level-based alert cannot see it otherwise", tc.status, strings.TrimSpace(line), tc.wantLevel)
+ }
+ if tc.wantReason && !strings.Contains(line, staleRefReason) {
+ t.Errorf("a %d logged without its reason: %s", tc.status, line)
+ }
+ if !tc.wantReason && (strings.Contains(line, "error=") || strings.Contains(line, "code=")) {
+ t.Errorf("a successful request logged an error-shaped line, so every line now reads like a failure: %s", line)
+ }
+ })
+ }
+}
+
+// The sinks get the SANITIZED message — byte-identical to the response body — because the
+// producer hands over what it serialized. A page-controlled error string reaching a
+// terminal raw is the hazard ErrorCode already closed for the response; recording it
+// would have re-opened it one frame away.
+func TestTheRecordedReasonIsTheSanitizedForm(t *testing.T) {
+ const raw = "action click: \x1b[31mred\x1b[0m \x07alert"
+
+ recorder := httptest.NewRecorder()
+ logs := &bytes.Buffer{}
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ handler := LoggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ httpx.ErrorCode(w, http.StatusInternalServerError, "action_failed", raw, false, nil)
+ }))
+ handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/action", nil))
+
+ var body struct {
+ Error string `json:"error"`
+ }
+ if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil {
+ t.Fatalf("decode response: %v", err)
+ }
+ if body.Error == raw {
+ t.Fatal("the response body is the raw message, so this fixture no longer drives anything sanitizable")
+ }
+ if !strings.Contains(logs.String(), body.Error) {
+ t.Errorf("the log does not carry the sanitized message the caller received (%q):\n%s", body.Error, logs.String())
+ }
+ if strings.Contains(logs.String(), "\x1b[31m") || strings.Contains(logs.String(), "\x07") {
+ t.Errorf("the log carries the RAW page-controlled string; a sink reading it in a terminal gets what the response body was cleaned of:\n%q", logs.String())
+ }
+}
+
+// Problem-Details responses must not be the one family without a reason.
+func TestProblemResponsesRecordTheirReason(t *testing.T) {
+ sw := &httpx.StatusWriter{ResponseWriter: httptest.NewRecorder(), Code: 200}
+ httpx.Problem(sw, http.StatusConflict, "tab_locked", "tab tab1 is locked by agent-7", false, nil)
+
+ if sw.FailureCode != "tab_locked" || sw.FailureMessage != "tab tab1 is locked by agent-7" {
+ t.Errorf("Problem recorded code=%q message=%q; the Problem serialiser holds both and must hand them over like ErrorCode does", sw.FailureCode, sw.FailureMessage)
+ }
+}
+
+// The reason travels OUTWARD through every wrapper: the handler holds the innermost
+// writer, and both the logging and the activity middleware wrap their own StatusWriter
+// around it. A recorder that only stamped the writer it was handed would leave whichever
+// sink sits further out reasonless.
+func TestTheReasonReachesEveryWrappedWriter(t *testing.T) {
+ outer := &httpx.StatusWriter{ResponseWriter: httptest.NewRecorder(), Code: 200}
+ inner := &httpx.StatusWriter{ResponseWriter: outer, Code: 200}
+
+ httpx.ErrorCode(inner, http.StatusInternalServerError, "action_failed", staleRefReason, false, nil)
+
+ if outer.FailureMessage != staleRefReason {
+ t.Errorf("the outer writer recorded %q; a sink one wrapper further out stays reasonless", outer.FailureMessage)
+ }
+ if inner.FailureMessage != staleRefReason {
+ t.Errorf("the writer the producer was handed recorded %q", inner.FailureMessage)
+ }
+}
diff --git a/internal/handlers/health_tabs.go b/internal/handlers/health_tabs.go
index 83c2b8c0..9515edc8 100644
--- a/internal/handlers/health_tabs.go
+++ b/internal/handlers/health_tabs.go
@@ -15,7 +15,7 @@ type tabHandoffReader interface {
func (h *Handlers) HandleHealth(w http.ResponseWriter, r *http.Request) {
if h.Bridge == nil {
- httpx.JSON(w, 503, map[string]any{"status": "error", "reason": "bridge not initialized"})
+ writeUnavailable(w, 503, "bridge_unavailable", "bridge not initialized")
return
}
if draining, retryAfter := h.bridgeRestartStatus(); draining {
@@ -24,7 +24,9 @@ func (h *Handlers) HandleHealth(w http.ResponseWriter, r *http.Request) {
seconds = 1
}
w.Header().Set("Retry-After", fmt.Sprintf("%d", seconds))
- httpx.JSON(w, http.StatusServiceUnavailable, map[string]any{"status": "draining", "retryAfterSeconds": seconds})
+ httpx.JSONError(w, http.StatusServiceUnavailable, "browser_draining",
+ fmt.Sprintf("browser is restarting; retry after %ds", seconds),
+ map[string]any{"status": "draining", "retryAfterSeconds": seconds})
return
}
@@ -32,12 +34,12 @@ func (h *Handlers) HandleHealth(w http.ResponseWriter, r *http.Request) {
if h.writeBridgeUnavailable(w, err) {
return
}
- httpx.JSON(w, 503, map[string]any{"status": "error", "reason": fmt.Sprintf("browser initialization failed: %v", err)})
+ writeUnavailable(w, 503, "browser_init_failed", fmt.Sprintf("browser initialization failed: %v", err))
return
}
targets, err := h.Bridge.ListTargets()
if err != nil {
- httpx.JSON(w, 503, map[string]any{"status": "error", "reason": err.Error()})
+ writeUnavailable(w, 503, "list_targets_failed", err.Error())
return
}
diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.go
index 2609eb72..ab079074 100644
--- a/internal/handlers/middleware.go
+++ b/internal/handlers/middleware.go
@@ -33,6 +33,20 @@ const (
backgroundHealthHeader = "PinchTab-Background-Marker"
)
+// requestLogLevel maps the answered status onto a severity an operator can route on. Every
+// request used to log at Info, so a server returning 500s looked exactly like a healthy one
+// to any level-based alert, dashboard or log shipper — `grep level=ERROR` found nothing.
+func requestLogLevel(status int) slog.Level {
+ switch {
+ case status >= 500:
+ return slog.LevelError
+ case status >= 400:
+ return slog.LevelWarn
+ default:
+ return slog.LevelInfo
+ }
+}
+
func LoggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
@@ -50,15 +64,21 @@ func LoggingMiddleware(next http.Handler) http.Handler {
Path: r.URL.Path,
Status: sw.Code,
Type: "http_error",
+ Code: sw.FailureCode,
+ Message: sw.FailureMessage,
})
}
- slog.Info("request",
+ attrs := []any{
"requestId", w.Header().Get("X-Request-Id"),
"method", r.Method,
"path", r.URL.Path,
"status", sw.Code,
"ms", ms,
- )
+ }
+ if sw.FailureMessage != "" {
+ attrs = append(attrs, "code", sw.FailureCode, "error", sw.FailureMessage)
+ }
+ slog.Log(r.Context(), requestLogLevel(sw.Code), "request", attrs...)
})
}
diff --git a/internal/handlers/network_export.go b/internal/handlers/network_export.go
index 30e1efee..8973fa00 100644
--- a/internal/handlers/network_export.go
+++ b/internal/handlers/network_export.go
@@ -175,9 +175,10 @@ func (h *Handlers) resolveExportContext(w http.ResponseWriter, r *http.Request,
}
factory := observe.GetFormat(formatName)
if factory == nil {
- httpx.JSON(w, 400, map[string]any{
+ message := fmt.Sprintf("unknown export format %q", formatName)
+ httpx.JSONError(w, 400, "unknown_format", message, map[string]any{
"code": "unknown_format",
- "error": fmt.Sprintf("unknown export format %q", formatName),
+ "error": message,
"available": observe.ListFormats(),
})
return exportContext{}, false
diff --git a/internal/handlers/observability.go b/internal/handlers/observability.go
index 23cac0af..c550eedd 100644
--- a/internal/handlers/observability.go
+++ b/internal/handlers/observability.go
@@ -1,11 +1,13 @@
package handlers
import (
+ "net/http"
"sync"
"sync/atomic"
"time"
"github.com/pinchtab/pinchtab/internal/bridge"
+ "github.com/pinchtab/pinchtab/internal/httpx"
)
const maxRecentFailures = 20
@@ -18,6 +20,11 @@ type FailureEvent struct {
Path string `json:"path"`
Status int `json:"status"`
Type string `json:"type"`
+ // Code and Message are the reason the request failed, taken from the producer that
+ // wrote the response rather than parsed back out of it. Type stays the constant
+ // "http_error": it names the KIND of record, and never said anything about this one.
+ Code string `json:"code,omitempty"`
+ Message string `json:"message,omitempty"`
}
var (
@@ -57,7 +64,7 @@ func FailureSnapshot(layer string) map[string]any {
events := make([]map[string]any, 0, len(recent))
for _, ev := range recent {
- events = append(events, map[string]any{
+ event := map[string]any{
"time": ev.Time,
"requestId": ev.RequestID,
"method": ev.Method,
@@ -65,7 +72,12 @@ func FailureSnapshot(layer string) map[string]any {
"status": ev.Status,
"type": ev.Type,
"layer": layer,
- })
+ }
+ if ev.Message != "" {
+ event["code"] = ev.Code
+ event["message"] = ev.Message
+ }
+ events = append(events, event)
}
return map[string]any{
@@ -113,3 +125,11 @@ func resetObservabilityForTests() {
recentFailures = nil
failureMu.Unlock()
}
+
+// writeUnavailable answers the {status,reason} shape that /health and /stealth/status
+// clients parse by key, and records the reason for the failure sinks with it. The shape
+// is a client contract, which is why these do not become ordinary error envelopes; the
+// reason travels either way.
+func writeUnavailable(w http.ResponseWriter, status int, code, reason string) {
+ httpx.JSONError(w, status, code, reason, map[string]any{"status": "error", "reason": reason})
+}
diff --git a/internal/handlers/stealth_status.go b/internal/handlers/stealth_status.go
index 7be08cc7..16b27853 100644
--- a/internal/handlers/stealth_status.go
+++ b/internal/handlers/stealth_status.go
@@ -9,10 +9,7 @@ import (
func (h *Handlers) HandleStealthStatus(w http.ResponseWriter, r *http.Request) {
status := h.Bridge.StealthStatus()
if status == nil {
- httpx.JSON(w, 503, map[string]any{
- "status": "error",
- "reason": "stealth bundle unavailable",
- })
+ writeUnavailable(w, 503, "stealth_unavailable", "stealth bundle unavailable")
return
}
if tabID := r.URL.Query().Get("tabId"); tabID != "" {
diff --git a/internal/httpx/httpx.go b/internal/httpx/httpx.go
index 13c18827..b2b3677b 100644
--- a/internal/httpx/httpx.go
+++ b/internal/httpx/httpx.go
@@ -30,6 +30,15 @@ type ProblemDetails struct {
Details map[string]any `json:"details,omitempty"`
}
+// JSONError writes a hand-shaped error body — one the standard envelope cannot express,
+// such as a health payload a client parses by key — and records the reason with it. Every
+// non-2xx JSON response goes through here or through ErrorCode, so no failure reaches the
+// log, the metrics or the activity record with the response body as its only witness.
+func JSONError(w http.ResponseWriter, status int, code, message string, payload any) {
+ RecordFailureReason(w, code, SanitizeErrorMessage(message))
+ JSON(w, status, payload)
+}
+
func JSON(w http.ResponseWriter, code int, data any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
@@ -50,8 +59,10 @@ func Error(w http.ResponseWriter, code int, err error) {
}
func ErrorCode(w http.ResponseWriter, status int, code, message string, retryable bool, details map[string]any) {
+ sanitized := SanitizeErrorMessage(message)
+ RecordFailureReason(w, code, sanitized)
payload := map[string]any{
- "error": SanitizeErrorMessage(message),
+ "error": sanitized,
"code": code,
}
if retryable {
@@ -120,11 +131,13 @@ func Problem(w http.ResponseWriter, status int, code, detail string, retryable b
title = "Error"
}
+ sanitized := SanitizeErrorMessage(detail)
+ RecordFailureReason(w, code, sanitized)
payload := ProblemDetails{
Type: "about:blank",
Title: title,
Status: status,
- Detail: SanitizeErrorMessage(detail),
+ Detail: sanitized,
Code: code,
Details: sanitizeDetails(details),
}
@@ -176,9 +189,38 @@ func ExtendWriteDeadline(w http.ResponseWriter, d time.Duration) {
_ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(d))
}
+// ReasonRecorder is implemented by a ResponseWriter that keeps the reason a failure was
+// written with, so an error producer can hand it back to the middlewares wrapped around
+// it. It is what makes the reason travel from the one frame that has it to the sinks,
+// instead of leaving the response body as its only copy.
+type ReasonRecorder interface {
+ RecordFailureReason(code, message string)
+}
+
+// RecordFailureReason gives w the code and the SANITIZED message that were just written,
+// and is a no-op for a plain ResponseWriter. Producers call it; nothing re-reads the
+// response body to recover what the producer already held.
+func RecordFailureReason(w http.ResponseWriter, code, message string) {
+ if recorder, ok := w.(ReasonRecorder); ok {
+ recorder.RecordFailureReason(code, message)
+ }
+}
+
type StatusWriter struct {
http.ResponseWriter
Code int
+
+ FailureCode string
+ FailureMessage string
+}
+
+// RecordFailureReason keeps the reason and passes it OUTWARD along the wrapper chain: the
+// request path stacks several StatusWriters (activity outside, logging inside), and each
+// one is a sink that needs the same fact. Whichever writer the handler holds, every
+// StatusWriter wrapped around it ends up with the reason.
+func (w *StatusWriter) RecordFailureReason(code, message string) {
+ w.FailureCode, w.FailureMessage = code, message
+ RecordFailureReason(w.ResponseWriter, code, message)
}
func (w *StatusWriter) Unwrap() http.ResponseWriter {
diff --git a/internal/httpx/json_error_census_test.go b/internal/httpx/json_error_census_test.go
new file mode 100644
index 00000000..a2ba78fa
--- /dev/null
+++ b/internal/httpx/json_error_census_test.go
@@ -0,0 +1,99 @@
+package httpx_test
+
+import (
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/srccensus"
+)
+
+// jsonCall matches a JSON write and captures the status it answers with. The status is
+// always on the call line even when the payload spans several, which is what makes a text
+// scan honest here.
+var jsonCall = regexp.MustCompile(`\bhttpx\.JSON\(\s*[^,]+,\s*([A-Za-z0-9_.]+)`)
+
+// A failure written as a hand-shaped JSON body carries its reason nowhere: the middlewares
+// see a status and nothing else, which is the defect this closed for the ~550 responses
+// that already went through ErrorCode. httpx.JSONError is the way to keep a bespoke body
+// AND record the reason, so a non-2xx httpx.JSON call anywhere in the module is a sink
+// going dark again.
+//
+// Module-wide rather than per-package: the four files this started with were found by
+// grep, and a fifth added in a package nobody thought to list is exactly what a
+// hand-listed scope misses.
+func TestNoHandWrittenJSONErrorEscapesTheReasonRecorder(t *testing.T) {
+ root := filepath.Join("..", "..")
+ files := srccensus.Tree(t, root, 200)
+
+ total := 0
+ used := map[string]bool{}
+ var offenders []string
+ for _, file := range files {
+ for i, line := range strings.Split(file.Text, "\n") {
+ match := jsonCall.FindStringSubmatch(line)
+ if match == nil {
+ continue
+ }
+ total++
+ site := file.Name + " | " + strings.TrimSpace(line)
+ if _, exempt := variableStatusExemptions[site]; exempt {
+ used[site] = true
+ continue
+ }
+ status, known := statusOf(match[1])
+ if !known {
+ offenders = append(offenders, file.Name+":"+strconv.Itoa(i+1)+" answers a status this census cannot read ("+match[1]+"); name it so the rule keeps applying:\n\t"+strings.TrimSpace(line))
+ continue
+ }
+ if status < 400 {
+ continue
+ }
+ offenders = append(offenders, file.Name+":"+strconv.Itoa(i+1)+" writes a "+match[1]+" through httpx.JSON, so its reason reaches no sink; use httpx.JSONError(w, status, code, message, payload):\n\t"+strings.TrimSpace(line))
+ }
+ }
+
+ if total < 20 {
+ t.Fatalf("found %d httpx.JSON call sites in the module, want many more; the pattern has stopped matching and this census guards nothing", total)
+ }
+ if len(offenders) > 0 {
+ t.Errorf("%d failure response(s) bypass the reason recorder:\n %s", len(offenders), strings.Join(offenders, "\n "))
+ }
+ for site := range variableStatusExemptions {
+ if !used[site] {
+ t.Errorf("the exemption for %q matches nothing any more; delete it rather than leaving a licence for a site that moved", site)
+ }
+ }
+}
+
+// variableStatusExemptions names the call sites whose status is a variable this text scan
+// cannot resolve, with the reason each is not a failure response. Checked in both
+// directions: an entry matching nothing fails too, so a moved site loses its licence
+// instead of keeping it silently.
+var variableStatusExemptions = map[string]string{
+ "internal/orchestrator/handlers_instances.go | httpx.JSON(w, status, inst)": "auditAndRespondAttach is a success responder — both call sites pass 200 or 201, and an attach that fails is written by httpx.Error long before it is reached",
+}
+
+// statusOf resolves the literal or net/http constant a call answers with. An unrecognised
+// spelling is reported rather than skipped: a silent skip is how a census stops covering
+// the site it was written for.
+func statusOf(token string) (int, bool) {
+ if n, err := strconv.Atoi(token); err == nil {
+ return n, true
+ }
+ name, found := strings.CutPrefix(token, "http.Status")
+ if !found {
+ return 0, false
+ }
+ status, known := map[string]int{
+ "OK": 200, "Created": 201, "Accepted": 202, "NoContent": 204,
+ "BadRequest": 400, "Unauthorized": 401, "Forbidden": 403, "NotFound": 404,
+ "MethodNotAllowed": 405, "Conflict": 409, "Gone": 410, "RequestEntityTooLarge": 413,
+ "UnprocessableEntity": 422, "Locked": 423, "TooManyRequests": 429,
+ "InternalServerError": 500, "NotImplemented": 501, "BadGateway": 502,
+ "ServiceUnavailable": 503, "GatewayTimeout": 504,
+ }[name]
+ return status, known
+}