Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(observability): record a failure reason for a multi-step run, so a batch or macro whose steps failed stops reading as healthy traffic (PIN-274)

Luigi Agostifde3a4d16ec1
diff --git a/docs/endpoints.md b/docs/endpoints.md
index 6ea8d6bc..c47e3544 100644
--- a/docs/endpoints.md
+++ b/docs/endpoints.md
@@ -161,6 +161,19 @@ POST /evaluate
POST /tabs/{id}/evaluate
```
+`POST /actions` and `POST /macro` answer **200 whatever their steps did**: the envelope
+reports the run, and each step's outcome is its own entry — `{"index", "success", "code",
+"error"}` — beside the top-level `total`, `successful` and `failed` counts. Read `failed`
+and the per-item entries; a 2xx does not mean the steps worked. A 4xx from these endpoints
+is about the request itself (an empty array, a bad body, a refused capability), never about
+a step.
+
+That is a deliberate contract and it does not make a failed run invisible: a run with any
+failed step publishes a failure reason on the server side, so it moves `requestsFailed`,
+appears in `failures.recent` with the failed count and the first step's code and message,
+logs at `WARN`, and carries `steps: {total, successful, failed}` plus the code and message
+on its activity record — see [reference/metrics.md](reference/metrics.md).
+
`/evaluate` is intentionally separate from selector frame scope. `GET/POST /frame` only affects selector-based `/snapshot` and `/action` calls, not arbitrary JavaScript evaluation.
`GET /action` decodes a subset of the action fields and refuses, with `400` naming the field, any parameter it cannot express rather than silently dropping it — so a modifier chord, a drag, `waitNav` or `humanize` must be sent as `POST /action` with a JSON body. A parameter the action request does not declare at all is refused the same way, with a `did you mean` hint for a near miss, so `?modifers=8` or `?Modifiers=8` no longer dispatches a plain click and answers `200`. The accepted set is the action request's own fields plus the parameters only the GET form carries, which today is `timeout` — a per-request action timeout in seconds, clamped to 0–60, that the POST form sends in its body instead. Cache-busters and stray parameters must be dropped from the URL.
diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md
index 2edc1557..1713b33b 100644
--- a/docs/reference/metrics.md
+++ b/docs/reference/metrics.md
@@ -33,6 +33,13 @@ Every response says which layer its numbers describe:
}
```
+A failure is not always a 4xx. `POST /actions` and `POST /macro` answer 200 with per-item
+results by design, so a run whose steps failed publishes a failure reason instead of a
+status: it counts in `requestsFailed`, appears in `failures.recent` with the run's path,
+`status: 200`, and a message of the form `N of M steps failed: <first step's error>`, and
+logs at `WARN`. Watching `requestsFailed` therefore sees an agent whose batches are all
+failing, which is the shape of traffic these endpoints exist to serve.
+
**The two layers are never added together.** A single `requestsTotal` covering both
would mean two things at once, and an operator cannot act on that number: a spike
would not say whether clients are being turned away at the door or the browser is
diff --git a/internal/activity/activity.go b/internal/activity/activity.go
index 4a0c5e15..3035c293 100644
--- a/internal/activity/activity.go
+++ b/internal/activity/activity.go
@@ -59,6 +59,20 @@ type Event struct {
Ref string `json:"ref,omitempty"`
Code string `json:"code,omitempty"`
Error string `json:"error,omitempty"`
+ // Steps is present only for a multi-step run (/actions, /macro), where the
+ // request's own status says nothing about how the steps went: the envelope
+ // answers 200 whatever happened inside it. The zeros inside are meaningful,
+ // so the whole block is a pointer rather than three omitempty ints.
+ Steps *StepCounts `json:"steps,omitempty"`
+}
+
+// StepCounts is how a multi-step run reports itself to the activity feed: the
+// counts the response body already carries, recorded so a reader of the feed can
+// see a run that failed entirely without re-fetching anything.
+type StepCounts struct {
+ Total int `json:"total"`
+ Successful int `json:"successful"`
+ Failed int `json:"failed"`
}
type Filter struct {
diff --git a/internal/activity/context.go b/internal/activity/context.go
index b9362fc0..9f6128e0 100644
--- a/internal/activity/context.go
+++ b/internal/activity/context.go
@@ -47,6 +47,7 @@ type Update struct {
Action string
Route *browserops.RouteMetadata
Ref string
+ Steps *StepCounts
}
func Middleware(rec Recorder, source string, next http.Handler) http.Handler {
@@ -166,6 +167,9 @@ func EnrichRequest(r *http.Request, update Update) {
if update.Action != "" {
state.event.Action = update.Action
}
+ if update.Steps != nil {
+ state.event.Steps = update.Steps
+ }
if update.Route != nil {
state.event.Route = update.Route
}
diff --git a/internal/handlers/actions.go b/internal/handlers/actions.go
index a8c3a479..cf5ca59d 100644
--- a/internal/handlers/actions.go
+++ b/internal/handlers/actions.go
@@ -892,15 +892,27 @@ func (h *Handlers) writeMultiStepActionResult(
route *browserops.RouteMetadata, extra map[string]any,
) {
successful := countSuccessful(results)
+ failed := total - successful
if successful > 0 {
h.maybeAutoSolve(ctx, resolvedTabID, autoSolverTriggerAction)
}
- h.recordActivity(r, activity.Update{Route: route})
+ h.recordActivity(r, activity.Update{
+ Route: route,
+ Steps: &activity.StepCounts{Total: total, Successful: successful, Failed: failed},
+ })
+ // The envelope answers 200 whatever happened inside it — that contract is
+ // deliberate and the per-item results are correct — so the failure signal
+ // travels on the channel built for it instead. Metrics, slog and the activity
+ // record all read the recorded reason, and without this a run in which every
+ // step failed is indistinguishable from one in which every step succeeded.
+ if reason := multiStepFailureReason(results, total, successful); reason.message != "" {
+ httpx.RecordFailureReason(w, reason.code, reason.message)
+ }
resp := map[string]any{
"results": results,
"total": total,
"successful": successful,
- "failed": total - successful,
+ "failed": failed,
"route": route,
}
for k, v := range extra {
@@ -909,6 +921,39 @@ func (h *Handlers) writeMultiStepActionResult(
httpx.JSON(w, 200, resp)
}
+type multiStepFailure struct {
+ code string
+ message string
+}
+
+// multiStepFailureReason summarises a run for the recording channels: how many of
+// how many steps failed, and the first failure's own code and message, which is
+// the one a reader needs to act — the later ones are usually its consequence.
+// A run with no failures records nothing, so a healthy batch stays at INFO and
+// moves no counter.
+func multiStepFailureReason(results []actionResult, total, successful int) multiStepFailure {
+ failed := total - successful
+ if failed <= 0 {
+ return multiStepFailure{}
+ }
+ first := multiStepFailure{code: "multi_step_failed"}
+ for _, result := range results {
+ if result.Success {
+ continue
+ }
+ if result.Code != "" {
+ first.code = result.Code
+ }
+ first.message = result.Error
+ break
+ }
+ summary := fmt.Sprintf("%d of %d steps failed", failed, total)
+ if first.message != "" {
+ summary += ": " + first.message
+ }
+ return multiStepFailure{code: first.code, message: summary}
+}
+
func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {
if !h.Config.AllowMacro {
h.writeCapabilityDisabled(w, routes.CapMacro)
diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.go
index a0c8bed8..2f4b0579 100644
--- a/internal/handlers/middleware.go
+++ b/internal/handlers/middleware.go
@@ -37,17 +37,29 @@ const (
// requestLogLevel maps the answered status onto a severity an operator can route on. Every
// request used to log at Info, so a server returning 500s looked exactly like a healthy one
// to any level-based alert, dashboard or log shipper — `grep level=ERROR` found nothing.
-func requestLogLevel(status int) slog.Level {
+//
+// A recorded reason raises the level on its own. Some endpoints answer 200 while
+// reporting a failure inside the envelope — a batch or macro run whose steps all
+// failed — and keying only on the status made those runs log as healthy traffic.
+func requestLogLevel(status int, reasonRecorded bool) slog.Level {
switch {
case status >= 500:
return slog.LevelError
- case status >= 400:
+ case status >= 400, reasonRecorded:
return slog.LevelWarn
default:
return slog.LevelInfo
}
}
+// requestFailed is the one definition of "this request failed" for the recording
+// channels: the status, or a reason the handler published beside a success status.
+// The producer that wrote the response is the only thing that knows the second
+// case, which is why it is a recorded fact rather than something re-derived here.
+func requestFailed(sw *httpx.StatusWriter) bool {
+ return sw.Code >= 400 || sw.FailureMessage != ""
+}
+
func LoggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
@@ -56,7 +68,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {
ms := uint64(time.Since(start).Milliseconds())
atomic.AddUint64(&metricRequestsTotal, 1)
atomic.AddUint64(&metricRequestLatencyN, ms)
- if sw.Code >= 400 {
+ if requestFailed(sw) {
atomic.AddUint64(&metricRequestsFailed, 1)
recordFailureEvent(FailureEvent{
Time: time.Now(),
@@ -79,7 +91,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {
if sw.FailureMessage != "" {
attrs = append(attrs, "code", sw.FailureCode, "error", sw.FailureMessage)
}
- slog.Log(r.Context(), requestLogLevel(sw.Code), "request", attrs...)
+ slog.Log(r.Context(), requestLogLevel(sw.Code, sw.FailureMessage != ""), "request", attrs...)
})
}
diff --git a/internal/handlers/multistep_failure_test.go b/internal/handlers/multistep_failure_test.go
new file mode 100644
index 00000000..a793ca5e
--- /dev/null
+++ b/internal/handlers/multistep_failure_test.go
@@ -0,0 +1,215 @@
+package handlers
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/activity"
+ "github.com/pinchtab/pinchtab/internal/browserops"
+ "github.com/pinchtab/pinchtab/internal/config"
+)
+
+// recordedRun is what the three durable channels held after one multi-step run.
+// They are collected together on purpose: the defect was that all three were
+// silent at once, and fixing one of them would look like progress while a run in
+// which every step failed still read as healthy traffic everywhere else.
+type recordedRun struct {
+ channels map[string]string
+ activity map[string]any
+ failures []map[string]any
+ failedDelta uint64
+}
+
+// runMultiStep drives the real chain — activity outside, logging inside, as
+// internal/server stacks them — around the ONE shared finalizer both /actions and
+// /macro reach. Macro cannot be exercised end to end with security.allowMacro off,
+// which is why the surface is a parameter here rather than a second handler.
+func runMultiStep(t *testing.T, path string, results []actionResult, extra map[string]any) recordedRun {
+ t.Helper()
+
+ logs := &bytes.Buffer{}
+ previous := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previous) })
+
+ logDir := t.TempDir()
+ store, err := activity.NewStore(logDir, 1)
+ if err != nil {
+ t.Fatalf("activity store: %v", err)
+ }
+ resetObservabilityForTests()
+
+ h := New(&mockBridge{}, &config.RuntimeConfig{}, nil, nil, nil)
+ route := &browserops.RouteMetadata{}
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ h.writeMultiStepActionResult(w, r, context.Background(), "tab1", results, len(results), route, extra)
+ })
+
+ chain := activity.Middleware(store, "client", LoggingMiddleware(handler))
+ req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`))
+ req.Header.Set("X-PinchTab-Source", "client")
+ w := httptest.NewRecorder()
+ chain.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("the shared writer answered %d; the 200-with-per-item-results contract is deliberate and this card must not change it", w.Code)
+ }
+
+ failed, _ := SnapshotMetrics()["requestsFailed"].(uint64)
+ event := lastActivityEvent(t, logDir)
+ encodedEvent, err := json.Marshal(event)
+ if err != nil {
+ t.Fatal(err)
+ }
+ failures := allRecentFailureEvents(t)
+ encodedFailures, err := json.Marshal(failures)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ return recordedRun{
+ channels: map[string]string{
+ "the server log": logs.String(),
+ "the activity record": string(encodedEvent),
+ "failures.recent": string(encodedFailures),
+ },
+ activity: event,
+ failures: failures,
+ failedDelta: failed,
+ }
+}
+
+// allRecentFailureEvents tolerates an empty list, unlike its sibling: the
+// successful run must record nothing, and that is an assertion rather than a
+// broken fixture.
+func allRecentFailureEvents(t *testing.T) []map[string]any {
+ t.Helper()
+ snapshot := FailureSnapshot(LayerInstance)
+ events, _ := snapshot["recent"].([]map[string]any)
+ return events
+}
+
+func failedStep(index int, code, message string) actionResult {
+ return actionResult{Index: index, Success: false, Code: code, Error: message}
+}
+
+func succeededStep(index int) actionResult {
+ return actionResult{Index: index, Success: true, Result: map[string]any{"ok": true}}
+}
+
+const firstBatchFailure = "ref e94 not found - take a /snapshot first"
+
+// The card's repro, as a differential: the same shared writer, one run in which
+// every step failed and one in which every step succeeded. Every channel is
+// asserted in BOTH directions from one table, so a channel added later inherits
+// the rule by being added to the map rather than by being remembered.
+func TestAFullyFailedMultiStepRunIsRecordedInEveryChannel(t *testing.T) {
+ for _, surface := range []struct {
+ name string
+ path string
+ extra map[string]any
+ }{
+ {"batch", "/tabs/tab1/actions", nil},
+ {"macro", "/macro", map[string]any{"kind": "macro"}},
+ } {
+ t.Run(surface.name, func(t *testing.T) {
+ failedRun := runMultiStep(t, surface.path, []actionResult{
+ failedStep(0, "ref_not_found", firstBatchFailure),
+ failedStep(1, "ref_not_found", "ref e95 not found - take a /snapshot first"),
+ failedStep(2, "ref_not_found", "ref e96 not found - take a /snapshot first"),
+ }, surface.extra)
+
+ for name, envelope := range failedRun.channels {
+ if !strings.Contains(envelope, "3 of 3 steps failed") {
+ t.Errorf("%s does not say how many steps failed:\n%s", name, envelope)
+ }
+ if !strings.Contains(envelope, firstBatchFailure) {
+ t.Errorf("%s does not carry the first failure's message:\n%s", name, envelope)
+ }
+ if !strings.Contains(envelope, "ref_not_found") {
+ t.Errorf("%s does not carry the first failure's code:\n%s", name, envelope)
+ }
+ }
+ if failedRun.failedDelta != 1 {
+ t.Errorf("requestsFailed moved by %d, want 1 — an operator watching this number saw a clean record", failedRun.failedDelta)
+ }
+ if !strings.Contains(failedRun.channels["the server log"], "level=WARN") {
+ t.Errorf("the run logged below WARN, so `grep level=WARN` finds nothing:\n%s", failedRun.channels["the server log"])
+ }
+ if len(failedRun.failures) == 0 {
+ t.Fatal("failures.recent has no entry for a run in which every step failed")
+ }
+ last := failedRun.failures[len(failedRun.failures)-1]
+ if got, _ := last["path"].(string); got != surface.path {
+ t.Errorf("failures.recent path = %q, want %q", got, surface.path)
+ }
+ if got, _ := last["status"].(float64); got != 200 {
+ if code, ok := last["status"].(int); !ok || code != 200 {
+ t.Errorf("failures.recent status = %v, want the 200 the endpoint actually answered", last["status"])
+ }
+ }
+ assertStepCounts(t, failedRun.activity, 3, 0, 3)
+
+ // The other half of the differential. Without it, a change that
+ // recorded every run as failed would satisfy every assertion above.
+ okRun := runMultiStep(t, surface.path, []actionResult{
+ succeededStep(0), succeededStep(1), succeededStep(2),
+ }, surface.extra)
+
+ for name, envelope := range okRun.channels {
+ for _, forbidden := range []string{"steps failed", firstBatchFailure, "ref_not_found"} {
+ if strings.Contains(envelope, forbidden) {
+ t.Errorf("%s reports %q for a run in which every step succeeded:\n%s", name, forbidden, envelope)
+ }
+ }
+ }
+ if okRun.failedDelta != 0 {
+ t.Errorf("a successful run moved requestsFailed by %d", okRun.failedDelta)
+ }
+ if len(okRun.failures) != 0 {
+ t.Errorf("a successful run left %d entries in failures.recent", len(okRun.failures))
+ }
+ if !strings.Contains(okRun.channels["the server log"], "level=INFO") {
+ t.Errorf("a successful run did not log at INFO:\n%s", okRun.channels["the server log"])
+ }
+ assertStepCounts(t, okRun.activity, 3, 3, 0)
+ })
+ }
+}
+
+// A partially failed run is still a failure: the successes already happened and
+// the body carries them, but an operator watching the failure record must see that
+// something did not.
+func TestAPartiallyFailedMultiStepRunIsRecordedToo(t *testing.T) {
+ run := runMultiStep(t, "/tabs/tab1/actions", []actionResult{
+ succeededStep(0),
+ failedStep(1, "action_failed", "click failed: element detached"),
+ }, nil)
+
+ if run.failedDelta != 1 {
+ t.Errorf("requestsFailed moved by %d, want 1", run.failedDelta)
+ }
+ if !strings.Contains(run.channels["failures.recent"], "1 of 2 steps failed") {
+ t.Errorf("failures.recent does not carry the partial count:\n%s", run.channels["failures.recent"])
+ }
+ assertStepCounts(t, run.activity, 2, 1, 1)
+}
+
+func assertStepCounts(t *testing.T, event map[string]any, total, successful, failed int) {
+ t.Helper()
+ steps, ok := event["steps"].(map[string]any)
+ if !ok {
+ t.Fatalf("the activity record carries no step counts: %v", event)
+ }
+ for field, want := range map[string]int{"total": total, "successful": successful, "failed": failed} {
+ if got, _ := steps[field].(float64); int(got) != want {
+ t.Errorf("activity steps.%s = %v, want %d", field, steps[field], want)
+ }
+ }
+}