High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(observability): record a failure reason for a multi-step run, so a batch or macro whose steps failed stops reading as healthy traffic (PIN-274)
Luigi Agostifde3a4d16ec1
diff --git a/docs/endpoints.md b/docs/endpoints.mdindex 6ea8d6bc..c47e3544 100644--- a/docs/endpoints.md+++ b/docs/endpoints.md@@ -161,6 +161,19 @@ POST /evaluatePOST /tabs/{id}/evaluate```+`POST /actions` and `POST /macro` answer **200 whatever their steps did**: the envelope+reports the run, and each step's outcome is its own entry — `{"index", "success", "code",+"error"}` — beside the top-level `total`, `successful` and `failed` counts. Read `failed`+and the per-item entries; a 2xx does not mean the steps worked. A 4xx from these endpoints+is about the request itself (an empty array, a bad body, a refused capability), never about+a step.++That is a deliberate contract and it does not make a failed run invisible: a run with any+failed step publishes a failure reason on the server side, so it moves `requestsFailed`,+appears in `failures.recent` with the failed count and the first step's code and message,+logs at `WARN`, and carries `steps: {total, successful, failed}` plus the code and message+on its activity record — see [reference/metrics.md](reference/metrics.md).+`/evaluate` is intentionally separate from selector frame scope. `GET/POST /frame` only affects selector-based `/snapshot` and `/action` calls, not arbitrary JavaScript evaluation.`GET /action` decodes a subset of the action fields and refuses, with `400` naming the field, any parameter it cannot express rather than silently dropping it — so a modifier chord, a drag, `waitNav` or `humanize` must be sent as `POST /action` with a JSON body. A parameter the action request does not declare at all is refused the same way, with a `did you mean` hint for a near miss, so `?modifers=8` or `?Modifiers=8` no longer dispatches a plain click and answers `200`. The accepted set is the action request's own fields plus the parameters only the GET form carries, which today is `timeout` — a per-request action timeout in seconds, clamped to 0–60, that the POST form sends in its body instead. Cache-busters and stray parameters must be dropped from the URL.diff --git a/docs/reference/metrics.md b/docs/reference/metrics.mdindex 2edc1557..1713b33b 100644--- a/docs/reference/metrics.md+++ b/docs/reference/metrics.md@@ -33,6 +33,13 @@ Every response says which layer its numbers describe:}```+A failure is not always a 4xx. `POST /actions` and `POST /macro` answer 200 with per-item+results by design, so a run whose steps failed publishes a failure reason instead of a+status: it counts in `requestsFailed`, appears in `failures.recent` with the run's path,+`status: 200`, and a message of the form `N of M steps failed: <first step's error>`, and+logs at `WARN`. Watching `requestsFailed` therefore sees an agent whose batches are all+failing, which is the shape of traffic these endpoints exist to serve.+**The two layers are never added together.** A single `requestsTotal` covering bothwould mean two things at once, and an operator cannot act on that number: a spikewould not say whether clients are being turned away at the door or the browser isdiff --git a/internal/activity/activity.go b/internal/activity/activity.goindex 4a0c5e15..3035c293 100644--- a/internal/activity/activity.go+++ b/internal/activity/activity.go@@ -59,6 +59,20 @@ type Event struct {Ref string `json:"ref,omitempty"`Code string `json:"code,omitempty"`Error string `json:"error,omitempty"`+ // Steps is present only for a multi-step run (/actions, /macro), where the+ // request's own status says nothing about how the steps went: the envelope+ // answers 200 whatever happened inside it. The zeros inside are meaningful,+ // so the whole block is a pointer rather than three omitempty ints.+ Steps *StepCounts `json:"steps,omitempty"`+}++// StepCounts is how a multi-step run reports itself to the activity feed: the+// counts the response body already carries, recorded so a reader of the feed can+// see a run that failed entirely without re-fetching anything.+type StepCounts struct {+ Total int `json:"total"`+ Successful int `json:"successful"`+ Failed int `json:"failed"`}type Filter struct {diff --git a/internal/activity/context.go b/internal/activity/context.goindex b9362fc0..9f6128e0 100644--- a/internal/activity/context.go+++ b/internal/activity/context.go@@ -47,6 +47,7 @@ type Update struct {Action stringRoute *browserops.RouteMetadataRef string+ Steps *StepCounts}func Middleware(rec Recorder, source string, next http.Handler) http.Handler {@@ -166,6 +167,9 @@ func EnrichRequest(r *http.Request, update Update) {if update.Action != "" {state.event.Action = update.Action}+ if update.Steps != nil {+ state.event.Steps = update.Steps+ }if update.Route != nil {state.event.Route = update.Route}diff --git a/internal/handlers/actions.go b/internal/handlers/actions.goindex a8c3a479..cf5ca59d 100644--- a/internal/handlers/actions.go+++ b/internal/handlers/actions.go@@ -892,15 +892,27 @@ func (h *Handlers) writeMultiStepActionResult(route *browserops.RouteMetadata, extra map[string]any,) {successful := countSuccessful(results)+ failed := total - successfulif successful > 0 {h.maybeAutoSolve(ctx, resolvedTabID, autoSolverTriggerAction)}- h.recordActivity(r, activity.Update{Route: route})+ h.recordActivity(r, activity.Update{+ Route: route,+ Steps: &activity.StepCounts{Total: total, Successful: successful, Failed: failed},+ })+ // The envelope answers 200 whatever happened inside it — that contract is+ // deliberate and the per-item results are correct — so the failure signal+ // travels on the channel built for it instead. Metrics, slog and the activity+ // record all read the recorded reason, and without this a run in which every+ // step failed is indistinguishable from one in which every step succeeded.+ if reason := multiStepFailureReason(results, total, successful); reason.message != "" {+ httpx.RecordFailureReason(w, reason.code, reason.message)+ }resp := map[string]any{"results": results,"total": total,"successful": successful,- "failed": total - successful,+ "failed": failed,"route": route,}for k, v := range extra {@@ -909,6 +921,39 @@ func (h *Handlers) writeMultiStepActionResult(httpx.JSON(w, 200, resp)}+type multiStepFailure struct {+ code string+ message string+}++// multiStepFailureReason summarises a run for the recording channels: how many of+// how many steps failed, and the first failure's own code and message, which is+// the one a reader needs to act — the later ones are usually its consequence.+// A run with no failures records nothing, so a healthy batch stays at INFO and+// moves no counter.+func multiStepFailureReason(results []actionResult, total, successful int) multiStepFailure {+ failed := total - successful+ if failed <= 0 {+ return multiStepFailure{}+ }+ first := multiStepFailure{code: "multi_step_failed"}+ for _, result := range results {+ if result.Success {+ continue+ }+ if result.Code != "" {+ first.code = result.Code+ }+ first.message = result.Error+ break+ }+ summary := fmt.Sprintf("%d of %d steps failed", failed, total)+ if first.message != "" {+ summary += ": " + first.message+ }+ return multiStepFailure{code: first.code, message: summary}+}+func (h *Handlers) HandleMacro(w http.ResponseWriter, r *http.Request) {if !h.Config.AllowMacro {h.writeCapabilityDisabled(w, routes.CapMacro)diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.goindex a0c8bed8..2f4b0579 100644--- a/internal/handlers/middleware.go+++ b/internal/handlers/middleware.go@@ -37,17 +37,29 @@ const (// requestLogLevel maps the answered status onto a severity an operator can route on. Every// request used to log at Info, so a server returning 500s looked exactly like a healthy one// to any level-based alert, dashboard or log shipper — `grep level=ERROR` found nothing.-func requestLogLevel(status int) slog.Level {+//+// A recorded reason raises the level on its own. Some endpoints answer 200 while+// reporting a failure inside the envelope — a batch or macro run whose steps all+// failed — and keying only on the status made those runs log as healthy traffic.+func requestLogLevel(status int, reasonRecorded bool) slog.Level {switch {case status >= 500:return slog.LevelError- case status >= 400:+ case status >= 400, reasonRecorded:return slog.LevelWarndefault:return slog.LevelInfo}}+// requestFailed is the one definition of "this request failed" for the recording+// channels: the status, or a reason the handler published beside a success status.+// The producer that wrote the response is the only thing that knows the second+// case, which is why it is a recorded fact rather than something re-derived here.+func requestFailed(sw *httpx.StatusWriter) bool {+ return sw.Code >= 400 || sw.FailureMessage != ""+}+func LoggingMiddleware(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {start := time.Now()@@ -56,7 +68,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {ms := uint64(time.Since(start).Milliseconds())atomic.AddUint64(&metricRequestsTotal, 1)atomic.AddUint64(&metricRequestLatencyN, ms)- if sw.Code >= 400 {+ if requestFailed(sw) {atomic.AddUint64(&metricRequestsFailed, 1)recordFailureEvent(FailureEvent{Time: time.Now(),@@ -79,7 +91,7 @@ func LoggingMiddleware(next http.Handler) http.Handler {if sw.FailureMessage != "" {attrs = append(attrs, "code", sw.FailureCode, "error", sw.FailureMessage)}- slog.Log(r.Context(), requestLogLevel(sw.Code), "request", attrs...)+ slog.Log(r.Context(), requestLogLevel(sw.Code, sw.FailureMessage != ""), "request", attrs...)})}diff --git a/internal/handlers/multistep_failure_test.go b/internal/handlers/multistep_failure_test.gonew file mode 100644index 00000000..a793ca5e--- /dev/null+++ b/internal/handlers/multistep_failure_test.go@@ -0,0 +1,215 @@+package handlers++import (+ "bytes"+ "context"+ "encoding/json"+ "log/slog"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/activity"+ "github.com/pinchtab/pinchtab/internal/browserops"+ "github.com/pinchtab/pinchtab/internal/config"+)++// recordedRun is what the three durable channels held after one multi-step run.+// They are collected together on purpose: the defect was that all three were+// silent at once, and fixing one of them would look like progress while a run in+// which every step failed still read as healthy traffic everywhere else.+type recordedRun struct {+ channels map[string]string+ activity map[string]any+ failures []map[string]any+ failedDelta uint64+}++// runMultiStep drives the real chain — activity outside, logging inside, as+// internal/server stacks them — around the ONE shared finalizer both /actions and+// /macro reach. Macro cannot be exercised end to end with security.allowMacro off,+// which is why the surface is a parameter here rather than a second handler.+func runMultiStep(t *testing.T, path string, results []actionResult, extra map[string]any) recordedRun {+ t.Helper()++ logs := &bytes.Buffer{}+ previous := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))+ t.Cleanup(func() { slog.SetDefault(previous) })++ logDir := t.TempDir()+ store, err := activity.NewStore(logDir, 1)+ if err != nil {+ t.Fatalf("activity store: %v", err)+ }+ resetObservabilityForTests()++ h := New(&mockBridge{}, &config.RuntimeConfig{}, nil, nil, nil)+ route := &browserops.RouteMetadata{}+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ h.writeMultiStepActionResult(w, r, context.Background(), "tab1", results, len(results), route, extra)+ })++ chain := activity.Middleware(store, "client", LoggingMiddleware(handler))+ req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(`{}`))+ req.Header.Set("X-PinchTab-Source", "client")+ w := httptest.NewRecorder()+ chain.ServeHTTP(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("the shared writer answered %d; the 200-with-per-item-results contract is deliberate and this card must not change it", w.Code)+ }++ failed, _ := SnapshotMetrics()["requestsFailed"].(uint64)+ event := lastActivityEvent(t, logDir)+ encodedEvent, err := json.Marshal(event)+ if err != nil {+ t.Fatal(err)+ }+ failures := allRecentFailureEvents(t)+ encodedFailures, err := json.Marshal(failures)+ if err != nil {+ t.Fatal(err)+ }++ return recordedRun{+ channels: map[string]string{+ "the server log": logs.String(),+ "the activity record": string(encodedEvent),+ "failures.recent": string(encodedFailures),+ },+ activity: event,+ failures: failures,+ failedDelta: failed,+ }+}++// allRecentFailureEvents tolerates an empty list, unlike its sibling: the+// successful run must record nothing, and that is an assertion rather than a+// broken fixture.+func allRecentFailureEvents(t *testing.T) []map[string]any {+ t.Helper()+ snapshot := FailureSnapshot(LayerInstance)+ events, _ := snapshot["recent"].([]map[string]any)+ return events+}++func failedStep(index int, code, message string) actionResult {+ return actionResult{Index: index, Success: false, Code: code, Error: message}+}++func succeededStep(index int) actionResult {+ return actionResult{Index: index, Success: true, Result: map[string]any{"ok": true}}+}++const firstBatchFailure = "ref e94 not found - take a /snapshot first"++// The card's repro, as a differential: the same shared writer, one run in which+// every step failed and one in which every step succeeded. Every channel is+// asserted in BOTH directions from one table, so a channel added later inherits+// the rule by being added to the map rather than by being remembered.+func TestAFullyFailedMultiStepRunIsRecordedInEveryChannel(t *testing.T) {+ for _, surface := range []struct {+ name string+ path string+ extra map[string]any+ }{+ {"batch", "/tabs/tab1/actions", nil},+ {"macro", "/macro", map[string]any{"kind": "macro"}},+ } {+ t.Run(surface.name, func(t *testing.T) {+ failedRun := runMultiStep(t, surface.path, []actionResult{+ failedStep(0, "ref_not_found", firstBatchFailure),+ failedStep(1, "ref_not_found", "ref e95 not found - take a /snapshot first"),+ failedStep(2, "ref_not_found", "ref e96 not found - take a /snapshot first"),+ }, surface.extra)++ for name, envelope := range failedRun.channels {+ if !strings.Contains(envelope, "3 of 3 steps failed") {+ t.Errorf("%s does not say how many steps failed:\n%s", name, envelope)+ }+ if !strings.Contains(envelope, firstBatchFailure) {+ t.Errorf("%s does not carry the first failure's message:\n%s", name, envelope)+ }+ if !strings.Contains(envelope, "ref_not_found") {+ t.Errorf("%s does not carry the first failure's code:\n%s", name, envelope)+ }+ }+ if failedRun.failedDelta != 1 {+ t.Errorf("requestsFailed moved by %d, want 1 — an operator watching this number saw a clean record", failedRun.failedDelta)+ }+ if !strings.Contains(failedRun.channels["the server log"], "level=WARN") {+ t.Errorf("the run logged below WARN, so `grep level=WARN` finds nothing:\n%s", failedRun.channels["the server log"])+ }+ if len(failedRun.failures) == 0 {+ t.Fatal("failures.recent has no entry for a run in which every step failed")+ }+ last := failedRun.failures[len(failedRun.failures)-1]+ if got, _ := last["path"].(string); got != surface.path {+ t.Errorf("failures.recent path = %q, want %q", got, surface.path)+ }+ if got, _ := last["status"].(float64); got != 200 {+ if code, ok := last["status"].(int); !ok || code != 200 {+ t.Errorf("failures.recent status = %v, want the 200 the endpoint actually answered", last["status"])+ }+ }+ assertStepCounts(t, failedRun.activity, 3, 0, 3)++ // The other half of the differential. Without it, a change that+ // recorded every run as failed would satisfy every assertion above.+ okRun := runMultiStep(t, surface.path, []actionResult{+ succeededStep(0), succeededStep(1), succeededStep(2),+ }, surface.extra)++ for name, envelope := range okRun.channels {+ for _, forbidden := range []string{"steps failed", firstBatchFailure, "ref_not_found"} {+ if strings.Contains(envelope, forbidden) {+ t.Errorf("%s reports %q for a run in which every step succeeded:\n%s", name, forbidden, envelope)+ }+ }+ }+ if okRun.failedDelta != 0 {+ t.Errorf("a successful run moved requestsFailed by %d", okRun.failedDelta)+ }+ if len(okRun.failures) != 0 {+ t.Errorf("a successful run left %d entries in failures.recent", len(okRun.failures))+ }+ if !strings.Contains(okRun.channels["the server log"], "level=INFO") {+ t.Errorf("a successful run did not log at INFO:\n%s", okRun.channels["the server log"])+ }+ assertStepCounts(t, okRun.activity, 3, 3, 0)+ })+ }+}++// A partially failed run is still a failure: the successes already happened and+// the body carries them, but an operator watching the failure record must see that+// something did not.+func TestAPartiallyFailedMultiStepRunIsRecordedToo(t *testing.T) {+ run := runMultiStep(t, "/tabs/tab1/actions", []actionResult{+ succeededStep(0),+ failedStep(1, "action_failed", "click failed: element detached"),+ }, nil)++ if run.failedDelta != 1 {+ t.Errorf("requestsFailed moved by %d, want 1", run.failedDelta)+ }+ if !strings.Contains(run.channels["failures.recent"], "1 of 2 steps failed") {+ t.Errorf("failures.recent does not carry the partial count:\n%s", run.channels["failures.recent"])+ }+ assertStepCounts(t, run.activity, 2, 1, 1)+}++func assertStepCounts(t *testing.T, event map[string]any, total, successful, failed int) {+ t.Helper()+ steps, ok := event["steps"].(map[string]any)+ if !ok {+ t.Fatalf("the activity record carries no step counts: %v", event)+ }+ for field, want := range map[string]int{"total": total, "successful": successful, "failed": failed} {+ if got, _ := steps[field].(float64); int(got) != want {+ t.Errorf("activity steps.%s = %v, want %d", field, steps[field], want)+ }+ }+}