High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(activity): stop one failing sink from silencing the others
Luigi Agosti91bbc9df0434
diff --git a/internal/activity/activity.go b/internal/activity/activity.goindex 93507cd6..4a0c5e15 100644--- a/internal/activity/activity.go+++ b/internal/activity/activity.go@@ -4,6 +4,7 @@ import ("bufio""encoding/json""fmt"+ "log/slog""os""path/filepath""sort"@@ -317,8 +318,14 @@ func (s *Store) Record(evt Event) error {}evt.URL = sanitizeActivityURL(evt.URL)+ // Retention is housekeeping, and housekeeping must not cost the event. A+ // failing prune — an unreadable directory, a file another process holds —+ // used to abort Record before the append, so a sweep that could not run threw+ // away an observation the append would have stored perfectly well. It is also+ // throttled: lastPruneTime advances whether or not the sweep succeeded, so+ // exactly one event per hour was lost to a fault that had nothing to do with it.if err := s.maybePrune(evt.Timestamp); err != nil {- return err+ slog.Warn("activity: retention sweep failed; the event is still recorded", "err", err)}// Marshal and append outside the lock: the only shared mutable state is thediff --git a/internal/activity/context.go b/internal/activity/context.goindex 3ae74452..b801eaa9 100644--- a/internal/activity/context.go+++ b/internal/activity/context.go@@ -5,9 +5,11 @@ import ("context""encoding/json""io"+ "log/slog""net/http""strings""sync"+ "sync/atomic""time""github.com/pinchtab/pinchtab/internal/authn"@@ -52,6 +54,13 @@ func Middleware(rec Recorder, source string, next http.Handler) http.Handler {return next}+ // Recording must never fail a request, so the error is swallowed — but+ // swallowing it silently is how the activity feed goes empty with nothing+ // anywhere saying why. Reported on the TRANSITION in each direction: a fault+ // that lasts warns once rather than once per request, which is the volume at+ // which an operator stops reading it, and the recovery says so too.+ var recordingBroken atomic.Bool+return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {start := time.Now()sw := &httpx.StatusWriter{ResponseWriter: w, Code: 200}@@ -94,7 +103,13 @@ func Middleware(rec Recorder, source string, next http.Handler) http.Handler {if evt.Method == "" {evt.Method = r.Method}- _ = rec.Record(evt)+ if err := rec.Record(evt); err != nil {+ if recordingBroken.CompareAndSwap(false, true) {+ slog.Warn("activity: recording failed; events are being lost until this recovers", "err", err)+ }+ } else if recordingBroken.CompareAndSwap(true, false) {+ slog.Info("activity: recording recovered")+ }})}diff --git a/internal/activity/sink_independence_test.go b/internal/activity/sink_independence_test.gonew file mode 100644index 00000000..88465234--- /dev/null+++ b/internal/activity/sink_independence_test.go@@ -0,0 +1,167 @@+package activity++import (+ "context"+ "errors"+ "log/slog"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "runtime"+ "strings"+ "testing"+ "time"+)++// writeOnlyDir returns a directory that can be written into but not enumerated,+// which is what makes the retention sweep fail while the append still works.+// Skips where that separation does not exist.+func writeOnlyDir(t *testing.T) string {+ t.Helper()+ if runtime.GOOS == "windows" {+ t.Skip("directory read and write permissions are not separable here")+ }+ if os.Getuid() == 0 {+ t.Skip("root bypasses the permission that arranges this")+ }+ dir := filepath.Join(t.TempDir(), "logs")+ if err := os.Mkdir(dir, 0o300); err != nil {+ t.Fatalf("create write-only dir: %v", err)+ }+ t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })+ if _, err := os.ReadDir(dir); err == nil {+ t.Skip("the directory is still enumerable, so this arranges no failing sweep")+ }+ return dir+}++// A retention sweep is housekeeping, and housekeeping must not cost the event.+// Aborting Record on the sweep's error threw away an observation the append would+// have stored perfectly well — and since the throttle advances whether or not the+// sweep succeeded, the loss was one event per hour, landing on whichever request+// happened to trip the timer.+func TestARetentionFailureStillRecordsTheEvent(t *testing.T) {+ dir := writeOnlyDir(t)+ store := &Store{dir: dir, retentionDays: 1, events: EventSourceConfig{Server: true}}++ if err := store.pruneExpiredFiles(time.Now().UTC()); err == nil {+ t.Fatal("the sweep succeeded, so this test no longer arranges the failure it is about")+ }++ store.lastPruneTime = time.Time{}+ if err := store.Record(Event{Source: "server", Method: "GET", Path: "/recorded"}); err != nil {+ t.Fatalf("Record returned %v; a failing retention sweep must not fail the event", err)+ }+ if !appended(t, dir, "/recorded") {+ t.Error("the event was not appended, so the sweep's failure cost the observation")+ }+}++// The append genuinely works in this directory, so the assertion above is about+// the sweep rather than about an unwritable directory.+func TestTheWriteOnlyDirectoryStillAcceptsAppends(t *testing.T) {+ dir := writeOnlyDir(t)+ if err := appendJSONL(filepath.Join(dir, "probe.jsonl"), []byte(`{"path":"/probe"}`)); err != nil {+ t.Fatalf("append into the write-only dir: %v; the retention test would pass for the wrong reason", err)+ }+ data, err := os.ReadFile(filepath.Join(dir, "probe.jsonl"))+ if err != nil {+ t.Fatalf("read back the probe: %v", err)+ }+ if !strings.Contains(string(data), "/probe") {+ t.Errorf("probe file holds %q, want the appended line", data)+ }+}++// appended reads the files Record writes BY NAME: the directory under test is+// deliberately not enumerable, so a listing would find nothing whatever was written.+func appended(t *testing.T, dir, path string) bool {+ t.Helper()+ day := time.Now().UTC().Format(time.DateOnly)+ for _, name := range []string{"events.jsonl", "events-server-" + day + ".jsonl"} {+ data, readErr := os.ReadFile(filepath.Join(dir, name))+ if readErr == nil && strings.Contains(string(data), `"`+path+`"`) {+ return true+ }+ }+ return false+}++// countingHandler counts log records at or above warn, so a test can assert how+// LOUD a sustained failure is without matching on message text.+type countingHandler struct {+ slog.Handler+ warns int+ infos int+}++func (h *countingHandler) Enabled(context.Context, slog.Level) bool { return true }++func (h *countingHandler) Handle(_ context.Context, r slog.Record) error {+ switch {+ case r.Level >= slog.LevelWarn:+ h.warns+++ case r.Level >= slog.LevelInfo:+ h.infos+++ }+ return nil+}++func (h *countingHandler) WithAttrs([]slog.Attr) slog.Handler { return h }+func (h *countingHandler) WithGroup(string) slog.Handler { return h }++type toggleRecorder struct{ failing bool }++func (t *toggleRecorder) Enabled() bool { return true }+func (t *toggleRecorder) Record(Event) error {+ if t.failing {+ return errors.New("cannot record")+ }+ return nil+}+func (t *toggleRecorder) Query(Filter) ([]Event, error) { return nil, nil }++// Recording must never fail a request, so the middleware swallows the error — but+// swallowing it silently is how the feed goes empty with nothing saying why, and+// warning on every request is how the warning stops being read. It reports the+// TRANSITION, in both directions: a latch that never resets is the easy wrong fix+// and passes a test that only fails once.+func TestRecordingFailureIsReportedOncePerTransition(t *testing.T) {+ handler := &countingHandler{}+ restore := slog.Default()+ slog.SetDefault(slog.New(handler))+ t.Cleanup(func() { slog.SetDefault(restore) })++ rec := &toggleRecorder{}+ handlerUnderTest := Middleware(rec, "server", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))+ serve := func() {+ handlerUnderTest.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/x", nil))+ }++ rec.failing = true+ for range 5 {+ serve()+ }+ if handler.warns != 1 {+ t.Errorf("five failing requests logged %d warnings, want 1; a per-request warning is the volume at which it stops being read", handler.warns)+ }++ rec.failing = false+ for range 3 {+ serve()+ }+ if handler.infos != 1 {+ t.Errorf("recovery logged %d times, want 1", handler.infos)+ }++ // It must be able to degrade AGAIN. A latch that never resets would report+ // the first fault of the process and stay quiet through every later one.+ rec.failing = true+ for range 3 {+ serve()+ }+ if handler.warns != 2 {+ t.Errorf("a second outage logged %d warnings in total, want 2; the report did not re-arm after recovery", handler.warns)+ }+}diff --git a/internal/server/activity_recorder.go b/internal/server/activity_recorder.goindex 207d111f..15fb431e 100644--- a/internal/server/activity_recorder.go+++ b/internal/server/activity_recorder.go@@ -21,16 +21,20 @@ func (r dashboardActivityRecorder) Enabled() bool {return r.dash != nil}+// Record fans one event out to two INDEPENDENT sinks: the on-disk store and the+// dashboard's live feed. Returning on the store's error skipped the broadcast, so+// a disk fault took the live feed — which touches no disk — dark with it. Each+// sink is now attempted regardless of the other, and the caller is told what+// failed rather than only that something did.func (r dashboardActivityRecorder) Record(evt activity.Event) error {+ var err errorif r.base != nil && r.base.Enabled() {- if err := r.base.Record(evt); err != nil {- return err- }+ err = r.base.Record(evt)}if r.dash != nil && shouldBroadcastDashboardActivity(evt) {r.dash.RecordActivityEvent(evt)}- return nil+ return err}func (r dashboardActivityRecorder) Query(filter activity.Filter) ([]activity.Event, error) {diff --git a/internal/server/activity_recorder_test.go b/internal/server/activity_recorder_test.goindex 46b955de..33072c84 100644--- a/internal/server/activity_recorder_test.go+++ b/internal/server/activity_recorder_test.go@@ -1,6 +1,7 @@package serverimport (+ "errors""net/http""testing""time"@@ -76,3 +77,45 @@ func TestDashboardActivityRecorderBroadcastsClientEvents(t *testing.T) {t.Fatalf("dashboard recent events = %d, want 1", len(dash.RecentEvents()))}}++// failingRecorder stands in for a store that cannot reach its disk.+type failingRecorder struct {+ calls int+}++func (f *failingRecorder) Enabled() bool { return true }++func (f *failingRecorder) Record(activity.Event) error {+ f.calls+++ return errors.New("disk is full")+}++func (f *failingRecorder) Query(activity.Filter) ([]activity.Event, error) { return nil, nil }++// The two sinks are independent: the store writes a file, the dashboard feed+// writes nothing at all. Returning on the store's error skipped the broadcast, so+// a disk fault took the live feed dark with it — the one sink that could still+// have worked, and the one an operator is watching while the disk fills.+func TestAStoreFailureStillBroadcastsToTheDashboard(t *testing.T) {+ base := &failingRecorder{}+ dash := dashboard.NewDashboard(nil)+ rec := newDashboardActivityRecorder(base, dash)++ err := rec.Record(activity.Event{+ Timestamp: time.Now().UTC(),+ Source: "client",+ Method: http.MethodPost,+ Path: "/tabs/tab_1/navigate",+ AgentID: "agent-1",+ })++ if err == nil {+ t.Error("Record reported success while the store failed; the caller cannot tell events are being lost")+ }+ if base.calls != 1 {+ t.Errorf("store was called %d times, want 1", base.calls)+ }+ if got := len(dash.RecentEvents()); got != 1 {+ t.Errorf("dashboard received %d events, want 1; the live feed does not depend on the disk", got)+ }+}