Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(activity): stop one failing sink from silencing the others

Luigi Agosti91bbc9df0434
diff --git a/internal/activity/activity.go b/internal/activity/activity.go
index 93507cd6..4a0c5e15 100644
--- a/internal/activity/activity.go
+++ b/internal/activity/activity.go
@@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"fmt"
+ "log/slog"
"os"
"path/filepath"
"sort"
@@ -317,8 +318,14 @@ func (s *Store) Record(evt Event) error {
}
evt.URL = sanitizeActivityURL(evt.URL)
+ // Retention is housekeeping, and housekeeping must not cost the event. A
+ // failing prune — an unreadable directory, a file another process holds —
+ // used to abort Record before the append, so a sweep that could not run threw
+ // away an observation the append would have stored perfectly well. It is also
+ // throttled: lastPruneTime advances whether or not the sweep succeeded, so
+ // exactly one event per hour was lost to a fault that had nothing to do with it.
if err := s.maybePrune(evt.Timestamp); err != nil {
- return err
+ slog.Warn("activity: retention sweep failed; the event is still recorded", "err", err)
}
// Marshal and append outside the lock: the only shared mutable state is the
diff --git a/internal/activity/context.go b/internal/activity/context.go
index 3ae74452..b801eaa9 100644
--- a/internal/activity/context.go
+++ b/internal/activity/context.go
@@ -5,9 +5,11 @@ import (
"context"
"encoding/json"
"io"
+ "log/slog"
"net/http"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/pinchtab/pinchtab/internal/authn"
@@ -52,6 +54,13 @@ func Middleware(rec Recorder, source string, next http.Handler) http.Handler {
return next
}
+ // Recording must never fail a request, so the error is swallowed — but
+ // swallowing it silently is how the activity feed goes empty with nothing
+ // anywhere saying why. Reported on the TRANSITION in each direction: a fault
+ // that lasts warns once rather than once per request, which is the volume at
+ // which an operator stops reading it, and the recovery says so too.
+ var recordingBroken atomic.Bool
+
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
sw := &httpx.StatusWriter{ResponseWriter: w, Code: 200}
@@ -94,7 +103,13 @@ func Middleware(rec Recorder, source string, next http.Handler) http.Handler {
if evt.Method == "" {
evt.Method = r.Method
}
- _ = rec.Record(evt)
+ if err := rec.Record(evt); err != nil {
+ if recordingBroken.CompareAndSwap(false, true) {
+ slog.Warn("activity: recording failed; events are being lost until this recovers", "err", err)
+ }
+ } else if recordingBroken.CompareAndSwap(true, false) {
+ slog.Info("activity: recording recovered")
+ }
})
}
diff --git a/internal/activity/sink_independence_test.go b/internal/activity/sink_independence_test.go
new file mode 100644
index 00000000..88465234
--- /dev/null
+++ b/internal/activity/sink_independence_test.go
@@ -0,0 +1,167 @@
+package activity
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+)
+
+// writeOnlyDir returns a directory that can be written into but not enumerated,
+// which is what makes the retention sweep fail while the append still works.
+// Skips where that separation does not exist.
+func writeOnlyDir(t *testing.T) string {
+ t.Helper()
+ if runtime.GOOS == "windows" {
+ t.Skip("directory read and write permissions are not separable here")
+ }
+ if os.Getuid() == 0 {
+ t.Skip("root bypasses the permission that arranges this")
+ }
+ dir := filepath.Join(t.TempDir(), "logs")
+ if err := os.Mkdir(dir, 0o300); err != nil {
+ t.Fatalf("create write-only dir: %v", err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(dir, 0o700) })
+ if _, err := os.ReadDir(dir); err == nil {
+ t.Skip("the directory is still enumerable, so this arranges no failing sweep")
+ }
+ return dir
+}
+
+// A retention sweep is housekeeping, and housekeeping must not cost the event.
+// Aborting Record on the sweep's error threw away an observation the append would
+// have stored perfectly well — and since the throttle advances whether or not the
+// sweep succeeded, the loss was one event per hour, landing on whichever request
+// happened to trip the timer.
+func TestARetentionFailureStillRecordsTheEvent(t *testing.T) {
+ dir := writeOnlyDir(t)
+ store := &Store{dir: dir, retentionDays: 1, events: EventSourceConfig{Server: true}}
+
+ if err := store.pruneExpiredFiles(time.Now().UTC()); err == nil {
+ t.Fatal("the sweep succeeded, so this test no longer arranges the failure it is about")
+ }
+
+ store.lastPruneTime = time.Time{}
+ if err := store.Record(Event{Source: "server", Method: "GET", Path: "/recorded"}); err != nil {
+ t.Fatalf("Record returned %v; a failing retention sweep must not fail the event", err)
+ }
+ if !appended(t, dir, "/recorded") {
+ t.Error("the event was not appended, so the sweep's failure cost the observation")
+ }
+}
+
+// The append genuinely works in this directory, so the assertion above is about
+// the sweep rather than about an unwritable directory.
+func TestTheWriteOnlyDirectoryStillAcceptsAppends(t *testing.T) {
+ dir := writeOnlyDir(t)
+ if err := appendJSONL(filepath.Join(dir, "probe.jsonl"), []byte(`{"path":"/probe"}`)); err != nil {
+ t.Fatalf("append into the write-only dir: %v; the retention test would pass for the wrong reason", err)
+ }
+ data, err := os.ReadFile(filepath.Join(dir, "probe.jsonl"))
+ if err != nil {
+ t.Fatalf("read back the probe: %v", err)
+ }
+ if !strings.Contains(string(data), "/probe") {
+ t.Errorf("probe file holds %q, want the appended line", data)
+ }
+}
+
+// appended reads the files Record writes BY NAME: the directory under test is
+// deliberately not enumerable, so a listing would find nothing whatever was written.
+func appended(t *testing.T, dir, path string) bool {
+ t.Helper()
+ day := time.Now().UTC().Format(time.DateOnly)
+ for _, name := range []string{"events.jsonl", "events-server-" + day + ".jsonl"} {
+ data, readErr := os.ReadFile(filepath.Join(dir, name))
+ if readErr == nil && strings.Contains(string(data), `"`+path+`"`) {
+ return true
+ }
+ }
+ return false
+}
+
+// countingHandler counts log records at or above warn, so a test can assert how
+// LOUD a sustained failure is without matching on message text.
+type countingHandler struct {
+ slog.Handler
+ warns int
+ infos int
+}
+
+func (h *countingHandler) Enabled(context.Context, slog.Level) bool { return true }
+
+func (h *countingHandler) Handle(_ context.Context, r slog.Record) error {
+ switch {
+ case r.Level >= slog.LevelWarn:
+ h.warns++
+ case r.Level >= slog.LevelInfo:
+ h.infos++
+ }
+ return nil
+}
+
+func (h *countingHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
+func (h *countingHandler) WithGroup(string) slog.Handler { return h }
+
+type toggleRecorder struct{ failing bool }
+
+func (t *toggleRecorder) Enabled() bool { return true }
+func (t *toggleRecorder) Record(Event) error {
+ if t.failing {
+ return errors.New("cannot record")
+ }
+ return nil
+}
+func (t *toggleRecorder) Query(Filter) ([]Event, error) { return nil, nil }
+
+// Recording must never fail a request, so the middleware swallows the error — but
+// swallowing it silently is how the feed goes empty with nothing saying why, and
+// warning on every request is how the warning stops being read. It reports the
+// TRANSITION, in both directions: a latch that never resets is the easy wrong fix
+// and passes a test that only fails once.
+func TestRecordingFailureIsReportedOncePerTransition(t *testing.T) {
+ handler := &countingHandler{}
+ restore := slog.Default()
+ slog.SetDefault(slog.New(handler))
+ t.Cleanup(func() { slog.SetDefault(restore) })
+
+ rec := &toggleRecorder{}
+ handlerUnderTest := Middleware(rec, "server", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {}))
+ serve := func() {
+ handlerUnderTest.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/x", nil))
+ }
+
+ rec.failing = true
+ for range 5 {
+ serve()
+ }
+ if handler.warns != 1 {
+ t.Errorf("five failing requests logged %d warnings, want 1; a per-request warning is the volume at which it stops being read", handler.warns)
+ }
+
+ rec.failing = false
+ for range 3 {
+ serve()
+ }
+ if handler.infos != 1 {
+ t.Errorf("recovery logged %d times, want 1", handler.infos)
+ }
+
+ // It must be able to degrade AGAIN. A latch that never resets would report
+ // the first fault of the process and stay quiet through every later one.
+ rec.failing = true
+ for range 3 {
+ serve()
+ }
+ if handler.warns != 2 {
+ t.Errorf("a second outage logged %d warnings in total, want 2; the report did not re-arm after recovery", handler.warns)
+ }
+}
diff --git a/internal/server/activity_recorder.go b/internal/server/activity_recorder.go
index 207d111f..15fb431e 100644
--- a/internal/server/activity_recorder.go
+++ b/internal/server/activity_recorder.go
@@ -21,16 +21,20 @@ func (r dashboardActivityRecorder) Enabled() bool {
return r.dash != nil
}
+// Record fans one event out to two INDEPENDENT sinks: the on-disk store and the
+// dashboard's live feed. Returning on the store's error skipped the broadcast, so
+// a disk fault took the live feed — which touches no disk — dark with it. Each
+// sink is now attempted regardless of the other, and the caller is told what
+// failed rather than only that something did.
func (r dashboardActivityRecorder) Record(evt activity.Event) error {
+ var err error
if r.base != nil && r.base.Enabled() {
- if err := r.base.Record(evt); err != nil {
- return err
- }
+ err = r.base.Record(evt)
}
if r.dash != nil && shouldBroadcastDashboardActivity(evt) {
r.dash.RecordActivityEvent(evt)
}
- return nil
+ return err
}
func (r dashboardActivityRecorder) Query(filter activity.Filter) ([]activity.Event, error) {
diff --git a/internal/server/activity_recorder_test.go b/internal/server/activity_recorder_test.go
index 46b955de..33072c84 100644
--- a/internal/server/activity_recorder_test.go
+++ b/internal/server/activity_recorder_test.go
@@ -1,6 +1,7 @@
package server
import (
+ "errors"
"net/http"
"testing"
"time"
@@ -76,3 +77,45 @@ func TestDashboardActivityRecorderBroadcastsClientEvents(t *testing.T) {
t.Fatalf("dashboard recent events = %d, want 1", len(dash.RecentEvents()))
}
}
+
+// failingRecorder stands in for a store that cannot reach its disk.
+type failingRecorder struct {
+ calls int
+}
+
+func (f *failingRecorder) Enabled() bool { return true }
+
+func (f *failingRecorder) Record(activity.Event) error {
+ f.calls++
+ return errors.New("disk is full")
+}
+
+func (f *failingRecorder) Query(activity.Filter) ([]activity.Event, error) { return nil, nil }
+
+// The two sinks are independent: the store writes a file, the dashboard feed
+// writes nothing at all. Returning on the store's error skipped the broadcast, so
+// a disk fault took the live feed dark with it — the one sink that could still
+// have worked, and the one an operator is watching while the disk fills.
+func TestAStoreFailureStillBroadcastsToTheDashboard(t *testing.T) {
+ base := &failingRecorder{}
+ dash := dashboard.NewDashboard(nil)
+ rec := newDashboardActivityRecorder(base, dash)
+
+ err := rec.Record(activity.Event{
+ Timestamp: time.Now().UTC(),
+ Source: "client",
+ Method: http.MethodPost,
+ Path: "/tabs/tab_1/navigate",
+ AgentID: "agent-1",
+ })
+
+ if err == nil {
+ t.Error("Record reported success while the store failed; the caller cannot tell events are being lost")
+ }
+ if base.calls != 1 {
+ t.Errorf("store was called %d times, want 1", base.calls)
+ }
+ if got := len(dash.RecentEvents()); got != 1 {
+ t.Errorf("dashboard received %d events, want 1; the live feed does not depend on the disk", got)
+ }
+}