Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep slow log viewers off the synchronous logging path
pulse-triage[bot]ca0d4fbdce90
diff --git a/internal/logging/broadcast.go b/internal/logging/broadcast.goindex 5f2a8c29e3..c574f16318 100644--- a/internal/logging/broadcast.go+++ b/internal/logging/broadcast.go@@ -2,9 +2,6 @@ package loggingimport ("container/ring"- "fmt"- "io"- "os""sync""github.com/google/uuid"@@ -20,9 +17,8 @@ const ()var (- broadcaster *LogBroadcaster- broadcastMu sync.Once- broadcastWarnWriter io.Writer = os.Stderr+ broadcaster *LogBroadcaster+ broadcastMu sync.Once)// LogBroadcaster captures log writes, buffers them, and broadcasts them to subscribers.@@ -46,13 +42,12 @@ func GetBroadcaster() *LogBroadcaster {// Write implements io.Writer. It writes to the internal buffer and notifies subscribers.func (b *LogBroadcaster) Write(p []byte) (n int, err error) {// Copy at most a bounded prefix so oversized log lines cannot inflate memory.- msg := string(p)+ var msg stringif len(p) > maxBroadcastMessageBytes {keep := maxBroadcastMessageBytes - len(broadcastTruncationTag)- if keep < 0 {- keep = 0- }msg = string(p[:keep]) + broadcastTruncationTag+ } else {+ msg = string(p)}b.mu.Lock()@@ -63,20 +58,15 @@ func (b *LogBroadcaster) Write(p []byte) (n int, err error) {b.buffer = b.buffer.Next()// 2. Broadcast to subscribers- for subscriberID, ch := range b.subscribers {+ for _, ch := range b.subscribers {select {case ch <- msg:// Sent successfullydefault:- // Channel blocked, too slow consumer.- // In a real production system we might drop the client,- // here we just skip this message for them to avoid blocking writer.- // Ideally we should warn or close their channel.- fmt.Fprintf(- broadcastWarnWriter,- "logging: subscriber_blocked subscriber_id=%s action=drop_message\n",- subscriberID,- )+ // A passive log viewer must not stall the producer or other viewers.+ // Skip only this delivery; bounded history and normal log sinks are+ // independent. Do not warn here: stderr can block too, and+ // doing so under this lock would also stall subscribe and shutdown.}}diff --git a/internal/logging/broadcast_test.go b/internal/logging/broadcast_test.goindex 6b0e927d9d..fd8da37786 100644--- a/internal/logging/broadcast_test.go+++ b/internal/logging/broadcast_test.go@@ -1,8 +1,20 @@package loggingimport (+ "bytes""container/ring"+ "context"+ "io"+ "os"+ "os/exec"+ "slices"+ "strconv"+ "strings"+ "sync""testing"+ "time"++ "github.com/rs/zerolog")func newTestBroadcaster(bufferSize int) *LogBroadcaster {@@ -136,6 +148,172 @@ func TestLogBroadcasterGetHistoryAfterWrap(t *testing.T) {}}+// Run the writer in a child whose stderr is an unread pipe. The configured+// sink remains writable. Any per-drop stderr diagnostic fills that pipe and+// stalls the logging caller, even though the viewer's channel is nonblocking.+// The timeout kills and reaps only this test child, not a service or a guest.+func TestLogBroadcasterSlowViewerDoesNotWriteStderr(t *testing.T) {+ const childEnv = "PULSE_TEST_SLOW_LOG_VIEWER_CHILD"+ if os.Getenv(childEnv) == "1" {+ b := newTestBroadcaster(4)+ fast := make(chan string, 1)+ slow := make(chan string, 1)+ slow <- "queued"+ b.subscribers["fast"] = fast+ b.subscribers["slow"] = slow++ var sink bytes.Buffer+ logger := zerolog.New(io.MultiWriter(&sink, b))+ originalLevel := zerolog.GlobalLevel()+ defer zerolog.SetGlobalLevel(originalLevel)+ zerolog.SetGlobalLevel(zerolog.DebugLevel)+ const line = "{\"level\":\"debug\",\"message\":\"normal polling fixture\"}\n"+ const records = 32768+ for i := 0; i < records; i++ {+ logger.Debug().Msg("normal polling fixture")+ select {+ case got := <-fast:+ if got != line {+ t.Fatalf("fast viewer received %q, want %q", got, line)+ }+ default:+ t.Fatal("slow viewer prevented delivery to fast viewer")+ }+ }+ if got, want := sink.String(), strings.Repeat(line, records); got != want {+ t.Fatal("slow viewer changed the configured log sink")+ }+ if got := <-slow; got != "queued" {+ t.Fatalf("slow viewer's queue was changed: %q", got)+ }+ if got, want := b.GetHistory(), []string{line, line, line, line}; !slices.Equal(got, want) {+ t.Fatalf("slow viewer changed bounded history: %#v", got)+ }+ b.Shutdown()+ return+ }++ readPipe, writePipe, err := os.Pipe()+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ _ = writePipe.Close()+ _ = readPipe.Close()+ })++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestLogBroadcasterSlowViewerDoesNotWriteStderr$", "-test.v")+ cmd.Env = append(os.Environ(), childEnv+"=1")+ var output bytes.Buffer+ cmd.Stdout = &output+ cmd.Stderr = writePipe+ cmd.WaitDelay = 2 * time.Second+ runErr := cmd.Run()+ _ = writePipe.Close()+ stderr, readErr := io.ReadAll(readPipe)+ if runErr != nil {+ t.Fatalf("logging caller stalled or child failed: %v (deadline: %v, stderr bytes: %d)\n%s", runErr, ctx.Err(), len(stderr), output.String())+ }+ if readErr != nil {+ t.Fatal(readErr)+ }+ if len(stderr) != 0 {+ t.Fatalf("dropping viewer deliveries wrote %d bytes to stderr", len(stderr))+ }+}++func TestLogBroadcasterWriteBoundsAndCopiesMessages(t *testing.T) {+ for _, size := range []int{0, 1, maxBroadcastMessageBytes, maxBroadcastMessageBytes + 1, 1024 * 1024} {+ t.Run(strconv.Itoa(size), func(t *testing.T) {+ b := newTestBroadcaster(2)+ ch := make(chan string, 1)+ b.subscribers["viewer"] = ch+ payload := bytes.Repeat([]byte{'x'}, size)+ want := string(payload)+ if size > maxBroadcastMessageBytes {+ want = want[:maxBroadcastMessageBytes-len(broadcastTruncationTag)] + broadcastTruncationTag+ }+ n, err := b.Write(payload)+ if err != nil || n != size {+ t.Fatalf("Write = %d, %v; want %d, nil", n, err, size)+ }+ for i := range payload {+ payload[i] = 'y'+ }+ if got := <-ch; got != want {+ t.Fatalf("viewer message size/content changed: length %d, want %d", len(got), len(want))+ }+ if got := b.GetHistory(); !slices.Equal(got, []string{want}) {+ t.Fatal("history did not retain the bounded independent copy")+ }+ })+ }+}++func TestLogBroadcasterSlowViewerDoesNotBlockConcurrentLifecycle(t *testing.T) {+ b := newTestBroadcaster(4)+ _, slow, _ := b.Subscribe()+ for i := 0; i < cap(slow); i++ {+ slow <- "queued"+ }++ var writers sync.WaitGroup+ for i := 0; i < 4; i++ {+ writers.Go(func() {+ for i := 0; i < 1000; i++ {+ n, err := b.Write([]byte("poll"))+ if err != nil || n != len("poll") {+ t.Errorf("Write = %d, %v", n, err)+ }+ }+ })+ }+ for i := 0; i < 2; i++ {+ writers.Go(func() {+ for i := 0; i < 100; i++ {+ id, _, _ := b.Subscribe()+ b.GetHistory()+ b.Unsubscribe(id)+ b.Shutdown()+ }+ })+ }+ writers.Wait()+ b.Shutdown()+ if len(b.subscribers) != 0 {+ t.Fatal("subscriber registrations survived shutdown")+ }+ for range slow {+ }+ if _, err := b.Write([]byte("after shutdown")); err != nil {+ t.Fatal(err)+ }+ if history := b.GetHistory(); history[len(history)-1] != "after shutdown" {+ t.Fatal("shutdown lost subsequent history")+ }+}++func BenchmarkLogBroadcasterOversizedMessage(b *testing.B) {+ writer := newTestBroadcaster(4)+ payload := bytes.Repeat([]byte{'x'}, 1024*1024)+ b.ReportAllocs()+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ _, _ = writer.Write(payload)+ }+}++func TestLogBroadcasterOversizedMessageAllocationBound(t *testing.T) {+ measured := testing.Benchmark(BenchmarkLogBroadcasterOversizedMessage)+ // Leave room for bookkeeping while rejecting a copy of the 1 MiB input.+ // This is an allocation bound, not a timing or process-RSS assertion.+ if got, limit := measured.AllocedBytesPerOp(), int64(2*maxBroadcastMessageBytes); got > limit {+ t.Fatalf("broadcaster allocated %d bytes per capped record, want <= %d", got, limit)+ }+}+func TestGlobalLevelSetAndGet(t *testing.T) {t.Cleanup(resetLoggingState)