Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep slow log viewers off the synchronous logging path

pulse-triage[bot]ca0d4fbdce90
diff --git a/internal/logging/broadcast.go b/internal/logging/broadcast.go
index 5f2a8c29e3..c574f16318 100644
--- a/internal/logging/broadcast.go
+++ b/internal/logging/broadcast.go
@@ -2,9 +2,6 @@ package logging
import (
"container/ring"
- "fmt"
- "io"
- "os"
"sync"
"github.com/google/uuid"
@@ -20,9 +17,8 @@ const (
)
var (
- broadcaster *LogBroadcaster
- broadcastMu sync.Once
- broadcastWarnWriter io.Writer = os.Stderr
+ broadcaster *LogBroadcaster
+ broadcastMu sync.Once
)
// LogBroadcaster captures log writes, buffers them, and broadcasts them to subscribers.
@@ -46,13 +42,12 @@ func GetBroadcaster() *LogBroadcaster {
// Write implements io.Writer. It writes to the internal buffer and notifies subscribers.
func (b *LogBroadcaster) Write(p []byte) (n int, err error) {
// Copy at most a bounded prefix so oversized log lines cannot inflate memory.
- msg := string(p)
+ var msg string
if len(p) > maxBroadcastMessageBytes {
keep := maxBroadcastMessageBytes - len(broadcastTruncationTag)
- if keep < 0 {
- keep = 0
- }
msg = string(p[:keep]) + broadcastTruncationTag
+ } else {
+ msg = string(p)
}
b.mu.Lock()
@@ -63,20 +58,15 @@ func (b *LogBroadcaster) Write(p []byte) (n int, err error) {
b.buffer = b.buffer.Next()
// 2. Broadcast to subscribers
- for subscriberID, ch := range b.subscribers {
+ for _, ch := range b.subscribers {
select {
case ch <- msg:
// Sent successfully
default:
- // Channel blocked, too slow consumer.
- // In a real production system we might drop the client,
- // here we just skip this message for them to avoid blocking writer.
- // Ideally we should warn or close their channel.
- fmt.Fprintf(
- broadcastWarnWriter,
- "logging: subscriber_blocked subscriber_id=%s action=drop_message\n",
- subscriberID,
- )
+ // A passive log viewer must not stall the producer or other viewers.
+ // Skip only this delivery; bounded history and normal log sinks are
+ // independent. Do not warn here: stderr can block too, and
+ // doing so under this lock would also stall subscribe and shutdown.
}
}
diff --git a/internal/logging/broadcast_test.go b/internal/logging/broadcast_test.go
index 6b0e927d9d..fd8da37786 100644
--- a/internal/logging/broadcast_test.go
+++ b/internal/logging/broadcast_test.go
@@ -1,8 +1,20 @@
package logging
import (
+ "bytes"
"container/ring"
+ "context"
+ "io"
+ "os"
+ "os/exec"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
"testing"
+ "time"
+
+ "github.com/rs/zerolog"
)
func newTestBroadcaster(bufferSize int) *LogBroadcaster {
@@ -136,6 +148,172 @@ func TestLogBroadcasterGetHistoryAfterWrap(t *testing.T) {
}
}
+// Run the writer in a child whose stderr is an unread pipe. The configured
+// sink remains writable. Any per-drop stderr diagnostic fills that pipe and
+// stalls the logging caller, even though the viewer's channel is nonblocking.
+// The timeout kills and reaps only this test child, not a service or a guest.
+func TestLogBroadcasterSlowViewerDoesNotWriteStderr(t *testing.T) {
+ const childEnv = "PULSE_TEST_SLOW_LOG_VIEWER_CHILD"
+ if os.Getenv(childEnv) == "1" {
+ b := newTestBroadcaster(4)
+ fast := make(chan string, 1)
+ slow := make(chan string, 1)
+ slow <- "queued"
+ b.subscribers["fast"] = fast
+ b.subscribers["slow"] = slow
+
+ var sink bytes.Buffer
+ logger := zerolog.New(io.MultiWriter(&sink, b))
+ originalLevel := zerolog.GlobalLevel()
+ defer zerolog.SetGlobalLevel(originalLevel)
+ zerolog.SetGlobalLevel(zerolog.DebugLevel)
+ const line = "{\"level\":\"debug\",\"message\":\"normal polling fixture\"}\n"
+ const records = 32768
+ for i := 0; i < records; i++ {
+ logger.Debug().Msg("normal polling fixture")
+ select {
+ case got := <-fast:
+ if got != line {
+ t.Fatalf("fast viewer received %q, want %q", got, line)
+ }
+ default:
+ t.Fatal("slow viewer prevented delivery to fast viewer")
+ }
+ }
+ if got, want := sink.String(), strings.Repeat(line, records); got != want {
+ t.Fatal("slow viewer changed the configured log sink")
+ }
+ if got := <-slow; got != "queued" {
+ t.Fatalf("slow viewer's queue was changed: %q", got)
+ }
+ if got, want := b.GetHistory(), []string{line, line, line, line}; !slices.Equal(got, want) {
+ t.Fatalf("slow viewer changed bounded history: %#v", got)
+ }
+ b.Shutdown()
+ return
+ }
+
+ readPipe, writePipe, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ _ = writePipe.Close()
+ _ = readPipe.Close()
+ })
+
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestLogBroadcasterSlowViewerDoesNotWriteStderr$", "-test.v")
+ cmd.Env = append(os.Environ(), childEnv+"=1")
+ var output bytes.Buffer
+ cmd.Stdout = &output
+ cmd.Stderr = writePipe
+ cmd.WaitDelay = 2 * time.Second
+ runErr := cmd.Run()
+ _ = writePipe.Close()
+ stderr, readErr := io.ReadAll(readPipe)
+ if runErr != nil {
+ t.Fatalf("logging caller stalled or child failed: %v (deadline: %v, stderr bytes: %d)\n%s", runErr, ctx.Err(), len(stderr), output.String())
+ }
+ if readErr != nil {
+ t.Fatal(readErr)
+ }
+ if len(stderr) != 0 {
+ t.Fatalf("dropping viewer deliveries wrote %d bytes to stderr", len(stderr))
+ }
+}
+
+func TestLogBroadcasterWriteBoundsAndCopiesMessages(t *testing.T) {
+ for _, size := range []int{0, 1, maxBroadcastMessageBytes, maxBroadcastMessageBytes + 1, 1024 * 1024} {
+ t.Run(strconv.Itoa(size), func(t *testing.T) {
+ b := newTestBroadcaster(2)
+ ch := make(chan string, 1)
+ b.subscribers["viewer"] = ch
+ payload := bytes.Repeat([]byte{'x'}, size)
+ want := string(payload)
+ if size > maxBroadcastMessageBytes {
+ want = want[:maxBroadcastMessageBytes-len(broadcastTruncationTag)] + broadcastTruncationTag
+ }
+ n, err := b.Write(payload)
+ if err != nil || n != size {
+ t.Fatalf("Write = %d, %v; want %d, nil", n, err, size)
+ }
+ for i := range payload {
+ payload[i] = 'y'
+ }
+ if got := <-ch; got != want {
+ t.Fatalf("viewer message size/content changed: length %d, want %d", len(got), len(want))
+ }
+ if got := b.GetHistory(); !slices.Equal(got, []string{want}) {
+ t.Fatal("history did not retain the bounded independent copy")
+ }
+ })
+ }
+}
+
+func TestLogBroadcasterSlowViewerDoesNotBlockConcurrentLifecycle(t *testing.T) {
+ b := newTestBroadcaster(4)
+ _, slow, _ := b.Subscribe()
+ for i := 0; i < cap(slow); i++ {
+ slow <- "queued"
+ }
+
+ var writers sync.WaitGroup
+ for i := 0; i < 4; i++ {
+ writers.Go(func() {
+ for i := 0; i < 1000; i++ {
+ n, err := b.Write([]byte("poll"))
+ if err != nil || n != len("poll") {
+ t.Errorf("Write = %d, %v", n, err)
+ }
+ }
+ })
+ }
+ for i := 0; i < 2; i++ {
+ writers.Go(func() {
+ for i := 0; i < 100; i++ {
+ id, _, _ := b.Subscribe()
+ b.GetHistory()
+ b.Unsubscribe(id)
+ b.Shutdown()
+ }
+ })
+ }
+ writers.Wait()
+ b.Shutdown()
+ if len(b.subscribers) != 0 {
+ t.Fatal("subscriber registrations survived shutdown")
+ }
+ for range slow {
+ }
+ if _, err := b.Write([]byte("after shutdown")); err != nil {
+ t.Fatal(err)
+ }
+ if history := b.GetHistory(); history[len(history)-1] != "after shutdown" {
+ t.Fatal("shutdown lost subsequent history")
+ }
+}
+
+func BenchmarkLogBroadcasterOversizedMessage(b *testing.B) {
+ writer := newTestBroadcaster(4)
+ payload := bytes.Repeat([]byte{'x'}, 1024*1024)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ _, _ = writer.Write(payload)
+ }
+}
+
+func TestLogBroadcasterOversizedMessageAllocationBound(t *testing.T) {
+ measured := testing.Benchmark(BenchmarkLogBroadcasterOversizedMessage)
+ // Leave room for bookkeeping while rejecting a copy of the 1 MiB input.
+ // This is an allocation bound, not a timing or process-RSS assertion.
+ if got, limit := measured.AllocedBytesPerOp(), int64(2*maxBroadcastMessageBytes); got > limit {
+ t.Fatalf("broadcaster allocated %d bytes per capped record, want <= %d", got, limit)
+ }
+}
+
func TestGlobalLevelSetAndGet(t *testing.T) {
t.Cleanup(resetLoggingState)