Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

fix(observability): flush telemetry on graceful shutdown, drain every sink

Sergey Kozyrenkobc8714db6e71
diff --git a/backend/cmd/pentagi/main.go b/backend/cmd/pentagi/main.go
index 046476b3..00ab153d 100644
--- a/backend/cmd/pentagi/main.go
+++ b/backend/cmd/pentagi/main.go
@@ -173,6 +173,28 @@ func main() {
select {
case <-ctx.Done():
logrus.Warn("Shutdown signal received, cleaning up resources...")
+ // ctx is already cancelled here, so drain telemetry on a fresh deadline.
+ // Flush before Shutdown: langfuse's Shutdown cancels without flushing, so its
+ // final batch drains only via Flush. The drain runs in a goroutine bounded by
+ // the select below because a provider Shutdown can ignore a spent context and
+ // block on its own — an unreachable collector must not stall process exit.
+ flushCtx, cancelFlush := context.WithTimeout(context.Background(), 5*time.Second)
+ drained := make(chan struct{})
+ go func() {
+ defer close(drained)
+ if err := obs.Observer.Flush(flushCtx); err != nil {
+ logrus.WithError(err).Warn("Telemetry flush incomplete")
+ }
+ if err := obs.Observer.Shutdown(flushCtx); err != nil {
+ logrus.WithError(err).Warn("Telemetry shutdown incomplete")
+ }
+ }()
+ select {
+ case <-drained:
+ case <-flushCtx.Done():
+ logrus.Warn("Telemetry cleanup exceeded its deadline")
+ }
+ cancelFlush()
case err := <-serverErrChan:
logrus.Fatalf("Server terminated unexpectedly: %v", err)
}
diff --git a/backend/pkg/observability/langfuse/observer_shutdown_test.go b/backend/pkg/observability/langfuse/observer_shutdown_test.go
new file mode 100644
index 00000000..95835470
--- /dev/null
+++ b/backend/pkg/observability/langfuse/observer_shutdown_test.go
@@ -0,0 +1,103 @@
+package langfuse
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "pentagi/pkg/observability/langfuse/api"
+)
+
+// Shutdown cancels without flushing, so the process shutdown path must ForceFlush
+// (Observer.Flush) before Shutdown or the last batch of observations is dropped.
+// These pin both halves so a change to either method surfaces here.
+func TestObserverShutdownDropsBatch_ForceFlushDrains(t *testing.T) {
+ // GET serves the project list that NewClient validates against; POST is the
+ // ingestion batch endpoint whose hits mean the batch was actually drained.
+ newSink := func() (*httptest.Server, *int32) {
+ var ingestHits int32
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if r.Method == http.MethodGet {
+ _, _ = w.Write([]byte(`{"data":[{"id":"proj","name":"test"}]}`))
+ return
+ }
+ atomic.AddInt32(&ingestHits, 1)
+ _, _ = w.Write([]byte(`{"successes":[],"errors":[]}`))
+ }))
+ return srv, &ingestHits
+ }
+
+ makeObserver := func(t *testing.T, url string) *observer {
+ t.Helper()
+ client, err := NewClient(
+ WithBaseURL(url),
+ WithPublicKey("pk"),
+ WithSecretKey("sk"),
+ WithProjectID("proj"),
+ )
+ if err != nil {
+ t.Fatalf("NewClient: %v", err)
+ }
+ // Huge interval + queue so nothing auto-flushes: the batch accumulates and
+ // is drained only by the method under test.
+ return NewObserver(client,
+ WithSendInterval(10*time.Minute),
+ WithSendTimeout(2*time.Second),
+ WithQueueSize(100),
+ ).(*observer)
+ }
+
+ newEvent := func() *api.IngestionEvent {
+ return &api.IngestionEvent{IngestionEventZero: &api.IngestionEventZero{
+ ID: newSpanID(),
+ Timestamp: getCurrentTimeString(),
+ }}
+ }
+
+ // Block until the sender goroutine has pulled the enqueued event out of the
+ // buffered queue and into its in-memory batch, so the subsequent action acts
+ // on a non-empty batch deterministically.
+ waitBatched := func(t *testing.T, o *observer) {
+ t.Helper()
+ for i := 0; i < 400; i++ {
+ if len(o.queue) == 0 {
+ return
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ t.Fatal("sender never consumed the queued event into its batch")
+ }
+
+ t.Run("Shutdown alone drops the buffered batch", func(t *testing.T) {
+ srv, hits := newSink()
+ defer srv.Close()
+ o := makeObserver(t, srv.URL)
+ o.enqueue(newEvent())
+ waitBatched(t, o)
+
+ _ = o.Shutdown(context.Background())
+
+ if got := atomic.LoadInt32(hits); got != 0 {
+ t.Fatalf("Shutdown must not flush; sink received %d requests", got)
+ }
+ })
+
+ t.Run("ForceFlush drains the buffered batch", func(t *testing.T) {
+ srv, hits := newSink()
+ defer srv.Close()
+ o := makeObserver(t, srv.URL)
+ o.enqueue(newEvent())
+ waitBatched(t, o)
+
+ _ = o.ForceFlush(context.Background())
+
+ if got := atomic.LoadInt32(hits); got != 1 {
+ t.Fatalf("ForceFlush must send the batch exactly once; sink received %d requests", got)
+ }
+ _ = o.Shutdown(context.Background())
+ })
+}
diff --git a/backend/pkg/observability/obs.go b/backend/pkg/observability/obs.go
index fd363272..d3095912 100644
--- a/backend/pkg/observability/obs.go
+++ b/backend/pkg/observability/obs.go
@@ -201,27 +201,33 @@ func InitObserver(ctx context.Context, lfclient LangfuseClient, otelclient Telem
}
func (obs *observer) Flush(ctx context.Context) error {
+ var errs []error
if obs.lfclient != nil {
- return obs.lfclient.ForceFlush(ctx)
+ if err := obs.lfclient.ForceFlush(ctx); err != nil {
+ errs = append(errs, err)
+ }
}
-
if obs.otelclient != nil {
- return obs.otelclient.ForceFlush(ctx)
+ if err := obs.otelclient.ForceFlush(ctx); err != nil {
+ errs = append(errs, err)
+ }
}
-
- return nil
+ return errors.Join(errs...)
}
func (obs *observer) Shutdown(ctx context.Context) error {
+ var errs []error
if obs.lfclient != nil {
- return obs.lfclient.Shutdown(ctx)
+ if err := obs.lfclient.Shutdown(ctx); err != nil {
+ errs = append(errs, err)
+ }
}
-
if obs.otelclient != nil {
- return obs.otelclient.Shutdown(ctx)
+ if err := obs.otelclient.Shutdown(ctx); err != nil {
+ errs = append(errs, err)
+ }
}
-
- return nil
+ return errors.Join(errs...)
}
func (obs *observer) StartProcessMetricCollect(attrs ...attribute.KeyValue) error {
diff --git a/backend/pkg/observability/obs_shutdown_test.go b/backend/pkg/observability/obs_shutdown_test.go
new file mode 100644
index 00000000..108728e3
--- /dev/null
+++ b/backend/pkg/observability/obs_shutdown_test.go
@@ -0,0 +1,117 @@
+package observability
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "pentagi/pkg/observability/langfuse"
+
+ otellog "go.opentelemetry.io/otel/log"
+ otelloggernoop "go.opentelemetry.io/otel/log/noop"
+ otelmetric "go.opentelemetry.io/otel/metric"
+ otelmetricnoop "go.opentelemetry.io/otel/metric/noop"
+ oteltrace "go.opentelemetry.io/otel/trace"
+ oteltracenoop "go.opentelemetry.io/otel/trace/noop"
+)
+
+type fakeLangfuseClient struct {
+ shutdownCalled bool
+ flushCalled bool
+ shutdownErr error
+ flushErr error
+}
+
+func (f *fakeLangfuseClient) API() langfuse.Client { return langfuse.Client{} }
+func (f *fakeLangfuseClient) Observer() langfuse.Observer { return langfuse.NewNoopObserver() }
+func (f *fakeLangfuseClient) Shutdown(context.Context) error {
+ f.shutdownCalled = true
+ return f.shutdownErr
+}
+func (f *fakeLangfuseClient) ForceFlush(context.Context) error {
+ f.flushCalled = true
+ return f.flushErr
+}
+
+type fakeTelemetryClient struct {
+ shutdownCalled bool
+ flushCalled bool
+ shutdownErr error
+ flushErr error
+}
+
+func (f *fakeTelemetryClient) Logger() otellog.LoggerProvider {
+ return otelloggernoop.NewLoggerProvider()
+}
+func (f *fakeTelemetryClient) Tracer() oteltrace.TracerProvider {
+ return oteltracenoop.NewTracerProvider()
+}
+func (f *fakeTelemetryClient) Meter() otelmetric.MeterProvider {
+ return otelmetricnoop.NewMeterProvider()
+}
+func (f *fakeTelemetryClient) Shutdown(context.Context) error {
+ f.shutdownCalled = true
+ return f.shutdownErr
+}
+func (f *fakeTelemetryClient) ForceFlush(context.Context) error {
+ f.flushCalled = true
+ return f.flushErr
+}
+
+func TestObserverShutdownDrainsBothClients(t *testing.T) {
+ lf := &fakeLangfuseClient{}
+ ot := &fakeTelemetryClient{}
+ obs := &observer{lfclient: lf, otelclient: ot}
+
+ if err := obs.Shutdown(context.Background()); err != nil {
+ t.Fatalf("Shutdown: %v", err)
+ }
+ if !lf.shutdownCalled {
+ t.Error("langfuse client was not shut down")
+ }
+ if !ot.shutdownCalled {
+ t.Error("otel client was not shut down")
+ }
+}
+
+func TestObserverFlushDrainsBothClients(t *testing.T) {
+ lf := &fakeLangfuseClient{}
+ ot := &fakeTelemetryClient{}
+ obs := &observer{lfclient: lf, otelclient: ot}
+
+ if err := obs.Flush(context.Background()); err != nil {
+ t.Fatalf("Flush: %v", err)
+ }
+ if !lf.flushCalled {
+ t.Error("langfuse client was not flushed")
+ }
+ if !ot.flushCalled {
+ t.Error("otel client was not flushed")
+ }
+}
+
+func TestObserverShutdownAggregatesErrorsAndDrainsBoth(t *testing.T) {
+ lfErr := errors.New("langfuse boom")
+ otErr := errors.New("otel boom")
+ lf := &fakeLangfuseClient{shutdownErr: lfErr}
+ ot := &fakeTelemetryClient{shutdownErr: otErr}
+ obs := &observer{lfclient: lf, otelclient: ot}
+
+ err := obs.Shutdown(context.Background())
+ if !errors.Is(err, lfErr) || !errors.Is(err, otErr) {
+ t.Fatalf("want both errors joined, got %v", err)
+ }
+ if !lf.shutdownCalled || !ot.shutdownCalled {
+ t.Fatal("a client was skipped despite the other erroring")
+ }
+}
+
+func TestObserverShutdownFlushNilClientsAreNoops(t *testing.T) {
+ obs := &observer{}
+ if err := obs.Shutdown(context.Background()); err != nil {
+ t.Errorf("Shutdown with no clients: %v", err)
+ }
+ if err := obs.Flush(context.Background()); err != nil {
+ t.Errorf("Flush with no clients: %v", err)
+ }
+}
diff --git a/backend/pkg/observability/otelclient.go b/backend/pkg/observability/otelclient.go
index 62a946fe..5c737f24 100644
--- a/backend/pkg/observability/otelclient.go
+++ b/backend/pkg/observability/otelclient.go
@@ -2,6 +2,7 @@ package observability
import (
"context"
+ "errors"
"fmt"
"strings"
"time"
@@ -62,29 +63,36 @@ func (c *telemetryClient) Meter() otelmetric.MeterProvider {
}
func (c *telemetryClient) Shutdown(ctx context.Context) error {
+ var errs []error
if err := c.logger.Shutdown(ctx); err != nil {
- return fmt.Errorf("failed to shutdown logger: %w", err)
+ errs = append(errs, fmt.Errorf("failed to shutdown logger: %w", err))
}
if err := c.meter.Shutdown(ctx); err != nil {
- return fmt.Errorf("failed to shutdown meter: %w", err)
+ errs = append(errs, fmt.Errorf("failed to shutdown meter: %w", err))
}
if err := c.tracer.Shutdown(ctx); err != nil {
- return fmt.Errorf("failed to shutdown tracer: %w", err)
+ errs = append(errs, fmt.Errorf("failed to shutdown tracer: %w", err))
}
- return c.conn.Close()
+ // Always close the connection, even if a provider shutdown failed above, so a
+ // stalled flush can't leak the grpc conn.
+ if err := c.conn.Close(); err != nil {
+ errs = append(errs, fmt.Errorf("failed to close telemetry connection: %w", err))
+ }
+ return errors.Join(errs...)
}
func (c *telemetryClient) ForceFlush(ctx context.Context) error {
+ var errs []error
if err := c.logger.ForceFlush(ctx); err != nil {
- return fmt.Errorf("failed to force flush logger: %w", err)
+ errs = append(errs, fmt.Errorf("failed to force flush logger: %w", err))
}
if err := c.meter.ForceFlush(ctx); err != nil {
- return fmt.Errorf("failed to force flush meter: %w", err)
+ errs = append(errs, fmt.Errorf("failed to force flush meter: %w", err))
}
if err := c.tracer.ForceFlush(ctx); err != nil {
- return fmt.Errorf("failed to force flush tracer: %w", err)
+ errs = append(errs, fmt.Errorf("failed to force flush tracer: %w", err))
}
- return nil
+ return errors.Join(errs...)
}
func NewTelemetryClient(ctx context.Context, cfg *config.Config) (TelemetryClient, error) {