Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
fix(observability): flush telemetry on graceful shutdown, drain every sink
Sergey Kozyrenkobc8714db6e71
diff --git a/backend/cmd/pentagi/main.go b/backend/cmd/pentagi/main.goindex 046476b3..00ab153d 100644--- a/backend/cmd/pentagi/main.go+++ b/backend/cmd/pentagi/main.go@@ -173,6 +173,28 @@ func main() {select {case <-ctx.Done():logrus.Warn("Shutdown signal received, cleaning up resources...")+ // ctx is already cancelled here, so drain telemetry on a fresh deadline.+ // Flush before Shutdown: langfuse's Shutdown cancels without flushing, so its+ // final batch drains only via Flush. The drain runs in a goroutine bounded by+ // the select below because a provider Shutdown can ignore a spent context and+ // block on its own — an unreachable collector must not stall process exit.+ flushCtx, cancelFlush := context.WithTimeout(context.Background(), 5*time.Second)+ drained := make(chan struct{})+ go func() {+ defer close(drained)+ if err := obs.Observer.Flush(flushCtx); err != nil {+ logrus.WithError(err).Warn("Telemetry flush incomplete")+ }+ if err := obs.Observer.Shutdown(flushCtx); err != nil {+ logrus.WithError(err).Warn("Telemetry shutdown incomplete")+ }+ }()+ select {+ case <-drained:+ case <-flushCtx.Done():+ logrus.Warn("Telemetry cleanup exceeded its deadline")+ }+ cancelFlush()case err := <-serverErrChan:logrus.Fatalf("Server terminated unexpectedly: %v", err)}diff --git a/backend/pkg/observability/langfuse/observer_shutdown_test.go b/backend/pkg/observability/langfuse/observer_shutdown_test.gonew file mode 100644index 00000000..95835470--- /dev/null+++ b/backend/pkg/observability/langfuse/observer_shutdown_test.go@@ -0,0 +1,103 @@+package langfuse++import (+ "context"+ "net/http"+ "net/http/httptest"+ "sync/atomic"+ "testing"+ "time"++ "pentagi/pkg/observability/langfuse/api"+)++// Shutdown cancels without flushing, so the process shutdown path must ForceFlush+// (Observer.Flush) before Shutdown or the last batch of observations is dropped.+// These pin both halves so a change to either method surfaces here.+func TestObserverShutdownDropsBatch_ForceFlushDrains(t *testing.T) {+ // GET serves the project list that NewClient validates against; POST is the+ // ingestion batch endpoint whose hits mean the batch was actually drained.+ newSink := func() (*httptest.Server, *int32) {+ var ingestHits int32+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ if r.Method == http.MethodGet {+ _, _ = w.Write([]byte(`{"data":[{"id":"proj","name":"test"}]}`))+ return+ }+ atomic.AddInt32(&ingestHits, 1)+ _, _ = w.Write([]byte(`{"successes":[],"errors":[]}`))+ }))+ return srv, &ingestHits+ }++ makeObserver := func(t *testing.T, url string) *observer {+ t.Helper()+ client, err := NewClient(+ WithBaseURL(url),+ WithPublicKey("pk"),+ WithSecretKey("sk"),+ WithProjectID("proj"),+ )+ if err != nil {+ t.Fatalf("NewClient: %v", err)+ }+ // Huge interval + queue so nothing auto-flushes: the batch accumulates and+ // is drained only by the method under test.+ return NewObserver(client,+ WithSendInterval(10*time.Minute),+ WithSendTimeout(2*time.Second),+ WithQueueSize(100),+ ).(*observer)+ }++ newEvent := func() *api.IngestionEvent {+ return &api.IngestionEvent{IngestionEventZero: &api.IngestionEventZero{+ ID: newSpanID(),+ Timestamp: getCurrentTimeString(),+ }}+ }++ // Block until the sender goroutine has pulled the enqueued event out of the+ // buffered queue and into its in-memory batch, so the subsequent action acts+ // on a non-empty batch deterministically.+ waitBatched := func(t *testing.T, o *observer) {+ t.Helper()+ for i := 0; i < 400; i++ {+ if len(o.queue) == 0 {+ return+ }+ time.Sleep(5 * time.Millisecond)+ }+ t.Fatal("sender never consumed the queued event into its batch")+ }++ t.Run("Shutdown alone drops the buffered batch", func(t *testing.T) {+ srv, hits := newSink()+ defer srv.Close()+ o := makeObserver(t, srv.URL)+ o.enqueue(newEvent())+ waitBatched(t, o)++ _ = o.Shutdown(context.Background())++ if got := atomic.LoadInt32(hits); got != 0 {+ t.Fatalf("Shutdown must not flush; sink received %d requests", got)+ }+ })++ t.Run("ForceFlush drains the buffered batch", func(t *testing.T) {+ srv, hits := newSink()+ defer srv.Close()+ o := makeObserver(t, srv.URL)+ o.enqueue(newEvent())+ waitBatched(t, o)++ _ = o.ForceFlush(context.Background())++ if got := atomic.LoadInt32(hits); got != 1 {+ t.Fatalf("ForceFlush must send the batch exactly once; sink received %d requests", got)+ }+ _ = o.Shutdown(context.Background())+ })+}diff --git a/backend/pkg/observability/obs.go b/backend/pkg/observability/obs.goindex fd363272..d3095912 100644--- a/backend/pkg/observability/obs.go+++ b/backend/pkg/observability/obs.go@@ -201,27 +201,33 @@ func InitObserver(ctx context.Context, lfclient LangfuseClient, otelclient Telem}func (obs *observer) Flush(ctx context.Context) error {+ var errs []errorif obs.lfclient != nil {- return obs.lfclient.ForceFlush(ctx)+ if err := obs.lfclient.ForceFlush(ctx); err != nil {+ errs = append(errs, err)+ }}-if obs.otelclient != nil {- return obs.otelclient.ForceFlush(ctx)+ if err := obs.otelclient.ForceFlush(ctx); err != nil {+ errs = append(errs, err)+ }}-- return nil+ return errors.Join(errs...)}func (obs *observer) Shutdown(ctx context.Context) error {+ var errs []errorif obs.lfclient != nil {- return obs.lfclient.Shutdown(ctx)+ if err := obs.lfclient.Shutdown(ctx); err != nil {+ errs = append(errs, err)+ }}-if obs.otelclient != nil {- return obs.otelclient.Shutdown(ctx)+ if err := obs.otelclient.Shutdown(ctx); err != nil {+ errs = append(errs, err)+ }}-- return nil+ return errors.Join(errs...)}func (obs *observer) StartProcessMetricCollect(attrs ...attribute.KeyValue) error {diff --git a/backend/pkg/observability/obs_shutdown_test.go b/backend/pkg/observability/obs_shutdown_test.gonew file mode 100644index 00000000..108728e3--- /dev/null+++ b/backend/pkg/observability/obs_shutdown_test.go@@ -0,0 +1,117 @@+package observability++import (+ "context"+ "errors"+ "testing"++ "pentagi/pkg/observability/langfuse"++ otellog "go.opentelemetry.io/otel/log"+ otelloggernoop "go.opentelemetry.io/otel/log/noop"+ otelmetric "go.opentelemetry.io/otel/metric"+ otelmetricnoop "go.opentelemetry.io/otel/metric/noop"+ oteltrace "go.opentelemetry.io/otel/trace"+ oteltracenoop "go.opentelemetry.io/otel/trace/noop"+)++type fakeLangfuseClient struct {+ shutdownCalled bool+ flushCalled bool+ shutdownErr error+ flushErr error+}++func (f *fakeLangfuseClient) API() langfuse.Client { return langfuse.Client{} }+func (f *fakeLangfuseClient) Observer() langfuse.Observer { return langfuse.NewNoopObserver() }+func (f *fakeLangfuseClient) Shutdown(context.Context) error {+ f.shutdownCalled = true+ return f.shutdownErr+}+func (f *fakeLangfuseClient) ForceFlush(context.Context) error {+ f.flushCalled = true+ return f.flushErr+}++type fakeTelemetryClient struct {+ shutdownCalled bool+ flushCalled bool+ shutdownErr error+ flushErr error+}++func (f *fakeTelemetryClient) Logger() otellog.LoggerProvider {+ return otelloggernoop.NewLoggerProvider()+}+func (f *fakeTelemetryClient) Tracer() oteltrace.TracerProvider {+ return oteltracenoop.NewTracerProvider()+}+func (f *fakeTelemetryClient) Meter() otelmetric.MeterProvider {+ return otelmetricnoop.NewMeterProvider()+}+func (f *fakeTelemetryClient) Shutdown(context.Context) error {+ f.shutdownCalled = true+ return f.shutdownErr+}+func (f *fakeTelemetryClient) ForceFlush(context.Context) error {+ f.flushCalled = true+ return f.flushErr+}++func TestObserverShutdownDrainsBothClients(t *testing.T) {+ lf := &fakeLangfuseClient{}+ ot := &fakeTelemetryClient{}+ obs := &observer{lfclient: lf, otelclient: ot}++ if err := obs.Shutdown(context.Background()); err != nil {+ t.Fatalf("Shutdown: %v", err)+ }+ if !lf.shutdownCalled {+ t.Error("langfuse client was not shut down")+ }+ if !ot.shutdownCalled {+ t.Error("otel client was not shut down")+ }+}++func TestObserverFlushDrainsBothClients(t *testing.T) {+ lf := &fakeLangfuseClient{}+ ot := &fakeTelemetryClient{}+ obs := &observer{lfclient: lf, otelclient: ot}++ if err := obs.Flush(context.Background()); err != nil {+ t.Fatalf("Flush: %v", err)+ }+ if !lf.flushCalled {+ t.Error("langfuse client was not flushed")+ }+ if !ot.flushCalled {+ t.Error("otel client was not flushed")+ }+}++func TestObserverShutdownAggregatesErrorsAndDrainsBoth(t *testing.T) {+ lfErr := errors.New("langfuse boom")+ otErr := errors.New("otel boom")+ lf := &fakeLangfuseClient{shutdownErr: lfErr}+ ot := &fakeTelemetryClient{shutdownErr: otErr}+ obs := &observer{lfclient: lf, otelclient: ot}++ err := obs.Shutdown(context.Background())+ if !errors.Is(err, lfErr) || !errors.Is(err, otErr) {+ t.Fatalf("want both errors joined, got %v", err)+ }+ if !lf.shutdownCalled || !ot.shutdownCalled {+ t.Fatal("a client was skipped despite the other erroring")+ }+}++func TestObserverShutdownFlushNilClientsAreNoops(t *testing.T) {+ obs := &observer{}+ if err := obs.Shutdown(context.Background()); err != nil {+ t.Errorf("Shutdown with no clients: %v", err)+ }+ if err := obs.Flush(context.Background()); err != nil {+ t.Errorf("Flush with no clients: %v", err)+ }+}diff --git a/backend/pkg/observability/otelclient.go b/backend/pkg/observability/otelclient.goindex 62a946fe..5c737f24 100644--- a/backend/pkg/observability/otelclient.go+++ b/backend/pkg/observability/otelclient.go@@ -2,6 +2,7 @@ package observabilityimport ("context"+ "errors""fmt""strings""time"@@ -62,29 +63,36 @@ func (c *telemetryClient) Meter() otelmetric.MeterProvider {}func (c *telemetryClient) Shutdown(ctx context.Context) error {+ var errs []errorif err := c.logger.Shutdown(ctx); err != nil {- return fmt.Errorf("failed to shutdown logger: %w", err)+ errs = append(errs, fmt.Errorf("failed to shutdown logger: %w", err))}if err := c.meter.Shutdown(ctx); err != nil {- return fmt.Errorf("failed to shutdown meter: %w", err)+ errs = append(errs, fmt.Errorf("failed to shutdown meter: %w", err))}if err := c.tracer.Shutdown(ctx); err != nil {- return fmt.Errorf("failed to shutdown tracer: %w", err)+ errs = append(errs, fmt.Errorf("failed to shutdown tracer: %w", err))}- return c.conn.Close()+ // Always close the connection, even if a provider shutdown failed above, so a+ // stalled flush can't leak the grpc conn.+ if err := c.conn.Close(); err != nil {+ errs = append(errs, fmt.Errorf("failed to close telemetry connection: %w", err))+ }+ return errors.Join(errs...)}func (c *telemetryClient) ForceFlush(ctx context.Context) error {+ var errs []errorif err := c.logger.ForceFlush(ctx); err != nil {- return fmt.Errorf("failed to force flush logger: %w", err)+ errs = append(errs, fmt.Errorf("failed to force flush logger: %w", err))}if err := c.meter.ForceFlush(ctx); err != nil {- return fmt.Errorf("failed to force flush meter: %w", err)+ errs = append(errs, fmt.Errorf("failed to force flush meter: %w", err))}if err := c.tracer.ForceFlush(ctx); err != nil {- return fmt.Errorf("failed to force flush tracer: %w", err)+ errs = append(errs, fmt.Errorf("failed to force flush tracer: %w", err))}- return nil+ return errors.Join(errs...)}func NewTelemetryClient(ctx context.Context, cfg *config.Config) (TelemetryClient, error) {