Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

5.1 KB
Raw Blame
package chat

import (
	"strings"
	"sync"

	"github.com/prometheus/client_golang/prometheus"
)

// maxLabelLen is the maximum length for a metric label value
const maxLabelLen = 64

// sanitizeLabel ensures a label value is safe for Prometheus:
// - Truncates to maxLabelLen
// - Replaces spaces with underscores
// - Returns "unknown" for empty values
func sanitizeLabel(s string) string {
	if s == "" {
		return "unknown"
	}
	s = strings.ReplaceAll(s, " ", "_")
	if len(s) > maxLabelLen {
		s = s[:maxLabelLen]
	}
	return s
}

// AIMetrics manages Prometheus instrumentation for AI chat safety/reliability.
// These metrics help prove the structural guarantees stay fixed over time.
type AIMetrics struct {
	// Strict resolution blocks - tracks when undiscovered resources are blocked
	strictResolutionBlock *prometheus.CounterVec

	// Routing mismatch blocks - tracks when operations target wrong layer
	routingMismatchBlock *prometheus.CounterVec

	// Phantom detection - tracks hallucinated tool execution claims
	phantomDetected *prometheus.CounterVec

	// Loop health - tracks agentic loop iterations
	agenticIterations *prometheus.CounterVec
}

var (
	aiMetricsInstance *AIMetrics
	aiMetricsOnce     sync.Once
)

// GetAIMetrics returns the singleton AI metrics instance.
// Call this to record metrics from anywhere in the chat package.
func GetAIMetrics() *AIMetrics {
	aiMetricsOnce.Do(func() {
		aiMetricsInstance = newAIMetrics()
	})
	return aiMetricsInstance
}

func newAIMetrics() *AIMetrics {
	m := &AIMetrics{
		strictResolutionBlock: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "pulse",
				Subsystem: "ai",
				Name:      "strict_resolution_block_total",
				Help:      "Total strict resolution blocks by tool and action",
			},
			[]string{"tool", "action"},
		),
		routingMismatchBlock: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "pulse",
				Subsystem: "ai",
				Name:      "routing_mismatch_block_total",
				Help:      "Total routing mismatch blocks when targeting parent host instead of child resource",
			},
			[]string{"tool", "target_kind", "child_kind"},
		),
		phantomDetected: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "pulse",
				Subsystem: "ai",
				Name:      "phantom_detected_total",
				Help:      "Total phantom execution detections by provider and model",
			},
			[]string{"provider", "model"},
		),
		agenticIterations: prometheus.NewCounterVec(
			prometheus.CounterOpts{
				Namespace: "pulse",
				Subsystem: "ai",
				Name:      "agentic_iterations_total",
				Help:      "Total agentic loop iterations by provider and model",
			},
			[]string{"provider", "model"},
		),
	}

	// Register all metrics
	prometheus.MustRegister(
		m.strictResolutionBlock,
		m.routingMismatchBlock,
		m.phantomDetected,
		m.agenticIterations,
	)

	return m
}

// RecordStrictResolutionBlock records when strict resolution blocks an action
// Note: tool should be the function name (e.g., "validateResolvedResource"), not user input
// Note: action should be a small enum (e.g., "restart", "exec"), not resource IDs
func (m *AIMetrics) RecordStrictResolutionBlock(tool, action string) {
	m.strictResolutionBlock.WithLabelValues(sanitizeLabel(tool), sanitizeLabel(action)).Inc()
}

// RecordRoutingMismatchBlock records when routing validation blocks an operation
// that targeted a parent host when the user recently referenced a child resource.
// Note: use small enums for kinds (node, system-container, vm, app-container), not resource IDs
func (m *AIMetrics) RecordRoutingMismatchBlock(tool, targetKind, childKind string) {
	m.routingMismatchBlock.WithLabelValues(sanitizeLabel(tool), sanitizeLabel(targetKind), sanitizeLabel(childKind)).Inc()
}

// RecordPhantomDetected records when phantom execution is detected
func (m *AIMetrics) RecordPhantomDetected(provider, model string) {
	m.phantomDetected.WithLabelValues(sanitizeLabel(provider), sanitizeLabel(model)).Inc()
}

// RecordAgenticIteration records an agentic loop iteration (one LLM call).
// This counts each turn in the agentic loop, not each tool call.
func (m *AIMetrics) RecordAgenticIteration(provider, model string) {
	m.agenticIterations.WithLabelValues(sanitizeLabel(provider), sanitizeLabel(model)).Inc()
}

// AIMetricsTelemetryCallback adapts AIMetrics to the tools.TelemetryCallback interface.
// This allows the tools package to record telemetry without importing the chat package.
type AIMetricsTelemetryCallback struct {
	metrics *AIMetrics
}

// NewAIMetricsTelemetryCallback creates a new telemetry callback adapter.
func NewAIMetricsTelemetryCallback() *AIMetricsTelemetryCallback {
	return &AIMetricsTelemetryCallback{
		metrics: GetAIMetrics(),
	}
}

// RecordStrictResolutionBlock implements tools.TelemetryCallback
func (c *AIMetricsTelemetryCallback) RecordStrictResolutionBlock(tool, action string) {
	if c.metrics != nil {
		c.metrics.RecordStrictResolutionBlock(tool, action)
	}
}

// RecordRoutingMismatchBlock implements tools.TelemetryCallback
func (c *AIMetricsTelemetryCallback) RecordRoutingMismatchBlock(tool, targetKind, childKind string) {
	if c.metrics != nil {
		c.metrics.RecordRoutingMismatchBlock(tool, targetKind, childKind)
	}
}