Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package cost
import (
"sort"
"strings"
"sync"
"time"
"github.com/rs/zerolog/log"
)
// UsageEvent represents a single AI provider call for cost/token tracking.
// It intentionally excludes prompt/response content for privacy.
type UsageEvent struct {
Timestamp time.Time `json:"timestamp"`
Provider string `json:"provider"`
RequestModel string `json:"request_model"`
ResponseModel string `json:"response_model,omitempty"`
UseCase string `json:"use_case,omitempty"` // "chat" or "patrol"
ContextScope string `json:"context_scope,omitempty"`
ToolCallCount int `json:"tool_call_count,omitempty"`
InputTokens int `json:"input_tokens,omitempty"`
OutputTokens int `json:"output_tokens,omitempty"`
// CacheCreationInputTokens and CacheReadInputTokens are prompt-cache
// buckets reported beside InputTokens (Anthropic); they never overlap it.
CacheCreationInputTokens int `json:"cache_creation_input_tokens,omitempty"`
CacheReadInputTokens int `json:"cache_read_input_tokens,omitempty"`
TargetType string `json:"target_type,omitempty"`
TargetID string `json:"target_id,omitempty"`
FindingID string `json:"finding_id,omitempty"`
// SessionID ties chat-session spend (turns, compaction, title calls)
// together so per-session cost can be summed for the drawer footer.
SessionID string `json:"session_id,omitempty"`
}
// TokenUsage returns the event's counts by billing bucket.
func (e UsageEvent) TokenUsage() TokenUsage {
return TokenUsage{
InputTokens: int64(e.InputTokens),
OutputTokens: int64(e.OutputTokens),
CacheCreationInputTokens: int64(e.CacheCreationInputTokens),
CacheReadInputTokens: int64(e.CacheReadInputTokens),
}
}
// Persistence defines the storage contract for usage history.
type Persistence interface {
SaveUsageHistory(events []UsageEvent) error
LoadUsageHistory() ([]UsageEvent, error)
}
// DefaultMaxDays is the default retention window for raw usage events.
const DefaultMaxDays = 365
// Store provides thread-safe usage tracking with optional persistence.
type Store struct {
mu sync.RWMutex
events []UsageEvent
maxDays int
persistence Persistence
// Debounced persistence to avoid frequent disk writes.
saveTimer *time.Timer
saveTimerDone chan struct{}
savePending bool
saveDebounce time.Duration
// Error tracking for persistence
lastSaveError error
lastSaveTime time.Time
onSaveError func(err error)
}
func normalizeUsageEvents(events []UsageEvent) []UsageEvent {
if events == nil {
return []UsageEvent{}
}
return events
}
// NewStore creates a new usage store.
func NewStore(maxDays int) *Store {
if maxDays <= 0 {
maxDays = DefaultMaxDays
}
return &Store{
events: make([]UsageEvent, 0),
maxDays: maxDays,
saveDebounce: 5 * time.Second,
}
}
func emptySummary(days, retentionDays, effectiveDays int, truncated bool) Summary {
return Summary{
Days: days,
RetentionDays: retentionDays,
EffectiveDays: effectiveDays,
Truncated: truncated,
PricingAsOf: PricingAsOf(),
ProviderModels: []ProviderModelSummary{},
UseCases: []UseCaseSummary{},
Targets: []TargetSummary{},
DailyTotals: []DailySummary{},
}
}
// EmptySummary returns a canonical empty summary for outward-facing cost
// responses when no backing store is available.
func EmptySummary(days, retentionDays, effectiveDays int, truncated bool) Summary {
return emptySummary(days, retentionDays, effectiveDays, truncated)
}
// SetPersistence sets persistence and loads any existing history.
func (s *Store) SetPersistence(p Persistence) error {
s.mu.Lock()
s.persistence = p
s.mu.Unlock()
if p == nil {
return nil
}
events, err := p.LoadUsageHistory()
if err != nil {
return err
}
s.mu.Lock()
s.events = normalizeUsageEvents(events)
s.trimLocked(time.Now())
s.mu.Unlock()
return nil
}
// Record appends a usage event and schedules persistence.
func (s *Store) Record(event UsageEvent) {
if event.Timestamp.IsZero() {
event.Timestamp = time.Now()
}
s.mu.Lock()
s.events = append(s.events, event)
s.trimLocked(time.Now())
s.scheduleSaveLocked()
s.mu.Unlock()
}
// SessionCostUSD sums the estimated spend for one chat session across its
// recorded events (turns, compaction, title calls). known is true only when
// the session has at least one event and every event priced against a known
// model, so callers never show a partial figure as the session total.
func (s *Store) SessionCostUSD(sessionID string) (usd float64, known bool) {
sessionID = strings.TrimSpace(sessionID)
if sessionID == "" {
return 0, false
}
s.mu.RLock()
defer s.mu.RUnlock()
matched := false
for _, e := range s.events {
if e.SessionID != sessionID {
continue
}
matched = true
provider, model := ResolveProviderAndModel(e.Provider, e.RequestModel, e.ResponseModel)
eventUSD, eventKnown, _ := EstimateUsageUSD(provider, model, e.TokenUsage())
if !eventKnown {
return 0, false
}
usd += eventUSD
}
if !matched {
return 0, false
}
return usd, true
}
// Clear removes all retained usage events and persists the empty history.
func (s *Store) Clear() error {
s.mu.Lock()
s.events = s.events[:0]
s.trimLocked(time.Now())
s.scheduleSaveLocked()
s.mu.Unlock()
return s.Flush()
}
// ListEvents returns a copy of usage events within the requested time window, applying retention.
func (s *Store) ListEvents(days int) []UsageEvent {
if days <= 0 {
days = 30
}
now := time.Now()
effectiveDays := days
if s.maxDays > 0 && effectiveDays > s.maxDays {
effectiveDays = s.maxDays
}
cutoff := now.AddDate(0, 0, -effectiveDays)
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]UsageEvent, 0, len(s.events))
for _, e := range s.events {
if !e.Timestamp.Before(cutoff) {
out = append(out, e)
}
}
return out
}
// GetSummary returns a rollup of usage over the last N days.
func (s *Store) GetSummary(days int) Summary {
if days <= 0 {
days = 30
}
now := time.Now()
retentionDays := s.maxDays
effectiveDays := days
truncated := false
if retentionDays > 0 && effectiveDays > retentionDays {
effectiveDays = retentionDays
truncated = true
}
cutoff := now.AddDate(0, 0, -effectiveDays)
s.mu.RLock()
events := make([]UsageEvent, 0, len(s.events))
for _, e := range s.events {
if !e.Timestamp.Before(cutoff) {
events = append(events, e)
}
}
s.mu.RUnlock()
type pmKey struct {
provider string
model string
}
pmTotals := make(map[pmKey]*ProviderModelSummary)
dailyTotals := make(map[string]*DailySummary)
var totalInput, totalOutput, totalCacheCreation, totalCacheRead int64
for _, e := range events {
provider, model := ResolveProviderAndModel(e.Provider, e.RequestModel, e.ResponseModel)
k := pmKey{provider: provider, model: model}
pm := pmTotals[k]
if pm == nil {
pm = &ProviderModelSummary{Provider: provider, Model: model}
pmTotals[k] = pm
}
pm.InputTokens += int64(e.InputTokens)
pm.OutputTokens += int64(e.OutputTokens)
pm.CacheCreationInputTokens += int64(e.CacheCreationInputTokens)
pm.CacheReadInputTokens += int64(e.CacheReadInputTokens)
totalInput += int64(e.InputTokens)
totalOutput += int64(e.OutputTokens)
totalCacheCreation += int64(e.CacheCreationInputTokens)
totalCacheRead += int64(e.CacheReadInputTokens)
usd, known, _ := EstimateUsageUSD(provider, model, e.TokenUsage())
if known {
pm.EstimatedUSD += usd
pm.PricingKnown = true
}
date := e.Timestamp.Format("2006-01-02")
ds := dailyTotals[date]
if ds == nil {
ds = &DailySummary{Date: date}
dailyTotals[date] = ds
}
ds.InputTokens += int64(e.InputTokens)
ds.OutputTokens += int64(e.OutputTokens)
ds.CacheCreationInputTokens += int64(e.CacheCreationInputTokens)
ds.CacheReadInputTokens += int64(e.CacheReadInputTokens)
if known {
ds.EstimatedUSD += usd
}
}
providerModels := make([]ProviderModelSummary, 0, len(pmTotals))
for _, pm := range pmTotals {
pm.TotalTokens = pm.InputTokens + pm.CacheCreationInputTokens + pm.CacheReadInputTokens + pm.OutputTokens
providerModels = append(providerModels, *pm)
}
sort.Slice(providerModels, func(i, j int) bool {
if providerModels[i].Provider == providerModels[j].Provider {
return providerModels[i].Model < providerModels[j].Model
}
return providerModels[i].Provider < providerModels[j].Provider
})
daily := make([]DailySummary, 0, len(dailyTotals))
for _, ds := range dailyTotals {
ds.TotalTokens = ds.InputTokens + ds.CacheCreationInputTokens + ds.CacheReadInputTokens + ds.OutputTokens
daily = append(daily, *ds)
}
sort.Slice(daily, func(i, j int) bool {
return daily[i].Date < daily[j].Date
})
totals := ProviderModelSummary{
Provider: "all",
InputTokens: totalInput,
OutputTokens: totalOutput,
CacheCreationInputTokens: totalCacheCreation,
CacheReadInputTokens: totalCacheRead,
TotalTokens: totalInput + totalCacheCreation + totalCacheRead + totalOutput,
}
for _, pm := range providerModels {
if pm.PricingKnown {
totals.EstimatedUSD += pm.EstimatedUSD
totals.PricingKnown = true
}
}
summary := emptySummary(days, retentionDays, effectiveDays, truncated)
summary.ProviderModels = providerModels
summary.UseCases = summarizeUseCases(events)
summary.Targets = summarizeTargets(events)
summary.DailyTotals = daily
summary.Totals = totals
return summary
}
// Flush immediately writes any pending changes to persistence.
func (s *Store) Flush() error {
s.mu.Lock()
timerDone := s.stopSaveTimerLocked()
s.savePending = false
events := make([]UsageEvent, len(s.events))
copy(events, s.events)
p := s.persistence
s.mu.Unlock()
if timerDone != nil {
<-timerDone
}
if p != nil {
return p.SaveUsageHistory(events)
}
return nil
}
func (s *Store) stopSaveTimerLocked() chan struct{} {
done := s.saveTimerDone
if s.saveTimer != nil {
if s.saveTimer.Stop() && done != nil {
close(done)
}
}
s.saveTimer = nil
s.saveTimerDone = nil
return done
}
func (s *Store) trimLocked(now time.Time) {
if s.maxDays <= 0 {
return
}
cutoff := now.AddDate(0, 0, -s.maxDays)
filtered := s.events[:0]
for _, e := range s.events {
if !e.Timestamp.Before(cutoff) {
filtered = append(filtered, e)
}
}
s.events = filtered
}
func (s *Store) scheduleSaveLocked() {
if s.persistence == nil {
return
}
_ = s.stopSaveTimerLocked()
s.savePending = true
done := make(chan struct{})
s.saveTimerDone = done
s.saveTimer = time.AfterFunc(s.saveDebounce, func() {
defer close(done)
s.mu.Lock()
if !s.savePending {
s.mu.Unlock()
return
}
s.savePending = false
events := make([]UsageEvent, len(s.events))
copy(events, s.events)
p := s.persistence
onErr := s.onSaveError
s.mu.Unlock()
if p != nil {
if err := p.SaveUsageHistory(events); err != nil {
log.Error().Err(err).Msg("failed to save AI usage history")
s.mu.Lock()
s.lastSaveError = err
s.mu.Unlock()
if onErr != nil {
onErr(err)
}
} else {
s.mu.Lock()
s.lastSaveError = nil
s.lastSaveTime = time.Now()
s.mu.Unlock()
}
}
})
}
// SetOnSaveError sets a callback that is called when persistence fails.
func (s *Store) SetOnSaveError(fn func(err error)) {
s.mu.Lock()
defer s.mu.Unlock()
s.onSaveError = fn
}
// GetPersistenceStatus returns the last save error, last save time, and whether persistence is configured.
func (s *Store) GetPersistenceStatus() (lastError error, lastSaveTime time.Time, hasPersistence bool) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.lastSaveError, s.lastSaveTime, s.persistence != nil
}
// ProviderModelSummary is a rollup for a provider/model pair.
type ProviderModelSummary struct {
Provider string `json:"provider"`
Model string `json:"model"`
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
// Prompt-cache buckets, disjoint from InputTokens; TotalTokens counts
// every token in and out, cached or not.
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens,omitempty"`
CacheReadInputTokens int64 `json:"cache_read_input_tokens,omitempty"`
TotalTokens int64 `json:"total_tokens"`
EstimatedUSD float64 `json:"estimated_usd,omitempty"`
PricingKnown bool `json:"pricing_known"`
}
// DailySummary is a rollup for a single day across all providers.
type DailySummary struct {
Date string `json:"date"`
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
// Prompt-cache buckets, disjoint from InputTokens; TotalTokens counts
// every token in and out, cached or not.
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens,omitempty"`
CacheReadInputTokens int64 `json:"cache_read_input_tokens,omitempty"`
TotalTokens int64 `json:"total_tokens"`
EstimatedUSD float64 `json:"estimated_usd,omitempty"`
}
// UseCaseSummary is a rollup for a use-case (e.g. "chat", "patrol").
type UseCaseSummary struct {
UseCase string `json:"use_case"`
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
// Prompt-cache buckets, disjoint from InputTokens; TotalTokens counts
// every token in and out, cached or not.
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens,omitempty"`
CacheReadInputTokens int64 `json:"cache_read_input_tokens,omitempty"`
TotalTokens int64 `json:"total_tokens"`
EstimatedUSD float64 `json:"estimated_usd,omitempty"`
PricingKnown bool `json:"pricing_known"`
}
// TargetSummary is a rollup for a Pulse target (e.g. vm/container/node).
type TargetSummary struct {
TargetType string `json:"target_type"`
TargetID string `json:"target_id"`
Calls int64 `json:"calls"`
InputTokens int64 `json:"input_tokens"`
OutputTokens int64 `json:"output_tokens"`
// Prompt-cache buckets, disjoint from InputTokens; TotalTokens counts
// every token in and out, cached or not.
CacheCreationInputTokens int64 `json:"cache_creation_input_tokens,omitempty"`
CacheReadInputTokens int64 `json:"cache_read_input_tokens,omitempty"`
TotalTokens int64 `json:"total_tokens"`
EstimatedUSD float64 `json:"estimated_usd,omitempty"`
PricingKnown bool `json:"pricing_known"`
}
// Summary is returned by the cost summary API.
type Summary struct {
Days int `json:"days"`
RetentionDays int `json:"retention_days"`
EffectiveDays int `json:"effective_days"`
Truncated bool `json:"truncated"`
PricingAsOf string `json:"pricing_as_of,omitempty"`
ProviderModels []ProviderModelSummary `json:"provider_models"`
UseCases []UseCaseSummary `json:"use_cases"`
Targets []TargetSummary `json:"targets"`
DailyTotals []DailySummary `json:"daily_totals"`
Totals ProviderModelSummary `json:"totals"`
}
func summarizeUseCases(events []UsageEvent) []UseCaseSummary {
type totals struct {
input int64
output int64
cacheCreation int64
cacheRead int64
usd float64
known bool
}
perUseCase := make(map[string]*totals)
for _, e := range events {
useCase := strings.TrimSpace(strings.ToLower(e.UseCase))
if useCase == "" {
useCase = "unknown"
}
t := perUseCase[useCase]
if t == nil {
t = &totals{}
perUseCase[useCase] = t
}
t.input += int64(e.InputTokens)
t.output += int64(e.OutputTokens)
t.cacheCreation += int64(e.CacheCreationInputTokens)
t.cacheRead += int64(e.CacheReadInputTokens)
provider, model := ResolveProviderAndModel(e.Provider, e.RequestModel, e.ResponseModel)
usd, known, _ := EstimateUsageUSD(provider, model, e.TokenUsage())
if known {
t.usd += usd
t.known = true
}
}
out := make([]UseCaseSummary, 0, len(perUseCase))
for useCase, t := range perUseCase {
out = append(out, UseCaseSummary{
UseCase: useCase,
InputTokens: t.input,
OutputTokens: t.output,
CacheCreationInputTokens: t.cacheCreation,
CacheReadInputTokens: t.cacheRead,
TotalTokens: t.input + t.cacheCreation + t.cacheRead + t.output,
EstimatedUSD: t.usd,
PricingKnown: t.known,
})
}
order := map[string]int{
"chat": 0,
"patrol": 1,
"unknown": 2,
}
sort.Slice(out, func(i, j int) bool {
oi, okI := order[out[i].UseCase]
oj, okJ := order[out[j].UseCase]
if okI && okJ && oi != oj {
return oi < oj
}
if okI != okJ {
return okI
}
return out[i].UseCase < out[j].UseCase
})
return out
}
func summarizeTargets(events []UsageEvent) []TargetSummary {
type key struct {
targetType string
targetID string
}
type totals struct {
calls int64
input int64
output int64
cacheCreation int64
cacheRead int64
usd float64
known bool
}
perTarget := make(map[key]*totals)
for _, e := range events {
targetType := strings.TrimSpace(strings.ToLower(e.TargetType))
targetID := strings.TrimSpace(e.TargetID)
if targetType == "" || targetID == "" {
continue
}
k := key{targetType: targetType, targetID: targetID}
t := perTarget[k]
if t == nil {
t = &totals{}
perTarget[k] = t
}
t.calls++
t.input += int64(e.InputTokens)
t.output += int64(e.OutputTokens)
t.cacheCreation += int64(e.CacheCreationInputTokens)
t.cacheRead += int64(e.CacheReadInputTokens)
provider, model := ResolveProviderAndModel(e.Provider, e.RequestModel, e.ResponseModel)
usd, known, _ := EstimateUsageUSD(provider, model, e.TokenUsage())
if known {
t.usd += usd
t.known = true
}
}
out := make([]TargetSummary, 0, len(perTarget))
for k, t := range perTarget {
out = append(out, TargetSummary{
TargetType: k.targetType,
TargetID: k.targetID,
Calls: t.calls,
InputTokens: t.input,
OutputTokens: t.output,
CacheCreationInputTokens: t.cacheCreation,
CacheReadInputTokens: t.cacheRead,
TotalTokens: t.input + t.cacheCreation + t.cacheRead + t.output,
EstimatedUSD: t.usd,
PricingKnown: t.known,
})
}
sort.Slice(out, func(i, j int) bool {
if out[i].EstimatedUSD != out[j].EstimatedUSD {
return out[i].EstimatedUSD > out[j].EstimatedUSD
}
if out[i].TotalTokens != out[j].TotalTokens {
return out[i].TotalTokens > out[j].TotalTokens
}
if out[i].TargetType != out[j].TargetType {
return out[i].TargetType < out[j].TargetType
}
return out[i].TargetID < out[j].TargetID
})
if len(out) > 20 {
out = out[:20]
}
return out
}