Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package ai
import (
"fmt"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/mockmode"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
"github.com/rs/zerolog/log"
)
// IsDemoMode returns true if mock/demo mode is enabled.
// Delegates to the build-tag-gated mockmode package (always false in release).
func IsDemoMode() bool {
return mockmode.IsEnabled()
}
// Demo provider identity reported by readiness and preflight surfaces while
// mock mode simulates Patrol. Matches the model name GenerateDemoAIResponse
// reports for assistant chat.
const (
DemoPatrolProvider = "demo"
DemoPatrolModel = "demo-model"
)
// IsDemoRuntimeIntended reports whether this process is meant to serve demo
// fixtures even when they are not enabled yet. Release demo instances boot
// with mock fixtures off until the license sync validates the demo_fixtures
// entitlement, so boot-time lifecycle decisions (whether to start the patrol
// loop at all) must look at operator intent, not the current gate state.
func IsDemoRuntimeIntended() bool {
return IsDemoMode() || mockmode.IsRequestedFromEnv()
}
const demoFindingIDPrefix = "demo-"
// isDemoFindingID reports whether a finding was synthesized by the demo
// patrol cycle. Real findings use hex hash IDs, so the prefix cannot collide.
func isDemoFindingID(id string) bool {
return strings.HasPrefix(strings.ToLower(strings.TrimSpace(id)), demoFindingIDPrefix)
}
// runDemoPatrolCycle executes one simulated patrol pass for demo/mock mode.
// It never contacts a provider: findings are synthesized from the current
// mock infrastructure state so every finding points at a resource the demo
// actually shows, then a matching run record is appended to check history.
// Repeated cycles refresh the same finding IDs (heartbeat semantics) and
// auto-resolve demo findings whose underlying mock condition went away.
// Returns true once a cycle has been recorded; false when it could not run
// yet (mock disabled again, another run in flight, or mock state not
// generated yet).
func (p *PatrolService) runDemoPatrolCycle(trigger TriggerReason) bool {
return p.runDemoPatrolCycleWithStart(trigger, nil)
}
func (p *PatrolService) runDemoPatrolCycleWithStart(trigger TriggerReason, acceptedStart *patrolRunStart) bool {
if p == nil || p.findings == nil || p.runHistoryStore == nil {
return false
}
if !IsDemoMode() {
return false
}
runStart := acceptedStart
if runStart == nil {
var accepted bool
runStart, accepted = p.beginRun("full")
if !accepted {
return false
}
}
defer p.endRun()
p.mu.Lock()
// Mock mode can enable after the boot-time config snapshot was taken
// (release demo instances authorize fixtures via the license sync).
// Repair the snapshot so status surfaces report an active patrol instead
// of the pre-demo disabled/blocked state.
p.config.Enabled = true
p.config.RuntimeBlockedReason = ""
p.config.RuntimeBlockedCause = PatrolFailureCauseNone
cfg := p.config
p.mu.Unlock()
p.clearBlockedReason()
now := time.Now()
state := p.currentPatrolRuntimeState()
findings := synthesizeDemoPatrolFindings(state, now)
counts := patrolRuntimeCountResources(state)
nodesChecked, guestsChecked, dockerChecked, storageChecked := 0, 0, 0, 0
hostsChecked, trueNASChecked, pbsChecked, pmgChecked, kubernetesChecked := 0, 0, 0, 0, 0
if cfg.AnalyzeNodes {
nodesChecked = counts.nodes
}
if cfg.AnalyzeGuests {
guestsChecked = counts.guests
}
if cfg.AnalyzeDocker {
dockerChecked = counts.docker
}
if cfg.AnalyzeStorage {
storageChecked = counts.storage
}
if cfg.AnalyzeHosts {
hostsChecked = counts.hosts
trueNASChecked = counts.truenas
}
if cfg.AnalyzePBS {
pbsChecked = counts.pbs
}
if cfg.AnalyzePMG {
pmgChecked = counts.pmg
}
if cfg.AnalyzeKubernetes {
kubernetesChecked = counts.kubernetes
}
resourceCount := nodesChecked + guestsChecked + dockerChecked + storageChecked +
hostsChecked + trueNASChecked + pbsChecked + pmgChecked + kubernetesChecked
if resourceCount == 0 && len(findings) == 0 {
// Mock state has not been generated yet (early boot). The warmup
// retry or the next scheduled cycle will populate the surface once
// state exists.
log.Debug().Msg("demo patrol: no mock state available yet, skipping cycle")
if acceptedStart != nil {
p.recordAcceptedRunFailure(runStart, trigger,
"Patrol demo state unavailable",
"Pulse accepted the run, but demo infrastructure state was not available.")
return true
}
return false
}
// Demo findings are deliberately not persisted, so a restart re-adds
// them as store-new. Findings already attested by prior demo run records
// still count as existing: the history says they were open before.
knownIDs := make(map[string]bool)
for _, run := range p.runHistoryStore.GetAll() {
if !isDemoPatrolRunRecord(run) {
continue
}
for _, id := range run.FindingIDs {
knownIDs[id] = true
}
}
newCount, existingCount := 0, 0
currentIDs := make(map[string]bool, len(findings))
findingIDs := make([]string, 0, len(findings))
for _, f := range findings {
currentIDs[f.ID] = true
if p.findings.Add(f) && !knownIDs[f.ID] {
newCount++
} else {
existingCount++
}
findingIDs = append(findingIDs, f.ID)
}
// Auto-resolve demo findings whose mock condition no longer holds.
resolvedCount := 0
for _, active := range p.findings.GetActive(FindingSeverityInfo) {
if active == nil || !isDemoFindingID(active.ID) || currentIDs[active.ID] {
continue
}
if p.findings.Resolve(active.ID, true) {
resolvedCount++
}
}
// Demo findings restate mock alerts on purpose (a storage pool near full
// beside its usage alert), so fold them the same way a real run does.
p.reconcileAlertMirrors()
// A simulated pass stands in for a healthy provider-backed run: clear any
// stale "Provider analysis error" finding left by real runs that failed
// before mock mode enabled.
p.resolvePatrolRuntimeFailureFinding("demo_patrol_cycle")
summary := p.findings.GetSummary()
findingsSummaryStr := "All healthy"
status := "healthy"
if summary.Critical+summary.Warning > 0 {
parts := []string{}
if summary.Critical > 0 {
parts = append(parts, fmt.Sprintf("%d critical", summary.Critical))
}
if summary.Warning > 0 {
parts = append(parts, fmt.Sprintf("%d warning", summary.Warning))
}
findingsSummaryStr = joinParts(parts)
if summary.Critical > 0 {
status = "critical"
} else {
status = "issues_found"
}
}
// Presentational duration and token counts: deterministic, scaled to the
// size of the mock fleet so the history reads like a real analysis pass.
duration := time.Duration(35+resourceCount%40) * time.Second
runID := fmt.Sprintf("demo-run-%d", now.UnixNano())
startedAt := now.Add(-duration)
if acceptedStart != nil {
runID = runStart.id
startedAt = runStart.startedAt
duration = now.Sub(startedAt)
}
record := PatrolRunRecord{
ID: runID,
Source: PatrolRunSourceDemo,
StartedAt: startedAt,
CompletedAt: now,
Duration: duration,
DurationMs: duration.Milliseconds(),
Type: "patrol",
TriggerReason: string(trigger),
ResourcesChecked: resourceCount,
NodesChecked: nodesChecked,
GuestsChecked: guestsChecked,
DockerChecked: dockerChecked,
StorageChecked: storageChecked,
HostsChecked: hostsChecked,
TrueNASChecked: trueNASChecked,
PBSChecked: pbsChecked,
PMGChecked: pmgChecked,
KubernetesChecked: kubernetesChecked,
NewFindings: newCount,
ExistingFindings: existingCount,
ResolvedFindings: resolvedCount,
FindingsSummary: findingsSummaryStr,
FindingIDs: findingIDs,
Status: status,
InputTokens: 1200 + 42*resourceCount,
OutputTokens: 380 + 60*len(findings),
AIAnalysis: fmt.Sprintf(
"Reviewed %d resources across %d nodes, %d guests, %d Docker hosts, and %d storage pools. %s.",
resourceCount, nodesChecked, guestsChecked, dockerChecked, storageChecked, findingsSummaryStr),
}
if p.backfillDemoRunHistory(record, now) {
// The seeded history attests these findings first appeared earlier,
// so the current run reports them as still open, not newly found.
record.ExistingFindings += record.NewFindings
record.NewFindings = 0
}
p.runHistoryStore.Add(record)
p.mu.Lock()
p.lastActivity = now
p.lastFullPatrol = now
p.lastDuration = duration
p.resourcesChecked = resourceCount
p.errorCount = 0
p.mu.Unlock()
log.Info().
Int("resources", resourceCount).
Int("findings", len(findings)).
Int("new", newCount).
Int("resolved", resolvedCount).
Str("trigger", string(trigger)).
Msg("demo patrol: completed simulated patrol cycle")
return true
}
// startDemoPatrolWarmup runs the first simulated patrol cycle once mock state
// is available. Mock fixtures generate concurrently with startup, so an
// immediate cycle can race an empty snapshot; retry briefly instead of
// waiting for the next scheduled tick, which the dev quota guard may never
// start.
func (p *PatrolService) startDemoPatrolWarmup() {
if p == nil {
return
}
go func() {
for attempt := 0; attempt < 24; attempt++ {
if p.runDemoPatrolCycle(TriggerReasonStartup) {
return
}
time.Sleep(5 * time.Second)
}
}()
}
// backfillDemoRunHistory seeds plausible past check history the first time a
// demo cycle runs, so the run history panel is not empty on a fresh demo
// instance. No-op once any demo record exists. Returns true when it seeded.
func (p *PatrolService) backfillDemoRunHistory(template PatrolRunRecord, now time.Time) bool {
for _, run := range p.runHistoryStore.GetAll() {
if isDemoPatrolRunRecord(run) {
return false
}
}
// Oldest first: the store prepends, so adding in reverse keeps newest on top.
for i := 10; i >= 1; i-- {
start := now.Add(-time.Duration(i*6) * time.Hour)
duration := time.Duration(38+(i*7)%31) * time.Second
rec := template
rec.ID = fmt.Sprintf("demo-run-%d", start.UnixNano())
rec.StartedAt = start
rec.CompletedAt = start.Add(duration)
rec.Duration = duration
rec.DurationMs = duration.Milliseconds()
rec.TriggerReason = string(TriggerReasonScheduled)
rec.NewFindings = 0
rec.ResolvedFindings = 0
rec.ExistingFindings = len(template.FindingIDs)
rec.AIAnalysis = ""
switch {
case i >= 8:
// The oldest runs predate the current issues.
rec.ExistingFindings = 0
rec.FindingIDs = nil
rec.FindingsSummary = "All healthy"
rec.Status = "healthy"
case i == 7:
// The run where the current issues first appeared.
rec.NewFindings = len(template.FindingIDs)
rec.ExistingFindings = 0
}
p.runHistoryStore.Add(rec)
}
return true
}
// demoResourceResolver names the snapshot records the demo synthesizer reads
// by the unified resources real Patrol findings name. The Inbox, resource
// drawers, the resource-scoped findings list and the alert-mirror reconcile
// all join a finding to its resource by that ID, so a demo finding stamped
// with a snapshot source ID joined none of them.
type demoResourceResolver struct {
readState unifiedresources.ReadState
subjects unifiedresources.AlertResourceReferenceResolver
}
// newDemoResourceResolver resolves against Patrol's read state. Without one,
// or while it holds none of the record kinds the synthesizer reads (a demo
// cycle can start before the registry's first ingest), it derives one from
// the snapshot, as the real producers do: the findings store keeps a
// finding's resource for the life of the process, so a first cycle must not
// settle for source IDs. The derived IDs match what the registry assigns on
// ingest unless a saved operator link folds the record into another
// resource; such a finding keeps the derived ID until restart. Once the
// registry holds such records it is the only authority, so a record it lacks
// or refuses (an operator link can fold a Docker host into a guest) never
// takes an ID from a registry without the operator's links.
func newDemoResourceResolver(snap patrolRuntimeState) *demoResourceResolver {
if snap.readState == nil || demoReadStateLacksRecords(snap.readState) {
snap = snap.withDerivedProviders()
}
return &demoResourceResolver{readState: snap.readState, subjects: patrolAlertReferenceResolver(snap)}
}
func demoReadStateLacksRecords(rs unifiedresources.ReadState) bool {
return len(rs.VMs()) == 0 && len(rs.Containers()) == 0 && len(rs.StoragePools()) == 0 &&
len(rs.DockerHosts()) == 0 && len(rs.DockerContainers()) == 0 &&
len(rs.PBSInstances()) == 0 && len(rs.PMGInstances()) == 0
}
// unifiedID resolves the reference an alert on the record carries the way
// the Inbox resolves an alert's subject (AttentionSubjectResourceID), so a
// demo finding names the resource an Inbox item about the same record names.
// A reference that binds nothing, or binds another type of resource, keeps
// the record's source ID.
func (r *demoResourceResolver) unifiedID(reference, sourceID string, resourceType unifiedresources.ResourceType) string {
binding, _ := r.subjects.ResolveAlertResourceReference(reference)
if binding.ResourceID == "" || binding.Component || unifiedresources.CanonicalResourceType(binding.ResourceType) != resourceType {
return sourceID
}
return binding.ResourceID
}
// containerID resolves an app container through the container views, by
// host and container ID: a Docker container alert reference binds only a
// complete container ID, and mock containers carry short ones.
func (r *demoResourceResolver) containerID(hostID, containerID string) string {
host, container := strings.TrimSpace(hostID), strings.TrimSpace(containerID)
for _, view := range r.readState.DockerContainers() {
if view != nil && view.HostSourceID() == host && view.ContainerID() == container {
return view.ID()
}
}
return containerID
}
// synthesizeDemoPatrolFindings derives demo findings from the live mock
// state. Every finding names a resource that exists in the demo dataset by
// its unified resource ID and quotes its actual observed values, so the
// patrol surface stays consistent with what the rest of the demo UI shows.
// The demo scenario data pins a set of deliberate anomalies (an offline
// Docker edge host, a degraded PBS instance, unhealthy containers), which
// gives the surface a stable core of findings across cycles.
func synthesizeDemoPatrolFindings(snap patrolRuntimeState, now time.Time) []*Finding {
var findings []*Finding
resources := newDemoResourceResolver(snap)
// Offline Docker host: the demo scenario keeps one edge host offline.
for _, host := range snap.DockerHosts {
if !strings.EqualFold(strings.TrimSpace(host.Status), "offline") {
continue
}
name := demoDockerHostDisplayName(host)
findings = append(findings, &Finding{
ID: demoFindingID("docker-host-offline", host.Hostname),
Key: fmt.Sprintf("docker:%s:offline", host.Hostname),
Severity: FindingSeverityCritical,
Category: FindingCategoryReliability,
ResourceID: resources.unifiedID(alerts.DockerHostResourceID(host.ID), host.ID, unifiedresources.ResourceTypeAgent),
ResourceName: name,
ResourceType: "docker-host",
Node: host.Hostname,
Title: fmt.Sprintf("Docker host %q is offline", name),
Description: fmt.Sprintf(
"The Pulse agent on %s has stopped reporting. %d containers on this host are unmonitored until the agent reconnects.",
name, len(host.Containers)),
Impact: "Workloads on this host may be down, and Pulse cannot see them while the agent is offline.",
Recommendation: "Confirm the machine has power and network connectivity, then check the agent with " +
"systemctl status pulse-agent and the Docker daemon with systemctl status docker. " +
"Recent agent logs (journalctl -u pulse-agent) usually show why reporting stopped.",
Evidence: fmt.Sprintf("status=offline containers=%d last_seen=%s", len(host.Containers), host.LastSeen.UTC().Format(time.RFC3339)),
DetectedAt: now.Add(-95 * time.Minute),
LastSeenAt: now,
Source: "patrol",
})
break
}
// Highest-utilization storage pool, quoting its actual numbers.
var topStorage *models.Storage
for i := range snap.Storage {
s := &snap.Storage[i]
if s.Total <= 0 || !s.Enabled || strings.EqualFold(strings.TrimSpace(s.Status), "offline") {
continue
}
if topStorage == nil || s.Usage > topStorage.Usage {
topStorage = s
}
}
if topStorage != nil && topStorage.Usage >= 60 {
severity := FindingSeverityWarning
urgency := "Growth at this level is worth reviewing before it becomes urgent."
if topStorage.Usage >= 85 {
severity = FindingSeverityCritical
urgency = "New snapshots, backups, and guest disk growth are at risk once the pool fills."
}
location := topStorage.Node
if topStorage.Shared || location == "" {
location = "the cluster"
}
findings = append(findings, &Finding{
ID: demoFindingID("storage-capacity", topStorage.ID),
Key: fmt.Sprintf("storage:%s:capacity", topStorage.Name),
Severity: severity,
Category: FindingCategoryCapacity,
ResourceID: resources.unifiedID(topStorage.ID, topStorage.ID, unifiedresources.ResourceTypeStorage),
ResourceName: topStorage.Name,
ResourceType: "storage",
Node: topStorage.Node,
Title: fmt.Sprintf("Storage %q is %.0f%% full", topStorage.Name, topStorage.Usage),
Description: fmt.Sprintf(
"Storage %s on %s has %s free of %s. It is the fullest pool in this environment. %s",
topStorage.Name, location, formatDemoBytes(topStorage.Free), formatDemoBytes(topStorage.Total), urgency),
Recommendation: "Review the largest guest disks and stale snapshots on this pool and reclaim what is no longer needed. " +
"Longer term, plan an expansion or migrate guests to a less utilized pool.",
Evidence: fmt.Sprintf("usage=%.1f%% used=%s total=%s", topStorage.Usage, formatDemoBytes(topStorage.Used), formatDemoBytes(topStorage.Total)),
DetectedAt: now.Add(-26 * time.Hour),
LastSeenAt: now,
Source: "patrol",
})
}
// Degraded backup server: the demo scenario keeps one PBS instance degraded.
for _, pbs := range snap.PBSInstances {
statusBad := !strings.EqualFold(strings.TrimSpace(pbs.Status), "online")
healthBad := pbs.ConnectionHealth != "" && !strings.EqualFold(strings.TrimSpace(pbs.ConnectionHealth), "healthy")
if !statusBad && !healthBad {
continue
}
findings = append(findings, &Finding{
ID: demoFindingID("pbs-degraded", pbs.Name),
Key: fmt.Sprintf("pbs:%s:connection", pbs.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryBackup,
ResourceID: resources.unifiedID(pbs.ID, pbs.ID, unifiedresources.ResourceTypePBS),
ResourceName: pbs.Name,
ResourceType: "pbs",
Node: pbs.Name,
Title: fmt.Sprintf("Backup server %q connection is degraded", pbs.Name),
Description: fmt.Sprintf(
"Pulse is reaching backup server %s intermittently. New backup and verify jobs cannot be confirmed while the connection is degraded.",
pbs.Name),
Impact: "If backups stop landing here, restore points age silently until the connection recovers.",
Recommendation: "Check the PBS service and datastore status on the server, confirm the network path from the " +
"Proxmox nodes, and review the most recent backup task logs for timeouts.",
Evidence: fmt.Sprintf("status=%s connection=%s", pbs.Status, pbs.ConnectionHealth),
DetectedAt: now.Add(-7 * time.Hour),
LastSeenAt: now,
Source: "patrol",
})
break
}
// Unhealthy or restart-looping containers on online Docker hosts.
containerFindings := 0
for _, host := range snap.DockerHosts {
if strings.EqualFold(strings.TrimSpace(host.Status), "offline") {
continue
}
hostName := demoDockerHostDisplayName(host)
for i := range host.Containers {
c := host.Containers[i]
unhealthy := strings.EqualFold(strings.TrimSpace(c.Health), "unhealthy")
flapping := c.RestartCount >= 3 && strings.EqualFold(strings.TrimSpace(c.State), "running")
if !unhealthy && !flapping {
continue
}
title := fmt.Sprintf("Container %q is failing its health check", c.Name)
description := fmt.Sprintf(
"Container %s on %s reports an unhealthy status from its Docker health check.",
c.Name, hostName)
if !unhealthy {
title = fmt.Sprintf("Container %q is restarting repeatedly", c.Name)
description = fmt.Sprintf(
"Container %s on %s has restarted %d times. Frequent restarts usually point at a crash loop or resource limits.",
c.Name, hostName, c.RestartCount)
}
findings = append(findings, &Finding{
ID: demoFindingID("docker-container", host.Hostname+"-"+c.Name),
Key: fmt.Sprintf("docker:%s/%s:health", host.Hostname, c.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryReliability,
ResourceID: resources.containerID(host.ID, c.ID),
ResourceName: c.Name,
ResourceType: "app-container",
Node: host.Hostname,
Title: title,
Description: description,
Recommendation: fmt.Sprintf(
"Check recent logs with docker logs %s --tail 100, inspect the health check definition with docker inspect, "+
"and watch docker stats for memory pressure.", c.Name),
Evidence: fmt.Sprintf("health=%s state=%s restarts=%d image=%s", c.Health, c.State, c.RestartCount, c.Image),
DetectedAt: now.Add(-4 * time.Hour),
LastSeenAt: now,
Source: "patrol",
})
containerFindings++
if containerFindings >= 2 {
break
}
}
if containerFindings >= 2 {
break
}
}
// Degraded mail gateway: the demo scenario keeps one PMG instance degraded.
for _, pmg := range snap.PMGInstances {
if strings.EqualFold(strings.TrimSpace(pmg.Status), "online") {
continue
}
findings = append(findings, &Finding{
ID: demoFindingID("pmg-degraded", pmg.Name),
Key: fmt.Sprintf("pmg:%s:connection", pmg.Name),
Severity: FindingSeverityWarning,
Category: FindingCategoryReliability,
ResourceID: resources.unifiedID(pmg.ID, pmg.ID, unifiedresources.ResourceTypePMG),
ResourceName: pmg.Name,
ResourceType: "pmg",
Node: pmg.Name,
Title: fmt.Sprintf("Mail gateway %q is degraded", pmg.Name),
Description: fmt.Sprintf(
"Mail gateway %s is responding intermittently. Mail flow statistics and quarantine data may be stale until it recovers.",
pmg.Name),
Recommendation: "Confirm the PMG services are running on the host, check connectivity from Pulse to the " +
"PMG API endpoint, and review the PMG task log for postfix or clamav errors.",
Evidence: fmt.Sprintf("status=%s", pmg.Status),
DetectedAt: now.Add(-3 * time.Hour),
LastSeenAt: now,
Source: "patrol",
})
break
}
// Highest-memory running guest, only when genuinely high.
type guestRef struct {
id, name, node, label string
resourceType unifiedresources.ResourceType
usage float64
used, total int64
}
var topGuest *guestRef
consider := func(id, name, node, label string, resourceType unifiedresources.ResourceType, status string, mem models.Memory) {
if !strings.EqualFold(strings.TrimSpace(status), "running") || mem.Total <= 0 {
return
}
if topGuest == nil || mem.Usage > topGuest.usage {
topGuest = &guestRef{id: id, name: name, node: node, label: label, resourceType: resourceType, usage: mem.Usage, used: mem.Used, total: mem.Total}
}
}
for i := range snap.VMs {
vm := &snap.VMs[i]
consider(vm.ID, vm.Name, vm.Node, "VM", unifiedresources.ResourceTypeVM, vm.Status, vm.Memory)
}
for i := range snap.Containers {
ct := &snap.Containers[i]
consider(ct.ID, ct.Name, ct.Node, "Container", unifiedresources.ResourceTypeSystemContainer, ct.Status, ct.Memory)
}
if topGuest != nil && topGuest.usage >= 85 {
findings = append(findings, &Finding{
ID: demoFindingID("guest-memory", topGuest.id),
Key: fmt.Sprintf("guest:%s:memory", topGuest.id),
Severity: FindingSeverityWarning,
Category: FindingCategoryPerformance,
ResourceID: resources.unifiedID(topGuest.id, topGuest.id, topGuest.resourceType),
ResourceName: topGuest.name,
ResourceType: string(topGuest.resourceType),
Node: topGuest.node,
Title: fmt.Sprintf("%s %q memory usage at %.0f%%", topGuest.label, topGuest.name, topGuest.usage),
Description: fmt.Sprintf(
"%s %s on %s is using %s of its %s allocation. Sustained usage at this level risks swapping and degraded performance under load.",
topGuest.label, topGuest.name, topGuest.node, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),
Recommendation: "Increase the memory allocation if the host has capacity, or check the workload for a leak " +
"and review application cache settings to reduce steady-state usage.",
Evidence: fmt.Sprintf("memory=%.1f%% used=%s total=%s", topGuest.usage, formatDemoBytes(topGuest.used), formatDemoBytes(topGuest.total)),
DetectedAt: now.Add(-5 * time.Hour),
LastSeenAt: now,
Source: "patrol",
})
}
return findings
}
func demoFindingID(kind, key string) string {
return demoFindingIDPrefix + kind + "-" + demoSlug(key)
}
func demoSlug(s string) string {
var b strings.Builder
for _, r := range strings.ToLower(strings.TrimSpace(s)) {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
b.WriteRune(r)
default:
b.WriteRune('-')
}
}
return strings.Trim(b.String(), "-")
}
func demoDockerHostDisplayName(host models.DockerHost) string {
if name := strings.TrimSpace(host.CustomDisplayName); name != "" {
return name
}
if name := strings.TrimSpace(host.DisplayName); name != "" {
return name
}
return strings.TrimSpace(host.Hostname)
}
func formatDemoBytes(bytes int64) string {
const gib = float64(1 << 30)
value := float64(bytes) / gib
if value >= 1024 {
return fmt.Sprintf("%.1f TiB", value/1024)
}
if value >= 10 {
return fmt.Sprintf("%.0f GiB", value)
}
return fmt.Sprintf("%.1f GiB", value)
}
// GenerateDemoAIResponse returns a realistic mock AI response for demo mode
// This allows the demo server to showcase the AI Assistant without a real API key
func GenerateDemoAIResponse(prompt string) *ExecuteResponse {
promptLower := strings.ToLower(prompt)
// Determine response based on prompt content
var response string
switch {
// Detect Patrol Analysis Request
case strings.Contains(promptLower, "analyze") && strings.Contains(promptLower, "infrastructure"):
response = `Based on the infrastructure data provided, I have identified the following issues:
[FINDING]
SEVERITY: critical
CATEGORY: capacity
KEY: storage:local-zfs:capacity
RESOURCE_ID: storage/pve1/local-zfs
RESOURCE_NAME: local-zfs
RESOURCE_TYPE: storage
NODE: pve1
TITLE: ZFS pool 'local-zfs' is 94% full
DESCRIPTION: Storage pool local-zfs on pve1 has critical capacity usage. This endangers VM stability and snapshot creation.
RECOMMENDATION: **Immediate actions:**
1. Identify large files: ` + "`zfs list -o name,used,refer -t all | sort -k2 -h`" + `
2. Check for orphaned VM disks
3. Remove old snapshots
EVIDENCE: Used: 94% (703GB/750GB)
[/FINDING]
[FINDING]
SEVERITY: warning
CATEGORY: performance
KEY: vm:vm-102:memory
RESOURCE_ID: qemu/102
RESOURCE_NAME: vm-database
RESOURCE_TYPE: vm
NODE: pve1
TITLE: High memory pressure on Database VM
DESCRIPTION: VM 'vm-database' is consistently using >90% RAM with significant swap activity.
RECOMMENDATION: Increase memory allocation to 16GB or enable ballooning.
EVIDENCE: Memory: 92% (7.4GB/8GB), Swap: 30%
[/FINDING]
[FINDING]
SEVERITY: warning
CATEGORY: security
KEY: agent:pve2:ssh
RESOURCE_ID: node/pve2
RESOURCE_NAME: pve2
RESOURCE_TYPE: node
NODE: pve2
TITLE: Root SSH login enabled
DESCRIPTION: Root SSH login is enabled on node pve2, which is a security risk.
RECOMMENDATION: Disable root login in /etc/ssh/sshd_config and use key-based authentication.
EVIDENCE: PermitRootLogin yes found in config
[/FINDING]
`
case strings.Contains(promptLower, "disk") || strings.Contains(promptLower, "storage") || strings.Contains(promptLower, "full"):
response = "## Disk Usage Analysis\n\n" +
"Based on the current metrics, I can see elevated disk usage. Here are my recommendations:\n\n" +
"### Immediate Actions\n" +
"1. **Check large files**: Run `du -sh /* | sort -rh | head -20` to find the largest directories\n" +
"2. **Review logs**: Old logs often consume significant space. Check `/var/log` and consider log rotation\n" +
"3. **Docker cleanup**: If using Docker, run `docker system prune -a` to remove unused images\n\n" +
"### Long-term Solutions\n" +
"- Set up automated log rotation with logrotate\n" +
"- Configure alerts at 80% to catch issues before they become critical\n" +
"- Consider expanding storage if usage is consistently high\n\n" +
"Would you like me to help investigate any specific directory?"
case strings.Contains(promptLower, "memory") || strings.Contains(promptLower, "ram") || strings.Contains(promptLower, "oom"):
response = "## Memory Analysis\n\n" +
"I can help analyze memory usage patterns. Here's what I recommend:\n\n" +
"### Quick Diagnostics\n" +
"1. **Current usage**: Check top consumers with `ps aux --sort=-%mem | head -15`\n" +
"2. **Memory pressure**: Review `/proc/meminfo` for swap usage\n" +
"3. **OOM events**: Check `dmesg | grep -i oom` for recent kills\n\n" +
"### Optimization Tips\n" +
"- Consider increasing VM memory allocation if the host has capacity\n" +
"- Review application memory limits (especially for Java apps with -Xmx)\n" +
"- Enable memory ballooning for better cluster-wide memory utilization\n\n" +
"This is a **demo instance** - in production, I can run these commands directly on your nodes."
case strings.Contains(promptLower, "backup") || strings.Contains(promptLower, "pbs"):
response = "## Backup Status Review\n\n" +
"Backups are critical for data protection. Here's my analysis:\n\n" +
"### Recommended Checks\n" +
"1. **PBS connectivity**: Verify the Proxmox Backup Server is reachable\n" +
"2. **Job schedules**: Review backup job configurations in Datacenter → Backup\n" +
"3. **Storage capacity**: Ensure PBS datastore has sufficient space for new backups\n\n" +
"### Best Practices\n" +
"- Schedule daily backups during low-usage periods\n" +
"- Keep at least 7 daily + 4 weekly retention\n" +
"- Test restores periodically to verify backup integrity\n\n" +
"Would you like me to help configure backup schedules for specific VMs?"
case strings.Contains(promptLower, "cpu") || strings.Contains(promptLower, "load") || strings.Contains(promptLower, "slow"):
response = "## CPU/Performance Analysis\n\n" +
"High CPU can indicate various issues. Let me help diagnose:\n\n" +
"### Diagnostic Steps\n" +
"1. **Top processes**: `top -bn1 | head -20` shows current CPU consumers\n" +
"2. **Load average**: Check if load > number of CPU cores\n" +
"3. **Per-VM usage**: Review individual guest CPU allocation\n\n" +
"### Common Causes\n" +
"- Overprovisioned guests (total vCPUs > physical cores)\n" +
"- Runaway processes within VMs\n" +
"- Background tasks like backups or replication\n\n" +
"### Quick Wins\n" +
"- Consider live-migrating busy VMs to less loaded nodes\n" +
"- Set CPU limits on non-critical guests\n" +
"- Schedule heavy tasks during off-peak hours"
case strings.Contains(promptLower, "hello") || strings.Contains(promptLower, "hi") || strings.Contains(promptLower, "help"):
response = "## Hello! 👋\n\n" +
"I'm the **Pulse Assistant**, here to help you manage your Proxmox infrastructure.\n\n" +
"### What I Can Help With\n" +
"- **Troubleshooting**: Diagnose disk, memory, CPU, and network issues\n" +
"- **Backups**: Review backup status and configure schedules\n" +
"- **Optimization**: Identify resource bottlenecks and optimization opportunities\n" +
"- **Commands**: Execute maintenance commands on your nodes (with your approval)\n\n" +
"### Try Asking\n" +
"- \"Why is my disk filling up?\"\n" +
"- \"Help me fix the backup failure on vm-102\"\n" +
"- \"Check memory usage on pve1\"\n\n" +
"*Note: This is a demo instance - command execution is disabled, but you can see how Pulse Assistant analysis works!*"
default:
response = "## Analysis\n\n" +
"I can help you with that! In a production environment with Pulse Assistant configured, I would:\n\n" +
"1. **Analyze** the current state of your infrastructure\n" +
"2. **Identify** potential issues or optimization opportunities\n" +
"3. **Recommend** specific actions with commands you can run\n" +
"4. **Execute** approved commands directly on your nodes\n\n" +
"### This Demo Shows\n" +
"- How Pulse Assistant analysis works in Pulse\n" +
"- The types of insights and recommendations you'll receive\n" +
"- Command approvals with manual confirmation\n\n" +
"To enable full Pulse Assistant capabilities in your own Pulse installation:\n" +
"1. Go to **Settings → Pulse Assistant**\n" +
"2. Add your API key (Anthropic, OpenAI, DeepSeek, or Ollama)\n" +
"3. Enable Pulse Assistant features\n\n" +
"*Visit [pulserelay.pro](https://pulserelay.pro) to get Pulse Pro!*"
}
return &ExecuteResponse{
Content: response,
Model: "demo-model",
InputTokens: 150,
OutputTokens: 400,
}
}
// GenerateDemoAIStream acts like GenerateDemoAIResponse but streams content via callback
func GenerateDemoAIStream(prompt string, callback StreamCallback) (*ExecuteResponse, error) {
resp := GenerateDemoAIResponse(prompt)
// Simulate streaming by sending chunks
chunkSize := 10
content := resp.Content
for i := 0; i < len(content); i += chunkSize {
end := i + chunkSize
if end > len(content) {
end = len(content)
}
callback(StreamEvent{
Type: "content",
Data: content[i:end],
})
// Tiny sleep to simulate generation speed
time.Sleep(10 * time.Millisecond)
}
callback(StreamEvent{
Type: "done",
})
return resp, nil
}