Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package monitoring
import (
"path"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)
const (
agentLXCFilesystemMinTTL = 3 * time.Minute
agentLXCFilesystemMaxTTL = 15 * time.Minute
agentLXCFilesystemMaxContainers = 128
agentLXCFilesystemMaxDisks = 257
agentLXCFilesystemMaxNameBytes = 256
agentLXCFilesystemMaxLabelBytes = 512
)
type agentLXCFilesystemCacheEntry struct {
agentID string
name string
disks []models.Disk
expiresAt time.Time
}
func agentLXCFilesystemCacheKey(instance, node string, vmid int) string {
return agentLXCFilesystemNodePrefix(instance, node) + strconv.Itoa(vmid)
}
func agentLXCFilesystemNodePrefix(instance, node string) string {
return strings.ToLower(strings.TrimSpace(instance)) + "\x00" +
strings.ToLower(strings.TrimSpace(node)) + "\x00"
}
func agentLXCFilesystemTTL(intervalSeconds int) time.Duration {
ttl := time.Duration(intervalSeconds) * 3 * time.Second
if ttl < agentLXCFilesystemMinTTL {
return agentLXCFilesystemMinTTL
}
if ttl > agentLXCFilesystemMaxTTL {
return agentLXCFilesystemMaxTTL
}
return ttl
}
// applyAgentLXCFilesystems admits node-local pct df data only when the
// reporting host agent is already linked to exactly one current Proxmox node.
// Receipt time, rather than the agent clock, controls freshness.
func (m *Monitor) applyAgentLXCFilesystems(
linkedNodeID string,
agentID string,
inventory *agentshost.ProxmoxLXCInventory,
receivedAt time.Time,
intervalSeconds int,
) {
agentID = strings.TrimSpace(agentID)
if m == nil || inventory == nil ||
strings.TrimSpace(linkedNodeID) == "" || agentID == "" {
return
}
if inventory.ValidateCollection() != nil {
return
}
// This is live agent admission, not a full inventory or mock-mode read.
// Capture the replaceable state once and copy only the unique linked node.
m.mu.RLock()
state := m.state
m.mu.RUnlock()
if state == nil {
return
}
node, unique := state.GetUniqueNode(linkedNodeID)
if !unique {
return
}
if strings.TrimSpace(node.Instance) == "" || strings.TrimSpace(node.Name) == "" {
return
}
if receivedAt.IsZero() {
receivedAt = time.Now()
}
expiresAt := receivedAt.Add(agentLXCFilesystemTTL(intervalSeconds))
limit := len(inventory.Containers)
if limit > agentLXCFilesystemMaxContainers {
limit = agentLXCFilesystemMaxContainers
}
updates := make(map[string]agentLXCFilesystemCacheEntry, limit)
for _, container := range inventory.Containers[:limit] {
name := strings.TrimSpace(container.Name)
if container.VMID < 100 || container.VMID > 999999999 ||
!safeAgentLXCFilesystemText(name, agentLXCFilesystemMaxNameBytes) {
continue
}
disks := normalizeAgentLXCFilesystems(container.Disks)
if len(disks) == 0 {
continue
}
updates[agentLXCFilesystemCacheKey(node.Instance, node.Name, container.VMID)] =
agentLXCFilesystemCacheEntry{
agentID: agentID,
name: name,
disks: disks,
expiresAt: expiresAt,
}
}
m.proxmoxLXCFilesystemsMu.Lock()
defer m.proxmoxLXCFilesystemsMu.Unlock()
if m.proxmoxLXCFilesystemsCache == nil {
m.proxmoxLXCFilesystemsCache = make(map[string]agentLXCFilesystemCacheEntry)
}
for key, entry := range m.proxmoxLXCFilesystemsCache {
if !receivedAt.Before(entry.expiresAt) ||
(inventory.Status == agentshost.ProxmoxLXCCollectionComplete && entry.agentID == agentID &&
strings.HasPrefix(key, agentLXCFilesystemNodePrefix(node.Instance, node.Name))) {
delete(m.proxmoxLXCFilesystemsCache, key)
}
}
for _, vmid := range inventory.OmittedVMIDs {
key := agentLXCFilesystemCacheKey(node.Instance, node.Name, vmid)
if entry, exists := m.proxmoxLXCFilesystemsCache[key]; exists && entry.agentID == agentID {
delete(m.proxmoxLXCFilesystemsCache, key)
}
}
for key, entry := range updates {
m.proxmoxLXCFilesystemsCache[key] = entry
}
}
func normalizeAgentLXCFilesystems(disks []agentshost.Disk) []models.Disk {
limit := len(disks)
if limit > agentLXCFilesystemMaxDisks {
limit = agentLXCFilesystemMaxDisks
}
result := make([]models.Disk, 0, limit)
seenMountpoints := make(map[string]struct{}, limit)
for _, disk := range disks[:limit] {
diskType := strings.ToLower(strings.TrimSpace(disk.Type))
mountpoint := strings.TrimSpace(disk.Mountpoint)
device := strings.TrimSpace(disk.Device)
if diskType != "rootfs" && !validAgentLXCMountKey(diskType) {
continue
}
if !validAgentLXCMountpoint(mountpoint) ||
!safeAgentLXCFilesystemText(device, agentLXCFilesystemMaxLabelBytes) ||
disk.TotalBytes <= 0 ||
disk.UsedBytes < 0 ||
disk.FreeBytes < 0 {
continue
}
mountpointKey := strings.ToLower(mountpoint)
if _, exists := seenMountpoints[mountpointKey]; exists {
continue
}
seenMountpoints[mountpointKey] = struct{}{}
used := disk.UsedBytes
if used > disk.TotalBytes {
used = disk.TotalBytes
}
free := disk.FreeBytes
if free > disk.TotalBytes {
free = disk.TotalBytes
}
result = append(result, models.Disk{
Total: disk.TotalBytes,
Used: used,
Free: free,
Usage: safePercentage(float64(used), float64(disk.TotalBytes)),
Mountpoint: mountpoint,
Type: diskType,
Device: device,
})
}
sort.SliceStable(result, func(i, j int) bool {
if result[i].Mountpoint == "/" {
return true
}
if result[j].Mountpoint == "/" {
return false
}
return result[i].Mountpoint < result[j].Mountpoint
})
return result
}
func (m *Monitor) enrichContainerWithAgentLXCFilesystems(
instance string,
node string,
container *models.Container,
now time.Time,
) {
if m == nil || container == nil || !strings.EqualFold(strings.TrimSpace(container.Status), "running") {
return
}
key := agentLXCFilesystemCacheKey(instance, node, container.VMID)
m.proxmoxLXCFilesystemsMu.RLock()
entry, exists := m.proxmoxLXCFilesystemsCache[key]
m.proxmoxLXCFilesystemsMu.RUnlock()
if !exists {
return
}
if now.IsZero() {
now = time.Now()
}
if !now.Before(entry.expiresAt) {
m.proxmoxLXCFilesystemsMu.Lock()
if current, ok := m.proxmoxLXCFilesystemsCache[key]; ok && !now.Before(current.expiresAt) {
delete(m.proxmoxLXCFilesystemsCache, key)
}
m.proxmoxLXCFilesystemsMu.Unlock()
return
}
if strings.TrimSpace(container.Name) != entry.name {
return
}
// Admission required the agent to be linked to this node. A reading cached
// before the operator split the node from the agent is not shown once they
// are split, whether or not a later report cleared it or even named a node.
if m.state.NodeSplitFromAgent(instance, node, entry.agentID) {
return
}
container.Disks = append([]models.Disk(nil), entry.disks...)
for _, disk := range container.Disks {
if disk.Mountpoint == "/" {
container.Disk = disk
break
}
}
}
func (m *Monitor) clearAgentLXCFilesystems(agentID string) {
if m == nil || strings.TrimSpace(agentID) == "" {
return
}
m.proxmoxLXCFilesystemsMu.Lock()
defer m.proxmoxLXCFilesystemsMu.Unlock()
for key, entry := range m.proxmoxLXCFilesystemsCache {
if entry.agentID == agentID {
delete(m.proxmoxLXCFilesystemsCache, key)
}
}
}
func validAgentLXCMountKey(value string) bool {
if !strings.HasPrefix(value, "mp") || len(value) < 3 || len(value) > 5 {
return false
}
index, err := strconv.Atoi(value[2:])
return err == nil && index >= 0 && index <= 255
}
func validAgentLXCMountpoint(value string) bool {
return safeAgentLXCFilesystemText(value, agentLXCFilesystemMaxLabelBytes) &&
strings.HasPrefix(value, "/") &&
path.Clean(value) == value
}
func safeAgentLXCFilesystemText(value string, maxBytes int) bool {
value = strings.TrimSpace(value)
if value == "" || len(value) > maxBytes || !utf8.ValidString(value) {
return false
}
for _, char := range value {
if char < 0x20 || char == 0x7f {
return false
}
}
return true
}