Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import type { FilterStack } from '@/utils/searchQuery';
export interface HysteresisThreshold {
trigger: number;
clear: number;
}
export interface AlertThresholds {
cpu?: HysteresisThreshold;
memory?: HysteresisThreshold;
disk?: HysteresisThreshold;
diskRead?: HysteresisThreshold;
diskWrite?: HysteresisThreshold;
networkIn?: HysteresisThreshold;
networkOut?: HysteresisThreshold;
temperature?: HysteresisThreshold;
diskTemperature?: HysteresisThreshold;
smartHealthFailure?: number;
smartReallocated?: number;
smartPending?: number;
smartUncorrectable?: number;
smartMediaErrors?: number;
smartCrcErrorDelta?: number;
smartLifeWarning?: number;
smartLifeCritical?: number;
smartSpareWarning?: number;
smartSpareCritical?: number;
usage?: HysteresisThreshold;
disableConnectivity?: boolean; // Disable connectivity/powered-off alerts
poweredOffSeverity?: 'warning' | 'critical';
// Allow indexing with string
[key: string]:
| HysteresisThreshold
| BackupAlertConfig
| SnapshotAlertConfig
| number
| boolean
| string
| undefined;
}
export type RawOverrideConfig = AlertThresholds & {
disabled?: boolean;
disableConnectivity?: boolean;
poweredOffSeverity?: 'warning' | 'critical';
note?: string;
backup?: BackupAlertConfig;
snapshot?: SnapshotAlertConfig;
// NOTE: To disable individual metrics, set threshold to -1
};
export interface CustomAlertRule {
id: string;
name: string;
description?: string;
filterConditions: FilterStack;
thresholds: AlertThresholds;
priority: number;
enabled: boolean;
notifications: {
email?: {
enabled: boolean;
recipients: string[];
};
webhook?: {
enabled: boolean;
url: string;
};
};
createdAt: string;
updatedAt: string;
}
export interface DockerThresholdConfig {
cpu?: HysteresisThreshold;
memory?: HysteresisThreshold;
disk?: HysteresisThreshold;
restartCount?: number;
restartWindow?: number;
memoryWarnPct?: number;
memoryCriticalPct?: number;
serviceWarnGapPercent?: number;
serviceCriticalGapPercent?: number;
updateAlertDelayHours?: number;
stateDisableConnectivity?: boolean;
statePoweredOffSeverity?: 'warning' | 'critical';
}
export interface PMGThresholdDefaults {
queueTotalWarning?: number;
queueTotalCritical?: number;
oldestMessageWarnMins?: number;
oldestMessageCritMins?: number;
deferredQueueWarn?: number;
deferredQueueCritical?: number;
holdQueueWarn?: number;
holdQueueCritical?: number;
quarantineSpamWarn?: number;
quarantineSpamCritical?: number;
quarantineVirusWarn?: number;
quarantineVirusCritical?: number;
quarantineGrowthWarnPct?: number;
quarantineGrowthWarnMin?: number;
quarantineGrowthCritPct?: number;
quarantineGrowthCritMin?: number;
}
export interface SnapshotAlertConfig {
enabled: boolean;
warningDays: number;
criticalDays: number;
warningSizeGiB?: number;
criticalSizeGiB?: number;
}
export interface BackupAlertConfig {
enabled: boolean;
warningDays: number;
criticalDays: number;
// Dashboard indicator thresholds (separate from alert thresholds)
freshHours?: number; // Backups newer than this show as green (default: 24)
staleHours?: number; // Backups older than freshHours but newer than this show as amber (default: 72)
// Global backup alert filters
alertOrphaned?: boolean; // Alert on backups that do not match a known guest (default: true)
ignoreVMIDs?: string[]; // Skip alerts for matching VMIDs (supports prefix*)
}
export type ActivationState = 'pending_review' | 'active' | 'snoozed';
export interface AlertConfig {
identitySchemaVersion?: number;
enabled: boolean;
activationState?: ActivationState;
observationWindowHours?: number;
activationTime?: string;
guestDefaults: AlertThresholds;
nodeDefaults: AlertThresholds;
agentDefaults?: AlertThresholds;
storageDefault: HysteresisThreshold;
dockerDefaults?: DockerThresholdConfig;
dockerIgnoredContainerPrefixes?: string[];
ignoredGuestPrefixes?: string[];
guestTagWhitelist?: string[];
guestTagBlacklist?: string[];
pmgDefaults?: PMGThresholdDefaults;
pbsDefaults?: AlertThresholds;
kubernetesDefaults?: AlertThresholds;
truenasDefaults?: AlertThresholds;
truenasDiskDefaults?: AlertThresholds;
// Set on every config written since an unset truenasDiskDefaults
// temperature means each TrueNAS disk follows Disk temperature by type.
truenasDiskTemperatureByType?: boolean;
vmwareDefaults?: AlertThresholds;
snapshotDefaults?: SnapshotAlertConfig;
backupDefaults?: BackupAlertConfig;
diskFillByType?: Record<string, HysteresisThreshold>;
diskTempByType?: Record<string, HysteresisThreshold>;
customRules?: CustomAlertRule[];
overrides: Record<string, RawOverrideConfig>; // key: resource ID
minimumDelta?: number;
suppressionWindow?: number;
hysteresisMargin?: number;
timeThresholds?: {
guest?: number;
node?: number;
storage?: number;
pbs?: number;
agent?: number;
'k8s-cluster'?: number;
'k8s-node'?: number;
'k8s-deployment'?: number;
'k8s-namespace'?: number;
pod?: number;
'truenas-system'?: number;
'truenas-pool'?: number;
'truenas-dataset'?: number;
'truenas-disk'?: number;
'vmware-host'?: number;
'vmware-vm'?: number;
'vmware-datastore'?: number;
'vmware-network'?: number;
};
metricTimeThresholds?: Record<string, Record<string, number>>;
metricEvaluationWindows?: Record<string, Record<string, number>>;
aggregation?: {
enabled: boolean;
timeWindow: number;
countThreshold: number;
similarityWindow: number;
};
flapping?: {
enabled: boolean;
threshold: number;
window: number;
suppressionTime: number;
minStability: number;
};
ioNormalization?: {
enabled: boolean;
vmDiskMax: number;
containerDiskMax: number;
networkMax: number;
};
notifications?: {
email?: {
server: string;
port: number;
username: string;
password: string;
from: string;
tls: boolean;
};
webhooks?: Array<{
id: string;
name: string;
url: string;
enabled: boolean;
}>;
};
schedule?: {
quietHours?: {
enabled: boolean;
start: string;
end: string;
timezone?: string;
days: number[] | Record<string, boolean>;
suppress?: {
performance?: boolean;
storage?: boolean;
offline?: boolean;
};
};
cooldown?: number;
maxAlertsHour?: number;
initialNotify?: string;
notifyOnResolve?: boolean;
grouping?: {
enabled: boolean;
window: number;
byNode?: boolean;
byGuest?: boolean;
};
escalation?: {
enabled: boolean;
levels?: Array<{ after: number; notify: string; destinationIds?: string[] }>;
repeatCritical?: boolean;
repeatEvery?: number;
};
};
disableAllNodes?: boolean;
disableAllGuests?: boolean;
disableAllStorage?: boolean;
disableAllPBS?: boolean;
disableAllPMG?: boolean;
disableAllAgents?: boolean;
disableAllDockerHosts?: boolean;
disableAllDockerContainers?: boolean;
disableAllDockerServices?: boolean;
disableAllKubernetes?: boolean;
disableAllTrueNAS?: boolean;
disableAllVMware?: boolean;
disableAllNodesOffline?: boolean;
disableAllGuestsOffline?: boolean;
disableAllPBSOffline?: boolean;
disableAllPMGOffline?: boolean;
disableAllAgentsOffline?: boolean;
disableAllDockerHostsOffline?: boolean;
}
export interface DeadManInterruption {
from: string;
to: string;
durationSeconds: number;
cleanShutdown: boolean;
}
export interface DeadManStatus {
configured: boolean;
state:
| 'disabled'
| 'starting'
| 'healthy'
| 'delivery_failed'
| 'monitor_stalled'
| 'misconfigured'
| 'configuration_unavailable';
heartbeatIntervalSeconds: number;
recommendedGraceSeconds: number;
lastMonitoringProgress?: string;
lastAttemptAt?: string;
lastSuccessAt?: string;
consecutiveFailures: number;
lastError?: string;
lastInterruption?: DeadManInterruption;
}
// Priority levels:
// 0: Global defaults
// 1-99: Reserved for system rules
// 100+: Custom user rules
// 1000+: Guest-specific overrides