Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

8.1 KB
Raw Blame
import type { FilterStack } from '@/utils/searchQuery';

export interface HysteresisThreshold {
  trigger: number;
  clear: number;
}

export interface AlertThresholds {
  cpu?: HysteresisThreshold;
  memory?: HysteresisThreshold;
  disk?: HysteresisThreshold;
  diskRead?: HysteresisThreshold;
  diskWrite?: HysteresisThreshold;
  networkIn?: HysteresisThreshold;
  networkOut?: HysteresisThreshold;
  temperature?: HysteresisThreshold;
  diskTemperature?: HysteresisThreshold;
  smartHealthFailure?: number;
  smartReallocated?: number;
  smartPending?: number;
  smartUncorrectable?: number;
  smartMediaErrors?: number;
  smartCrcErrorDelta?: number;
  smartLifeWarning?: number;
  smartLifeCritical?: number;
  smartSpareWarning?: number;
  smartSpareCritical?: number;
  usage?: HysteresisThreshold;
  disableConnectivity?: boolean; // Disable connectivity/powered-off alerts
  poweredOffSeverity?: 'warning' | 'critical';
  // Allow indexing with string
  [key: string]:
    | HysteresisThreshold
    | BackupAlertConfig
    | SnapshotAlertConfig
    | number
    | boolean
    | string
    | undefined;
}

export type RawOverrideConfig = AlertThresholds & {
  disabled?: boolean;
  disableConnectivity?: boolean;
  poweredOffSeverity?: 'warning' | 'critical';
  note?: string;
  backup?: BackupAlertConfig;
  snapshot?: SnapshotAlertConfig;
  // NOTE: To disable individual metrics, set threshold to -1
};

export interface CustomAlertRule {
  id: string;
  name: string;
  description?: string;
  filterConditions: FilterStack;
  thresholds: AlertThresholds;
  priority: number;
  enabled: boolean;
  notifications: {
    email?: {
      enabled: boolean;
      recipients: string[];
    };
    webhook?: {
      enabled: boolean;
      url: string;
    };
  };
  createdAt: string;
  updatedAt: string;
}

export interface DockerThresholdConfig {
  cpu?: HysteresisThreshold;
  memory?: HysteresisThreshold;
  disk?: HysteresisThreshold;
  restartCount?: number;
  restartWindow?: number;
  memoryWarnPct?: number;
  memoryCriticalPct?: number;
  serviceWarnGapPercent?: number;
  serviceCriticalGapPercent?: number;
  updateAlertDelayHours?: number;
  stateDisableConnectivity?: boolean;
  statePoweredOffSeverity?: 'warning' | 'critical';
}

export interface PMGThresholdDefaults {
  queueTotalWarning?: number;
  queueTotalCritical?: number;
  oldestMessageWarnMins?: number;
  oldestMessageCritMins?: number;
  deferredQueueWarn?: number;
  deferredQueueCritical?: number;
  holdQueueWarn?: number;
  holdQueueCritical?: number;
  quarantineSpamWarn?: number;
  quarantineSpamCritical?: number;
  quarantineVirusWarn?: number;
  quarantineVirusCritical?: number;
  quarantineGrowthWarnPct?: number;
  quarantineGrowthWarnMin?: number;
  quarantineGrowthCritPct?: number;
  quarantineGrowthCritMin?: number;
}

export interface SnapshotAlertConfig {
  enabled: boolean;
  warningDays: number;
  criticalDays: number;
  warningSizeGiB?: number;
  criticalSizeGiB?: number;
}

export interface BackupAlertConfig {
  enabled: boolean;
  warningDays: number;
  criticalDays: number;
  // Dashboard indicator thresholds (separate from alert thresholds)
  freshHours?: number; // Backups newer than this show as green (default: 24)
  staleHours?: number; // Backups older than freshHours but newer than this show as amber (default: 72)
  // Global backup alert filters
  alertOrphaned?: boolean; // Alert on backups that do not match a known guest (default: true)
  ignoreVMIDs?: string[]; // Skip alerts for matching VMIDs (supports prefix*)
}

export type ActivationState = 'pending_review' | 'active' | 'snoozed';

export interface AlertConfig {
  identitySchemaVersion?: number;
  enabled: boolean;
  activationState?: ActivationState;
  observationWindowHours?: number;
  activationTime?: string;
  guestDefaults: AlertThresholds;
  nodeDefaults: AlertThresholds;
  agentDefaults?: AlertThresholds;
  storageDefault: HysteresisThreshold;
  dockerDefaults?: DockerThresholdConfig;
  dockerIgnoredContainerPrefixes?: string[];
  ignoredGuestPrefixes?: string[];
  guestTagWhitelist?: string[];
  guestTagBlacklist?: string[];
  pmgDefaults?: PMGThresholdDefaults;
  pbsDefaults?: AlertThresholds;
  kubernetesDefaults?: AlertThresholds;
  truenasDefaults?: AlertThresholds;
  truenasDiskDefaults?: AlertThresholds;
  // Set on every config written since an unset truenasDiskDefaults
  // temperature means each TrueNAS disk follows Disk temperature by type.
  truenasDiskTemperatureByType?: boolean;
  vmwareDefaults?: AlertThresholds;
  snapshotDefaults?: SnapshotAlertConfig;
  backupDefaults?: BackupAlertConfig;
  diskFillByType?: Record<string, HysteresisThreshold>;
  diskTempByType?: Record<string, HysteresisThreshold>;
  customRules?: CustomAlertRule[];
  overrides: Record<string, RawOverrideConfig>; // key: resource ID
  minimumDelta?: number;
  suppressionWindow?: number;
  hysteresisMargin?: number;
  timeThresholds?: {
    guest?: number;
    node?: number;
    storage?: number;
    pbs?: number;
    agent?: number;
    'k8s-cluster'?: number;
    'k8s-node'?: number;
    'k8s-deployment'?: number;
    'k8s-namespace'?: number;
    pod?: number;
    'truenas-system'?: number;
    'truenas-pool'?: number;
    'truenas-dataset'?: number;
    'truenas-disk'?: number;
    'vmware-host'?: number;
    'vmware-vm'?: number;
    'vmware-datastore'?: number;
    'vmware-network'?: number;
  };
  metricTimeThresholds?: Record<string, Record<string, number>>;
  metricEvaluationWindows?: Record<string, Record<string, number>>;
  aggregation?: {
    enabled: boolean;
    timeWindow: number;
    countThreshold: number;
    similarityWindow: number;
  };
  flapping?: {
    enabled: boolean;
    threshold: number;
    window: number;
    suppressionTime: number;
    minStability: number;
  };
  ioNormalization?: {
    enabled: boolean;
    vmDiskMax: number;
    containerDiskMax: number;
    networkMax: number;
  };
  notifications?: {
    email?: {
      server: string;
      port: number;
      username: string;
      password: string;
      from: string;
      tls: boolean;
    };
    webhooks?: Array<{
      id: string;
      name: string;
      url: string;
      enabled: boolean;
    }>;
  };
  schedule?: {
    quietHours?: {
      enabled: boolean;
      start: string;
      end: string;
      timezone?: string;
      days: number[] | Record<string, boolean>;
      suppress?: {
        performance?: boolean;
        storage?: boolean;
        offline?: boolean;
      };
    };
    cooldown?: number;
    maxAlertsHour?: number;
    initialNotify?: string;
    notifyOnResolve?: boolean;
    grouping?: {
      enabled: boolean;
      window: number;
      byNode?: boolean;
      byGuest?: boolean;
    };
    escalation?: {
      enabled: boolean;
      levels?: Array<{ after: number; notify: string; destinationIds?: string[] }>;
      repeatCritical?: boolean;
      repeatEvery?: number;
    };
  };
  disableAllNodes?: boolean;
  disableAllGuests?: boolean;
  disableAllStorage?: boolean;
  disableAllPBS?: boolean;
  disableAllPMG?: boolean;
  disableAllAgents?: boolean;
  disableAllDockerHosts?: boolean;
  disableAllDockerContainers?: boolean;
  disableAllDockerServices?: boolean;
  disableAllKubernetes?: boolean;
  disableAllTrueNAS?: boolean;
  disableAllVMware?: boolean;
  disableAllNodesOffline?: boolean;
  disableAllGuestsOffline?: boolean;
  disableAllPBSOffline?: boolean;
  disableAllPMGOffline?: boolean;
  disableAllAgentsOffline?: boolean;
  disableAllDockerHostsOffline?: boolean;
}

export interface DeadManInterruption {
  from: string;
  to: string;
  durationSeconds: number;
  cleanShutdown: boolean;
}

export interface DeadManStatus {
  configured: boolean;
  state:
    | 'disabled'
    | 'starting'
    | 'healthy'
    | 'delivery_failed'
    | 'monitor_stalled'
    | 'misconfigured'
    | 'configuration_unavailable';
  heartbeatIntervalSeconds: number;
  recommendedGraceSeconds: number;
  lastMonitoringProgress?: string;
  lastAttemptAt?: string;
  lastSuccessAt?: string;
  consecutiveFailures: number;
  lastError?: string;
  lastInterruption?: DeadManInterruption;
}

// Priority levels:
// 0: Global defaults
// 1-99: Reserved for system rules
// 100+: Custom user rules
// 1000+: Guest-specific overrides