Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

18.9 KB
Raw Blame
import type { AlertIntentSignal } from '@/api/alertIntentPolicies';
import type { Resource as UnifiedResource, ResourcePolicy } from '@/types/resource';
import type { PlatformTableColumnKind } from '@/features/platformPage/columnAlignment';
import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity';
import { getPulseRelaxedGuestTrigger, type MetricType } from '@/utils/metricThresholds';

const COLUMN_TOOLTIP_LOOKUP: Record<string, string> = {
  'cpu %': 'Percent CPU utilization allowed before an alert fires.',
  'memory %': 'Percent memory usage threshold for triggering alerts.',
  'disk %': 'Percent disk usage threshold for this resource.',
  'disk r mb/s': 'Maximum sustained disk read throughput before alerting.',
  'disk w mb/s': 'Maximum sustained disk write throughput before alerting.',
  'net in mb/s': 'Inbound network throughput threshold for alerts.',
  'net out mb/s': 'Outbound network throughput threshold for alerts.',
  'usage %': 'Storage capacity usage percentage that triggers an alert.',
  'temp °c': 'CPU temperature limit for node alerts.',
  'temperature °c': 'CPU temperature limit for node alerts.',
  temperature: 'CPU temperature limit for node alerts.',
  'disk temp °c': 'Individual disk temperature threshold for agents.',
  'restart count': 'Maximum container restarts within the evaluation window.',
  'restart window': 'Time window used to evaluate the restart count threshold.',
  'restart window (s)': 'Time window used to evaluate the restart count threshold.',
  'memory warn %': 'Warning threshold for container memory usage.',
  'memory critical %': 'Critical threshold for container memory usage.',
  'queue warn': 'Early warning when total mail queue exceeds this message count.',
  'queue crit': 'Critical alert requiring urgent action when queue reaches this size.',
  'deferred warn':
    'Early warning for messages stuck in deferred queue (waiting to retry delivery).',
  'deferred crit': 'Critical threshold for deferred messages indicating serious delivery problems.',
  'hold warn': 'Early warning when administratively held messages exceed this count.',
  'hold crit': 'Critical alert for held messages requiring immediate moderation attention.',
  'oldest warn (min)': 'Early warning when oldest queued message exceeds this age in minutes.',
  'oldest crit (min)': 'Critical alert when message queue age indicates delivery has stalled.',
  'spam warn': 'Early warning for spam messages accumulating in quarantine.',
  'spam crit': 'Critical spam quarantine level requiring urgent intervention.',
  'virus warn': 'Early warning for virus-positive messages in quarantine.',
  'virus crit': 'Critical virus quarantine threshold indicating potential outbreak.',
  'growth warn %': 'Early warning when quarantine growth rate exceeds this percentage.',
  'growth warn min': 'Minimum new messages required before growth percentage triggers warning.',
  'growth crit %': 'Critical quarantine growth rate requiring immediate investigation.',
  'growth crit min':
    'Minimum new messages required before growth percentage triggers critical alert.',
  'warning size (gib)': 'Total snapshot size in GiB that raises a warning.',
  'critical size (gib)': 'Total snapshot size in GiB that raises a critical alert.',
};

export const ALERT_RESOURCE_TABLE_SLIDER_METRICS = new Set([
  'cpu',
  'memory',
  'disk',
  'temperature',
  'diskTemperature',
]);

export type AlertResourceThresholdMap = Record<string, number | undefined>;

export interface AlertResourceTableResourceLike {
  id: string;
  name: string;
  displayName?: string;
  policy?: ResourcePolicy;
  aiSafeSummary?: string;
  rawName?: string;
  type?: string;
  thresholds?: AlertResourceThresholdMap;
  defaults?: AlertResourceThresholdMap;
  note?: string;
  [key: string]: unknown;
}

export function flattenAlertResourceTableResources<T extends AlertResourceTableResourceLike>(
  resources?: T[],
  groupedResources?: Record<string, T[]>,
): T[] {
  if (groupedResources) {
    return Object.values(groupedResources).flat();
  }
  return resources ?? [];
}

export function hasAlertResourceTableRows<T extends AlertResourceTableResourceLike>(
  resources?: T[],
  groupedResources?: Record<string, T[]>,
  globalDefaults?: AlertResourceThresholdMap,
): boolean {
  if (flattenAlertResourceTableResources(resources, groupedResources).length > 0) {
    return true;
  }
  if (groupedResources && Object.keys(groupedResources).length > 0) {
    return true;
  }
  return Boolean(globalDefaults);
}

export function hasCustomAlertResourceGlobalDefaults(
  globalDefaults?: AlertResourceThresholdMap,
  factoryDefaults?: AlertResourceThresholdMap,
): boolean {
  if (!globalDefaults || !factoryDefaults) {
    return false;
  }

  return Object.keys(factoryDefaults).some((key) => {
    const current = globalDefaults[key];
    const factory = factoryDefaults[key];
    return current !== undefined && !alertResourceThresholdsEquivalent(current, factory);
  });
}

export function normalizeAlertResourceMetricKey(column: string): string {
  const key = column.trim().toLowerCase();
  const mapped = new Map<string, string>([
    ['cpu %', 'cpu'],
    ['memory %', 'memory'],
    ['disk %', 'disk'],
    ['disk r mb/s', 'diskRead'],
    ['disk w mb/s', 'diskWrite'],
    ['net in mb/s', 'networkIn'],
    ['net out mb/s', 'networkOut'],
    ['usage %', 'usage'],
    ['temp °c', 'temperature'],
    ['temperature °c', 'temperature'],
    ['temperature', 'temperature'],
    ['restart count', 'restartCount'],
    ['restart window', 'restartWindow'],
    ['restart window (s)', 'restartWindow'],
    ['memory warn %', 'memoryWarnPct'],
    ['memory critical %', 'memoryCriticalPct'],
    ['warning size (gib)', 'warningSizeGiB'],
    ['critical size (gib)', 'criticalSizeGiB'],
    ['disk temp °c', 'diskTemperature'],
    ['backup', 'backup'],
    ['snapshot', 'snapshot'],
  ]).get(key);

  if (mapped) {
    return mapped;
  }

  return key
    .replace(' %', '')
    .replace(' °c', '')
    .replace(' mb/s', '')
    .replace('disk r', 'diskRead')
    .replace('disk w', 'diskWrite')
    .replace('net in', 'networkIn')
    .replace('net out', 'networkOut');
}

export function getAlertResourceMetricBounds(metric: string): { min: number; max: number } {
  if (metric === 'temperature' || metric === 'diskTemperature') {
    return { min: 1, max: 150 };
  }
  if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
    return { min: 0.01, max: 10000 };
  }
  if (['cpu', 'memory', 'disk', 'usage', 'memoryWarnPct', 'memoryCriticalPct'].includes(metric)) {
    return { min: 1, max: 100 };
  }
  if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
    return { min: 0.01, max: 100000 };
  }
  if (metric === 'restartCount') {
    return { min: 1, max: 50 };
  }
  if (metric === 'restartWindow') {
    return { min: 1, max: 86400 };
  }
  return { min: 1, max: 10000 };
}

export function getAlertResourceMetricStep(metric: string): string | number {
  if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
    return 'any';
  }
  if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
    return 'any';
  }
  return 1;
}

export function getAlertResourceEnabledDefault(metric: string): number {
  if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
    return 100;
  }
  if (metric === 'temperature') {
    return 80;
  }
  if (metric === 'diskTemperature') {
    return 55;
  }
  if (metric === 'restartCount') {
    return 3;
  }
  if (metric === 'restartWindow') {
    return 300;
  }
  if (metric === 'memoryWarnPct') {
    return 90;
  }
  if (metric === 'memoryCriticalPct') {
    return 95;
  }
  return 80;
}

export const ALERT_RESOURCE_METRIC_OFF_VALUE = -1;

/**
 * The alert engine disables a metric whenever its trigger is `<= 0`
 * (`internal/alerts/canonical_metric.go`), so every editor surface has to read
 * `0` as Off too: older builds advertised `0` as the disable value and those
 * overrides are still on disk. Editors always stage the canonical `-1`; the
 * save payload writes an Off global default as `0`, because the backend reads
 * a negative default as unset.
 */
export function isAlertResourceMetricOff(
  value: number | undefined | null,
  inheritedDefault?: number,
): boolean {
  const effectiveValue = value ?? inheritedDefault;
  return (
    typeof effectiveValue === 'number' && Number.isFinite(effectiveValue) && effectiveValue <= 0
  );
}

/**
 * Whether two thresholds alert the same way. Every value `<= 0` is Off, and a
 * global default saves Off as `0` while the editors and factory defaults use
 * `-1`, so the two compare equal.
 */
export function alertResourceThresholdsEquivalent(
  left: number | undefined,
  right: number | undefined,
): boolean {
  return left === right || (isAlertResourceMetricOff(left) && isAlertResourceMetricOff(right));
}

/**
 * Value an override editor must stage when a metric is switched back on.
 * Clearing the override (`undefined`) only re-inherits the default, so it
 * re-enables nothing when the inherited default is itself off; in that case the
 * editor has to stage an explicit enabled threshold instead.
 */
export function resolveAlertResourceMetricEnableValue(
  inheritedDefault: number | undefined,
  metric: string,
): number | undefined {
  if (inheritedDefault === undefined || isAlertResourceMetricOff(inheritedDefault)) {
    return getAlertResourceEnabledDefault(metric);
  }
  return undefined;
}

/**
 * A global default that may stay unset because another setting decides it,
 * such as TrueNAS disk temperature following Disk temperature by type. Unset,
 * the cell is empty with `label` as its placeholder and `title` explaining
 * where the value comes from. `off` says the other setting is itself off, so
 * the unset cell reads Off and switching it on has to stage `enableValue`.
 */
export interface AlertResourceGlobalDefaultFallback {
  label: string;
  title: string;
  off?: boolean;
  enableValue?: number;
}

export interface AlertResourceGlobalDefaultCell {
  /** Unset and decided by the fallback. */
  follows: boolean;
  isOff: boolean;
  /** Value staged when the cell is switched back on from Off. */
  enableValue: number | undefined;
}

/**
 * State of one Global Defaults metric cell. An unset default is Off, as the
 * alert engine reads it, unless the metric has a fallback; then it reads as
 * the fallback does. Switching it on from Off returns it to unset when the
 * fallback is on, and stages an explicit value when the fallback is off,
 * since unset would still be off.
 */
export function resolveAlertResourceGlobalDefaultCell(
  metric: string,
  value: number | undefined,
  fallback: AlertResourceGlobalDefaultFallback | undefined,
): AlertResourceGlobalDefaultCell {
  const follows = value === undefined && fallback !== undefined;
  const explicitEnableValue = fallback?.enableValue ?? getAlertResourceEnabledDefault(metric);
  return {
    follows,
    isOff: follows ? Boolean(fallback?.off) : isAlertResourceMetricOff(value ?? 0),
    enableValue: fallback && !fallback.off ? undefined : explicitEnableValue,
  };
}

export function getAlertResourceMetricDelayOverride(
  metricDelaySeconds: Record<string, number> | undefined,
  metric: string,
): number | undefined {
  const normalized = metric.trim().toLowerCase();
  const value = metricDelaySeconds?.[normalized] ?? metricDelaySeconds?.[metric];
  if (typeof value !== 'number' || !Number.isFinite(value)) {
    return undefined;
  }
  return value;
}

export function getAlertResourceColumnHeaderTooltip(column: string): string | undefined {
  const normalized = column.trim().toLowerCase();
  return COLUMN_TOOLTIP_LOOKUP[column] ?? COLUMN_TOOLTIP_LOOKUP[normalized];
}

export function getAlertResourceColumnKind(column: string): PlatformTableColumnKind {
  const metric = normalizeAlertResourceMetricKey(column);
  if (metric === 'backup' || metric === 'snapshot') {
    return 'badge';
  }
  return 'numeric-value';
}

export function getAlertResourceInitialEditMetric(columns: string[], resourceType?: string) {
  return columns
    .map(normalizeAlertResourceMetricKey)
    .find(
      (metric) =>
        metric !== 'backup' &&
        metric !== 'snapshot' &&
        alertResourceSupportsMetric(resourceType, metric),
    );
}

// The alert signal a row's delay action configures: its first threshold
// metric the resource supports, else its offline state.
export function getAlertResourcePreferredIntentSignal(
  columns: readonly string[],
  resourceType: string | undefined,
): AlertIntentSignal {
  for (const metric of ['cpu', 'memory', 'disk', 'temperature'] as const) {
    if (
      columns.some((column) => normalizeAlertResourceMetricKey(column) === metric) &&
      alertResourceSupportsMetric(resourceType, metric)
    ) {
      return `metric.${metric}`;
    }
  }
  return 'state.offline';
}

// How the delay panel names a row it may not list itself, such as a machine
// disk or sensor: the row's label and, when known, its machine or node.
export function getAlertResourceIntentTargetLabel(resource: {
  name: string;
  displayName?: string;
  node?: string;
}): string {
  const label = resource.displayName?.trim() || resource.name;
  const node = resource.node?.trim();
  return node && node !== label ? `${label} · ${node}` : label;
}

export function alertResourceSupportsMetric(
  resourceType: string | undefined,
  metric: string,
): boolean {
  if (!resourceType) return true;
  if (
    ['node', 'agent'].includes(resourceType) &&
    ['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)
  ) {
    return false;
  }
  if (resourceType === 'pbs') {
    return ['cpu', 'memory'].includes(metric);
  }
  if (resourceType === 'storage') {
    return metric === 'usage';
  }
  if (resourceType === 'agentSensor') {
    return metric === 'temperature';
  }
  if (resourceType === 'kubernetesNamespace') {
    return false;
  }
  if (
    resourceType === 'kubernetesCluster' ||
    resourceType === 'kubernetesDeployment' ||
    resourceType === 'kubernetesPod'
  ) {
    return ['cpu', 'memory', 'disk', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(
      metric,
    );
  }
  if (resourceType === 'kubernetesNode') {
    return ['cpu', 'memory', 'disk'].includes(metric);
  }
  if (resourceType === 'truenasSystem') {
    return [
      'cpu',
      'memory',
      'disk',
      'diskRead',
      'diskWrite',
      'networkIn',
      'networkOut',
      'temperature',
    ].includes(metric);
  }
  if (resourceType === 'truenasPool' || resourceType === 'truenasDataset') {
    return metric === 'usage';
  }
  if (resourceType === 'truenasDisk') {
    return metric === 'temperature';
  }
  if (resourceType === 'vmwareHost') {
    return ['cpu', 'memory', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric);
  }
  if (resourceType === 'vmwareVm') {
    return ['cpu', 'memory', 'disk', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(
      metric,
    );
  }
  if (resourceType === 'vmwareDatastore') {
    return metric === 'usage';
  }
  if (resourceType === 'vmwareNetwork') {
    return false;
  }
  // A per-resource override stores only the metrics in the backend's
  // ThresholdConfig. Restart and memory-limit rules exist only as Docker
  // defaults, and PMG thresholds only as PMG defaults, so the backend dropped
  // per-container and per-server edits of them on save.
  if (resourceType === 'dockerContainer') {
    return ['cpu', 'memory'].includes(metric);
  }
  if (resourceType === 'pmg') {
    return false;
  }
  return true;
}

export function getAlertResourceLabel(resource: AlertResourceTableResourceLike): string {
  // Operator-local /alerts table; redaction is a transmission-boundary policy
  // (docs/PRIVACY.md). Use the raw infra display name to match /infrastructure.
  return getPreferredInfrastructureDisplayName(resource as unknown as UnifiedResource);
}

function parseAlertMetricNumber(value: unknown): number | undefined {
  if (value === undefined || value === null) return undefined;
  if (typeof value === 'number') return value;
  const parsed = Number(value);
  return Number.isFinite(parsed) ? parsed : undefined;
}

export function getAlertResourceMetricDisplayValue(
  resource: AlertResourceTableResourceLike,
  metric: string,
  editingThresholds?: AlertResourceThresholdMap,
  isEditing = false,
): number {
  const extract = (source: Record<string, unknown> | undefined) =>
    parseAlertMetricNumber(source?.[metric]);
  const defaults = resource.defaults as Record<string, unknown> | undefined;

  if (isEditing) {
    const edited = extract(editingThresholds as Record<string, unknown> | undefined);
    if (edited !== undefined) {
      return edited;
    }
    const fallback = extract(defaults);
    return fallback !== undefined ? fallback : 0;
  }

  const liveValue = extract(resource.thresholds as Record<string, unknown> | undefined);
  if (liveValue !== undefined) {
    return liveValue;
  }

  const fallback = extract(defaults);
  return fallback !== undefined ? fallback : 0;
}

const RELAXED_TRIGGER_METRIC_LABELS: Record<MetricType, string> = {
  cpu: 'CPU',
  memory: 'Memory',
  disk: 'Disk',
};

export interface AlertResourceRelaxedTriggerSummary {
  /** Raised thresholds in metric order, e.g. "CPU 95%". */
  raised: string[];
  title: string;
}

/**
 * The alert thresholds a guest's pulse-relaxed tag raises above the values
 * its row shows, as the alert engine applies them, or null when the tag
 * raises none. Reads the values being edited, so the line follows an edit
 * before it is saved.
 */
export function getAlertResourceRelaxedTriggerSummary(
  resource: AlertResourceTableResourceLike,
  editingThresholds?: AlertResourceThresholdMap,
  isEditing = false,
): AlertResourceRelaxedTriggerSummary | null {
  if (resource.pulseRelaxed !== true || resource.disabled === true) {
    return null;
  }
  const rowThresholds = isEditing ? editingThresholds : resource.thresholds;
  const raised = (['cpu', 'memory', 'disk'] as const).flatMap((metric) => {
    const configured =
      parseAlertMetricNumber(rowThresholds?.[metric]) ??
      parseAlertMetricNumber(resource.defaults?.[metric]);
    const trigger = getPulseRelaxedGuestTrigger(metric, configured);
    if (trigger === null || (configured !== undefined && trigger <= configured)) {
      return [];
    }
    return [`${RELAXED_TRIGGER_METRIC_LABELS[metric]} ${trigger}%`];
  });
  if (raised.length === 0) {
    return null;
  }
  return {
    raised,
    title:
      'This guest is tagged pulse-relaxed in Proxmox, which raises these alert thresholds ' +
      'above the values in this row. Remove the tag in Proxmox to alert at the row values.',
  };
}

export function isAlertResourceMetricOverridden(
  resource: AlertResourceTableResourceLike,
  metric: string,
): boolean {
  return resource.thresholds?.[metric] !== undefined && resource.thresholds?.[metric] !== null;
}

export function buildAlertResourceEditPayload(resource: AlertResourceTableResourceLike) {
  return {
    thresholds: resource.thresholds ? { ...resource.thresholds } : {},
    defaults: resource.defaults ? { ...resource.defaults } : {},
    note: typeof resource.note === 'string' ? resource.note : undefined,
  };
}