Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.6 KB
Raw Blame
import type { Alert } from '@/types/api';
import type { Resource } from '@/types/resource';
import { getActionableAgentIdFromResource } from '@/utils/agentResources';
import { isAlertsDetectionEnabled } from '@/utils/alertsActivation';

const noAlertStyles = {
  rowClass: '',
  indicatorClass: '',
  badgeClass: '',
  hasAlert: false,
  alertCount: 0,
  severity: null as 'critical' | 'warning' | 'info' | null,
  hasPoweredOffAlert: false,
  hasNonPoweredOffAlert: false,
  hasUnacknowledgedAlert: false,
  unacknowledgedCount: 0,
  acknowledgedCount: 0,
  hasAcknowledgedOnlyAlert: false,
};

// Get alert highlighting styles based on active alerts for a resource.
// When nodeMatch is provided, also includes alerts whose `node` field matches
// (covers storage/topology/disk alerts that belong to a node but have a
// different resourceId than the node itself).
export const getAlertStyles = (
  resourceId: string | string[],
  activeAlerts: Record<string, Alert>,
  alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
  nodeMatch?: string,
) => {
  if (!alertsEnabled) {
    return noAlertStyles;
  }

  return getAlertStylesForAlerts(
    getAlertsForResource(
      Array.isArray(resourceId) ? resourceId : [resourceId],
      activeAlerts,
      alertsEnabled,
      nodeMatch,
    ),
  );
};

// Alert highlighting for a unified resource's row, from the open alerts its
// drawer lists under "Needs attention" (getAlertsForUnifiedResource). Matching
// by display name misses machines whose alerts are keyed on the agent or
// Docker host id, and a row that disagrees with its own drawer hides the
// problem until someone opens it.
export const getUnifiedResourceAlertStyles = (
  resource: Resource,
  activeAlerts: Record<string, Alert>,
  alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
) => {
  if (!alertsEnabled) {
    return noAlertStyles;
  }

  return getAlertStylesForAlerts(
    getAlertsForUnifiedResource(resource, activeAlerts, alertsEnabled),
  );
};

function getAlertStylesForAlerts(alertsForResource: Alert[]) {
  const unacknowledgedAlerts = alertsForResource.filter((alert) => !alert.acknowledged);
  const acknowledgedAlerts = alertsForResource.filter((alert) => alert.acknowledged);

  let highestSeverity: 'critical' | 'warning' | 'info' | null = null;
  let hasPoweredOffAlert = false;
  let hasNonPoweredOffAlert = false;

  unacknowledgedAlerts.forEach((alert) => {
    if (
      alert.level === 'critical' ||
      (alert.level === 'warning' && highestSeverity !== 'critical') ||
      (alert.level === 'info' && highestSeverity === null)
    ) {
      highestSeverity = alert.level;
    }

    if (alert.type === 'powered-off') {
      hasPoweredOffAlert = true;
    } else {
      hasNonPoweredOffAlert = true;
    }
  });

  const alertCount = alertsForResource.length;
  const unacknowledgedCount = unacknowledgedAlerts.length;
  const acknowledgedCount = acknowledgedAlerts.length;
  const hasUnacknowledgedAlert = unacknowledgedCount > 0;
  const hasAlert = alertCount > 0;

  if (highestSeverity === 'critical') {
    return {
      rowClass: 'bg-red-50 dark:bg-red-950/25 border-l-4 border-red-500 dark:border-red-400',
      indicatorClass: 'bg-red-500',
      badgeClass: 'bg-red-100 text-red-800 dark:bg-red-900/25 dark:text-red-200',
      hasAlert,
      alertCount,
      severity: 'critical' as const,
      hasPoweredOffAlert,
      hasNonPoweredOffAlert,
      hasUnacknowledgedAlert,
      unacknowledgedCount,
      acknowledgedCount,
      hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
    };
  }

  if (highestSeverity === 'warning') {
    return {
      rowClass:
        'bg-yellow-50 dark:bg-yellow-950/25 border-l-4 border-yellow-500 dark:border-yellow-400',
      indicatorClass: 'bg-yellow-500',
      badgeClass: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/25 dark:text-yellow-200',
      hasAlert,
      alertCount,
      severity: 'warning' as const,
      hasPoweredOffAlert,
      hasNonPoweredOffAlert,
      hasUnacknowledgedAlert,
      unacknowledgedCount,
      acknowledgedCount,
      hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
    };
  }

  if (highestSeverity === 'info') {
    return {
      rowClass: 'bg-blue-50 dark:bg-blue-950/25 border-l-4 border-blue-500 dark:border-blue-400',
      indicatorClass: 'bg-blue-500',
      badgeClass: 'bg-blue-100 text-blue-800 dark:bg-blue-900/25 dark:text-blue-200',
      hasAlert,
      alertCount,
      severity: 'info' as const,
      hasPoweredOffAlert,
      hasNonPoweredOffAlert,
      hasUnacknowledgedAlert,
      unacknowledgedCount,
      acknowledgedCount,
      hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
    };
  }

  return {
    rowClass: '',
    indicatorClass: '',
    badgeClass: '',
    hasAlert,
    alertCount,
    severity: null,
    hasPoweredOffAlert,
    hasNonPoweredOffAlert,
    hasUnacknowledgedAlert,
    unacknowledgedCount,
    acknowledgedCount,
    hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
  };
}

export function getAlertsForResource(
  resourceIds: string[],
  activeAlerts: Record<string, Alert>,
  alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
  nodeMatch?: string,
): Alert[] {
  if (!alertsEnabled) return [];
  const ids = new Set(resourceIds.filter(Boolean));
  return Object.values(activeAlerts).filter(
    (alert) => ids.has(alert.resourceId) || (nodeMatch !== undefined && alert.node === nodeMatch),
  );
}

const ALERT_LEVEL_RANK: Record<Alert['level'], number> = { critical: 0, warning: 1, info: 2 };

// The alert keys one unified resource answers to. Alerts raised on the unified
// resource itself (provider incidents, availability, TrueNAS, vSphere,
// Kubernetes) carry its id; poller alerts carry the canonical primary id, the
// Proxmox source id or the metrics target (PBS, PMG, storage). A machine also
// answers to its agent ("agent:<id>") and Docker runtime ("docker:<id>") keys,
// and its components' alerts nest under those: "agent:<id>/disk:<mount>",
// "agent:<id>/raid:<device>", "docker:<id>/<container>".
const getUnifiedResourceAlertKeys = (
  resource: Resource,
): {
  exact: Set<string>;
  componentPrefixes: string[];
  localStorage?: { node: string; instance?: string };
} => {
  const exact = new Set<string>();
  const add = (value: string | undefined) => {
    const trimmed = value?.trim();
    if (trimmed) exact.add(trimmed);
  };
  add(resource.id);
  add(resource.canonicalIdentity?.primaryId);
  resource.canonicalIdentity?.supersededIds?.forEach(add);
  add(resource.proxmox?.sourceId);
  add(resource.metricsTarget?.resourceId);
  // PBS and PMG alerts are keyed on the instance id (internal/alerts pmg.go,
  // pbs.go), which the platform tables match too; the drawer must not depend
  // on a metrics target being set to find them.
  add(resource.pbs?.instanceId);
  add(resource.pmg?.instanceId);
  // The alert engine keys a Docker container's alerts on its host and
  // container ID ("docker:<host>/<container ID>"), or on its name when it has
  // no ID (internal/alerts DockerContainerResourceID).
  if (resource.type === 'app-container') {
    const dockerHostId = resource.docker?.hostSourceId?.trim();
    const containerId = resource.docker?.containerId?.trim();
    const containerName = resource.name?.trim().replace(/^\/+/, '');
    if (dockerHostId && containerId) add(`docker:${dockerHostId}/${containerId}`);
    else if (dockerHostId && containerName) add(`docker:${dockerHostId}/name:${containerName}`);
  }

  const machine = new Set<string>();
  const componentPrefixes: string[] = [];
  let localStorage: { node: string; instance?: string } | undefined;
  if (resource.type === 'agent' || resource.type === 'docker-host') {
    const agentId = getActionableAgentIdFromResource(resource);
    if (agentId) machine.add(agentId.startsWith('agent:') ? agentId : `agent:${agentId}`);
    // The backend writes the agent key of a machine reported through a linked
    // agent (a Proxmox node, a Docker host) into the canonical aliases.
    resource.canonicalIdentity?.aliases
      ?.filter((alias) => alias.startsWith('agent:'))
      .forEach((alias) => machine.add(alias));
    // Docker hosts are "docker-host" resources or agents with a Docker runtime;
    // the backend keys their alerts on the Docker host's source id.
    const dockerHostId =
      resource.docker?.hostSourceId?.trim() ||
      (resource.metricsTarget?.resourceType === 'docker-host'
        ? resource.metricsTarget.resourceId?.trim()
        : undefined);
    if (dockerHostId) machine.add(`docker:${dockerHostId}`);
    // A Proxmox node's physical disks raise health and wearout alerts on
    // "<instance>:<node>:disk:<device>" (ProxmoxPhysicalDiskAlertResourceID).
    const instance = resource.proxmox?.instance?.trim();
    const nodeName = resource.proxmox?.nodeName?.trim();
    if (instance && nodeName) componentPrefixes.push(`${instance}:${nodeName}:disk:`);
    // Its own storage (local, local-zfs) raises usage, offline and ZFS alerts
    // on the storage id under the node's name. Storage shared across a cluster
    // is recorded under "cluster", so it never matches one node.
    if (nodeName) localStorage = { node: nodeName, instance };
  }
  machine.forEach(add);
  componentPrefixes.push(...[...machine].map((key) => `${key}/`));
  return { exact, componentPrefixes, localStorage };
};

const isLocalStorageAlert = (
  alert: Alert,
  localStorage: { node: string; instance?: string } | undefined,
): boolean => {
  if (!localStorage || alert.node !== localStorage.node) return false;
  const resourceType = alert.metadata?.resourceType;
  if (typeof resourceType !== 'string' || resourceType.trim().toLowerCase() !== 'storage') {
    return false;
  }
  return !localStorage.instance || !alert.instance || alert.instance === localStorage.instance;
};

/**
 * The resource IDs a unified resource's own alerts are raised on: its unified
 * id, the poller's source or metrics-target id, and a machine's "agent:<id>"
 * and "docker:<id>" keys. Alerts on its components nest under the latter.
 */
export const getUnifiedResourceAlertResourceIds = (resource: Resource): string[] => [
  ...getUnifiedResourceAlertKeys(resource).exact,
];

/**
 * Open alerts for one unified resource, read from the websocket's active
 * alert map, most severe first. This is the canonical source for a resource
 * drawer's "Needs attention" list: resources do not embed their alerts.
 */
export function getAlertsForUnifiedResource(
  resource: Resource,
  activeAlerts: Record<string, Alert>,
  alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
): Alert[] {
  if (!alertsEnabled) return [];
  const { exact, componentPrefixes, localStorage } = getUnifiedResourceAlertKeys(resource);
  return Object.values(activeAlerts)
    .filter(
      (alert) =>
        exact.has(alert.resourceId) ||
        componentPrefixes.some((prefix) => alert.resourceId?.startsWith(prefix)) ||
        isLocalStorageAlert(alert, localStorage),
    )
    .sort(
      (a, b) =>
        (ALERT_LEVEL_RANK[a.level] ?? 3) - (ALERT_LEVEL_RANK[b.level] ?? 3) ||
        Date.parse(a.startTime) - Date.parse(b.startTime),
    );
}

/**
 * Alerts raised on the resource itself, not on anything nested under it (a
 * Docker host's containers and services, a machine's disks). A drawer chart
 * draws only these: a container's memory trigger is not its host's.
 */
export function getOwnAlertsForUnifiedResource(
  resource: Resource,
  activeAlerts: Record<string, Alert>,
  alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
): Alert[] {
  if (!alertsEnabled) return [];
  const { exact } = getUnifiedResourceAlertKeys(resource);
  return Object.values(activeAlerts).filter((alert) => exact.has(alert.resourceId));
}