Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

24.6 KB
Raw Blame
import type { Node, PBSInstance } from '@/types/api';
import type { RawOverrideConfig } from '@/types/alerts';
import type { Resource } from '@/types/resource';
import { getActionableAgentIdFromResource, isTrueNASSystemResource } from '@/utils/agentResources';
import { isAppContainerDiscoveryResourceType } from '@/utils/discoveryTarget';

import { parseAgentSensorResourceId } from './agentSensorIdentity';
import {
  extractTriggerValues,
  getAlertResourceDisplayLabel,
  guessNumericId,
  platformData,
} from './helpers';
import {
  getGuestOverrideIdentity,
  parseGuestDiskOverrideId,
  guestOverrideIdCandidates,
  guestOverrideStorageId,
  normalizeGuestOverrideKey,
} from './guestOverrideIdentity';
import type { Override } from './types';

const asRecord = (value: unknown): Record<string, unknown> | undefined =>
  value && typeof value === 'object' ? (value as Record<string, unknown>) : undefined;

const asString = (value: unknown): string | undefined =>
  typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;

const uniqueIds = (...values: unknown[]): string[] => {
  const ids: string[] = [];
  const seen = new Set<string>();

  values.forEach((value) => {
    const normalized = asString(value);
    if (!normalized || seen.has(normalized)) return;
    seen.add(normalized);
    ids.push(normalized);
  });

  return ids;
};

const buildSharedStorageLegacyKeyMap = (storageResources: Resource[]): Map<string, string> => {
  const legacyToCanonical = new Map<string, string>();

  storageResources.forEach((resource) => {
    const storageMeta = resource.storage;
    const resourceName = asString(resource.name);
    const proxmox = resource.proxmox;
    const instance = asString(proxmox?.instance) || asString(resource.platformId);
    const canonicalID = storageOverrideActionId(resource);

    if (!storageMeta?.shared || !resourceName || !canonicalID) {
      return;
    }

    const clusterNodes = Array.isArray(storageMeta.nodes) ? storageMeta.nodes : [];
    clusterNodes.forEach((node) => {
      const normalizedNode = asString(node);
      if (!normalizedNode) {
        return;
      }

      const prefix =
        instance && !normalizedNode.toLowerCase().startsWith(`${instance.toLowerCase()}-`)
          ? `${instance}-${normalizedNode}`
          : normalizedNode;
      legacyToCanonical.set(`${prefix}-${resourceName}`, canonicalID);
    });
  });

  return legacyToCanonical;
};

// PBS datastore overrides were historically stored under the poller's
// "<instance-id>-<name>" storage ID while the canonical resource ID is
// "<instance-id>/<name>". Datastore names cannot contain "/", so mapping the
// last slash to a dash reconstructs the legacy key exactly; carrying it as a
// trailing candidate keeps pre-existing overrides bound to the single card
// and lets the next save re-home them onto the canonical key (#1591).
const legacyPBSDatastoreOverrideId = (
  resource: Resource,
  canonicalId?: string,
): string | undefined => {
  if (resource.storage?.platform !== 'pbs' || !canonicalId) return undefined;
  const slash = canonicalId.lastIndexOf('/');
  if (slash <= 0) return undefined;
  return `${canonicalId.slice(0, slash)}-${canonicalId.slice(slash + 1)}`;
};

export const storageOverrideIdCandidates = (resource: Resource): string[] => {
  const canonicalId =
    resource.metricsTarget?.resourceType === 'storage'
      ? resource.metricsTarget.resourceId
      : undefined;
  return uniqueIds(canonicalId, resource.id, legacyPBSDatastoreOverrideId(resource, canonicalId));
};

export const storageOverrideActionId = (resource: Resource): string =>
  storageOverrideIdCandidates(resource)[0] || resource.id;

const buildStorageOverrideKeyMap = (storageResources: Resource[]): Map<string, string> => {
  const keyMap = new Map<string, string>();

  storageResources.forEach((resource) => {
    const canonicalID = storageOverrideActionId(resource);
    if (!canonicalID) return;

    storageOverrideIdCandidates(resource).forEach((candidate) => {
      if (candidate !== canonicalID) {
        keyMap.set(candidate, canonicalID);
      }
    });
  });

  buildSharedStorageLegacyKeyMap(storageResources).forEach((canonicalID, legacyID) => {
    keyMap.set(legacyID, canonicalID);
  });

  return keyMap;
};

export const normalizeRawOverridesConfig = (
  rawOverrides: Record<string, RawOverrideConfig>,
  storageResources: Resource[] = [],
): Record<string, RawOverrideConfig> => {
  const cleanedOverrides: Record<string, RawOverrideConfig> = {};
  const priorityByKey = new Map<string, number>();
  const storageOverrideKeyMap = buildStorageOverrideKeyMap(storageResources);

  for (const [key, value] of Object.entries(rawOverrides)) {
    const normalizedGuestKey = normalizeGuestOverrideKey(key);
    const diskMatch = normalizedGuestKey.match(/^(agent:.+\/disk:)(.+)$/);
    let normalizedKey = normalizedGuestKey;
    if (diskMatch) {
      const normalized =
        diskMatch[2]
          .toLowerCase()
          .replace(/[^a-z0-9]/g, '-')
          .replace(/-{2,}/g, '-')
          .replace(/^-|-$/g, '') || 'unknown';
      normalizedKey = diskMatch[1] + normalized;
    }
    const storageOverrideKey = storageOverrideKeyMap.get(normalizedKey);
    if (storageOverrideKey) {
      normalizedKey = storageOverrideKey;
    }

    const priority = normalizedKey === key ? 1 : 0;
    const existingPriority = priorityByKey.get(normalizedKey);
    if (existingPriority !== undefined && existingPriority > priority) {
      continue;
    }

    priorityByKey.set(normalizedKey, priority);
    cleanedOverrides[normalizedKey] = value;
  }

  return cleanedOverrides;
};

export const hostOverrideIdCandidates = (resource: Resource): string[] => {
  const data = platformData(resource);
  const agent = asRecord(data?.agent);
  return uniqueIds(
    getActionableAgentIdFromResource(resource),
    resource.discoveryTarget?.agentId,
    resource.agent?.agentId,
    agent?.agentId,
    data?.agentId,
    resource.proxmox?.sourceId,
    resource.metricsTarget?.resourceId,
    resource.canonicalIdentity?.primaryId,
    ...(resource.canonicalIdentity?.aliases ?? []),
    ...(resource.canonicalIdentity?.supersededIds ?? []),
    resource.id,
  );
};

export const nodeOverrideIdCandidates = (
  node: Pick<Node, 'id' | 'name' | 'instance' | 'host' | 'linkedAgentId'>,
): string[] =>
  uniqueIds(
    node.linkedAgentId,
    node.id,
    node.instance && node.name ? `${node.instance}-${node.name}` : undefined,
    node.instance && node.name ? `${node.instance}:${node.name}` : undefined,
    node.name,
    node.host,
  );

export const dockerHostOverrideIdCandidates = (resource: Resource): string[] => {
  const data = platformData(resource);
  const docker = asRecord(data?.docker);
  const discoveryTarget = resource.discoveryTarget;
  return uniqueIds(
    isAppContainerDiscoveryResourceType(discoveryTarget?.resourceType)
      ? discoveryTarget?.resourceId
      : undefined,
    docker?.hostSourceId,
    data?.hostSourceId,
    discoveryTarget?.agentId,
    resource.id,
  );
};

// Container override keys must survive container recreates: the durable key
// is docker:{hostId}/{containerName}, which is what the backend evaluator
// resolves first. ID-based forms trail as lookup candidates so overrides
// written under the runtime container ID (the legacy backend key) or the
// unified hash id (what this UI keyed on before #1601) still bind to the row
// and re-home onto the stable name key on the next save.
export const dockerContainerOverrideIdCandidates = (
  host: Resource,
  container: Resource,
): string[] => {
  const containerId =
    asString(container.docker?.containerId) ?? asString(platformData(container)?.containerId);
  const keyParts = uniqueIds(
    asString(container.name)?.replace(/^\/+/, ''),
    containerId,
    containerId && containerId.length > 12 ? containerId.slice(0, 12) : undefined,
    container.id,
    // Historical slash-form resource ids stored their tail as the key part.
    container.id.includes('/') ? container.id.split('/').pop() : undefined,
  );
  return uniqueIds(
    ...keyParts.flatMap((part) =>
      dockerHostOverrideIdCandidates(host).map((hostId) => `docker:${hostId}/${part}`),
    ),
  );
};

// Kubernetes / TrueNAS / VMware resources are keyed by their unified resource
// IDs in the overrides map (the same candidates buildProjectedOverrides
// indexes platform alert rows under).
export const unifiedPlatformOverrideIdCandidates = (resource: Resource): string[] =>
  uniqueIds(
    resource.id,
    ...(resource.canonicalIdentity?.supersededIds ?? []),
    resource.metricsTarget?.resourceId,
    resource.discoveryTarget?.resourceId,
    resource.platformId,
  );

export const buildContainerRuntimeResources = ({
  allResources,
  dockerHostResources,
}: {
  allResources: Resource[];
  dockerHostResources: Resource[];
}): Resource[] => {
  const resourceById = new Map(allResources.map((resource) => [resource.id, resource]));
  const runtimes: Resource[] = [];
  const seen = new Set<string>();

  const addRuntime = (resource: Resource | undefined) => {
    if (!resource || seen.has(resource.id)) return;
    seen.add(resource.id);
    runtimes.push(resource);
  };

  dockerHostResources.forEach(addRuntime);
  allResources.forEach((resource) => {
    if (isTrueNASSystemResource(resource)) {
      addRuntime(resource);
    }
  });
  allResources.forEach((resource) => {
    if (resource.type !== 'app-container' || !resource.parentId) return;
    addRuntime(resourceById.get(resource.parentId));
  });

  return runtimes;
};

interface BuildProjectedOverridesArgs {
  rawConfig: Record<string, RawOverrideConfig>;
  nodeResources: Resource[];
  vmResources: Resource[];
  containerResources: Resource[];
  storageResources: Resource[];
  agentResourceList: Resource[];
  containerRuntimeResources: Resource[];
  getChildren: (resourceId: string) => Resource[];
  pbsInstanceById: Map<string, PBSInstance>;
  allResources?: Resource[];
}

export const buildProjectedOverrides = ({
  rawConfig,
  nodeResources,
  vmResources,
  containerResources,
  storageResources,
  agentResourceList,
  containerRuntimeResources,
  getChildren,
  pbsInstanceById,
  allResources = [],
}: BuildProjectedOverridesArgs): Override[] => {
  const overridesList: Override[] = [];
  const overrideIndexByID = new Map<string, number>();
  const dockerHostMap = new Map<string, Resource>();
  const dockerContainerMap = new Map<
    string,
    { host: Resource; container: Resource; containerShortId: string }
  >();
  const agentMap = new Map<string, Resource>();
  const guestMap = new Map<string, Resource>();
  const storageMap = new Map<string, Resource>();
  const alertPlatformMap = new Map<
    string,
    { resource: Resource; type: Override['type']; resourceType: string }
  >();

  // Every projection is keyed by its raw override key, so the stored note is
  // carried here once for all resource types. Without it a reloaded editor
  // opens with a blank note and the next save drops the stored one.
  const upsertProjectedOverride = (projected: Override) => {
    const storedNote = rawConfig[projected.id]?.note;
    const override =
      projected.note === undefined && typeof storedNote === 'string' && storedNote.trim()
        ? { ...projected, note: storedNote }
        : projected;
    const existingIndex = overrideIndexByID.get(override.id);
    if (existingIndex !== undefined) {
      overridesList[existingIndex] = override;
      return;
    }
    overrideIndexByID.set(override.id, overridesList.length);
    overridesList.push(override);
  };

  const alertResourceIdCandidates = unifiedPlatformOverrideIdCandidates;

  const isTrueNASResource = (resource: Resource): boolean =>
    isTrueNASSystemResource(resource) ||
    resource.platformType === 'truenas' ||
    resource.sources?.includes('truenas') ||
    resource.storage?.platform === 'truenas';

  const isVMwareResource = (resource: Resource): boolean =>
    resource.platformType === 'vmware-vsphere' ||
    resource.sources?.includes('vmware') ||
    Boolean(resource.vmware) ||
    resource.storage?.platform === 'vmware-vsphere';

  allResources.forEach((resource) => {
    let type: Override['type'] | undefined;
    let resourceType = '';
    switch (resource.type) {
      case 'k8s-cluster':
        type = 'kubernetesCluster';
        resourceType = 'Kubernetes Cluster';
        break;
      case 'k8s-node':
        type = 'kubernetesNode';
        resourceType = 'Kubernetes Node';
        break;
      case 'k8s-namespace':
        type = 'kubernetesNamespace';
        resourceType = 'Kubernetes Namespace';
        break;
      case 'k8s-deployment':
        type = 'kubernetesDeployment';
        resourceType = 'Kubernetes Deployment';
        break;
      case 'pod':
        type = 'kubernetesPod';
        resourceType = 'Kubernetes Pod';
        break;
      case 'agent':
        if (isTrueNASResource(resource)) {
          type = 'truenasSystem';
          resourceType = 'TrueNAS System';
        } else if (isVMwareResource(resource)) {
          type = 'vmwareHost';
          resourceType = 'vSphere Host';
        }
        break;
      case 'vm':
        if (isVMwareResource(resource)) {
          type = 'vmwareVm';
          resourceType = 'vSphere VM';
        }
        break;
      case 'physical_disk':
        if (isTrueNASResource(resource)) {
          type = 'truenasDisk';
          resourceType = 'TrueNAS Disk';
        }
        break;
      case 'storage':
      case 'pool':
      case 'dataset':
        if (isTrueNASResource(resource)) {
          const topology = resource.storage?.topology || resource.type;
          type = topology === 'dataset' ? 'truenasDataset' : 'truenasPool';
          resourceType = topology === 'dataset' ? 'TrueNAS Dataset' : 'TrueNAS Pool';
        } else if (isVMwareResource(resource)) {
          type = 'vmwareDatastore';
          resourceType = 'vSphere Datastore';
        }
        break;
      case 'network':
        if (isVMwareResource(resource)) {
          type = 'vmwareNetwork';
          resourceType = 'vSphere Network';
        }
        break;
    }
    if (!type) return;
    alertResourceIdCandidates(resource).forEach((id) => {
      alertPlatformMap.set(id, { resource, type, resourceType });
    });
  });

  const storageCoords = (resource: Resource): { node: string; instance: string } => {
    const data = platformData(resource);
    if (resource.type === 'datastore') {
      const instance =
        (data?.pbsInstanceId as string | undefined) ||
        resource.parentId ||
        resource.platformId ||
        'pbs';
      const node = (data?.pbsInstanceName as string | undefined) || instance;
      return { node, instance };
    }

    return {
      node: (data?.node as string | undefined) || '',
      instance: (data?.instance as string | undefined) || resource.platformId || '',
    };
  };

  containerRuntimeResources.forEach((host) => {
    dockerHostOverrideIdCandidates(host).forEach((id) => {
      dockerHostMap.set(id, host);
    });

    const containers = getChildren(host.id).filter((resource) => resource.type === 'app-container');

    containers.forEach((container) => {
      const containerId = asString(container.docker?.containerId);
      const shortId = containerId ? containerId.slice(0, 12) : container.id;
      dockerContainerOverrideIdCandidates(host, container).forEach((resourceId) => {
        dockerContainerMap.set(resourceId, { host, container, containerShortId: shortId });
      });
    });
  });

  agentResourceList.forEach((agentResource) => {
    hostOverrideIdCandidates(agentResource).forEach((id) => {
      agentMap.set(id, agentResource);
    });
  });

  storageResources.forEach((storageResource) => {
    const canonicalID = storageOverrideActionId(storageResource);
    if (canonicalID) {
      storageMap.set(canonicalID, storageResource);
    }
    storageOverrideIdCandidates(storageResource).forEach((candidate) => {
      storageMap.set(candidate, storageResource);
    });
  });

  buildSharedStorageLegacyKeyMap(storageResources).forEach((canonicalID, legacyID) => {
    const storageResource = storageMap.get(canonicalID);
    if (storageResource) {
      storageMap.set(legacyID, storageResource);
    }
  });

  [...vmResources, ...containerResources].forEach((guest) => {
    guestOverrideIdCandidates(guest).forEach((candidate) => {
      guestMap.set(candidate, guest);
    });
  });

  Object.entries(rawConfig).forEach(([key, thresholds]) => {
    const guestDiskIdentity = parseGuestDiskOverrideId(key);
    if (guestDiskIdentity) {
      const guest = guestMap.get(guestDiskIdentity.guestKey);
      const diskName = guestDiskIdentity.diskKey === 'root' ? '/' : guestDiskIdentity.diskKey;
      upsertProjectedOverride({
        id: key,
        name: diskName,
        type: 'guestDisk',
        resourceType: 'Guest Filesystem',
        node: guest ? getAlertResourceDisplayLabel(guest) : guestDiskIdentity.guestKey,
        instance: guestDiskIdentity.guestKey,
        disabled: thresholds.disabled || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const alertPlatformResource = alertPlatformMap.get(key);
    if (alertPlatformResource) {
      const { resource, type, resourceType } = alertPlatformResource;
      upsertProjectedOverride({
        id: key,
        name: getAlertResourceDisplayLabel(resource),
        type,
        resourceType,
        node:
          resource.parentName ||
          resource.kubernetes?.nodeName ||
          resource.truenas?.hostname ||
          resource.vmware?.runtimeHostName ||
          resource.vmware?.clusterName ||
          resource.vmware?.datacenterName ||
          resource.vmware?.vcenterHost,
        instance:
          resource.kubernetes?.clusterName ||
          resource.kubernetes?.namespace ||
          (resource.truenas ? 'TrueNAS' : undefined) ||
          resource.vmware?.connectionName ||
          resource.vmware?.vcenterHost ||
          (resource.vmware ? 'vSphere' : undefined),
        disabled: thresholds.disabled || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const dockerHost = dockerHostMap.get(key);
    if (dockerHost) {
      upsertProjectedOverride({
        id: key,
        name: getAlertResourceDisplayLabel(dockerHost),
        type: 'dockerHost',
        resourceType: 'Container Runtime',
        disableConnectivity: thresholds.disableConnectivity || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const dockerContainer = dockerContainerMap.get(key);
    if (dockerContainer) {
      const { host, container, containerShortId } = dockerContainer;
      const containerName = getAlertResourceDisplayLabel(container, containerShortId);
      upsertProjectedOverride({
        id: key,
        name: containerName,
        type: 'dockerContainer',
        resourceType: 'Container',
        node: getAlertResourceDisplayLabel(host),
        instance: getAlertResourceDisplayLabel(host),
        disabled: thresholds.disabled || false,
        disableConnectivity: thresholds.disableConnectivity || false,
        poweredOffSeverity:
          thresholds.poweredOffSeverity === 'critical'
            ? 'critical'
            : thresholds.poweredOffSeverity === 'warning'
              ? 'warning'
              : undefined,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    if (key.startsWith('docker:')) {
      const [, rest] = key.split(':', 2);
      const [hostId, containerId] = (rest || '').split('/', 2);
      if (containerId) {
        upsertProjectedOverride({
          id: key,
          name: containerId,
          type: 'dockerContainer',
          resourceType: 'Container',
          node: hostId,
          disabled: thresholds.disabled || false,
          disableConnectivity: thresholds.disableConnectivity || false,
          poweredOffSeverity:
            thresholds.poweredOffSeverity === 'critical'
              ? 'critical'
              : thresholds.poweredOffSeverity === 'warning'
                ? 'warning'
                : undefined,
          thresholds: extractTriggerValues(thresholds),
        });
        return;
      }

      upsertProjectedOverride({
        id: key,
        name: hostId || key,
        type: 'dockerHost',
        resourceType: 'Container Runtime',
        disableConnectivity: thresholds.disableConnectivity || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const sensorKey = parseAgentSensorResourceId(key);
    if (sensorKey) {
      const agent = agentMap.get(sensorKey.hostId);
      upsertProjectedOverride({
        id: key,
        name: sensorKey.sensorId,
        type: 'agentSensor',
        resourceType: 'Machine Sensor',
        node: agent ? getAlertResourceDisplayLabel(agent) : sensorKey.hostId,
        disabled: thresholds.disabled || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const diskMatch = key.match(/^agent:(.+)\/disk:(.+)$/);
    if (diskMatch) {
      const [, agentId, diskLabel] = diskMatch;
      const agent = agentMap.get(agentId);
      upsertProjectedOverride({
        id: key,
        name: diskLabel.replace(/-/g, '/'),
        type: 'agentDisk',
        resourceType: 'Agent Disk',
        node: agent ? getAlertResourceDisplayLabel(agent) : agentId,
        disabled: thresholds.disabled || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const agentResource = agentMap.get(key);
    if (agentResource) {
      const displayName = getAlertResourceDisplayLabel(agentResource);
      const data = platformData(agentResource);
      const agent = asRecord(data?.agent);
      upsertProjectedOverride({
        id: key,
        name: displayName,
        type: 'agent',
        resourceType: 'Agent',
        node: displayName,
        instance:
          asString(agent?.platform) ||
          asString(agent?.osName) ||
          asString(data?.platform) ||
          asString(data?.osName) ||
          '',
        disabled: thresholds.disabled || false,
        disableConnectivity: thresholds.disableConnectivity || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    // PBS datastore override keys are also "pbs-" prefixed (the canonical
    // "<instance-id>/<name>" and legacy "<instance-id>-<name>" forms), so an
    // instance miss must fall through to the storage lookup below instead of
    // swallowing the key (#1591).
    if (key.startsWith('pbs-')) {
      const pbs = pbsInstanceById.get(key);
      if (pbs) {
        upsertProjectedOverride({
          id: key,
          name: pbs.name,
          type: 'pbs',
          resourceType: 'PBS',
          disableConnectivity: thresholds.disableConnectivity || false,
          thresholds: extractTriggerValues(thresholds),
        });
        return;
      }
    }

    const node = nodeResources.find((resource) => resource.id === key);
    if (node) {
      upsertProjectedOverride({
        id: key,
        name: getAlertResourceDisplayLabel(node),
        type: 'agent',
        resourceType: 'Agent',
        disableConnectivity: thresholds.disableConnectivity || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const storage = storageMap.get(key);
    if (storage) {
      const coords = storageCoords(storage);
      upsertProjectedOverride({
        id: storageOverrideActionId(storage),
        name: getAlertResourceDisplayLabel(storage),
        type: 'storage',
        resourceType: 'Storage',
        node: coords.node,
        instance: coords.instance,
        disabled: thresholds.disabled || false,
        thresholds: extractTriggerValues(thresholds),
      });
      return;
    }

    const guest =
      guestMap.get(key) ||
      vmResources.find((resource) => resource.id === key) ||
      containerResources.find((resource) => resource.id === key);
    if (!guest) {
      return;
    }

    const guestIdentity = getGuestOverrideIdentity(guest);
    upsertProjectedOverride({
      id: guestOverrideStorageId(guest) || key,
      name: getAlertResourceDisplayLabel(guest),
      type: 'guest',
      resourceType: guest.type === 'vm' ? 'VM' : 'Container',
      vmid: guestIdentity?.vmid ?? guessNumericId(guest.id),
      node: guestIdentity?.node ?? '',
      instance: guestIdentity?.instance ?? guest.platformId,
      disabled: thresholds.disabled || false,
      disableConnectivity: thresholds.disableConnectivity || false,
      poweredOffSeverity:
        thresholds.poweredOffSeverity === 'critical'
          ? 'critical'
          : thresholds.poweredOffSeverity === 'warning'
            ? 'warning'
            : undefined,
      thresholds: extractTriggerValues(thresholds),
      backup: thresholds.backup,
      snapshot: thresholds.snapshot,
    });
  });

  return overridesList;
};