Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import { readMemoryObservation } from '@/utils/memoryObservation';
import type {
Disk,
HostNetworkInterface,
Memory,
Node,
PBSBackupJob,
PBSGarbageJob,
PBSInstance,
PBSNamespace,
PBSPruneJob,
PBSSyncJob,
PBSDatastore,
PBSVerifyJob,
PMGDomainStat,
PMGInstance,
PMGMailCountPoint,
PMGMailStats,
PMGNodeStatus,
PMGQuarantineTotals,
PMGRelayDomain,
PMGSpamBucket,
Temperature,
} from '@/types/api';
import { $RAW } from 'solid-js/store';
import type { Resource } from '@/types/resource';
import {
getExplicitAgentIdFromResource,
getAgentFirstReportStatusIndicator,
getExplicitResourceClusterName,
hasDockerFacetEvidence,
} from '@/utils/agentResources';
import {
getPreferredInfrastructureDisplayName,
getPreferredResourceClusterName,
getPreferredResourceHostname,
resolveGuestUrlWithIdentity,
} from '@/utils/resourceIdentity';
import {
normalizeSourcePlatformScopes,
normalizeSourcePlatformKey,
resolvePlatformTypeFromSources,
resolveSourceTypeFromSources,
} from '@/utils/sourcePlatforms';
type JsonRecord = Record<string, unknown>;
const asRecord = (value: unknown): JsonRecord | undefined =>
value && typeof value === 'object' ? (value as JsonRecord) : undefined;
const asString = (value: unknown): string | undefined =>
typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
const asNumber = (value: unknown): number | undefined =>
typeof value === 'number' && Number.isFinite(value) ? value : undefined;
const asBoolean = (value: unknown): boolean | undefined =>
typeof value === 'boolean' ? value : undefined;
const asArray = (value: unknown): unknown[] => (Array.isArray(value) ? value : []);
const toISOTime = (value: unknown, fallbackMs?: number): string => {
const asStr = asString(value);
if (asStr) return asStr;
if (typeof fallbackMs === 'number' && Number.isFinite(fallbackMs)) {
return new Date(fallbackMs).toISOString();
}
return new Date(0).toISOString();
};
const getCanonicalPlatformId = (resource: Resource): string | undefined => {
const platformId = resource.canonicalIdentity?.platformId;
return typeof platformId === 'string' && platformId.trim().length > 0
? platformId.trim()
: undefined;
};
export const resourcePlatformData = (resource: Resource): Record<string, unknown> | undefined =>
asRecord(resource.platformData);
const mergeStringArrays = (incoming?: string[], existing?: string[]): string[] | undefined => {
const merged = [...(incoming ?? []), ...(existing ?? [])]
.map((value) => asString(value))
.filter((value): value is string => Boolean(value));
return merged.length > 0 ? Array.from(new Set(merged)) : undefined;
};
const readStringArray = (value: unknown): string[] | undefined => {
if (!Array.isArray(value)) return undefined;
const normalized = value
.map((entry) => asString(entry))
.filter((entry): entry is string => Boolean(entry));
return normalized.length > 0 ? Array.from(new Set(normalized)) : undefined;
};
const readExplicitPlatformScopes = (resource: Resource): string[] | undefined =>
readStringArray(getResourceRecord(resource).platformScopes) ??
readStringArray(asRecord(resource.platformData)?.platformScopes);
const normalizeSourceToken = (value: string): string =>
normalizeSourcePlatformKey(value) || value.trim().toLowerCase();
const sourceListHas = (sources: string[] | undefined, ...candidates: string[]): boolean => {
if (!sources || sources.length === 0) return false;
const sourceSet = new Set(sources.map((source) => normalizeSourceToken(source)));
return candidates.some((candidate) => sourceSet.has(normalizeSourceToken(candidate)));
};
const shouldKeepSourceFacet = (
authoritativeSources: string[] | undefined,
...sourceCandidates: string[]
): boolean => !authoritativeSources || sourceListHas(authoritativeSources, ...sourceCandidates);
const mergeRecord = <T extends JsonRecord>(incoming?: T, existing?: T): T | undefined => {
if (!incoming) return existing;
if (!existing) return incoming;
return { ...existing, ...incoming };
};
// Native guest runtime facets omit backupInProgress when false. Their positive
// VMID and runtime state distinguish them from thin/legacy enrichment. This
// applies to LXC too, independently of any QEMU guest-agent outcome.
// Only an explicit guest-agent outcome also withdraws its optional read reason,
// lock, false expected flag and filesystems. Neither withdrawal establishes thaw.
// Partial/legacy facets without the relevant evidence retain the prior facts.
const mergeProxmoxFacet = <T extends JsonRecord>(incoming?: T, existing?: T): T | undefined => {
const merged = mergeRecord(incoming, existing);
if (
!incoming ||
!merged ||
typeof incoming.vmid !== 'number' ||
!Number.isInteger(incoming.vmid) ||
incoming.vmid <= 0 ||
typeof incoming.runtimeStatus !== 'string' ||
!incoming.runtimeStatus.trim()
)
return merged;
const next = { ...merged };
if (!Object.prototype.hasOwnProperty.call(incoming, 'backupInProgress')) {
delete next.backupInProgress;
}
if (typeof incoming.guestAgentStatus !== 'string' || !incoming.guestAgentStatus.trim()) {
return next;
}
for (const key of ['diskStatusReason', 'guestAgentExpected', 'lock', 'disks']) {
if (!Object.prototype.hasOwnProperty.call(incoming, key)) delete next[key];
}
return next;
};
// Native agent facets always carry `stale`; facets synthesized from legacy
// platform data never do. Their filesystem list is omitempty on wire, so a
// native report without `disks` has withdrawn them, while a partial or legacy
// facet keeps the prior list.
const mergeAgentFacet = <T extends JsonRecord>(incoming?: T, existing?: T): T | undefined => {
const merged = mergeRecord(incoming, existing);
if (
!incoming ||
!merged ||
typeof incoming.stale !== 'boolean' ||
Object.prototype.hasOwnProperty.call(incoming, 'disks')
)
return merged;
const next = { ...merged };
delete next.disks;
return next;
};
// Thin producers can omit source status, so it survives as REST enrichment.
// Entries of sources the merged row no
// longer lists are stale: they would keep a split-off provider claiming the row.
const pruneSourceStatus = (
value: unknown,
sources: string[] | undefined,
): JsonRecord | undefined => {
const status = asRecord(value);
if (!status || !sources || sources.length === 0) return status;
const entries = Object.entries(status);
const kept = entries.filter(([source]) => sourceListHas(sources, source));
if (kept.length === entries.length) return status;
return kept.length > 0 ? Object.fromEntries(kept) : undefined;
};
const mergePlatformData = (
incomingValue: Resource['platformData'],
existingValue: Resource['platformData'],
): Resource['platformData'] => {
const incoming = asRecord(incomingValue);
const existing = asRecord(existingValue);
if (!incoming) return existingValue;
if (!existing) return incomingValue;
const incomingSources = readStringArray(incoming.sources);
const existingSources = readStringArray(existing.sources);
const sources = incomingSources ?? existingSources;
const merged: JsonRecord = { ...existing, ...incoming };
for (const [key, ...sourceCandidates] of [
['agent', 'agent'],
['docker', 'docker'],
['proxmox', 'proxmox-pve'],
['pbs', 'proxmox-pbs'],
['pmg', 'proxmox-pmg'],
['kubernetes', 'kubernetes'],
['vmware', 'vmware-vsphere'],
['truenas', 'truenas'],
['availability', 'availability'],
] as const) {
if (!shouldKeepSourceFacet(incomingSources, ...sourceCandidates)) {
delete merged[key];
continue;
}
const nested =
key === 'proxmox'
? mergeProxmoxFacet(asRecord(incoming[key]), asRecord(existing[key]))
: key === 'agent'
? mergeAgentFacet(asRecord(incoming[key]), asRecord(existing[key]))
: key === 'availability'
? (asRecord(incoming[key]) ?? asRecord(existing[key]))
: mergeRecord(asRecord(incoming[key]), asRecord(existing[key]));
if (key === 'docker') {
if (hasDockerFacetEvidence(nested)) {
merged[key] = nested;
} else {
delete merged[key];
}
continue;
}
if (nested) {
merged[key] = nested;
}
}
// A departed availability source must not resurrect its check set from the mirror.
if (!shouldKeepSourceFacet(incomingSources, 'availability')) {
delete merged.availabilityChecks;
}
for (const key of ['storage', 'physicalDisk', 'ceph', 'metrics', 'discoveryTarget']) {
const nested = mergeRecord(asRecord(incoming[key]), asRecord(existing[key]));
if (nested) {
merged[key] = nested;
}
}
const sourceStatus = pruneSourceStatus(
mergeRecord(asRecord(incoming.sourceStatus), asRecord(existing.sourceStatus)),
sources,
);
if (sourceStatus) {
merged.sourceStatus = sourceStatus;
} else {
delete merged.sourceStatus;
}
if (sources) {
merged.sources = sources;
}
return merged;
};
const getResourceRecord = (resource: Resource): JsonRecord => resource as unknown as JsonRecord;
const getExplicitResourceSources = (resource: Resource): string[] | undefined =>
readStringArray(getResourceRecord(resource).sources);
const getFacetRecord = (
resource: Resource,
platformData: Resource['platformData'] | undefined,
key: string,
): JsonRecord | undefined => {
const resourceRecord = getResourceRecord(resource);
const platformRecord = asRecord(platformData);
return asRecord(resourceRecord[key]) || asRecord(platformRecord?.[key]);
};
const deriveLegacySourceList = (
resource: Resource,
platformData: Resource['platformData'] | undefined = resource.platformData,
): string[] | undefined => {
const resourceSources = getExplicitResourceSources(resource);
if (resourceSources && resourceSources.length > 0) {
return resourceSources;
}
const sources: string[] = [];
if (getFacetRecord(resource, platformData, 'proxmox')) sources.push('proxmox');
if (getFacetRecord(resource, platformData, 'pbs')) sources.push('pbs');
if (getFacetRecord(resource, platformData, 'pmg')) sources.push('pmg');
if (getFacetRecord(resource, platformData, 'vmware')) sources.push('vmware');
if (getFacetRecord(resource, platformData, 'truenas')) sources.push('truenas');
if (getFacetRecord(resource, platformData, 'kubernetes')) sources.push('kubernetes');
if (hasDockerFacetEvidence(getFacetRecord(resource, platformData, 'docker'))) {
sources.push('docker');
}
if (getFacetRecord(resource, platformData, 'availability')) sources.push('availability');
if (getFacetRecord(resource, platformData, 'agent')) sources.push('agent');
if (sources.length > 0) {
return Array.from(new Set(sources));
}
if (
resource.type === 'network-endpoint' ||
resource.platformType === 'availability' ||
Boolean(resource.availability) ||
Boolean(asRecord(resource.platformData)?.availability)
) {
return ['availability'];
}
switch (resource.platformType) {
case 'proxmox-pve':
return resource.sourceType === 'hybrid' ? ['proxmox', 'agent'] : ['proxmox'];
case 'docker':
return ['docker'];
case 'kubernetes':
return resource.sourceType === 'hybrid' ? ['agent', 'kubernetes'] : ['kubernetes'];
case 'proxmox-pbs':
return ['pbs'];
case 'proxmox-pmg':
return ['pmg'];
case 'truenas':
return ['truenas'];
case 'vmware-vsphere':
return ['vmware'];
default:
return resource.sourceType === 'agent' ? ['agent'] : undefined;
}
};
const hasLegacyProxmoxShape = (
resource: Resource,
platformData: JsonRecord,
sources?: string[],
): boolean => {
// Flat instance/node fields also describe storage from other providers.
// Current source ownership takes precedence over the legacy field shape.
if (sources && sources.length > 0) {
return sourceListHas(sources, 'proxmox-pve');
}
return (
resource.platformType === 'proxmox-pve' ||
[
'instance',
'node',
'clusterName',
'vmid',
'cpus',
'template',
'swapUsed',
'swapTotal',
'balloon',
].some((key) => platformData[key] !== undefined)
);
};
const canonicalizeLegacyPlatformData = (resource: Resource): Resource['platformData'] => {
const platformData = asRecord(resource.platformData);
if (!platformData) {
const normalizedSources = deriveLegacySourceList(resource);
if (!normalizedSources || normalizedSources.length === 0) {
return resource.platformData;
}
const normalized: JsonRecord = { sources: normalizedSources };
const resourceRecord = getResourceRecord(resource);
for (const [key, value] of [
['agent', resourceRecord.agent],
['docker', resourceRecord.docker],
['proxmox', resourceRecord.proxmox],
['pbs', resourceRecord.pbs],
['pmg', resourceRecord.pmg],
['kubernetes', resourceRecord.kubernetes],
['vmware', resourceRecord.vmware],
['truenas', resourceRecord.truenas],
['storage', resourceRecord.storage],
['availability', resourceRecord.availability],
['physicalDisk', resourceRecord.physicalDisk],
] as const) {
if (key === 'docker' && !hasDockerFacetEvidence(value)) {
continue;
}
if (value !== undefined) {
normalized[key] = value;
}
}
return normalized;
}
const normalized: JsonRecord = { ...platformData };
if (asRecord(normalized.docker) && !hasDockerFacetEvidence(normalized.docker)) {
delete normalized.docker;
}
const resourceRecord = getResourceRecord(resource);
for (const key of [
'agent',
'docker',
'proxmox',
'pbs',
'pmg',
'kubernetes',
'vmware',
'truenas',
'storage',
'availability',
'physicalDisk',
] as const) {
if (
key === 'docker' &&
!asRecord(normalized[key]) &&
hasDockerFacetEvidence(resourceRecord[key])
) {
normalized[key] = resourceRecord[key];
continue;
}
if (key === 'docker') {
continue;
}
if (!asRecord(normalized[key]) && asRecord(resourceRecord[key])) {
normalized[key] = resourceRecord[key];
}
}
const explicitResourceSources = getExplicitResourceSources(resource);
const normalizedSources =
explicitResourceSources ??
(Array.isArray(platformData.sources) && platformData.sources.length > 0
? (platformData.sources as string[])
: deriveLegacySourceList(resource, platformData));
if (normalizedSources && normalizedSources.length > 0) {
normalized.sources = normalizedSources;
}
if (!asRecord(normalized.agent)) {
const agentPayload: JsonRecord = {};
for (const [legacyKey, nextKey] of [
['agentId', 'agentId'],
['agentVersion', 'agentVersion'],
['hostname', 'hostname'],
['platform', 'platform'],
['osName', 'osName'],
['osVersion', 'osVersion'],
['kernelVersion', 'kernelVersion'],
['architecture', 'architecture'],
['commandsEnabled', 'commandsEnabled'],
] as const) {
if (platformData[legacyKey] !== undefined) {
agentPayload[nextKey] = platformData[legacyKey];
}
}
if (platformData.memory !== undefined) agentPayload.memory = platformData.memory;
if (platformData.interfaces !== undefined)
agentPayload.networkInterfaces = platformData.interfaces;
if (platformData.disks !== undefined) agentPayload.disks = platformData.disks;
if (Object.keys(agentPayload).length > 0) {
normalized.agent = agentPayload;
}
}
if (!asRecord(normalized.docker)) {
const dockerPayload: JsonRecord = {};
for (const [legacyKey, nextKey] of [
['agentId', 'agentId'],
['runtime', 'runtime'],
['runtimeVersion', 'runtimeVersion'],
['dockerVersion', 'dockerVersion'],
['os', 'os'],
['kernelVersion', 'kernelVersion'],
['architecture', 'architecture'],
['agentVersion', 'agentVersion'],
['hostname', 'hostname'],
['displayName', 'displayName'],
['machineId', 'machineId'],
['containerCount', 'containerCount'],
['uptimeSeconds', 'uptimeSeconds'],
['intervalSeconds', 'intervalSeconds'],
['temperature', 'temperature'],
['hostSourceId', 'hostSourceId'],
] as const) {
if (platformData[legacyKey] !== undefined) {
dockerPayload[nextKey] = platformData[legacyKey];
}
}
if (platformData.swarm !== undefined) dockerPayload.swarm = platformData.swarm;
if (platformData.interfaces !== undefined)
dockerPayload.networkInterfaces = platformData.interfaces;
if (platformData.disks !== undefined) dockerPayload.disks = platformData.disks;
if (Object.keys(dockerPayload).length > 0) {
normalized.docker = dockerPayload;
}
}
if (
!asRecord(normalized.proxmox) &&
hasLegacyProxmoxShape(resource, platformData, normalizedSources)
) {
const proxmoxPayload: JsonRecord = {};
for (const [legacyKey, nextKey] of [
['instance', 'instance'],
['node', 'nodeName'],
['clusterName', 'clusterName'],
['vmid', 'vmid'],
['cpus', 'cpus'],
['template', 'template'],
['swapUsed', 'swapUsed'],
['swapTotal', 'swapTotal'],
['balloon', 'balloon'],
] as const) {
if (platformData[legacyKey] !== undefined) {
proxmoxPayload[nextKey] = platformData[legacyKey];
}
}
if (platformData.disks !== undefined) proxmoxPayload.disks = platformData.disks;
if (typeof platformData.backupInProgress === 'boolean') {
proxmoxPayload.backupInProgress = platformData.backupInProgress;
}
if (Object.keys(proxmoxPayload).length > 0) {
normalized.proxmox = proxmoxPayload;
}
}
if (!asRecord(normalized.pbs)) {
const pbsPayload: JsonRecord = {};
if (platformData.host !== undefined) pbsPayload.hostname = platformData.host;
if (platformData.version !== undefined) pbsPayload.version = platformData.version;
if (platformData.connectionHealth !== undefined) {
pbsPayload.connectionHealth = platformData.connectionHealth;
}
if (platformData.numDatastores !== undefined) {
pbsPayload.datastoreCount = platformData.numDatastores;
}
if (Object.keys(pbsPayload).length > 0) {
normalized.pbs = pbsPayload;
}
}
if (!asRecord(normalized.pmg)) {
const pmgPayload: JsonRecord = {};
if (platformData.host !== undefined) pmgPayload.hostname = platformData.host;
if (platformData.version !== undefined) pmgPayload.version = platformData.version;
if (platformData.connectionHealth !== undefined) {
pmgPayload.connectionHealth = platformData.connectionHealth;
}
for (const [legacyKey, nextKey] of [
['nodeCount', 'nodeCount'],
['queueActive', 'queueActive'],
['queueDeferred', 'queueDeferred'],
['queueHold', 'queueHold'],
['queueIncoming', 'queueIncoming'],
['queueTotal', 'queueTotal'],
] as const) {
if (platformData[legacyKey] !== undefined) {
pmgPayload[nextKey] = platformData[legacyKey];
}
}
if (Object.keys(pmgPayload).length > 0) {
normalized.pmg = pmgPayload;
}
}
if (!asRecord(normalized.kubernetes)) {
const kubernetesPayload: JsonRecord = {};
for (const [legacyKey, nextKey] of [
['agentId', 'agentId'],
['clusterId', 'clusterId'],
['context', 'context'],
['nodeName', 'nodeName'],
['namespace', 'namespace'],
['clusterName', 'clusterName'],
['pendingUninstall', 'pendingUninstall'],
] as const) {
if (platformData[legacyKey] !== undefined) {
kubernetesPayload[nextKey] = platformData[legacyKey];
}
}
if (Object.keys(kubernetesPayload).length > 0) {
normalized.kubernetes = kubernetesPayload;
}
}
return normalized;
};
const getCanonicalSourceList = (
resource: Resource,
platformData?: Resource['platformData'],
): string[] | undefined => {
const resourceSources = getExplicitResourceSources(resource);
if (resourceSources && resourceSources.length > 0) return resourceSources;
const platformRecord = asRecord(platformData);
return Array.isArray(platformRecord?.sources) && platformRecord.sources.length > 0
? (platformRecord.sources as string[])
: deriveLegacySourceList(resource, platformData);
};
const hasAvailabilityFacet = (
resource: Resource,
platformData?: Resource['platformData'],
): boolean => {
const platformRecord = asRecord(platformData);
const sources = getCanonicalSourceList(resource, platformData);
return (
resource.type === 'network-endpoint' ||
resource.platformType === 'availability' ||
Boolean(resource.availability) ||
Boolean(platformRecord?.availability) ||
Boolean(sources?.some((source) => source.trim().toLowerCase() === 'availability'))
);
};
export const canonicalizeRealtimeResource = (
resource: Resource,
options: { synthesizePlatformScopes?: boolean } = {},
): Resource => {
const platformData = canonicalizeLegacyPlatformData(resource);
const platformRecord = asRecord(platformData);
const sources = getCanonicalSourceList(resource, platformData);
const docker = hasDockerFacetEvidence(resource.docker)
? resource.docker
: hasDockerFacetEvidence(platformRecord?.docker)
? (platformRecord?.docker as Resource['docker'])
: undefined;
const platformType =
resolvePlatformTypeFromSources(sources) ||
(hasAvailabilityFacet(resource, platformData) ? 'availability' : resource.platformType);
const explicitPlatformScopes = readExplicitPlatformScopes(resource);
const platformScopes =
explicitPlatformScopes !== undefined
? normalizeSourcePlatformScopes(explicitPlatformScopes, platformType)
: options.synthesizePlatformScopes === false
? undefined
: normalizeSourcePlatformScopes(undefined, platformType);
const sourceType =
sources && sources.length > 0 ? resolveSourceTypeFromSources(sources) : resource.sourceType;
const normalizedBase = {
...resource,
platformType,
platformScopes,
sourceType,
platformData,
};
return {
...normalizedBase,
clusterId: resource.clusterId ?? getExplicitResourceClusterName(normalizedBase),
platformData,
agent: resource.agent ?? (platformRecord?.agent as Resource['agent']),
proxmox: resource.proxmox ?? (platformRecord?.proxmox as Resource['proxmox']),
pbs: resource.pbs ?? (platformRecord?.pbs as Resource['pbs']),
kubernetes: resource.kubernetes ?? (platformRecord?.kubernetes as Resource['kubernetes']),
docker,
vmware: resource.vmware ?? (platformRecord?.vmware as Resource['vmware']),
truenas: resource.truenas ?? (platformRecord?.truenas as Resource['truenas']),
storage: resource.storage ?? (platformRecord?.storage as Resource['storage']),
availability:
resource.availability ?? (platformRecord?.availability as Resource['availability']),
availabilityChecks:
resource.availabilityChecks ??
(platformRecord?.availabilityChecks as Resource['availabilityChecks']),
physicalDisk:
resource.physicalDisk ?? (platformRecord?.physicalDisk as Resource['physicalDisk']),
};
};
const mergeCanonicalIdentity = (
incoming?: Resource['canonicalIdentity'],
existing?: Resource['canonicalIdentity'],
): Resource['canonicalIdentity'] => {
if (!incoming) return existing;
if (!existing) return incoming;
const aliases = mergeStringArrays(incoming.aliases, existing.aliases);
const supersededIds = mergeStringArrays(incoming.supersededIds, existing.supersededIds);
return {
...existing,
...incoming,
aliases,
supersededIds,
};
};
const mergeCanonicalSourceFacet = <T extends JsonRecord>(
incomingFacet: T | undefined,
existingFacet: T | undefined,
incomingSources: string[] | undefined,
...sourceCandidates: string[]
): T | undefined =>
shouldKeepSourceFacet(incomingSources, ...sourceCandidates)
? sourceCandidates.includes('proxmox-pve')
? mergeProxmoxFacet(incomingFacet, existingFacet)
: sourceCandidates.includes('agent')
? mergeAgentFacet(incomingFacet, existingFacet)
: mergeRecord(incomingFacet, existingFacet)
: incomingFacet;
// Availability summaries/check sets are complete records in REST and in the
// websocket baseline rebuilt from merge patches. Their optional failure,
// location and certificate fields are withdrawn by omission on recovery.
// Only an absent record retains enrichment, and only while its source remains.
const replaceCanonicalSourceFacet = <T>(
incoming: T | undefined,
existing: T | undefined,
sources: string[] | undefined,
...sourceCandidates: string[]
): T | undefined =>
incoming ?? (shouldKeepSourceFacet(sources, ...sourceCandidates) ? existing : undefined);
// A missing field alone can be a partial snapshot. Explicit unavailable raw
// evidence plus no canonical metric means the producer has withdrawn memory;
// retaining the previous display metric would hide that state.
const hasWithdrawnProxmoxMemory = (incoming: Resource): boolean =>
incoming.memory == null &&
asBoolean(asRecord(asRecord(incoming.proxmox)?.memory)?.usageUnavailable) === true;
// The source list a row carries itself: top-level on realtime rows, in
// platformData on REST rows. Never one inferred from the facets present.
const getOwnSourceList = (resource: Resource): string[] | undefined =>
getExplicitResourceSources(resource) ?? readStringArray(asRecord(resource.platformData)?.sources);
// Every source a display row claims through a list it carries itself: the
// top-level `sources` of a realtime row and the `platformData.sources` of a REST
// row. A row whose two lists disagree (an enrichment from another generation
// landed on it, or a shared array changed under it) claims both compositions.
const getClaimedSources = (resource: Resource): string[] | undefined =>
mergeStringArrays(
getExplicitResourceSources(resource),
readStringArray(asRecord(resource.platformData)?.sources),
);
// Sources the previous display row of an id listed that the incoming realtime
// row no longer does: a split took a provider away, or a module or check
// stopped reporting. Only the realtime row's own top-level list is evidence;
// one inferred from the facets a thin row happens to carry is not.
const getDepartedSources = (incoming: Resource, existing: Resource): string[] => {
const incomingSources = getExplicitResourceSources(incoming);
const existingSources = getClaimedSources(existing);
if (!incomingSources || !existingSources) return [];
return existingSources.filter((source) => !sourceListHas(incomingSources, source));
};
// Facets a platform scope can rest on besides a source of that name; the
// server derives scopes from sources, from facets, and from a Docker host
// running inside a Proxmox container (platform_scopes.go).
const SCOPE_FACET_KEYS: Record<string, string> = {
'proxmox-pve': 'proxmox',
'proxmox-pbs': 'pbs',
'proxmox-pmg': 'pmg',
docker: 'docker',
kubernetes: 'kubernetes',
'vmware-vsphere': 'vmware',
truenas: 'truenas',
agent: 'agent',
};
const PROXMOX_LXC_DOCKER_HOST_PREFIX = 'proxmox-lxc-docker:';
// Whether the incoming row still holds evidence for a platform scope that a
// departed source also names: a surviving source of that platform, a Docker
// host running in a Proxmox container (the server places it on the Proxmox
// scope by its host ID whatever sources the row lists, platform_scopes.go), or,
// on a row that is not a host, a facet of that platform (an agent-reported disk
// under a Proxmox node carries a Proxmox ownership facet and lists only the
// agent). A host row's facets are no evidence: canonicalization synthesizes
// them from flattened platform data (a node's kernel version reads as an agent
// facet), and the snapshot wrappers canonicalize before this runs.
const scopeStillSupported = (scope: string, incoming: Resource): boolean => {
if (sourceListHas(getExplicitResourceSources(incoming), scope)) return true;
const record = getResourceRecord(incoming);
const platform = normalizeSourceToken(scope);
const hostSourceId = asString(asRecord(record.docker)?.hostSourceId);
if (platform === 'proxmox-pve' && hostSourceId?.startsWith(PROXMOX_LXC_DOCKER_HOST_PREFIX)) {
return true;
}
const facetKey = SCOPE_FACET_KEYS[platform];
return incoming.type !== 'agent' && Boolean(facetKey && asRecord(record[facetKey]));
};
// A row whose sources shrank describes a different composition, and the
// per-id merge would keep what the previous row carried for the departed
// source wherever the incoming row omits it: flattened provider facts in
// platformData (from which the provider facet is rebuilt), cluster, identity
// fields, canonical aliases, metrics. The row is rebuilt from the incoming one.
// The REST-only enrichments the realtime payload never carries are kept where
// the departed sources did not own them: platform scopes, minus the departed
// sources' own that no surviving source, Docker LXC host or non-host facet supports, source status
// of the sources that remain, discovery readiness unless its target coordinates now differ
// from the row's, and action readiness unless the agent left. Everything else
// REST adds returns with the next REST refresh.
const replaceRecomposedResource = (
incoming: Resource,
existingCanonical: Resource,
departedSources: string[],
): Resource => {
const replacement = canonicalizeRealtimeResource(incoming);
const keptScopes =
readExplicitPlatformScopes(incoming) === undefined
? existingCanonical.platformScopes?.filter(
(scope) => !sourceListHas(departedSources, scope) || scopeStillSupported(scope, incoming),
)
: undefined;
const survivingSources = getOwnSourceList(replacement);
const keepStatus = (own: unknown, previous: unknown): JsonRecord | undefined =>
pruneSourceStatus(mergeRecord(asRecord(own), asRecord(previous)), survivingSources);
const sourceStatus = keepStatus(
(replacement as unknown as JsonRecord).sourceStatus,
existingCanonical.sourceStatus,
);
const platformData = asRecord(replacement.platformData);
const platformStatus = keepStatus(
platformData?.sourceStatus,
asRecord(existingCanonical.platformData)?.sourceStatus,
);
const agentLeft = sourceListHas(departedSources, 'agent');
const rebuilt: Resource = {
...replacement,
platformScopes:
keptScopes && keptScopes.length > 0
? normalizeSourcePlatformScopes(keptScopes, replacement.platformType)
: replacement.platformScopes,
actionReadiness:
replacement.actionReadiness ?? (agentLeft ? undefined : existingCanonical.actionReadiness),
};
if (sourceStatus) {
rebuilt.sourceStatus = sourceStatus as Resource['sourceStatus'];
} else {
delete rebuilt.sourceStatus;
}
if (platformData) {
const { sourceStatus: _inherited, ...rest } = platformData;
rebuilt.platformData = platformStatus ? { ...rest, sourceStatus: platformStatus } : rest;
}
return rebuilt;
};
export const mergeCanonicalResource = (incoming: Resource, existing?: Resource): Resource => {
if (!existing) {
return canonicalizeRealtimeResource(incoming);
}
const existingCanonical = canonicalizeRealtimeResource(existing);
const departedSources = getDepartedSources(incoming, existing);
if (departedSources.length > 0) {
return replaceRecomposedResource(incoming, existingCanonical, departedSources);
}
const incomingSources = getCanonicalSourceList(incoming, incoming.platformData);
const merged: Resource = {
...existingCanonical,
...incoming,
...(hasWithdrawnProxmoxMemory(incoming) ? { memory: undefined } : {}),
clusterId: incoming.clusterId ?? existingCanonical.clusterId,
platformScopes: normalizeSourcePlatformScopes(
incoming.platformScopes ?? existingCanonical.platformScopes,
incoming.platformType ?? existingCanonical.platformType,
),
discoveryTarget: incoming.discoveryTarget ?? existingCanonical.discoveryTarget,
metricsTarget: incoming.metricsTarget ?? existingCanonical.metricsTarget,
canonicalIdentity: mergeCanonicalIdentity(
incoming.canonicalIdentity,
existingCanonical.canonicalIdentity,
),
policy: incoming.policy ?? existingCanonical.policy,
aiSafeSummary: incoming.aiSafeSummary ?? existingCanonical.aiSafeSummary,
recentChanges: incoming.recentChanges ?? existingCanonical.recentChanges,
facetCounts: incoming.facetCounts ?? existingCanonical.facetCounts,
diskIO: incoming.diskIO ?? existingCanonical.diskIO,
agent: mergeCanonicalSourceFacet(
incoming.agent as JsonRecord | undefined,
existingCanonical.agent as JsonRecord | undefined,
incomingSources,
'agent',
) as Resource['agent'],
proxmox: mergeCanonicalSourceFacet(
incoming.proxmox as JsonRecord | undefined,
existingCanonical.proxmox as JsonRecord | undefined,
incomingSources,
'proxmox-pve',
) as Resource['proxmox'],
pbs: mergeCanonicalSourceFacet(
incoming.pbs as JsonRecord | undefined,
existingCanonical.pbs as JsonRecord | undefined,
incomingSources,
'proxmox-pbs',
) as Resource['pbs'],
kubernetes: mergeCanonicalSourceFacet(
incoming.kubernetes as JsonRecord | undefined,
existingCanonical.kubernetes as JsonRecord | undefined,
incomingSources,
'kubernetes',
) as Resource['kubernetes'],
docker: mergeCanonicalSourceFacet(
hasDockerFacetEvidence(incoming.docker)
? (incoming.docker as JsonRecord | undefined)
: undefined,
hasDockerFacetEvidence(existingCanonical.docker)
? (existingCanonical.docker as JsonRecord | undefined)
: undefined,
incomingSources,
'docker',
) as Resource['docker'],
vmware: mergeCanonicalSourceFacet(
incoming.vmware as JsonRecord | undefined,
existingCanonical.vmware as JsonRecord | undefined,
incomingSources,
'vmware-vsphere',
) as Resource['vmware'],
truenas: mergeCanonicalSourceFacet(
incoming.truenas as JsonRecord | undefined,
existingCanonical.truenas as JsonRecord | undefined,
incomingSources,
'truenas',
) as Resource['truenas'],
availability: replaceCanonicalSourceFacet(
incoming.availability,
existingCanonical.availability,
incomingSources,
'availability',
),
availabilityChecks: replaceCanonicalSourceFacet(
incoming.availabilityChecks,
existingCanonical.availabilityChecks,
incomingSources,
'availability',
),
storage: mergeRecord(
incoming.storage as JsonRecord | undefined,
existingCanonical.storage as JsonRecord | undefined,
) as Resource['storage'],
physicalDisk: mergeRecord(
incoming.physicalDisk as JsonRecord | undefined,
existingCanonical.physicalDisk as JsonRecord | undefined,
) as Resource['physicalDisk'],
identity: mergeRecord(
incoming.identity as JsonRecord | undefined,
existingCanonical.identity as JsonRecord | undefined,
) as Resource['identity'],
platformData: mergePlatformData(incoming.platformData, existingCanonical.platformData),
tags: incoming.tags && incoming.tags.length > 0 ? incoming.tags : existingCanonical.tags,
labels:
incoming.labels && Object.keys(incoming.labels).length > 0
? incoming.labels
: existingCanonical.labels,
};
// Only a list a row carries itself prunes: one inferred from the facets a
// thin row happens to carry is no evidence that a source left.
const status = pruneSourceStatus(
merged.sourceStatus,
getOwnSourceList(incoming) ?? getOwnSourceList(existingCanonical),
);
if (status) {
merged.sourceStatus = status as Resource['sourceStatus'];
} else if (merged.sourceStatus !== undefined) {
delete merged.sourceStatus;
}
if (getAgentFirstReportStatusIndicator(incoming)) {
// A new monitor generation has no accepted telemetry for this enrollment.
// Its explicit delivery state withdraws the previous browser observation.
merged.cpu = undefined;
merged.memory = undefined;
merged.disk = undefined;
merged.network = undefined;
merged.diskIO = undefined;
merged.temperature = undefined;
merged.uptime = undefined;
merged.agent = incoming.agent;
merged.platformData = incoming.platformData;
}
return merged;
};
// Every realtime snapshot (the websocket payload, its deltas, and the
// `/api/state` recovery) carries the server's presentation rows: the server
// already coalesced host views, honouring operator splits and provider scope
// the browser cannot see. Each incoming row is therefore one display row; the
// browser only canonicalizes it and merges it with the previous row of its id.
// Re-joining same-hostname host rows here undid server splits (#1753, and a
// node and agent the operator split apart).
export const mergeCanonicalResourceSnapshot = (
incoming: Resource[],
existing: Resource[],
): Resource[] => {
if (incoming.length === 0) {
return [];
}
const existingById = new Map(existing.map((resource) => [resource.id, resource] as const));
return incoming.map((resource) =>
mergeCanonicalResource(
canonicalizeRealtimeResource(resource, { synthesizePlatformScopes: false }),
existingById.get(resource.id),
),
);
};
// Per-resource top-level keys touched by a server delta's JSON merge patches.
// `platformData` is expanded one level into `platformData.<leaf>` entries. A
// `null` value means the change shape is unknown (row added/removed, whole
// subtree replaced, deferred ticks) and the row must take the full merge path.
export type ResourceChangedKeys = ReadonlyMap<string, readonly string[] | null>;
// Union of two per-resource changed-key lists across ticks. Unknown (`null`)
// contaminates: once a tick could not describe a row's change shape, no later
// tick can restore fast-path eligibility for that row.
export const unionResourceChangedKeys = (
first: readonly string[] | null | undefined,
second: readonly string[] | null | undefined,
): readonly string[] | null => {
if (first === undefined) return second ?? null;
if (second === undefined) return first ?? null;
if (first === null || second === null) return null;
return Array.from(new Set([...first, ...second]));
};
// Top-level Resource fields the canonical merge takes verbatim from the
// incoming row (plain `...incoming` spread, no special handling) and that the
// canonicalization pass never reads. A patch confined to these fields cannot
// change platform/source resolution, facet keeps, or identity merging, so the
// merged display row is the previous display row with just these subtrees
// replaced. `diskIO` is special-cased in the merge (`incoming ?? existing`)
// but a merge-patch either sets it (incoming wins) or deletes it (existing
// survives both paths), so it stays equivalent.
const FAST_MERGE_TOP_KEYS = new Set([
'cpu',
'memory',
'disk',
'network',
'diskIO',
'temperature',
'uptime',
'lastSeen',
'status',
]);
const FAST_MERGE_PLATFORM_DATA_PREFIX = 'platformData.';
// platformData leaves the metric mirror writes touch every tick. None of them
// are read by canonicalizeLegacyPlatformData or the source-list derivation, so
// patching them cannot alter canonicalization output beyond the leaf values.
const FAST_MERGE_PLATFORM_DATA_KEYS = new Set(['diskRead', 'diskWrite', 'networkIn', 'networkOut']);
// O(1) escape from a Solid store proxy; identity for plain values. Never use
// store `unwrap` here: it deep-walks plain objects, which is the cost this
// path exists to avoid.
const rawStoreValue = <T>(value: T): T =>
(value != null && ((value as { [$RAW]?: T })[$RAW] as T)) || value;
// Returns the validated changed-key list when `id`'s row can skip the full
// clone+canonicalize+merge, or null when it must take the full path. The same
// predicate gates the per-key store commits, so it must stay conservative:
// anything it accepts is asserted to leave every other field of the merged
// display row untouched.
export const getFastResourceMergePatchKeys = (
changedKeys: ResourceChangedKeys | undefined,
id: string,
existing: Resource | undefined,
): readonly string[] | null => {
if (!changedKeys || !existing) return null;
// Agent rows keep the full merge. The exclusion dates from browser host
// coalescing, which made an agent's output depend on its group; equivalence
// with the full path has not been proven for agent rows since.
if (existing.type === 'agent') return null;
const keys = changedKeys.get(id);
if (!keys || keys.length === 0) return null;
let touchesPlatformData = false;
for (const key of keys) {
if (FAST_MERGE_TOP_KEYS.has(key) || key === 'proxmox') continue;
if (key.startsWith(FAST_MERGE_PLATFORM_DATA_PREFIX)) {
if (!FAST_MERGE_PLATFORM_DATA_KEYS.has(key.slice(FAST_MERGE_PLATFORM_DATA_PREFIX.length))) {
return null;
}
touchesPlatformData = true;
continue;
}
return null;
}
if (
touchesPlatformData &&
!asRecord(rawStoreValue(existing as unknown as JsonRecord).platformData)
) {
return null;
}
return keys;
};
// Patched subtrees are JSON-derived plain data and typically tiny (a handful
// of numeric leaves). A manual walk clones them several times faster than
// structuredClone, whose per-invocation setup dominates at this size; symbol
// keys (Solid's internal store markers on raw nodes) are skipped by design.
const clonePlainValue = <T>(value: T): T => {
if (value === null || typeof value !== 'object') return value;
if (Array.isArray(value)) return value.map(clonePlainValue) as unknown as T;
const source = value as Record<string, unknown>;
const out: Record<string, unknown> = {};
for (const key of Object.keys(source)) out[key] = clonePlainValue(source[key]);
return out as T;
};
const cloneFastPatchValue = (value: unknown): unknown => clonePlainValue(value);
// Fast-path counterpart of mergeCanonicalResource for rows whose patch passed
// getFastResourceMergePatchKeys: the previous display row with the patched
// subtrees cloned in. Content-equivalent to the full path; unpatched subtrees
// keep their object identity so downstream reconciles short-circuit on them.
const applyFastResourceMergePatch = (
incoming: Resource,
existing: Resource,
keys: readonly string[],
): Resource => {
const rawIncoming = rawStoreValue(incoming as unknown as JsonRecord);
const rawExisting = rawStoreValue(existing as unknown as JsonRecord);
const next: JsonRecord = { ...rawExisting };
let platformDataLeaves: string[] | null = null;
for (const key of keys) {
if (key.startsWith(FAST_MERGE_PLATFORM_DATA_PREFIX)) {
(platformDataLeaves ??= []).push(key.slice(FAST_MERGE_PLATFORM_DATA_PREFIX.length));
continue;
}
// A merge-patch deletion normally preserves the richer display value.
// Explicitly withdrawn Proxmox memory is cleared below, as in the full merge.
if (!(key in rawIncoming)) continue;
const value = rawIncoming[key];
if (key === 'proxmox') {
const incomingFacet = asRecord(value);
if (!incomingFacet) continue;
const cloned = clonePlainValue(incomingFacet);
const existingFacet = asRecord(next.proxmox);
// Mirror mergeCanonicalSourceFacet: the facet merges with the existing
// one only while the (unchanged) source list keeps it authoritative.
next.proxmox =
existingFacet &&
shouldKeepSourceFacet(
getCanonicalSourceList(existing, existing.platformData),
'proxmox-pve',
)
? mergeProxmoxFacet(cloned, existingFacet)
: cloned;
continue;
}
next[key] = cloneFastPatchValue(value);
}
if (platformDataLeaves) {
const incomingPlatformData = asRecord(rawIncoming.platformData);
const nextPlatformData: JsonRecord = { ...(asRecord(next.platformData) ?? {}) };
for (const leaf of platformDataLeaves) {
if (!incomingPlatformData || !(leaf in incomingPlatformData)) continue;
nextPlatformData[leaf] = cloneFastPatchValue(incomingPlatformData[leaf]);
}
next.platformData = nextPlatformData;
}
if (hasWithdrawnProxmoxMemory(incoming)) next.memory = undefined;
return next as unknown as Resource;
};
export type FastResourceStorePatchOp = {
key: string;
// Set for platformData leaf writes; the value then targets platformData[leaf].
leaf?: string;
value: unknown;
// Records diff via a nested reconcile at the key path; primitives replace.
mode: 'set' | 'reconcile';
};
// Store-commit counterpart of the fast merge: instead of a full-row reconcile
// (whose unwrap deep-walks every subtree), a fast row commits as a handful of
// per-key writes. Callers apply `reconcile` ops with a subtree reconcile and
// `set` ops as plain path sets.
export const buildFastResourceStorePatchOps = (
row: Resource,
keys: readonly string[],
): FastResourceStorePatchOp[] => {
const record = rawStoreValue(row as unknown as JsonRecord);
const ops: FastResourceStorePatchOp[] = [];
// Unavailability can also arrive with a raw Proxmox facet change rather
// than a memory key (the canonical metric may already be absent on wire).
if (!keys.includes('memory') && hasWithdrawnProxmoxMemory(row)) {
ops.push({ key: 'memory', value: undefined, mode: 'set' });
}
const platformData = asRecord(record.platformData);
for (const key of keys) {
if (key.startsWith(FAST_MERGE_PLATFORM_DATA_PREFIX)) {
const leaf = key.slice(FAST_MERGE_PLATFORM_DATA_PREFIX.length);
if (!platformData || !(leaf in platformData)) continue;
const value = platformData[leaf];
ops.push({
key: 'platformData',
leaf,
value,
mode: value !== null && typeof value === 'object' ? 'reconcile' : 'set',
});
continue;
}
if (!(key in record)) continue;
const value = record[key];
ops.push({
key,
value,
mode: value !== null && typeof value === 'object' ? 'reconcile' : 'set',
});
}
return ops;
};
// Incremental counterpart to mergeCanonicalResourceSnapshot. Server delta
// application preserves the raw object identity of untouched resources, and
// each server row is one display row (the server coalesces host views), so
// only changed and newly added rows need to be cloned and canonicalized.
// Resources outside the delta retain their exact display objects, preventing
// an estate-wide reactive invalidation on every metrics tick while producing
// the same rows as the full-snapshot path. Rows whose per-key change shape
// passes getFastResourceMergePatchKeys skip the clone+canonicalize+merge
// entirely and patch the previous display row.
export const mergeCanonicalResourceDeltaSnapshot = (
incoming: Resource[],
existing: Resource[],
changedIds: ReadonlySet<string>,
changedKeys?: ResourceChangedKeys,
): Resource[] => {
if (incoming.length === 0) {
return [];
}
const existingById = new Map(existing.map((resource) => [resource.id, resource] as const));
return incoming.map((resource) => {
const existingResource = existingById.get(resource.id);
if (existingResource === undefined) {
return mergeCanonicalResource(
canonicalizeRealtimeResource(structuredClone(resource), {
synthesizePlatformScopes: false,
}),
);
}
if (!changedIds.has(resource.id)) {
return existingResource;
}
const fastKeys = getFastResourceMergePatchKeys(changedKeys, resource.id, existingResource);
if (fastKeys) {
return applyFastResourceMergePatch(resource, existingResource, fastKeys);
}
return mergeCanonicalResource(
canonicalizeRealtimeResource(structuredClone(resource), {
synthesizePlatformScopes: false,
}),
existingResource,
);
});
};
const buildMemory = (
metric: Resource['memory'],
fallback?: Record<string, unknown>,
proxmoxMeta?: Record<string, unknown>,
): Memory => {
const total = metric?.total ?? asNumber(fallback?.total) ?? 0;
const observation = readMemoryObservation(
metric == null ? fallback?.observation : metric.observation,
);
const usageUnavailable = metric == null && asBoolean(fallback?.usageUnavailable) === true;
const used = metric?.used ?? asNumber(fallback?.used) ?? 0;
const cache = asNumber(proxmoxMeta?.memoryCache) ?? asNumber(fallback?.cache) ?? 0;
// The metric ships no free bytes for PVE payloads; total-used is the
// reclaimable-inclusive available, so carve the cache back out when known.
const free =
metric?.free ??
asNumber(fallback?.free) ??
(usageUnavailable ? 0 : Math.max(total - used - cache, 0));
const usage =
metric?.current ??
(usageUnavailable ? 0 : total > 0 ? (used / total) * 100 : (asNumber(fallback?.usage) ?? 0));
return {
total,
used,
free,
usage,
usageUnavailable,
...(observation ? { observation } : {}),
cache: cache > 0 ? cache : undefined,
swapUsed: asNumber(proxmoxMeta?.swapUsed) ?? asNumber(fallback?.swapUsed),
swapTotal: asNumber(proxmoxMeta?.swapTotal) ?? asNumber(fallback?.swapTotal),
balloon: asNumber(fallback?.balloon),
};
};
const buildDisk = (metric: Resource['disk'], fallback?: Record<string, unknown>): Disk => {
const total = metric?.total ?? asNumber(fallback?.total) ?? 0;
const used = metric?.used ?? asNumber(fallback?.used) ?? 0;
const free = metric?.free ?? asNumber(fallback?.free) ?? Math.max(total - used, 0);
const usage =
metric?.current ?? (total > 0 ? (used / total) * 100 : (asNumber(fallback?.usage) ?? 0));
return {
total,
used,
free,
usage,
mountpoint: asString(fallback?.mountpoint),
type: asString(fallback?.type),
device: asString(fallback?.device),
};
};
const temperatureFromRecord = (
raw: Record<string, unknown>,
resource: Resource,
): Temperature | undefined => {
const available = asBoolean(raw.available);
const cpuPackage = asNumber(raw.cpuPackage) ?? asNumber(raw.temperature) ?? asNumber(raw.cpu);
const lastUpdate = toISOTime(raw.lastUpdate, resource.lastSeen);
if (available || typeof cpuPackage === 'number') {
return {
cpuPackage,
cpuMax: asNumber(raw.cpuMax),
cpuMin: asNumber(raw.cpuMin),
cpuMaxRecord: asNumber(raw.cpuMaxRecord),
minRecorded: asString(raw.minRecorded),
maxRecorded: asString(raw.maxRecorded),
cores: asArray(raw.cores)
.map((entry) => {
const rec = asRecord(entry);
if (!rec) return null;
const core = asNumber(rec.core);
const temp = asNumber(rec.temp);
if (typeof core !== 'number' || typeof temp !== 'number') return null;
return { core, temp };
})
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry)),
gpu: asArray(raw.gpu)
.map((entry) => {
const rec = asRecord(entry);
if (!rec) return null;
const device = asString(rec.device);
if (!device) return null;
return {
device,
edge: asNumber(rec.edge),
junction: asNumber(rec.junction),
mem: asNumber(rec.mem),
};
})
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry)),
nvme: asArray(raw.nvme)
.map((entry) => {
const rec = asRecord(entry);
if (!rec) return null;
const device = asString(rec.device);
const temp = asNumber(rec.temp);
if (!device || typeof temp !== 'number') return null;
return { device, temp };
})
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry)),
available: available ?? true,
hasCPU: asBoolean(raw.hasCPU) ?? (typeof cpuPackage === 'number' ? true : undefined),
hasGPU: asBoolean(raw.hasGPU),
hasNVMe: asBoolean(raw.hasNVMe),
lastUpdate,
};
}
return undefined;
};
const hasCPUTemperatureReading = (temperature: Temperature): boolean =>
(temperature.cpuPackage ?? 0) > 0 ||
(temperature.cpuMax ?? 0) > 0 ||
(temperature.cores ?? []).some((core) => core.temp > 0);
const buildTemperature = (
resource: Resource,
nodeMeta?: Record<string, unknown>,
): Temperature | undefined => {
const platform = resourcePlatformData(resource);
const raw =
asRecord(platform?.temperature) ||
asRecord(nodeMeta?.temperature) ||
asRecord(platform?.agent) ||
undefined;
const fromRecord = raw ? temperatureFromRecord(raw, resource) : undefined;
if (fromRecord) return fromRecord;
// A Proxmox node without a host agent publishes its full reading, including
// the CPU low and record values the poller tracks, only as
// proxmox.temperatureDetails (proxmox.temperature is the scalar maximum). Use
// it ahead of the bare-reading fallback when it holds a current CPU reading.
const details = asRecord(nodeMeta?.temperatureDetails);
const fromDetails = details ? temperatureFromRecord(details, resource) : undefined;
if (fromDetails?.available && hasCPUTemperatureReading(fromDetails)) return fromDetails;
if (typeof resource.temperature === 'number' && Number.isFinite(resource.temperature)) {
const temp = resource.temperature;
// A bare reading carries no low or record history. Leave them unset so the
// drawer omits those rows instead of repeating the current value as a record.
return {
cpuPackage: temp,
cpuMax: temp,
available: true,
hasCPU: true,
lastUpdate: toISOTime(undefined, resource.lastSeen),
};
}
return undefined;
};
export const nodeFromResource = (resource: Resource): Node | null => {
if (resource.type !== 'agent') return null;
const platform = resourcePlatformData(resource);
const proxmox =
asRecord(platform?.proxmox) ||
(resource.proxmox as unknown as Record<string, unknown> | undefined);
const cpuInfo = asRecord(proxmox?.cpuInfo);
const preferredHostLabel =
getPreferredResourceHostname(resource) ||
getPreferredInfrastructureDisplayName(resource) ||
resource.id;
const instance =
asString(proxmox?.instance) ||
resource.platformId ||
getCanonicalPlatformId(resource) ||
preferredHostLabel;
const name = asString(proxmox?.nodeName) || asString(proxmox?.node) || preferredHostLabel;
const linkedAgentId =
asString(platform?.linkedAgentId) || getExplicitAgentIdFromResource(resource);
const agentFacet = resource.agent;
const agentNetworkInterfaces = agentFacet?.networkInterfaces;
const proxmoxNetworkInterfaces = Array.isArray(proxmox?.networkInterfaces)
? (proxmox.networkInterfaces as HostNetworkInterface[])
: [];
const pveVersion =
asString(proxmox?.pveVersion) ||
((agentFacet?.osName || '').toLowerCase().includes('proxmox')
? asString(agentFacet?.osVersion)
: '') ||
'Unknown';
return {
id: resource.id,
metricsTarget: resource.metricsTarget,
name,
displayName: getPreferredInfrastructureDisplayName(resource),
instance,
host: name || preferredHostLabel,
// proxmox.guestUrl is the operator-set link override and proxmox.host the
// PVE API connection URL; `host` above stays a hostname label for display.
guestURL: resolveGuestUrlWithIdentity(
asString(proxmox?.guestUrl) ||
asString((resource as unknown as Record<string, unknown>).customURL) ||
asString((resource as unknown as Record<string, unknown>).customUrl) ||
asString(proxmox?.host) ||
'',
resource,
),
status: resource.status || 'unknown',
type: resource.type,
cpu: resource.cpu?.current ?? 0,
memory: buildMemory(resource.memory, asRecord(proxmox?.memory), proxmox),
disk: buildDisk(resource.disk, asRecord(proxmox?.disk)),
networkIn: resource.network?.rxBytes,
networkOut: resource.network?.txBytes,
diskRead: resource.diskIO?.readRate,
diskWrite: resource.diskIO?.writeRate,
networkInterfaces:
agentNetworkInterfaces && agentNetworkInterfaces.length > 0
? agentNetworkInterfaces
: proxmoxNetworkInterfaces,
uptime: resource.uptime ?? asNumber(proxmox?.uptime) ?? 0,
loadAverage: asArray(proxmox?.loadAverage)
.map((value) => asNumber(value))
.filter((value): value is number => typeof value === 'number'),
kernelVersion: asString(proxmox?.kernelVersion) || 'Unknown',
pveVersion,
cpuInfo: {
model: asString(cpuInfo?.model) || 'Unknown',
cores: asNumber(cpuInfo?.cores) ?? 0,
sockets: asNumber(cpuInfo?.sockets) ?? 0,
mhz: asString(cpuInfo?.mhz) || '0',
},
sensors: agentFacet?.sensors,
sensorsStale: agentFacet?.stale === true,
temperature: buildTemperature(resource, proxmox),
// The backend's primary CPU reading: the agent's when one is linked, the
// same source the node's history line reads.
cpuTemperature:
typeof resource.temperature === 'number' &&
Number.isFinite(resource.temperature) &&
resource.temperature > 0
? resource.temperature
: undefined,
temperatureMonitoringEnabled:
asBoolean(platform?.temperatureMonitoringEnabled) ??
asBoolean(proxmox?.temperatureMonitoringEnabled) ??
null,
pendingUpdates: asNumber(proxmox?.pendingUpdates),
pendingUpdatesCheckedAt: asString(proxmox?.pendingUpdatesCheckedAt),
pendingUpdatesStatus: asString(proxmox?.pendingUpdatesStatus) as Node['pendingUpdatesStatus'],
pendingUpdatesReason: asString(proxmox?.pendingUpdatesReason) as Node['pendingUpdatesReason'],
lastSeen: toISOTime(undefined, resource.lastSeen),
connectionHealth: asString(proxmox?.connectionHealth) || resource.status || 'unknown',
isClusterMember: asBoolean(proxmox?.isClusterMember),
clusterName: getPreferredResourceClusterName(resource),
linkedAgentId,
};
};
const mapPBSNamespace = (value: unknown): PBSNamespace | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
path: asString(rec.path) || '',
parent: asString(rec.parent) || '',
depth: asNumber(rec.depth) ?? 0,
};
};
const mapPBSDatastore = (value: unknown): PBSDatastore | null => {
const rec = asRecord(value);
if (!rec) return null;
const total = asNumber(rec.total) ?? 0;
const used = asNumber(rec.used) ?? 0;
const free = asNumber(rec.free) ?? asNumber(rec.available) ?? Math.max(total - used, 0);
const usage =
asNumber(rec.usage) ?? asNumber(rec.usagePercent) ?? (total > 0 ? (used / total) * 100 : 0);
return {
name: asString(rec.name) || '',
total,
used,
free,
usage,
status: asString(rec.status) || '',
error: asString(rec.error) || '',
namespaces: asArray(rec.namespaces)
.map(mapPBSNamespace)
.filter((entry): entry is PBSNamespace => Boolean(entry)),
deduplicationFactor: asNumber(rec.deduplicationFactor),
};
};
const mapPBSBackupJob = (value: unknown): PBSBackupJob | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
id: asString(rec.id) || '',
store: asString(rec.store) || '',
type: asString(rec.type) || '',
vmid: asString(rec.vmid) || '',
lastBackup: asString(rec.lastBackup) || '',
nextRun: asString(rec.nextRun) || '',
status: asString(rec.status) || '',
error: asString(rec.error) || '',
};
};
const mapPBSSyncJob = (value: unknown): PBSSyncJob | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
id: asString(rec.id) || '',
store: asString(rec.store) || '',
remote: asString(rec.remote) || '',
status: asString(rec.status) || '',
lastSync: asString(rec.lastSync) || '',
nextRun: asString(rec.nextRun) || '',
error: asString(rec.error) || '',
};
};
const mapPBSVerifyJob = (value: unknown): PBSVerifyJob | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
id: asString(rec.id) || '',
store: asString(rec.store) || '',
status: asString(rec.status) || '',
lastVerify: asString(rec.lastVerify) || '',
nextRun: asString(rec.nextRun) || '',
error: asString(rec.error) || '',
};
};
const mapPBSPruneJob = (value: unknown): PBSPruneJob | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
id: asString(rec.id) || '',
store: asString(rec.store) || '',
status: asString(rec.status) || '',
lastPrune: asString(rec.lastPrune) || '',
nextRun: asString(rec.nextRun) || '',
error: asString(rec.error) || '',
};
};
const mapPBSGarbageJob = (value: unknown): PBSGarbageJob | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
id: asString(rec.id) || '',
store: asString(rec.store) || '',
status: asString(rec.status) || '',
lastGarbage: asString(rec.lastGarbage) || '',
nextRun: asString(rec.nextRun) || '',
removedBytes: asNumber(rec.removedBytes) ?? 0,
error: asString(rec.error) || '',
};
};
export const pbsInstanceFromResource = (resource: Resource): PBSInstance | null => {
if (resource.type !== 'pbs') return null;
const platform = resourcePlatformData(resource);
const pbs = asRecord(platform?.pbs);
const memoryTotal = resource.memory?.total ?? asNumber(pbs?.memoryTotal) ?? 0;
const memoryUsed = resource.memory?.used ?? asNumber(pbs?.memoryUsed) ?? 0;
const cpu = resource.cpu?.current ?? asNumber(pbs?.cpuPercent) ?? 0;
const memoryPercent =
resource.memory?.current ?? (memoryTotal > 0 ? (memoryUsed / memoryTotal) * 100 : 0);
const hostName = getPreferredResourceHostname(resource) || resource.id;
const host = resource.platformId || `https://${hostName}:8007`;
return {
id: asString(pbs?.instanceId) || resource.id,
name: getPreferredInfrastructureDisplayName(resource),
host,
guestURL:
asString((resource as unknown as Record<string, unknown>).customURL) ||
asString((resource as unknown as Record<string, unknown>).customUrl),
status: resource.status || 'unknown',
version: asString(pbs?.version) || '',
cpu,
memory: memoryPercent,
memoryUsed,
memoryTotal,
uptime: resource.uptime ?? asNumber(pbs?.uptimeSeconds) ?? 0,
datastores: asArray(pbs?.datastores)
.map(mapPBSDatastore)
.filter((entry): entry is PBSDatastore => Boolean(entry)),
backupJobs: asArray(pbs?.backupJobs)
.map(mapPBSBackupJob)
.filter((entry): entry is PBSBackupJob => Boolean(entry)),
syncJobs: asArray(pbs?.syncJobs)
.map(mapPBSSyncJob)
.filter((entry): entry is PBSSyncJob => Boolean(entry)),
verifyJobs: asArray(pbs?.verifyJobs)
.map(mapPBSVerifyJob)
.filter((entry): entry is PBSVerifyJob => Boolean(entry)),
pruneJobs: asArray(pbs?.pruneJobs)
.map(mapPBSPruneJob)
.filter((entry): entry is PBSPruneJob => Boolean(entry)),
garbageJobs: asArray(pbs?.garbageJobs)
.map(mapPBSGarbageJob)
.filter((entry): entry is PBSGarbageJob => Boolean(entry)),
connectionHealth: asString(pbs?.connectionHealth) || resource.status || 'unknown',
lastSeen: toISOTime(undefined, resource.lastSeen),
};
};
const mapPMGNodeStatus = (value: unknown): PMGNodeStatus | null => {
const rec = asRecord(value);
if (!rec) return null;
const queue = asRecord(rec.queueStatus);
return {
name: asString(rec.name) || '',
status: asString(rec.status) || '',
role: asString(rec.role),
uptime: asNumber(rec.uptime),
loadAvg: asString(rec.loadAvg),
queueStatus: queue
? {
active: asNumber(queue.active) ?? 0,
deferred: asNumber(queue.deferred) ?? 0,
hold: asNumber(queue.hold) ?? 0,
incoming: asNumber(queue.incoming) ?? 0,
total: asNumber(queue.total) ?? 0,
oldestAge: asNumber(queue.oldestAge) ?? 0,
updatedAt: asString(queue.updatedAt) || '',
}
: undefined,
};
};
const mapPMGMailStats = (value: unknown): PMGMailStats | undefined => {
const rec = asRecord(value);
if (!rec) return undefined;
return {
timeframe: asString(rec.timeframe) || '',
countTotal: asNumber(rec.countTotal) ?? 0,
countIn: asNumber(rec.countIn) ?? 0,
countOut: asNumber(rec.countOut) ?? 0,
spamIn: asNumber(rec.spamIn) ?? 0,
spamOut: asNumber(rec.spamOut) ?? 0,
virusIn: asNumber(rec.virusIn) ?? 0,
virusOut: asNumber(rec.virusOut) ?? 0,
bouncesIn: asNumber(rec.bouncesIn) ?? 0,
bouncesOut: asNumber(rec.bouncesOut) ?? 0,
bytesIn: asNumber(rec.bytesIn) ?? 0,
bytesOut: asNumber(rec.bytesOut) ?? 0,
greylistCount: asNumber(rec.greylistCount) ?? 0,
junkIn: asNumber(rec.junkIn) ?? 0,
averageProcessTimeMs: asNumber(rec.averageProcessTimeMs) ?? 0,
rblRejects: asNumber(rec.rblRejects) ?? 0,
pregreetRejects: asNumber(rec.pregreetRejects) ?? 0,
updatedAt: toISOTime(rec.updatedAt),
};
};
const mapPMGQuarantine = (value: unknown): PMGQuarantineTotals | undefined => {
const rec = asRecord(value);
if (!rec) return undefined;
return {
spam: asNumber(rec.spam) ?? 0,
virus: asNumber(rec.virus) ?? 0,
attachment: asNumber(rec.attachment) ?? 0,
blacklisted: asNumber(rec.blacklisted) ?? 0,
};
};
const mapPMGSpamBucket = (value: unknown): PMGSpamBucket | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
score: asString(rec.score) || asString(rec.bucket) || '',
count: asNumber(rec.count) ?? 0,
};
};
const mapPMGRelayDomain = (value: unknown): PMGRelayDomain | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
domain: asString(rec.domain) || '',
comment: asString(rec.comment),
};
};
const mapPMGDomainStat = (value: unknown): PMGDomainStat | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
domain: asString(rec.domain) || '',
mailCount: asNumber(rec.mailCount) ?? 0,
spamCount: asNumber(rec.spamCount) ?? 0,
virusCount: asNumber(rec.virusCount) ?? 0,
bytes: asNumber(rec.bytes),
};
};
const mapPMGMailCountPoint = (value: unknown): PMGMailCountPoint | null => {
const rec = asRecord(value);
if (!rec) return null;
return {
timestamp: toISOTime(rec.timestamp),
count: asNumber(rec.count) ?? 0,
countIn: asNumber(rec.countIn) ?? 0,
countOut: asNumber(rec.countOut) ?? 0,
spamIn: asNumber(rec.spamIn) ?? 0,
spamOut: asNumber(rec.spamOut) ?? 0,
virusIn: asNumber(rec.virusIn) ?? 0,
virusOut: asNumber(rec.virusOut) ?? 0,
rblRejects: asNumber(rec.rblRejects) ?? 0,
pregreet: asNumber(rec.pregreet) ?? 0,
bouncesIn: asNumber(rec.bouncesIn) ?? 0,
bouncesOut: asNumber(rec.bouncesOut) ?? 0,
greylist: asNumber(rec.greylist) ?? 0,
index: asNumber(rec.index) ?? 0,
timeframe: asString(rec.timeframe) || '',
windowStart: asString(rec.windowStart),
windowEnd: asString(rec.windowEnd),
};
};
export const pmgInstanceFromResource = (resource: Resource): PMGInstance | null => {
if (resource.type !== 'pmg') return null;
const platform = resourcePlatformData(resource);
const pmg = asRecord(platform?.pmg);
const hostName = getPreferredResourceHostname(resource) || resource.id;
const host = asString(pmg?.hostUrl) || resource.platformId || `https://${hostName}:8006`;
const lastSeen = toISOTime(undefined, resource.lastSeen);
const mailStats =
mapPMGMailStats(pmg?.mailStats) ||
mapPMGMailStats({
countTotal: asNumber(pmg?.mailCountTotal),
spamIn: asNumber(pmg?.spamIn),
virusIn: asNumber(pmg?.virusIn),
updatedAt: pmg?.lastUpdated,
});
return {
id: asString(pmg?.instanceId) || resource.id,
name: getPreferredInfrastructureDisplayName(resource),
host,
guestURL:
asString(pmg?.guestUrl) ||
asString((resource as unknown as Record<string, unknown>).customURL) ||
asString((resource as unknown as Record<string, unknown>).customUrl),
status: resource.status || 'unknown',
version: asString(pmg?.version) || '',
nodes: asArray(pmg?.nodes)
.map(mapPMGNodeStatus)
.filter((entry): entry is PMGNodeStatus => Boolean(entry)),
mailStats,
mailCount: asArray(pmg?.mailCount)
.map(mapPMGMailCountPoint)
.filter((entry): entry is PMGMailCountPoint => Boolean(entry)),
spamDistribution: asArray(pmg?.spamDistribution)
.map(mapPMGSpamBucket)
.filter((entry): entry is PMGSpamBucket => Boolean(entry)),
quarantine: mapPMGQuarantine(pmg?.quarantine),
relayDomains: asArray(pmg?.relayDomains)
.map(mapPMGRelayDomain)
.filter((entry): entry is PMGRelayDomain => Boolean(entry)),
domainStats: asArray(pmg?.domainStats)
.map(mapPMGDomainStat)
.filter((entry): entry is PMGDomainStat => Boolean(entry)),
domainStatsAsOf: toISOTime(pmg?.domainStatsAsOf),
connectionHealth: asString(pmg?.connectionHealth) || resource.status || 'unknown',
lastSeen,
lastUpdated: toISOTime(pmg?.lastUpdated, resource.lastSeen),
};
};