Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

12.3 KB
Raw Blame
import { createEffect, createMemo, createSignal, onCleanup, type Accessor } from 'solid-js';

import {
  ChartsAPI,
  type AllMetricsHistoryResponse,
  type ChartData,
  type ResourceType,
  type SingleMetricHistoryResponse,
  type WorkloadChartsResponse,
} from '@/api/charts';
import type { Node } from '@/types/api';
import type { WorkloadGuest } from '@/types/workloads';
import { createNonSuspendingQuery } from '@/hooks/createNonSuspendingQuery';
import { getCanonicalWorkloadId } from '@/utils/workloads';
import { getWorkloadMetricHistoryTarget } from '@/utils/workloadMetricHistoryTarget';
import { getWorkloadHistoryNodeId } from './workloadTopology';

import {
  buildInfrastructureHistoryChartMap,
  findChartDataForCandidates,
  getMetricSparklineSeriesFromChartData,
  getNodeChartKeyCandidates,
  getWorkloadChartKeyCandidates,
  WORKLOAD_TABLE_HISTORY_INFRA_METRICS,
  WORKLOAD_TABLE_HISTORY_MAX_POINTS,
  WORKLOAD_TABLE_HISTORY_POLL_MS,
  type WorkloadMetricHistoryReader,
  type WorkloadTableMetricHistoryRange,
  type WorkloadTableMetric,
} from './workloadMetricHistoryModel';

interface WorkloadTableMetricHistoryOptions {
  activeGuest?: Accessor<WorkloadGuest | null>;
  enabled: Accessor<boolean>;
  onDemand?: Accessor<boolean>;
  prefetchGuests?: Accessor<readonly WorkloadGuest[]>;
  range: Accessor<WorkloadTableMetricHistoryRange>;
  selectedNode?: Accessor<string | null | undefined>;
  nodes?: Accessor<Node[]>;
  // The rows this caller draws while `enabled`. Each estate-wide history is a
  // polled summary read, so a reader polls only the one its rows consume:
  // guest rows read the workloads summary, node rows the infrastructure
  // summary. Readers do not share in-flight requests, so a second reader on
  // the same page pays for its own reads.
  series: 'guests' | 'nodes';
}

interface ActiveGuestHistoryQueryKey {
  guestId: string;
  range: WorkloadTableMetricHistoryRange;
  resourceId: string;
  resourceType: ResourceType;
}

const EMPTY_WORKLOAD_CHARTS_RESPONSE: WorkloadChartsResponse = {
  data: {},
  dockerData: {},
  guestTypes: {},
  timestamp: 0,
  stats: { oldestDataTimestamp: 0 },
};

const EMPTY_INFRASTRUCTURE_CHART_MAP = new Map<string, ChartData>();

const ROW_HISTORY_PREFETCH_CONCURRENCY = 4;
const ROW_HISTORY_INITIAL_PREFETCH_COUNT = 6;
const ROW_HISTORY_ADJACENT_PREFETCH_COUNT = 4;
const ROW_HISTORY_CACHE_MAX_ENTRIES = 18;

const normalizeActiveGuestHistoryResponse = (
  response: AllMetricsHistoryResponse | SingleMetricHistoryResponse,
): AllMetricsHistoryResponse =>
  'metrics' in response
    ? response
    : {
        resourceType: response.resourceType,
        resourceId: response.resourceId,
        range: response.range,
        start: response.start,
        end: response.end,
        metrics: { [response.metric]: response.points },
        source: response.source,
      };

const normalizeNodeScope = (value: string | null | undefined): string => value?.trim() ?? '';
const buildHistoryScope = (range: WorkloadTableMetricHistoryRange, nodeScope = ''): string =>
  `${range}::${nodeScope || '__all__'}`;
const parseHistoryScope = (scope: string) => {
  const [range, nodeScope = '__all__'] = scope.split('::');
  return {
    range: range as WorkloadTableMetricHistoryRange,
    nodeScope,
  };
};

const buildActiveGuestHistoryQueryKey = (
  guest: WorkloadGuest,
  range: WorkloadTableMetricHistoryRange,
): ActiveGuestHistoryQueryKey | null => {
  const target = getWorkloadMetricHistoryTarget(guest);
  if (!target) return null;
  return {
    guestId: getCanonicalWorkloadId(guest),
    range,
    resourceId: target.resourceId,
    resourceType: target.resourceType,
  };
};

const buildActiveGuestHistoryCacheKey = (key: ActiveGuestHistoryQueryKey): string =>
  `${key.resourceType}:${key.resourceId}:${key.range}:${WORKLOAD_TABLE_HISTORY_MAX_POINTS}`;

export function useWorkloadTableMetricHistory(
  options: WorkloadTableMetricHistoryOptions,
): WorkloadMetricHistoryReader {
  const selectedNodeScope = createMemo(() => normalizeNodeScope(options.selectedNode?.()));
  const workloadHistoryScope = createMemo(() => {
    if (options.series !== 'guests' || !options.enabled()) return null;
    return buildHistoryScope(
      options.range(),
      getWorkloadHistoryNodeId(selectedNodeScope(), options.nodes?.()),
    );
  });
  const infrastructureHistoryScope = createMemo(() =>
    options.series === 'nodes' && options.enabled() ? buildHistoryScope(options.range()) : null,
  );

  const workloadHistory = createNonSuspendingQuery<WorkloadChartsResponse, string>({
    source: workloadHistoryScope,
    fetcher: (scope, signal) => {
      const parsed = parseHistoryScope(scope);
      return ChartsAPI.getWorkloadCharts(parsed.range, signal, {
        maxPoints: WORKLOAD_TABLE_HISTORY_MAX_POINTS,
        nodeId: parsed.nodeScope === '__all__' ? undefined : parsed.nodeScope,
      });
    },
    initialValue: EMPTY_WORKLOAD_CHARTS_RESPONSE,
    cacheKey: (scope) => `workload-table-history:${scope}`,
    pollMs: WORKLOAD_TABLE_HISTORY_POLL_MS,
    retainPreviousValueOnSourceChange: false,
  });

  const infrastructureHistory = createNonSuspendingQuery<Map<string, ChartData>, string>({
    source: infrastructureHistoryScope,
    fetcher: (scope, signal) => {
      const parsed = parseHistoryScope(scope);
      return ChartsAPI.getInfrastructureSummaryCharts(parsed.range, signal, {
        metrics: WORKLOAD_TABLE_HISTORY_INFRA_METRICS,
      }).then(buildInfrastructureHistoryChartMap);
    },
    initialValue: EMPTY_INFRASTRUCTURE_CHART_MAP,
    cacheKey: (scope) => `workload-table-infra-history:${scope}`,
    pollMs: WORKLOAD_TABLE_HISTORY_POLL_MS,
    retainPreviousValueOnSourceChange: false,
  });

  const [rowHistoryRevision, setRowHistoryRevision] = createSignal(0);
  const rowHistoryCache = new Map<string, AllMetricsHistoryResponse>();
  const pendingRowHistory: ActiveGuestHistoryQueryKey[] = [];
  const inFlightRowHistory = new Map<string, { controller: AbortController; generation: number }>();
  let rowHistoryGeneration = 0;
  let scheduledRange: WorkloadTableMetricHistoryRange | null = null;
  let rowHistoryWarmingActive = false;
  let disposed = false;

  const cancelRowHistoryQueue = () => {
    rowHistoryGeneration += 1;
    pendingRowHistory.splice(0, pendingRowHistory.length);
    inFlightRowHistory.forEach(({ controller }) => controller.abort());
    inFlightRowHistory.clear();
  };

  const trimRowHistoryCache = () => {
    while (rowHistoryCache.size > ROW_HISTORY_CACHE_MAX_ENTRIES) {
      const oldestKey = rowHistoryCache.keys().next().value as string | undefined;
      if (!oldestKey) return;
      rowHistoryCache.delete(oldestKey);
    }
  };

  const pumpRowHistoryQueue = () => {
    while (
      !disposed &&
      inFlightRowHistory.size < ROW_HISTORY_PREFETCH_CONCURRENCY &&
      pendingRowHistory.length > 0
    ) {
      const key = pendingRowHistory.shift();
      if (!key) break;
      const cacheKey = buildActiveGuestHistoryCacheKey(key);
      if (rowHistoryCache.has(cacheKey) || inFlightRowHistory.has(cacheKey)) continue;

      const controller = new AbortController();
      const generation = rowHistoryGeneration;
      inFlightRowHistory.set(cacheKey, { controller, generation });
      void ChartsAPI.getMetricsHistory({
        resourceType: key.resourceType,
        resourceId: key.resourceId,
        range: key.range,
        maxPoints: WORKLOAD_TABLE_HISTORY_MAX_POINTS,
        signal: controller.signal,
      })
        .then((response) => {
          if (disposed || generation !== rowHistoryGeneration || controller.signal.aborted) return;
          rowHistoryCache.set(cacheKey, normalizeActiveGuestHistoryResponse(response));
          trimRowHistoryCache();
          setRowHistoryRevision((revision) => revision + 1);
        })
        .catch(() => {
          // Hover warming is opportunistic. The live bars remain available if
          // history is missing or the request is cancelled by a range change.
        })
        .finally(() => {
          const current = inFlightRowHistory.get(cacheKey);
          if (current?.generation === generation) {
            inFlightRowHistory.delete(cacheKey);
          }
          pumpRowHistoryQueue();
        });
    }
  };

  const enqueueRowHistory = (keys: ActiveGuestHistoryQueryKey[], prioritize: boolean) => {
    const uniqueKeys = keys.filter((key, index) => {
      const cacheKey = buildActiveGuestHistoryCacheKey(key);
      return (
        keys.findIndex((candidate) => buildActiveGuestHistoryCacheKey(candidate) === cacheKey) ===
          index &&
        !rowHistoryCache.has(cacheKey) &&
        !inFlightRowHistory.has(cacheKey)
      );
    });
    if (uniqueKeys.length === 0) return;

    const uniqueCacheKeys = new Set(uniqueKeys.map(buildActiveGuestHistoryCacheKey));
    for (let index = pendingRowHistory.length - 1; index >= 0; index -= 1) {
      if (uniqueCacheKeys.has(buildActiveGuestHistoryCacheKey(pendingRowHistory[index]))) {
        pendingRowHistory.splice(index, 1);
      }
    }
    if (prioritize) {
      pendingRowHistory.unshift(...uniqueKeys);
    } else {
      pendingRowHistory.push(...uniqueKeys);
    }
    pumpRowHistoryQueue();
  };

  createEffect(() => {
    const range = options.range();
    const onDemand = options.onDemand?.() ?? false;
    const candidates = options.prefetchGuests?.() ?? [];
    const activeGuest = options.activeGuest?.() ?? null;

    if (scheduledRange !== range) {
      cancelRowHistoryQueue();
      rowHistoryCache.clear();
      setRowHistoryRevision((revision) => revision + 1);
      scheduledRange = range;
    }

    if (!onDemand) {
      if (rowHistoryWarmingActive) cancelRowHistoryQueue();
      rowHistoryWarmingActive = false;
      return;
    }
    rowHistoryWarmingActive = true;

    if (!activeGuest) {
      enqueueRowHistory(
        candidates
          .slice(0, ROW_HISTORY_INITIAL_PREFETCH_COUNT)
          .map((guest) => buildActiveGuestHistoryQueryKey(guest, range))
          .filter((key): key is ActiveGuestHistoryQueryKey => key !== null),
        false,
      );
      return;
    }

    const activeId = getCanonicalWorkloadId(activeGuest);
    const activeIndex = candidates.findIndex((guest) => getCanonicalWorkloadId(guest) === activeId);
    const adjacentGuests =
      activeIndex >= 0
        ? [
            activeGuest,
            ...candidates.slice(
              activeIndex + 1,
              activeIndex + 1 + ROW_HISTORY_ADJACENT_PREFETCH_COUNT,
            ),
            ...(activeIndex > 0 ? [candidates[activeIndex - 1]] : []),
          ]
        : [activeGuest];
    enqueueRowHistory(
      adjacentGuests
        .map((guest) => buildActiveGuestHistoryQueryKey(guest, range))
        .filter((key): key is ActiveGuestHistoryQueryKey => key !== null),
      true,
    );
  });

  onCleanup(() => {
    disposed = true;
    cancelRowHistoryQueue();
  });

  const getCachedGuestHistory = (guest: WorkloadGuest): AllMetricsHistoryResponse | null => {
    rowHistoryRevision();
    const key = buildActiveGuestHistoryQueryKey(guest, options.range());
    return key ? (rowHistoryCache.get(buildActiveGuestHistoryCacheKey(key)) ?? null) : null;
  };

  const getGuestMetricSeries: WorkloadMetricHistoryReader['getGuestMetricSeries'] = (
    guest,
    metric,
    seriesOptions,
  ) => {
    const activeGuest = options.onDemand?.() ? options.activeGuest?.() : null;
    if (activeGuest) {
      if (getCanonicalWorkloadId(activeGuest) !== getCanonicalWorkloadId(guest)) {
        return [];
      }
      return getMetricSparklineSeriesFromChartData(
        getCachedGuestHistory(guest)?.metrics,
        metric,
        seriesOptions,
      );
    }

    const response = workloadHistory.value();
    const chartData = findChartDataForCandidates(getWorkloadChartKeyCandidates(guest), [
      response.data,
      response.dockerData,
    ]);
    return getMetricSparklineSeriesFromChartData(chartData, metric, seriesOptions);
  };

  const getNodeMetricSeries = (node: Node, metric: WorkloadTableMetric) => {
    const chartData = findChartDataForCandidates(getNodeChartKeyCandidates(node), [
      infrastructureHistory.value(),
    ]);
    return getMetricSparklineSeriesFromChartData(chartData, metric);
  };

  return {
    hasGuestHistory: (guest) => {
      const response = getCachedGuestHistory(guest);
      return (
        response !== null && Object.values(response.metrics).some((points) => points.length > 0)
      );
    },
    getGuestMetricSeries,
    getNodeMetricSeries,
  };
}