Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.0 KB
Raw Blame
import { cleanup, fireEvent, render, screen, within } from '@solidjs/testing-library';
import { createSignal } from 'solid-js';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { MaintenanceVerificationReport } from '@/api/maintenanceVerification';
import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery';
import { RELATIVE_TIME_TICK_MS } from '@/utils/relativeTimeClock';

const api = vi.hoisted(() => ({ apiFetchJSON: vi.fn() }));
vi.mock('@/utils/apiClient', async (importOriginal) => ({
  ...(await importOriginal<typeof import('@/utils/apiClient')>()),
  apiFetchJSON: api.apiFetchJSON,
}));
vi.mock('@/stores/notifications', () => ({
  notificationStore: { error: vi.fn(), success: vi.fn() },
}));

import { MaintenanceVerificationSection } from '../MaintenanceVerificationSection';

const start = Date.parse('2026-10-11T04:00:00Z');
const iso = (offset: number) => new Date(start + offset).toISOString();
const resourceId = 'cluster:pve-a:101';
const report = (
  overrides: Partial<MaintenanceVerificationReport> = {},
): MaintenanceVerificationReport => ({
  id: 'report:window',
  resourceId,
  trigger: 'maintenance_window_ended',
  status: 'healthy',
  startedAt: iso(-30_000),
  completedAt: iso(-30_000),
  windowStartedAt: iso(-3_600_000),
  windowEndedAt: iso(-30_000),
  evidence: {
    activeCriticalAlerts: 0,
    activeWarningAlerts: 2,
    activeCriticalFindings: 0,
    activeWarningFindings: 1,
    failedActionsSinceWindowStart: 0,
    metricRecovery: { samplesAfterEnd: 3, trend: 'improving' },
  },
  linkedFindingIds: [],
  linkedAlertIds: [],
  linkedActionIds: [],
  recommendation: 'Recorded recovery evidence',
  ...overrides,
});
const response = (reports: MaintenanceVerificationReport[]) => ({ data: reports });
const mount = async (reports: MaintenanceVerificationReport[]) => {
  api.apiFetchJSON.mockResolvedValue(response(reports));
  const rendered = render(() => <MaintenanceVerificationSection resourceId={resourceId} />);
  await vi.advanceTimersByTimeAsync(0);
  return rendered;
};
const mutations = () =>
  api.apiFetchJSON.mock.calls.filter(([, options]) => options?.method === 'POST');

describe('durable maintenance verification evidence', () => {
  beforeEach(() => {
    vi.useFakeTimers();
    resetCreateNonSuspendingQueryCacheForTest();
    vi.setSystemTime(start);
    api.apiFetchJSON.mockReset();
  });
  afterEach(() => {
    cleanup();
    resetCreateNonSuspendingQueryCacheForTest();
    vi.useRealTimers();
  });

  it('ages held window and review events without fetching, rewriting or rerunning the report', async () => {
    const saved = report({
      userOutcome: 'reviewed',
      reviewedAt: iso(-30_000),
      reviewedBy: 'fixture-admin',
    });
    const original = JSON.stringify(saved);
    await mount([saved]);
    expect(screen.getByText('Window ended 30s ago')).toBeInTheDocument();
    expect(screen.getByText('Reviewed 30s ago by fixture-admin')).toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(RELATIVE_TIME_TICK_MS);
    expect(screen.getByText('Window ended 1 min ago')).toBeInTheDocument();
    expect(screen.getByText('Reviewed 1 min ago by fixture-admin')).toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(3_600_000);
    expect(screen.getByText('Window ended 1 hour ago')).toBeInTheDocument();
    expect(screen.getByText('Reviewed 1 hour ago by fixture-admin')).toBeInTheDocument();
    expect(screen.getByTestId('maintenance-verification-status-healthy')).toHaveTextContent(
      'Healthy',
    );
    expect(screen.queryByTestId('maintenance-verification-review')).not.toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(1);
    expect(mutations()).toEqual([]);
    expect(JSON.stringify(saved)).toBe(original);
  });

  it('shows when a check completed, separately from the earlier maintenance window and review', async () => {
    await mount([
      report({
        windowEndedAt: iso(-86_400_000),
        userOutcome: 'reviewed',
        reviewedAt: iso(-10_000),
      }),
    ]);
    expect(screen.getByText('Checked 30s ago')).toBeInTheDocument();
    expect(screen.getByText('Window ended 1 day ago')).toBeInTheDocument();
    expect(screen.getByText('Reviewed 10s ago')).toBeInTheDocument();
    expect(
      screen.getByText(/Reports record past checks, not current health\./),
    ).toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.getByText('Checked 1 min ago')).toBeInTheDocument();
    expect(screen.getByText('Reviewed 1 min ago')).toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(1);
  });

  it('keeps every recorded verdict and its counts unchanged as several reports age', async () => {
    const states = ['pending', 'healthy', 'needs_review', 'failed_verification'] as const;
    const records = states.map((status) => report({ id: `report:${status}`, status }));
    const original = JSON.stringify(records);
    await mount(records);
    await vi.advanceTimersByTimeAsync(120_000);
    for (const status of states) {
      const row = screen.getByTestId(`maintenance-verification-status-${status}`).closest('li')!;
      expect(within(row).getByText('Window ended 2 mins ago')).toBeInTheDocument();
      expect(within(row).getByText('Checked 2 mins ago')).toBeInTheDocument();
      expect(within(row).getByText('Warning alerts').nextElementSibling).toHaveTextContent('2');
      expect(within(row).getByText('Metric samples').nextElementSibling).toHaveTextContent('3');
      expect(within(row).getByTestId('maintenance-verification-review')).toBeEnabled();
    }
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(1);
    expect(JSON.stringify(records)).toBe(original);
  });

  it.each(['', 'invalid', '0001-01-01T00:00:00Z'])(
    'keeps absent or malformed event times unknown (%j)',
    async (time) => {
      await mount([
        report({
          windowEndedAt: time,
          completedAt: time,
          userOutcome: 'reviewed',
          reviewedAt: time,
        }),
      ]);
      await vi.advanceTimersByTimeAsync(60_000);
      expect(screen.getByText('Window ended unknown')).toBeInTheDocument();
      expect(screen.getByText('Checked unknown')).toBeInTheDocument();
      expect(screen.getByText('Reviewed unknown')).toBeInTheDocument();
      expect(api.apiFetchJSON).toHaveBeenCalledTimes(1);
    },
  );

  it('keeps clock-skewed past events as ages rather than inventing a scheduled operation', async () => {
    await mount([report({ completedAt: iso(30_000), windowEndedAt: iso(30_000) })]);
    expect(screen.getByText('Checked 0s ago')).toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(90_000);
    expect(screen.getByText('Checked 1 min ago')).toBeInTheDocument();
    expect(screen.getByText('Window ended 1 min ago')).toBeInTheDocument();
    expect(mutations()).toEqual([]);
  });

  it('withdraws the old resource reports and ignores a late old-resource response across ticks', async () => {
    let finishOld!: (value: ReturnType<typeof response>) => void;
    api.apiFetchJSON.mockImplementation((path: string) =>
      path.includes('old-resource')
        ? new Promise((resolve) => {
            finishOld = resolve;
          })
        : Promise.resolve(response([])),
    );
    const [id, setId] = createSignal('old-resource');
    render(() => <MaintenanceVerificationSection resourceId={id()} />);
    await vi.advanceTimersByTimeAsync(0);
    setId('empty-resource');
    await vi.advanceTimersByTimeAsync(0);
    finishOld(response([report()]));
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.queryByTestId('maintenance-verification-report')).not.toBeInTheDocument();
    expect(screen.getByText(/No verification reports yet/)).toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(2);
  });

  it('does not revive withdrawn reports after an access-denied resource read', async () => {
    api.apiFetchJSON
      .mockResolvedValueOnce(response([report()]))
      .mockRejectedValueOnce(new Error('Access denied'));
    const [id, setId] = createSignal(resourceId);
    render(() => <MaintenanceVerificationSection resourceId={id()} />);
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.getByTestId('maintenance-verification-report')).toBeInTheDocument();
    setId('denied-resource');
    await vi.advanceTimersByTimeAsync(0);
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.queryByTestId('maintenance-verification-report')).not.toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(2);
  });

  it('requires an explicit review, keeps pending admission disabled and uses the returned review fields', async () => {
    const saved = report();
    await mount([saved]);
    let finishReview!: (value: MaintenanceVerificationReport) => void;
    api.apiFetchJSON.mockImplementationOnce(
      () =>
        new Promise((resolve) => {
          finishReview = resolve;
        }),
    );
    const button = screen.getByTestId('maintenance-verification-review');
    fireEvent.click(button);
    await vi.advanceTimersByTimeAsync(60_000);
    expect(button).toBeDisabled();
    expect(button).toHaveTextContent('Marking…');
    fireEvent.click(button);
    expect(mutations()).toHaveLength(1);
    expect(mutations()[0][0]).toBe('/api/maintenance-verifications/report%3Awindow/review');
    expect(screen.getByTestId('maintenance-verification-status-healthy')).toHaveTextContent(
      'Healthy',
    );
    expect(screen.queryByText(/Reviewed/)).not.toBeInTheDocument();
    const reviewed = {
      ...saved,
      userOutcome: 'reviewed' as const,
      reviewedAt: iso(60_000),
      reviewedBy: 'fixture-admin',
    };
    api.apiFetchJSON.mockResolvedValue(response([reviewed]));
    finishReview(reviewed);
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.queryByTestId('maintenance-verification-review')).not.toBeInTheDocument();
    expect(screen.getByText('Reviewed 0s ago by fixture-admin')).toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.getByText('Reviewed 1 min ago by fixture-admin')).toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(3);
  });

  it('reruns only after a click, keeps the pending action disabled and retains the old report beside the new check', async () => {
    const saved = report();
    await mount([saved]);
    let finishRerun!: (value: MaintenanceVerificationReport) => void;
    api.apiFetchJSON.mockImplementationOnce(
      () =>
        new Promise((resolve) => {
          finishRerun = resolve;
        }),
    );
    const button = screen.getByTestId('maintenance-verification-rerun');
    fireEvent.click(button);
    await vi.advanceTimersByTimeAsync(60_000);
    expect(button).toBeDisabled();
    fireEvent.click(button);
    expect(mutations()).toHaveLength(1);
    expect(mutations()[0][0]).toBe(
      '/api/resources/cluster%3Apve-a%3A101/maintenance-verifications/rerun',
    );
    const rerun = report({
      id: 'report:window-rerun-1',
      completedAt: iso(60_000),
      status: 'needs_review',
    });
    api.apiFetchJSON.mockResolvedValue(response([rerun, saved]));
    finishRerun(rerun);
    await vi.advanceTimersByTimeAsync(0);
    expect(button).toBeEnabled();
    expect(screen.getAllByTestId('maintenance-verification-report')).toHaveLength(2);
    expect(screen.getByText('Checked 0s ago')).toBeInTheDocument();
    expect(screen.getByTestId('maintenance-verification-status-healthy')).toHaveTextContent(
      'Healthy',
    );
    expect(screen.getByTestId('maintenance-verification-status-needs_review')).toHaveTextContent(
      'Needs review',
    );
    await vi.advanceTimersByTimeAsync(60_000);
    expect(mutations()).toHaveLength(1);
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(3);
  });

  it.each(['review', 'rerun'])(
    'does not retry a refused %s on a clock tick or alter the report',
    async (action) => {
      await mount([report()]);
      api.apiFetchJSON.mockRejectedValueOnce(new Error('Synthetic write refusal'));
      fireEvent.click(screen.getByTestId(`maintenance-verification-${action}`));
      await vi.advanceTimersByTimeAsync(60_000);
      expect(screen.getByTestId(`maintenance-verification-${action}`)).toBeEnabled();
      expect(screen.getByTestId('maintenance-verification-status-healthy')).toHaveTextContent(
        'Healthy',
      );
      expect(screen.getByText('Window ended 1 min ago')).toBeInTheDocument();
      expect(mutations()).toHaveLength(1);
      expect(api.apiFetchJSON).toHaveBeenCalledTimes(2);
    },
  );

  it('does not describe a pending first read as an empty report history', async () => {
    let finishRead!: (value: ReturnType<typeof response>) => void;
    api.apiFetchJSON.mockImplementationOnce(
      () =>
        new Promise((resolve) => {
          finishRead = resolve;
        }),
    );
    render(() => <MaintenanceVerificationSection resourceId={resourceId} />);
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.getByText('Loading verification reports…')).toBeInTheDocument();
    expect(screen.queryByText(/No verification reports yet/)).not.toBeInTheDocument();
    finishRead(response([]));
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.getByText(/No verification reports yet/)).toBeInTheDocument();
  });

  it('withdraws loaded reports immediately while another resource read is pending', async () => {
    api.apiFetchJSON
      .mockResolvedValueOnce(response([report()]))
      .mockImplementationOnce(() => new Promise(() => {}));
    const [id, setId] = createSignal(resourceId);
    render(() => <MaintenanceVerificationSection resourceId={id()} />);
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.getByTestId('maintenance-verification-report')).toBeInTheDocument();
    setId('new-resource-pending');
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.queryByTestId('maintenance-verification-report')).not.toBeInTheDocument();
    expect(screen.getByText('Loading verification reports…')).toBeInTheDocument();
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(2);
  });

  it('distinguishes a failed first read from empty history and retries only after an explicit read request', async () => {
    api.apiFetchJSON.mockRejectedValueOnce(
      Object.assign(new Error('Synthetic server internals must not render'), { status: 503 }),
    );
    render(() => <MaintenanceVerificationSection resourceId={resourceId} />);
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.getByTestId('maintenance-verification-load-error')).toHaveTextContent(
      'Verification reports unavailable',
    );
    expect(screen.queryByText(/No verification reports yet/)).not.toBeInTheDocument();
    expect(screen.queryByText(/Synthetic server internals/)).not.toBeInTheDocument();
    await vi.advanceTimersByTimeAsync(60_000);
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(1);
    let finishRetry!: (value: ReturnType<typeof response>) => void;
    api.apiFetchJSON.mockImplementationOnce(
      () =>
        new Promise((resolve) => {
          finishRetry = resolve;
        }),
    );
    fireEvent.click(screen.getByRole('button', { name: 'Retry reports' }));
    expect(screen.getByRole('button', { name: 'Retry reports' })).toBeDisabled();
    await vi.advanceTimersByTimeAsync(60_000);
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(2);
    finishRetry(response([report()]));
    await vi.advanceTimersByTimeAsync(0);
    expect(screen.queryByTestId('maintenance-verification-load-error')).not.toBeInTheDocument();
    expect(screen.getByTestId('maintenance-verification-report')).toBeInTheDocument();
    expect(mutations()).toEqual([]);
  });

  it('retains qualified historical reports after a same-resource refresh fails, without inventing a successful review', async () => {
    const saved = report();
    await mount([saved]);
    api.apiFetchJSON
      .mockResolvedValueOnce(saved)
      .mockRejectedValueOnce(Object.assign(new Error('Synthetic read failure'), { status: 503 }));
    fireEvent.click(screen.getByTestId('maintenance-verification-review'));
    await vi.advanceTimersByTimeAsync(60_000);
    expect(screen.getByTestId('maintenance-verification-load-error')).toHaveTextContent(
      'Last loaded reports remain below.',
    );
    expect(screen.getByTestId('maintenance-verification-status-healthy')).toHaveTextContent(
      'Healthy',
    );
    expect(screen.getByText('Window ended 1 min ago')).toBeInTheDocument();
    expect(screen.queryByText(/Reviewed/)).not.toBeInTheDocument();
    expect(mutations()).toHaveLength(1);
    expect(api.apiFetchJSON).toHaveBeenCalledTimes(3);
  });

  it.each([401, 403])(
    'withdraws reports on a denied same-resource refresh (%s), rather than offering them as authorised history',
    async (status) => {
      const saved = report();
      await mount([saved]);
      api.apiFetchJSON
        .mockResolvedValueOnce(saved)
        .mockRejectedValueOnce(
          Object.assign(new Error('Synthetic access internals must not render'), { status }),
        );
      fireEvent.click(screen.getByTestId('maintenance-verification-rerun'));
      await vi.advanceTimersByTimeAsync(60_000);
      expect(screen.queryByTestId('maintenance-verification-report')).not.toBeInTheDocument();
      expect(screen.queryByText(/No verification reports yet/)).not.toBeInTheDocument();
      expect(screen.getByTestId('maintenance-verification-load-error')).toHaveTextContent(
        status === 401
          ? 'Sign in again or check your API token.'
          : 'Access denied. Check your permissions and license plan.',
      );
      expect(screen.queryByText(/Synthetic access internals/)).not.toBeInTheDocument();
      expect(api.apiFetchJSON).toHaveBeenCalledTimes(3);
      const reviewed = report({ userOutcome: 'reviewed', reviewedAt: iso(60_000) });
      api.apiFetchJSON.mockResolvedValueOnce(response([reviewed]));
      fireEvent.click(screen.getByRole('button', { name: 'Retry reports' }));
      await vi.advanceTimersByTimeAsync(0);
      expect(screen.getByText('Reviewed 0s ago')).toBeInTheDocument();
      expect(screen.queryByTestId('maintenance-verification-load-error')).not.toBeInTheDocument();
      expect(api.apiFetchJSON).toHaveBeenCalledTimes(4);
      expect(mutations()).toHaveLength(1);
    },
  );

  it('requests the resource reports by encoded id and lists every evidence count with the Patrol follow-up', async () => {
    await mount([
      report({
        evidence: {
          ...report().evidence,
          activeCriticalAlerts: 4,
          activeCriticalFindings: 5,
          activeWarningFindings: 6,
          failedActionsSinceWindowStart: 7,
          patrolRunTodo: 'Run a scoped Patrol check for this resource',
        },
      }),
    ]);
    expect(api.apiFetchJSON.mock.calls[0][0]).toBe(
      '/api/resources/cluster%3Apve-a%3A101/maintenance-verifications?limit=25',
    );
    for (const [label, count] of [
      ['Critical alerts', '4'],
      ['Warning alerts', '2'],
      ['Critical findings', '5'],
      ['Warning findings', '6'],
      ['Failed actions', '7'],
    ]) {
      expect(screen.getByText(label).nextElementSibling).toHaveTextContent(count);
    }
    expect(screen.getByText('Run a scoped Patrol check for this resource')).toBeInTheDocument();
  });
});