Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

13.3 KB
Raw Blame
import fs from 'node:fs';
import { spawn } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

import { clearRuntimeState, readRuntimeState, writeRuntimeState } from './runtime-state.mjs';

const truthy = (value) =>
  ['1', 'true', 'yes', 'on'].includes(String(value || '').trim().toLowerCase());

const trim = (value) => String(value || '').trim();
const browserHostForBind = (host) => {
  const normalized = trim(host).toLowerCase().replace(/^\[/, '').replace(/\]$/, '');
  if (normalized === '' || normalized === '0.0.0.0' || normalized === '::') {
    return '127.0.0.1';
  }
  return trim(host);
};
const HOT_DEV_DEFAULT_AUTH_USER = 'admin';
const HOT_DEV_DEFAULT_AUTH_HASH = '$2a$12$J/Vu6FlBUJDTK.VAkysjB.AnvFOcijDbETyumhCB.nJVes5gvpiI6';

function repoRootFromEnv(env = process.env) {
  const scriptDir = path.dirname(fileURLToPath(import.meta.url));
  return trim(env.PULSE_E2E_REPO_ROOT) || path.resolve(scriptDir, '..', '..', '..');
}

function hotDevBgScriptPath(env = process.env) {
  return path.join(repoRootFromEnv(env), 'scripts', 'hot-dev-bg.sh');
}

function managedVerifyLockPath(env = process.env) {
  return trim(env.HOT_DEV_VERIFY_LOCK_FILE) || path.join(repoRootFromEnv(env), 'tmp', 'hot-dev.verify.lock');
}

function hotDevBrowserURL(env = process.env) {
  const host = browserHostForBind(env.FRONTEND_DEV_HOST || '127.0.0.1');
  const port = trim(env.FRONTEND_DEV_PORT) || '5173';
  return `http://${host}:${port}`;
}

function hotDevBackendURL(env = process.env) {
  const host = trim(env.PULSE_DEV_API_HOST) || '127.0.0.1';
  const port = trim(env.PULSE_DEV_API_PORT) || '7655';
  return `http://${host}:${port}`;
}

function hotDevRuntimeEnvPath(env = process.env) {
  return path.join(repoRootFromEnv(env), 'tmp', 'dev-config', '.env');
}

function unquoteDotEnvValue(value) {
  const trimmed = trim(value);
  if (
    (trimmed.startsWith("'") && trimmed.endsWith("'")) ||
    (trimmed.startsWith('"') && trimmed.endsWith('"'))
  ) {
    return trimmed.slice(1, -1);
  }
  return trimmed;
}

function readHotDevRuntimeAuth(env = process.env) {
  try {
    const contents = fs.readFileSync(hotDevRuntimeEnvPath(env), 'utf8');
    const auth = {};
    for (const line of contents.split(/\r?\n/)) {
      const trimmed = line.trim();
      if (!trimmed || trimmed.startsWith('#')) {
        continue;
      }
      const separator = trimmed.indexOf('=');
      if (separator === -1) {
        continue;
      }
      auth[trimmed.slice(0, separator).trim()] = unquoteDotEnvValue(
        trimmed.slice(separator + 1),
      );
    }
    return {
      user: trim(auth.PULSE_AUTH_USER),
      pass: trim(auth.PULSE_AUTH_PASS),
    };
  } catch {
    return null;
  }
}

function hotDevDesiredRuntimeAuth(env = process.env) {
  return {
    user: trim(env.HOT_DEV_AUTH_USER) || HOT_DEV_DEFAULT_AUTH_USER,
    pass: trim(env.HOT_DEV_AUTH_PASS) || HOT_DEV_DEFAULT_AUTH_HASH,
  };
}

function managedRuntimeAuthMatchesDesired(env = process.env) {
  const currentAuth = readHotDevRuntimeAuth(env);
  if (!currentAuth) {
    return false;
  }
  const desiredAuth = hotDevDesiredRuntimeAuth(env);
  return currentAuth.user === desiredAuth.user && currentAuth.pass === desiredAuth.pass;
}

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function run(command, args, options = {}) {
  return new Promise((resolve, reject) => {
    const child = spawn(command, args, { ...options });
    let stdout = '';
    let stderr = '';

    child.stdout?.on('data', (chunk) => {
      stdout += chunk.toString();
    });
    child.stderr?.on('data', (chunk) => {
      stderr += chunk.toString();
    });
    child.on('error', reject);
    child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr }));
  });
}

async function runHotDevBg(args, env = process.env) {
  const mergedEnv = { ...process.env, ...env };
  return run(hotDevBgScriptPath(mergedEnv), args, {
    cwd: repoRootFromEnv(mergedEnv),
    env: mergedEnv,
  });
}

function statusReportsRunning(output) {
  return output.includes('[hot-dev-bg] Running');
}

export function managedVerifyLockActive(env = process.env) {
  try {
    const raw = fs.readFileSync(managedVerifyLockPath(env), 'utf8');
    const ownerPid = Number.parseInt(
      raw
        .split('\n')
        .find((line) => line.startsWith('pid='))?.slice(4) || '',
      10,
    );
    if (!Number.isInteger(ownerPid) || ownerPid <= 0) {
      return false;
    }
    process.kill(ownerPid, 0);
    return true;
  } catch {
    return false;
  }
}

export function shouldRestartManagedDevRuntimeForVerification({
  env = process.env,
  wasRunning,
}) {
  if (!wasRunning) {
    return false;
  }
  return managedVerifyLockActive(env) || !managedRuntimeAuthMatchesDesired(env);
}

async function managedRuntimeStatusOutput(env = process.env) {
  const status = await runHotDevBg(['status'], env);
  if (status.code !== 0) {
    throw new Error(`hot-dev-bg status failed: ${status.stderr || status.stdout}`);
  }

  return `${status.stdout}${status.stderr}`;
}

function managedSupervisorPidFromStatus(output) {
  const match = output.match(/\[hot-dev-bg\] Running \(pid: (\d+)\)/);
  return match?.[1] || '';
}

function managedListenerPidFromStatus(output, port) {
  const escapedPort = String(port).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
  const match = output.match(new RegExp(`\\[hot-dev-bg\\] Port ${escapedPort}: managed listener pid=(\\d+)`));
  return match?.[1] || '';
}

async function managedBackendPid(env = process.env) {
  const statusOutput = await managedRuntimeStatusOutput(env);
  return managedListenerPidFromStatus(statusOutput, trim(env.PULSE_DEV_API_PORT) || '7655');
}

async function managedHotDevOwnerPid(env = process.env) {
  const statusOutput = await managedRuntimeStatusOutput(env);
  const supervisorPid = managedSupervisorPidFromStatus(statusOutput);
  if (!supervisorPid) {
    throw new Error(`Managed dev runtime is not running:\n${statusOutput}`);
  }

  const psResult = await run(
    'ps',
    ['-axo', 'pid=,ppid=,pgid=,command='],
    { env: { ...process.env, ...env } },
  );
  if (psResult.code !== 0) {
    throw new Error(`ps failed while locating managed hot-dev owner: ${psResult.stderr || psResult.stdout}`);
  }

  const ownerLines = `${psResult.stdout}${psResult.stderr}`
    .split('\n')
    .map((line) => line.trim())
    .filter((line) => {
      const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/);
      if (!match) {
        return false;
      }
      const [, , ppid, pgid, command] = match;
      if (!command.includes('/scripts/hot-dev.sh')) {
        return false;
      }
      return ppid === supervisorPid || pgid === supervisorPid;
    })
    .sort((left, right) => {
      const leftPid = Number(left.match(/^(\d+)/)?.[1] || '0');
      const rightPid = Number(right.match(/^(\d+)/)?.[1] || '0');
      return leftPid - rightPid;
    });

  const ownerLine = ownerLines[0] || '';
  if (!ownerLine) {
    throw new Error(
      `Managed hot-dev owner process was not found under supervisor ${supervisorPid}:\n${psResult.stdout}`,
    );
  }

  const ownerMatch = ownerLine.match(/^(\d+)\s+/);
  return ownerMatch?.[1] || '';
}

async function ensureHealthyStatus(env = process.env) {
  const output = await managedRuntimeStatusOutput(env);
  const requiredMarkers = [
    '[hot-dev-bg] Frontend shell HTTP: 200',
    '[hot-dev-bg] Frontend proxy /api/health: 200',
    '[hot-dev-bg] Backend /api/health: 200',
  ];
  for (const marker of requiredMarkers) {
    if (!output.includes(marker)) {
      throw new Error(`Managed dev runtime is not healthy enough for browser proof:\n${output}`);
    }
  }

  return output;
}

async function waitForStableManagedRuntime({
  env = process.env,
  timeoutMs = 180_000,
  sampleIntervalMs = 2_000,
  requiredStableSamples = 2,
} = {}) {
  const startedAt = Date.now();
  let stableSamples = 0;
  let previousSnapshot = null;
  let lastError = null;

  while (Date.now() - startedAt < timeoutMs) {
    try {
      const statusOutput = await ensureHealthyStatus(env);
      const supervisorPid = managedSupervisorPidFromStatus(statusOutput);
      const frontendPid = managedListenerPidFromStatus(
        statusOutput,
        trim(env.FRONTEND_DEV_PORT) || '5173',
      );
      const backendPid = managedListenerPidFromStatus(
        statusOutput,
        trim(env.PULSE_DEV_API_PORT) || '7655',
      );
      const ownerPid = await managedHotDevOwnerPid(env);
      const snapshot = {
        supervisorPid,
        ownerPid,
        frontendPid,
        backendPid,
      };

      if (!supervisorPid || !ownerPid || !frontendPid || !backendPid) {
        throw new Error(`Managed dev runtime is healthy but missing stable pid markers:\n${statusOutput}`);
      }

      if (
        previousSnapshot &&
        previousSnapshot.supervisorPid === snapshot.supervisorPid &&
        previousSnapshot.ownerPid === snapshot.ownerPid &&
        previousSnapshot.frontendPid === snapshot.frontendPid &&
        previousSnapshot.backendPid === snapshot.backendPid
      ) {
        stableSamples += 1;
      } else {
        stableSamples = 1;
      }

      previousSnapshot = snapshot;
      if (stableSamples >= requiredStableSamples) {
        return snapshot;
      }
    } catch (error) {
      lastError = error;
      stableSamples = 0;
      previousSnapshot = null;
    }

    await sleep(sampleIntervalMs);
  }

  const reason =
    lastError instanceof Error
      ? lastError.message
      : String(lastError || 'runtime never reached a stable healthy state');
  throw new Error(
    `Managed dev runtime did not stabilize within ${timeoutMs}ms: ${reason}`,
  );
}

export async function startManagedDevRuntime({
  env = process.env,
  logger = console,
} = {}) {
  await clearRuntimeState(env);

  const statusBefore = await runHotDevBg(['status'], env);
  if (statusBefore.code !== 0) {
    throw new Error(`hot-dev-bg status failed before start: ${statusBefore.stderr || statusBefore.stdout}`);
  }
  const wasRunning = statusReportsRunning(`${statusBefore.stdout}${statusBefore.stderr}`);
  const shouldRestartForVerification = shouldRestartManagedDevRuntimeForVerification({
    env,
    wasRunning,
  });

  const startArgs = [shouldRestartForVerification ? 'restart' : 'start'];
  if (truthy(env.PULSE_E2E_HOT_DEV_TAKEOVER)) {
    startArgs.push('--takeover');
  }
  if (shouldRestartForVerification) {
    logger.log('[integration] Restarting managed dev runtime before browser verification to pick up current source changes');
  }
  const startResult = await runHotDevBg(startArgs, env);
  if (startResult.code !== 0) {
    throw new Error(`hot-dev-bg ${startArgs[0]} failed: ${startResult.stderr || startResult.stdout}`);
  }

  await waitForStableManagedRuntime({ env });

  const runtimeState = {
    managedDevRuntime: true,
    baseURL: hotDevBrowserURL(env),
    browserURL: hotDevBrowserURL(env),
    backendURL: hotDevBackendURL(env),
    startedByHarness: !wasRunning,
  };
  await writeRuntimeState(runtimeState, env);
  logger.log(
    `[integration] Managed dev runtime ready at ${runtimeState.browserURL} (backend ${runtimeState.backendURL})`,
  );
  return runtimeState;
}

export async function restartManagedDevRuntimeBackend({
  env = process.env,
} = {}) {
  const beforeRuntime = await waitForStableManagedRuntime({ env });
  const beforePid = beforeRuntime.backendPid;
  const result = await runHotDevBg(['backend-restart'], env);
  if (result.code !== 0) {
    throw new Error(`hot-dev-bg backend-restart failed: ${result.stderr || result.stdout}`);
  }
  const stableRuntime = await waitForStableManagedRuntime({ env });
  const afterPid = stableRuntime.backendPid;
  if (!beforePid || !afterPid || beforePid === afterPid) {
    throw new Error(`Managed backend restart did not replace the backend listener (before=${beforePid || 'missing'}, after=${afterPid || 'missing'})`);
  }
  return { beforePid, afterPid };
}

export async function killManagedDevRuntimeOwnerProcess({
  env = process.env,
} = {}) {
  const beforeRuntime = await waitForStableManagedRuntime({ env });
  const beforeOwnerPid = beforeRuntime.ownerPid;
  if (!beforeOwnerPid) {
    throw new Error('Managed hot-dev owner pid could not be resolved');
  }

  const killResult = await run(
    'kill',
    ['-KILL', beforeOwnerPid],
    { env: { ...process.env, ...env } },
  );
  if (killResult.code !== 0) {
    throw new Error(`kill -KILL ${beforeOwnerPid} failed: ${killResult.stderr || killResult.stdout}`);
  }

  const stableRuntime = await waitForStableManagedRuntime({
    env,
    timeoutMs: 90_000,
    sampleIntervalMs: 3_000,
    requiredStableSamples: 3,
  });
  const afterOwnerPid = stableRuntime.ownerPid;
  if (!afterOwnerPid || beforeOwnerPid === afterOwnerPid) {
    throw new Error(`Managed hot-dev owner was not replaced after kill (before=${beforeOwnerPid}, after=${afterOwnerPid || 'missing'})`);
  }
  return { beforeOwnerPid, afterOwnerPid };
}

export async function stopManagedDevRuntime({
  env = process.env,
  logger = console,
} = {}) {
  const runtimeState = await readRuntimeState(env);
  if (!runtimeState?.managedDevRuntime) {
    await clearRuntimeState(env);
    return false;
  }

  if (runtimeState.startedByHarness) {
    const result = await runHotDevBg(['stop'], env);
    if (result.code !== 0) {
      throw new Error(`hot-dev-bg stop failed: ${result.stderr || result.stdout}`);
    }
    logger.log('[integration] Stopped managed dev runtime started by harness');
  }

  await clearRuntimeState(env);
  return true;
}