Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.5 KB
Raw Blame
import { readFileSync } from 'node:fs';
import { createHash } from 'node:crypto';
import { createRequire } from 'node:module';
import { join } from 'node:path';
import { z } from 'zod';
import { PATHS } from '../lib/paths.js';
import { POOL_CONFIG } from '../lib/db.js';
import { acquireBackupSnapshotCut } from '../lib/backupSnapshotBoundary.js';
import { assertDatabaseAdmission, createDatabaseMaintenanceJournal, databaseMaintenanceEndpointSchema } from '../lib/databaseMaintenanceJournal.js';
import { databaseMaintenancePreflightSchema } from '../lib/validation.js';
import { ServerError } from '../lib/errorHandler.js';
import { classifyWriterQuiescence, createDatabaseWriterRegistry } from '../lib/databaseWriterRegistry.js';
import { summarizeSystemActivity } from '../lib/systemIdle.js';

const require = createRequire(import.meta.url);
const refuse = (code, message) => new ServerError(message, { status: 409, code });
const unavailable = () => refuse('DATABASE_PREFLIGHT_UNTRUSTED', 'Database configuration or work state could not be verified.');
const stale = () => refuse('DATABASE_PREFLIGHT_STALE', 'Requested direction does not match the current database configuration and running pool.');
const busy = () => refuse('DATABASE_PREFLIGHT_BUSY', 'PortOS has active or queued work. Wait for it to finish before database maintenance.');
const ACCEPT_DRAIN_TIMEOUT_MS = 30_000;
const count = z.number().int().nonnegative();
// The dashboard summarizer intentionally accepts partial snapshots. Maintenance
// cannot use a missing slice as proof that there are no writers in that domain.
const activitySchema = z.object({
  jobs: z.array(z.object({ status: z.enum(['running', 'queued']) })),
  extras: z.object({ imageTo3d: z.array(z.unknown()) }),
  agents: z.object({ trusted: z.literal(true), active: count, queued: count }),
  mind: z.object({ trusted: z.literal(true), thinking: z.boolean(), queued: count }),
  llm: z.object({ trusted: z.literal(true), active: count }),
  appOperations: z.array(z.unknown()),
  update: z.object({ inProgress: z.boolean() }),
  backup: z.object({ inProgress: z.boolean() }),
});
const normalizeHost = host => ['localhost', '127.0.0.1', '::1', '[::1]'].includes(host.toLowerCase()) ? 'loopback' : host.toLowerCase();

function readConfiguration() {
  // Ecosystem exports are evaluated once by require. Re-evaluate on each read
  // so an operator's saved mode change cannot validate against cached settings.
  // Check readability separately: ecosystem's boot defaults deliberately mask
  // a failed .env read, which cannot be treated as evidence for a cutover.
  const envPath = join(PATHS.installRoot, '.env');
  let content;
  try { content = readFileSync(envPath, 'utf8'); } catch { throw unavailable(); }
  const configPath = join(PATHS.installRoot, 'ecosystem.config.cjs');
  let configuration;
  try {
    delete require.cache[require.resolve(configPath)];
    configuration = require(configPath);
  } catch { throw unavailable(); }
  const { DATABASE_MODE: mode, DATABASE_ENDPOINTS: endpoints } = configuration ?? {};
  const source = databaseMaintenanceEndpointSchema.safeParse(endpoints?.[mode]);
  const target = databaseMaintenanceEndpointSchema.safeParse(endpoints?.[mode === 'native' ? 'docker' : 'native']);
  if (!source.success || !target.success || source.data.mode !== mode || target.data.mode === mode) throw unavailable();
  if (normalizeHost(source.data.host) === normalizeHost(target.data.host) && source.data.port === target.data.port) throw stale();
  return { source: source.data, target: target.data, revision: createHash('sha256').update(JSON.stringify(content)).digest('hex') };
}

function validateDirection(direction, configuration) {
  if (direction.source !== configuration.source.mode || direction.target !== configuration.target.mode
    || ['host', 'port', 'database', 'user'].some(key => configuration.source[key] !== POOL_CONFIG[key])) throw stale();
}

// Trusted idle: every activity slice present and quiet. Reads process/state
// files only, so it stays valid after the admission fence closes.
async function assertTrustedIdle() {
  // Keep heavy activity/CoS graphs off ordinary database admin imports. Use the
  // lifecycle-only reader, not GPU telemetry or external Ollama probes.
  const [snapshot, activeAgents, mindGuard] = await Promise.all([
    import('./activeProcessing.js'), import('./updatePreflight.js'),
  ]).then(([processing, preflight]) => Promise.all([
    processing.getSystemActivity(), preflight.countActiveCosAgents(), preflight.getPersistentMindImageWorkGuard(),
  ])).catch(() => { throw unavailable(); });
  const activity = activitySchema.safeParse(snapshot);
  if (!activity.success || !Number.isInteger(activeAgents) || activeAgents < 0
    || mindGuard?.trusted !== true || typeof mindGuard.safe !== 'boolean') throw unavailable();
  if (!summarizeSystemActivity(activity.data).idle || activeAgents > 0 || !mindGuard.safe) throw busy();
}

async function assertReconcilableWriters() {
  let rows, processes;
  try {
    const { snapshotProcesses } = await import('../lib/detachedSpawn.js');
    processes = process.platform === 'win32' ? [] : await snapshotProcesses();
    // Read after the asynchronous snapshot so newly admitted reservations are
    // included. Acceptance repeats this under the fence before worker launch.
    rows = createDatabaseWriterRegistry().read();
  } catch { throw unavailable(); }
  if (rows.some(row => process.platform === 'win32'
    ? row.state !== 'abandoned'
    : ['pending', 'ambiguous', 'orphaned'].includes(classifyWriterQuiescence(row, processes).verdict))) throw busy();
}

/** Non-mutating advice only; no operation token, reservation, or writer drain. */
export async function preflightDatabaseMaintenance(input) {
  const direction = databaseMaintenancePreflightSchema.parse(input);
  assertDatabaseAdmission();
  const initial = readConfiguration();
  validateDirection(direction, initial);
  await assertTrustedIdle();
  await assertReconcilableWriters();
  // No await between the final fence/configuration checks and the response.
  // This narrows the observation race, but is deliberately NOT acceptance.
  assertDatabaseAdmission();
  const final = readConfiguration();
  validateDirection(direction, final);
  if (JSON.stringify(initial) !== JSON.stringify(final)) throw stale();
  return { source: direction.source, target: direction.target, advisory: true, accepted: false };
}

/**
 * Accept a cutover. Repeats the advisory validation, then publishes the
 * operation — which closes admission synchronously — and repeats the idle and
 * configuration checks UNDER the fence: work admitted just before publication
 * must already be finished, and nothing new can be admitted. Only then does a
 * coordinator take ownership. A failed final check cancels the still-unowned
 * operation — but only while the saved source still equals the recorded one;
 * otherwise the fence stays for the operator.
 *
 * Returns `{ accepted, launch }`. The caller runs `launch()` only after its
 * response is sent: the worker's first act is stopping this server. An owner
 * whose worker never launched is resumable with same-operation recovery.
 */
export async function acceptDatabaseMaintenance(input) {
  const direction = databaseMaintenancePreflightSchema.parse(input);
  await preflightDatabaseMaintenance(direction);
  const configuration = readConfiguration();
  validateDirection(direction, configuration);
  const journal = createDatabaseMaintenanceJournal();
  // Publish the fence only at a whole-workflow boundary: admitted file-plus-row
  // publications finish first, and a backup that owns the cut is never fenced
  // out mid-snapshot. After the fence exists, new publications refuse on their
  // own, so the cut is released as soon as begin() returns or fails.
  const releaseSnapshotCut = await acquireBackupSnapshotCut({ timeoutMs: ACCEPT_DRAIN_TIMEOUT_MS }).catch(err => {
    throw err.code === 'BACKUP_SNAPSHOT_BUSY' ? busy() : err;
  });
  let operation;
  try {
    await assertReconcilableWriters();
    try {
      operation = journal.begin({ source: configuration.source, target: configuration.target });
    } catch {
      throw refuse('DATABASE_MAINTENANCE', 'Another database maintenance operation is active or requires recovery.');
    }
  } finally {
    releaseSnapshotCut();
  }
  try {
    await assertTrustedIdle();
    await assertReconcilableWriters();
    const final = readConfiguration();
    validateDirection(direction, final);
    if (final.revision !== configuration.revision) throw stale();
  } catch (err) {
    // cancel() itself refuses when the saved source no longer matches the
    // record; a changed or unreadable configuration keeps the fence closed.
    let current = null;
    try { current = readConfiguration().source; } catch { /* keep the fence */ }
    if (current) {
      try { journal.cancel(operation.id, current); } catch { /* keep the fence */ }
    }
    throw err;
  }
  const token = journal.acquireCoordinator(operation.id);
  const { spawnDatabaseMaintenanceWorker } = await import('../lib/detachedSpawn.js');
  const launch = async () => {
    try {
      const worker = await spawnDatabaseMaintenanceWorker(operation.id, token);
      worker.on('error', () => console.error('❌ Database cutover worker failed to launch; run scripts/database-maintenance.mjs recover'));
    } catch {
      console.error('❌ Database cutover worker failed to launch; run scripts/database-maintenance.mjs recover');
    }
  };
  return { accepted: { id: operation.id, stage: 'accepted', source: direction.source, target: direction.target, accepted: true }, launch };
}