Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.0 KB
Raw Blame
/** Machine-local graceful drain. The same journal fences the server and runner. */
import { AsyncLocalStorage } from 'node:async_hooks';
import { EventEmitter } from 'node:events';
import { randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import { join } from 'node:path';
import { z } from 'zod';
import { PATHS } from './paths.js';
import { assertNotRealDataWrite } from './testDataIsolation.js';
import { createMaintenanceExclusive, maintenanceExclusiveSchema, maintenanceExclusiveReceiptSchema } from './maintenanceExclusive.js';

const operationSchema = z.object({
  id: z.string().uuid(), kind: z.string().min(1), resource: z.string().max(256),
  pid: z.number().int().positive(), startedAt: z.string().datetime(), unsettled: z.boolean().optional(),
  uncertaintyStamp: z.string().uuid().optional(),
}).strict();
export const maintenanceJournalSchema = z.object({
  version: z.literal(1), revision: z.number().int().nonnegative(),
  hold: z.object({ id: z.string().uuid(), revision: z.number().int().positive(),
    reason: z.string().min(1).max(500), owner: z.string().min(1).max(128), requestedAt: z.string().datetime(),
  }).strict().nullable(),
  operations: z.array(operationSchema),
  exclusive: maintenanceExclusiveSchema.nullable().optional(),
  exclusiveReceipt: maintenanceExclusiveReceiptSchema.optional(),
}).strict().refine(state => !state.exclusive || (state.hold?.id === state.exclusive.holdId
  && state.hold?.revision === state.exclusive.holdRevision));
export const maintenanceBeginSchema = z.object({ reason: z.string().trim().min(1).max(500) }).strict();
export const maintenanceResumeSchema = z.object({ id: z.string().uuid(), revision: z.number().int().positive() }).strict();
const empty = () => ({ version: 1, revision: 0, hold: null, operations: [] });
const error = (code = 'MAINTENANCE_HELD', message = 'Maintenance is holding new work. Existing work can finish.') =>
  Object.assign(new Error(message), { status: code === 'MAINTENANCE_STALE' ? 409 : 503, code });
export const isMaintenanceHold = err => ['MAINTENANCE_HELD', 'MAINTENANCE_UNAVAILABLE'].includes(err?.code);

export function syncMaintenanceDirectory(path, io = fs) {
  if (process.platform === 'win32') return;
  const fd = io.openSync(path, 'r');
  try { io.fsyncSync(fd); } finally { io.closeSync(fd); }
}

export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, assertWrite = assertNotRealDataWrite,
  makeId = randomUUID, now = Date.now, verifyExclusiveReceipt = () => false } = {}) {
  const directory = join(dataDir, 'workflow-maintenance');
  const file = join(directory, 'state.json');
  const lock = join(directory, 'transaction');
  const context = new AsyncLocalStorage();
  const events = new EventEmitter();
  // A failed publication must not reopen admission in this process, even if
  // an operator removes an interrupted transaction before restarting it.
  let uncertain = false;
  let failureReported = false;
  const reportFailure = err => {
    if (failureReported) return;
    failureReported = true;
    while (err.cause instanceof Error) err = err.cause;
    console.error(`❌ Maintenance transaction failed (${err.code ?? 'UNKNOWN'}): ${String(err.message).replace(/\s+/g, ' ')}`);
  };
  const unavailable = cause => Object.assign(error('MAINTENANCE_UNAVAILABLE', 'Maintenance state needs recovery; new work remains held.'), cause ? { cause } : {});
  const exists = path => {
    try { io.lstatSync(path); return true; } catch (err) { if (err.code === 'ENOENT') return false; throw err; }
  };
  const read = () => {
    if (uncertain) throw unavailable();
    try {
      if (!exists(file)) return empty();
      if (!io.lstatSync(file).isFile()) throw unavailable();
      return maintenanceJournalSchema.parse(JSON.parse(io.readFileSync(file, 'utf8')));
    } catch (err) { throw unavailable(err); }
  };
  // status()/held() are polled several times a second (media drain loop, spawn
  // gates); re-parsing a journal that carries every settled operation allocated
  // ~400 KB per call. Writers publish by renaming a new file into place, so any
  // change gives it a new identity; the stat is taken before the read, so a
  // racing publish can only cost a re-parse. Read-only: transactions mutate the
  // state they read and must keep calling read() for a private copy.
  let statusCache = null;
  const readForStatus = () => {
    if (uncertain) throw unavailable();
    let stats;
    try { stats = io.lstatSync(file); }
    catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(err); }
    if (!stats.isFile()) throw unavailable();
    const key = `${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;
    if (statusCache?.key !== key) statusCache = { key, state: read() };
    return statusCache.state;
  };
  const project = state => ({
    state: state.hold ? (state.operations.length || state.exclusive ? 'draining' : 'ready') : 'normal',
    revision: state.revision, hold: state.hold && { ...state.hold },
    blockers: [...state.operations.map(({ kind, resource, startedAt, pid, unsettled }) => ({ kind, resource, startedAt, pid, unsettled })),
      ...(state.exclusive ? [{ kind: 'exclusive-maintenance', resource: state.exclusive.operation.operationId,
        startedAt: state.exclusive.claimedAt, unsettled: state.exclusive.phase === 'uncertain' }] : [])],
    scope: 'PortOS agents, mind turns, provider runs, media renders and scheduled shell/script work, through cleanup and saving. Unrelated host applications are outside this scope.',
  });
  const status = () => {
    let cause = 'unreadable-journal';
    let since;
    try {
      if (exists(lock)) {
        cause = 'interrupted-transaction';
        since = io.lstatSync(lock).mtime.toISOString();
        throw unavailable();
      }
      if (uncertain) { cause = 'process-uncertain'; throw unavailable(); }
      return project(readForStatus());
    } catch {
      return { state: 'unavailable', revision: null, hold: null, blockers: [],
        cause, ...(since ? { since } : {}),
        error: 'Maintenance state needs recovery. New work is held; readiness is unknown.' };
    }
  };
  // Synchronous mkdir + read/replace is the shared linearization point. Never
  // steal this lock by age/PID: a crash or failed fsync stays fail-closed.
  const transaction = change => {
    assertWrite(directory, 'workflow maintenance transaction');
    if (uncertain) throw unavailable();
    try {
      io.mkdirSync(directory, { recursive: true, mode: 0o700 });
      // Writers do no async work while holding this lock. Same-process calls
      // cannot contend; briefly yield the CPU to a competing server/runner.
      // A crashed writer remains closed, rather than stealing its lock.
      for (let attempt = 0; ; attempt++) {
        try { io.mkdirSync(lock, { mode: 0o700 }); break; }
        catch (err) {
          if (err.code !== 'EEXIST' || attempt === 99) throw err;
          Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5);
        }
      }
    } catch (err) { reportFailure(err); throw unavailable(); }
    let published = false;
    let result;
    try {
      const state = read();
      result = change(state);
      state.revision++;
      maintenanceJournalSchema.parse(state);
      const pending = join(directory, `pending-${makeId()}.json`);
      const fd = io.openSync(pending, 'wx', 0o600);
      try { io.writeFileSync(fd, JSON.stringify(state) + '\n'); io.fsyncSync(fd); } finally { io.closeSync(fd); }
      io.renameSync(pending, file);
      syncMaintenanceDirectory(directory, io);
      published = true;
    } catch (err) {
      // Refused admission/stale requests didn't attempt a publication.
      if (['MAINTENANCE_HELD', 'MAINTENANCE_STALE'].includes(err.code)) published = true;
      else { uncertain = true; reportFailure(err); }
      throw err.code?.startsWith('MAINTENANCE_') ? err : unavailable();
    } finally {
      if (published) {
        try { io.rmdirSync(lock); syncMaintenanceDirectory(directory, io); }
        catch (err) { uncertain = true; reportFailure(err); }
      }
    }
    if (uncertain) throw unavailable();
    // Listener errors must never turn a committed reservation into a failed
    // admission whose caller cannot own/release it.
    queueMicrotask(() => { for (const listener of events.listeners('changed')) {
      try { listener(status()); } catch (err) { console.error(`❌ Maintenance notification failed: ${err.message}`); }
    } });
    return result;
  };
  const exclusive = createMaintenanceExclusive({ transaction, read, lockExists: () => exists(lock), makeId, error, now,
    verifyReceipt: verifyExclusiveReceipt });
  const held = () => status().state !== 'normal';
  const assertOpen = () => { if (held()) throw error(status().state === 'unavailable' ? 'MAINTENANCE_UNAVAILABLE' : 'MAINTENANCE_HELD'); };
  const currentId = () => context.getStore() ?? null;
  const admit = (kind, resource = '', { continuation = false, parentId = null, parentKinds = null, reconnect = false } = {}) => {
    const inheritedId = parentId ?? (continuation ? currentId() : null);
    let id = makeId();
    let recovery;
    transaction(state => {
      const inherited = inheritedId && state.operations.some(op => op.id === inheritedId && (!op.unsettled || kind === 'settlement') && (!parentKinds || parentKinds.includes(op.kind)));
      if (state.hold && !inherited) throw error();
      // An explicitly supplied expired parent must never become new work.
      if (parentId && !inherited) throw error('MAINTENANCE_HELD', 'The admitted parent operation has already settled.');
      // Reconciliation may submit an idempotent replay, so it still obeys the
      // admission fence above; reuse ownership only after that fence passes.
      const existing = reconnect && state.operations.find(op => op.kind === kind && op.resource === resource);
      if (existing) { id = existing.id; recovery = recoverySnapshot(existing); return; }
      state.operations.push({ id, kind, resource, pid: process.pid, startedAt: new Date().toISOString() });
    });
    return permitFor(id, recovery);
  };
  // A capability binds one previously observed operation, including every
  // uncertainty mark made before reconciliation. A legacy unsettled record
  // has no stamp to compare and cannot be completed through this capability.
  const recoverySnapshot = op => Object.freeze({ id: op.id, kind: op.kind, resource: op.resource,
    unsettled: op.unsettled === true, uncertaintyStamp: op.uncertaintyStamp });
  const completeRecovery = expected => {
    if (expected.unsettled && !expected.uncertaintyStamp) return false;
    return transaction(state => {
      const index = state.operations.findIndex(op => op.id === expected.id && op.kind === expected.kind
        && op.resource === expected.resource && (op.unsettled === true) === expected.unsettled
        && op.uncertaintyStamp === expected.uncertaintyStamp);
      if (index < 0) return false;
      state.operations.splice(index, 1);
      return true;
    });
  };
  // Local owning-service recovery after explicit operator disposition of a
  // legacy pre-admission refusal. The service must preserve inspected evidence
  // in publish(), durably and replayably, before this exact owner is retired.
  const reconcilePublicationRefusal = ({ hold, expected, publish, replay }) => transaction(state => {
    if (!hold || state.hold?.id !== hold.id || state.hold?.revision !== hold.revision || state.exclusive)
      throw error('MAINTENANCE_STALE', 'The maintenance hold changed.');
    if (expected?.kind !== 'settlement' || expected.resource !== 'Output publication'
      || expected.unsettled !== true || !expected.uncertaintyStamp)
      throw error('MAINTENANCE_STALE', 'An exact uncertain publication reservation is required.');
    const index = state.operations.findIndex(op => op.id === expected.id);
    if (index < 0) return replay();
    if (JSON.stringify(state.operations[index]) !== JSON.stringify(expected))
      throw error('MAINTENANCE_STALE', 'The publication reservation changed.');
    const receipt = publish();
    state.operations.splice(index, 1);
    return receipt;
  });
  const permitFor = (id, recovery) => ({
    id,
    run: fn => context.run(id, fn),
    finish: () => finish(id),
    markUnsettled: () => markUnsettled(op => op.id === id),
    identify: resource => transaction(state => { const op = state.operations.find(op => op.id === id); if (op) op.resource = resource; }),
    ...(recovery ? { completeRecovery: () => completeRecovery(recovery) } : {}),
  });
  // Trusted recovery only: this records observed existing work, never starts it.
  const recoverOwned = (kind, resource) => {
    let recovery;
    const id = transaction(state => {
      const existing = state.operations.find(op => op.kind === kind && op.resource === resource);
      if (existing) { recovery = recoverySnapshot(existing); return existing.id; }
      const id = makeId();
      exclusive.observeExistingWork(state);
      state.operations.push({ id, kind, resource, pid: process.pid, startedAt: new Date().toISOString() });
      return id;
    });
    return permitFor(id, recovery);
  };
  const tryAdmit = (...args) => { try { return admit(...args); } catch (err) { if (isMaintenanceHold(err)) return null; throw err; } };
  const finishing = new Map();
  const finish = id => {
    if (finishing.has(id)) return finishing.get(id);
    // A runner and server can finish in the same millisecond. Contention is
    // retried; never lose a completion solely because another writer owns mkdir.
    const settle = async () => {
      for (let attempt = 0; attempt < 10; attempt++) {
        try {
          transaction(state => { state.operations = state.operations.filter(op => op.id !== id || op.unsettled); });
          return true;
        } catch {
          if (uncertain || !exists(lock)) break;
          await new Promise(resolve => { const timer = setTimeout(resolve, 50); timer.unref?.(); });
        }
      }
      console.error('❌ Maintenance settlement needs recovery; preserving its blocker.');
      return false;
    };
    const pending = settle().finally(() => finishing.delete(id));
    finishing.set(id, pending);
    return pending;
  };
  // Only trusted lifecycle code calls this after terminal persistence/cleanup.
  // A server restarted during a runner job can finish the old owner's record.
  const resourceOperations = (kind, resource) => read().operations.filter(op => op.kind === kind && op.resource === resource);
  const finishResource = (kind, resource) => Promise.all(resourceOperations(kind, resource).map(op => finish(op.id)));
  const withResource = (kind, resource, fn) => {
    const operation = resourceOperations(kind, resource).find(op => !op.unsettled);
    return operation ? context.run(operation.id, fn) : fn();
  };
  const markUnsettled = matches => {
    try { transaction(state => { for (const op of state.operations) if (matches(op)) {
      op.unsettled = true;
      op.uncertaintyStamp = randomUUID();
    } }); }
    catch { uncertain = true; console.error('❌ Maintenance recovery record could not be saved; readiness remains unknown.'); }
  };
  const markResourceUnsettled = (kind, resource) => markUnsettled(op => op.kind === kind && op.resource === resource);
  const markCurrentUnsettled = () => {
    const id = currentId();
    if (id) markUnsettled(op => op.id === id);
  };
  // A publication boundary may prove its callback never started. Keep the
  // reservation while it waits, but do not mistake refused admission for an
  // uncertain write. Callers without that proof retain the conservative default.
  const continueSettlement = (fn, { hasStarted } = {}) => currentId()
    ? run('settlement', 'Output publication', fn, { continuation: true, hasStarted }) : fn();
  const run = async (kind, resource, fn, options) => {
    const permit = admit(kind, resource, options);
    try { return await permit.run(fn); }
    catch (err) {
      if (kind === 'settlement') {
        let started = true;
        try { started = options?.hasStarted?.() !== false; } catch { /* Missing proof remains uncertain. */ }
        if (started) permit.markUnsettled();
      }
      throw err;
    }
    finally { await permit.finish(); }
  };
  const begin = ({ reason, owner }) => {
    const input = maintenanceBeginSchema.parse({ reason });
    transaction(state => {
      if (state.hold) throw error('MAINTENANCE_STALE', 'A maintenance hold already exists. Refresh its status.');
      state.hold = { id: makeId(), revision: state.revision + 1, ...input, owner,
        requestedAt: new Date().toISOString() };
    });
    return status();
  };
  const resume = input => {
    const expected = maintenanceResumeSchema.parse(input);
    transaction(state => {
      if (state.hold?.id !== expected.id || state.hold?.revision !== expected.revision)
        throw error('MAINTENANCE_STALE', 'This maintenance hold changed. Refresh before resuming.');
      if (state.exclusive)
        throw error('MAINTENANCE_STALE', 'An exclusive maintenance operation owns this hold until verified reconciliation.');
      state.hold = null;
    });
    return status();
  };
  // Cached read-only snapshot for trusted reconcilers (runs on every zombie sweep); releaseOperations re-verifies under the lock.
  const operations = () => readForStatus().operations.map(op => ({ ...op }));
  // Removes only settled records still identical to what the caller inspected;
  // anything that changed (or settled elsewhere) since then is left alone.
  const releaseOperations = expected => transaction(state => {
    const before = state.operations.length;
    state.operations = state.operations.filter(op => op.unsettled
      || !expected.some(seen => seen.id === op.id && seen.kind === op.kind
        && seen.resource === op.resource && seen.pid === op.pid && seen.startedAt === op.startedAt));
    return before - state.operations.length;
  });
  // Fail-closed stamp for records the caller already inspected. Already-unsettled
  // rows keep their stamp: rotating it would invalidate an operator receipt in flight.
  const markOperationsUncertain = expected => transaction(state => {
    let marked = 0;
    for (const op of state.operations) {
      if (op.unsettled) continue;
      const seen = expected.some(item => item.id === op.id && item.kind === op.kind
        && item.resource === op.resource && item.pid === op.pid && item.startedAt === op.startedAt);
      if (!seen) continue;
      op.unsettled = true;
      op.uncertaintyStamp = randomUUID();
      marked += 1;
    }
    return marked;
  });
  // Owning-service recovery only. Validation and durable receipt publication
  // run under the same journal lock as the exact reservation removal.
  const reconcileAbandonedAgent = ({ hold, agentId, validate, publish, replay }) => transaction(state => {
    if (state.hold?.id !== hold.id || state.hold?.revision !== hold.revision || state.exclusive)
      throw error('MAINTENANCE_STALE', 'The recovery hold changed or is exclusively owned.');
    const matches = state.operations.filter(op => op.resource === agentId);
    if (matches.length === 0) return replay();
    if (matches.length !== 1 || matches[0].kind !== 'agent' || matches[0].unsettled)
      throw error('MAINTENANCE_STALE', 'Recovery requires exactly one settled agent reservation.');
    const operation = { ...matches[0] };
    validate(operation);
    const receipt = publish(operation);
    state.operations = state.operations.filter(op => op.id !== operation.id);
    return receipt;
  });
  const { observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability } = exclusive;
  return { directory, status, held, assertOpen, admit, tryAdmit, recoverOwned, reconcilePublicationRefusal, operations, releaseOperations, markOperationsUncertain, run, currentId, finish, finishResource, withResource, markResourceUnsettled, markCurrentUnsettled, continueSettlement, begin, resume, events,
    observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability, reconcileAbandonedAgent };
}

// Resolve the configured data root on first use, after host/test setup.
let defaultAdmission;
export const maintenance = new Proxy({}, {
  get(_target, property) {
    defaultAdmission ??= createMaintenanceAdmission();
    return defaultAdmission[property];
  },
});