Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.6 KB
Raw Blame
/**
 * Receiver-local durable consumption. A record is neither a grant nor launch
 * authorization; the receiver verifies authority, ownership and adapter evidence.
 */
import { createHash, randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import { z } from 'zod';
import { peerAdminIntentSchema, peerAdminActionSchema } from '../lib/peerAdminValidation.js';
import { createPeerExecutionAuthority, executionAuthorityError } from '../lib/peerExecutionAuthority.js';
import { toPgJson } from '../lib/pgJson.js';

const uuid = z.string().uuid().toLowerCase();
const digest = z.string().regex(/^[a-f0-9]{64}$/);
const integer = z.number().int().nonnegative().safe();
const positive = integer.positive();
export const peerExecutionBindingSchema = z.object({
  hostInstanceId: uuid, peerInstanceId: uuid, requestId: uuid, grantId: uuid,
  grantGeneration: positive, scope: z.literal('execution-v1'), pairBinding: digest,
  intent: peerAdminIntentSchema, receiverVersion: z.string().min(1).max(100), evidenceDigest: digest,
  executionEpoch: uuid,
}).strict().refine(value => value.hostInstanceId !== value.peerInstanceId);
const states = ['queued', 'draining', 'in-flight', 'awaiting-reconnect', 'succeeded', 'failed', 'uncertain'];
const terminal = state => ['succeeded', 'failed'].includes(state);
const claimSchema = z.object({ id: uuid, revision: positive, fingerprint: digest }).strict();
const receiptSchema = z.object({ outcome: z.enum(['succeeded', 'failed']),
  code: z.string().min(1).max(100).regex(/^[A-Z0-9_]+$/), evidenceDigest: digest.nullable(), executionEpoch: uuid }).strict();
const operationSchema = z.object({
  operationId: uuid, binding: peerExecutionBindingSchema, fingerprint: digest,
  state: z.enum(states), revision: positive, claim: claimSchema.nullable(), receipt: receiptSchema.nullable(),
}).strict().refine(row => row.fingerprint === fingerprint(row.binding))
  .refine(row => terminal(row.state) === Boolean(row.receipt) && (!row.receipt || row.receipt.outcome === row.state));
const floorSchema = z.object({ hostInstanceId: uuid, peerInstanceId: uuid,
  action: peerAdminActionSchema, generation: integer }).strict();
const factSchema = z.discriminatedUnion('kind', [
  z.object({ kind: z.literal('operation'), value: operationSchema }).strict(),
  z.object({ kind: z.literal('floor'), value: floorSchema }).strict(),
]);
const fingerprint = binding => createHash('sha256').update(JSON.stringify(binding)).digest('hex');
const fail = (code, message) => { throw Object.assign(new Error(message), { code, status: 409 }); };
const fromRow = row => {
  const operation = operationSchema.parse({ operationId: row.operation_id, binding: row.binding,
    fingerprint: row.fingerprint, state: row.state, revision: Number(row.revision), claim: row.claim, receipt: row.receipt });
  if (row.host_instance_id !== operation.binding.hostInstanceId || row.peer_instance_id !== operation.binding.peerInstanceId
    || row.request_id !== operation.binding.requestId || row.execution_epoch !== operation.binding.executionEpoch)
    throw executionAuthorityError('Execution consumption identity columns conflict with their immutable binding.');
  return operation;
};
const LOCK_KEY = 90011027;
const ACTIVE_LIMIT = 32;
const PAGE_SIZE = 100;
const MAX_FACT_BYTES = 8192;

/** Inject the guarded db.js query/withTransaction and a fixture or receiver data root. */
export function createPeerExecutionLedger({ db, dataDir, authority = createPeerExecutionAuthority(dataDir), makeId = randomUUID }) {
  const locked = fn => db.withTransaction(async client => {
    await client.query('SELECT pg_advisory_xact_lock($1)', [LOCK_KEY]);
    return fn(client);
  });
  const initialize = () => locked(async client => {
    if (!authority.read()) {
      const { rows: [row] } = await client.query('SELECT EXISTS(SELECT 1 FROM peer_execution_operations) OR EXISTS(SELECT 1 FROM peer_execution_generation_floors) AS occupied');
      if (row.occupied) throw executionAuthorityError('Existing consumption has no machine authority; reconcile it instead of resetting it.');
      authority.initialize();
    }
    return authority.read();
  });
  const find = async (client, binding) => {
    const { rows: [row] } = await client.query(`SELECT * FROM peer_execution_operations
      WHERE host_instance_id = $1 AND peer_instance_id = $2 AND request_id = $3`,
    [binding.hostInstanceId, binding.peerInstanceId, binding.requestId]);
    return row ? fromRow(row) : null;
  };
  const consume = async raw => {
    const binding = peerExecutionBindingSchema.parse(raw);
    authority.requireReady(binding.executionEpoch);
    return locked(async client => {
      authority.requireReady(binding.executionEpoch); // Also after waiting for the shared writer lock.
      const previous = await find(client, binding);
      if (previous) {
        if (previous.fingerprint !== fingerprint(binding)) fail('PEER_EXECUTION_REPLAY_CONFLICT', 'This authenticated request identity was already consumed with different evidence.');
        return { operation: previous, isNew: false };
      }
      const { rows: [floor] } = await client.query(`SELECT generation FROM peer_execution_generation_floors
        WHERE host_instance_id = $1 AND peer_instance_id = $2 AND action = $3`,
      [binding.hostInstanceId, binding.peerInstanceId, binding.intent.action]);
      if (floor && Number(floor.generation) > binding.grantGeneration)
        fail('PEER_EXECUTION_GENERATION_STALE', 'The execution grant generation was invalidated.');
      const { rows: [count] } = await client.query(`SELECT count(*)::int AS count FROM peer_execution_operations
        WHERE state IN ('queued', 'draining', 'in-flight', 'awaiting-reconnect', 'uncertain')`);
      if (count.count >= ACTIVE_LIMIT) fail('PEER_EXECUTION_QUEUE_FULL', 'Unresolved execution records must settle before accepting more.');
      const { rows: [row] } = await client.query(`INSERT INTO peer_execution_operations
        (operation_id, host_instance_id, peer_instance_id, request_id, fingerprint, binding, execution_epoch, state)
        VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, 'queued') RETURNING *`,
      [makeId(), binding.hostInstanceId, binding.peerInstanceId, binding.requestId, fingerprint(binding), toPgJson(binding), binding.executionEpoch]);
      await client.query(`INSERT INTO peer_execution_generation_floors (host_instance_id, peer_instance_id, action, generation)
        VALUES ($1, $2, $3, $4) ON CONFLICT (host_instance_id, peer_instance_id, action)
        DO UPDATE SET generation = GREATEST(peer_execution_generation_floors.generation, EXCLUDED.generation)`,
      [binding.hostInstanceId, binding.peerInstanceId, binding.intent.action, binding.grantGeneration]);
      authority.requireReady(binding.executionEpoch);
      return { operation: fromRow(row), isNew: true };
    });
  };
  // A status recovery may retire a signed request that never reached consume.
  // Use the same DB lock as consume: either the launch owns the row already,
  // or this permanent terminal tombstone wins and every delayed launch loses.
  const retireUnconsumed = async raw => {
    const binding = peerExecutionBindingSchema.parse(raw);
    return locked(async client => {
      const epoch = authority.read()?.epoch;
      authority.requireReady(epoch);
      const previous = await find(client, binding);
      if (previous) return previous;
      const receipt = { outcome: 'failed', code: 'PEER_EXECUTION_NOT_ACCEPTED', evidenceDigest: null, executionEpoch: epoch };
      const { rows: [row] } = await client.query(`INSERT INTO peer_execution_operations
        (operation_id, host_instance_id, peer_instance_id, request_id, fingerprint, binding, execution_epoch, state, receipt)
        VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, 'failed', $8::jsonb) RETURNING *`,
      [makeId(), binding.hostInstanceId, binding.peerInstanceId, binding.requestId, fingerprint(binding), toPgJson(binding), binding.executionEpoch, toPgJson(receipt)]);
      authority.requireReady(epoch);
      return fromRow(row);
    });
  };
  const readRequest = async ({ hostInstanceId, peerInstanceId, requestId }) => {
    uuid.parse(hostInstanceId); uuid.parse(peerInstanceId); uuid.parse(requestId);
    return find(db, { hostInstanceId, peerInstanceId, requestId });
  };
  const generationFloor = async ({ hostInstanceId, peerInstanceId, action }) => {
    floorSchema.omit({ generation: true }).parse({ hostInstanceId, peerInstanceId, action });
    const { rows: [row] } = await db.query(`SELECT generation FROM peer_execution_generation_floors
      WHERE host_instance_id = $1 AND peer_instance_id = $2 AND action = $3`, [hostInstanceId, peerInstanceId, action]);
    return row ? Number(row.generation) : 0;
  };
  const read = async id => {
    uuid.parse(id);
    const { rows: [row] } = await db.query('SELECT * FROM peer_execution_operations WHERE operation_id = $1', [id]);
    return row ? fromRow(row) : null;
  };
  const advanceGenerationFloor = async raw => {
    const input = floorSchema.extend({ executionEpoch: uuid }).parse(raw);
    authority.requireReady(input.executionEpoch);
    return locked(async client => {
      authority.requireReady(input.executionEpoch);
      const { rows: [row] } = await client.query(`INSERT INTO peer_execution_generation_floors
        (host_instance_id, peer_instance_id, action, generation) VALUES ($1, $2, $3, $4)
        ON CONFLICT (host_instance_id, peer_instance_id, action) DO UPDATE
        SET generation = GREATEST(peer_execution_generation_floors.generation, EXCLUDED.generation) RETURNING generation`,
      [input.hostInstanceId, input.peerInstanceId, input.action, input.generation]);
      authority.requireReady(input.executionEpoch);
      return Number(row.generation);
    });
  };
  // Persistence CAS, not a dispatcher. The receiver must own and verify
  // the coordinator claim/adapter evidence before calling this internal method.
  const transition = (id, revision, state, { claim = null, receipt = null, authorityEpoch = null } = {}) => {
    uuid.parse(id); positive.parse(revision); z.enum(states).parse(state);
    if (claim) claimSchema.parse(claim);
    if (receipt) receiptSchema.parse(receipt);
    uuid.nullable().parse(authorityEpoch);
    return locked(async client => {
      const { rows: [row] } = await client.query('SELECT * FROM peer_execution_operations WHERE operation_id = $1 FOR UPDATE', [id]);
      if (!row) fail('PEER_EXECUTION_NOT_FOUND', 'The execution operation does not exist.');
      const current = fromRow(row);
      const epoch = authorityEpoch ?? current.binding.executionEpoch;
      authority.requireReady(epoch);
      if (current.revision !== revision) fail('PEER_EXECUTION_STALE', 'Refresh the current operation revision.');
      const allowed = { queued: ['draining', 'failed'], draining: ['in-flight', 'failed', 'uncertain'],
        'in-flight': ['awaiting-reconnect', 'succeeded', 'failed', 'uncertain'],
        'awaiting-reconnect': ['succeeded', 'failed', 'uncertain'], uncertain: ['succeeded', 'failed'], succeeded: [], failed: [] };
      if (!allowed[current.state].includes(state)) fail('PEER_EXECUTION_TRANSITION_REFUSED', 'An operation cannot move backwards or launch again.');
      if (state === 'in-flight' && !claim) fail('PEER_EXECUTION_CLAIM_REQUIRED', 'Persist the exact coordinator ownership before recording a start.');
      if (current.claim && claim && (current.claim.id !== claim.id || current.claim.fingerprint !== claim.fingerprint || claim.revision < current.claim.revision))
        fail('PEER_EXECUTION_CLAIM_CHANGED', 'A different coordinator owner cannot replace the recorded start.');
      if (terminal(state) && receipt?.executionEpoch !== epoch)
        fail('PEER_EXECUTION_RECEIPT_STALE', 'Terminal evidence must bind the current non-rewound authority epoch.');
      const next = operationSchema.parse({ ...current, state, revision: revision + 1, claim: claim ?? current.claim, receipt });
      const { rows: [updated] } = await client.query(`UPDATE peer_execution_operations
        SET state = $2, revision = $3, claim = $4::jsonb, receipt = $5::jsonb, updated_at = NOW()
        WHERE operation_id = $1 RETURNING *`, [id, next.state, next.revision, toPgJson(next.claim), toPgJson(next.receipt)]);
      authority.requireReady(epoch);
      return fromRow(updated);
    });
  };
  const list = async ({ after = null, limit = PAGE_SIZE } = {}) => {
    uuid.nullable().parse(after); z.number().int().min(1).max(PAGE_SIZE).parse(limit);
    const { rows } = await db.query(`SELECT * FROM peer_execution_operations
      WHERE ($1::uuid IS NULL OR operation_id > $1) ORDER BY operation_id LIMIT $2`, [after, limit]);
    return rows.map(fromRow);
  };
  const listActive = async () => {
    const { rows } = await db.query(`SELECT * FROM peer_execution_operations
      WHERE state IN ('queued', 'draining', 'in-flight', 'awaiting-reconnect', 'uncertain') ORDER BY created_at LIMIT 32`);
    return rows.map(fromRow);
  };
  const captureRestore = async (client, id, started) => {
      if (started.phase !== 'capturing') return started; // Never recapture a rewound database on retry.
      const pending = `${authority.recoveryPath}.${makeId()}.pending`;
      authority.assertWrite(pending, 'peer execution restore evidence');
      const fd = fs.openSync(pending, 'wx', 0o600);
      const hash = createHash('sha256');
      let count = 0;
      const write = fact => {
        factSchema.parse(fact);
        const bytes = Buffer.from(JSON.stringify(fact) + '\n');
        if (bytes.length > MAX_FACT_BYTES) throw executionAuthorityError('A recovery fact exceeds the bounded record size.');
        fs.writeFileSync(fd, bytes); hash.update(bytes); count++;
      };
      try {
        let after = null;
        for (;;) {
          const { rows } = await client.query(`SELECT * FROM peer_execution_operations
            WHERE ($1::uuid IS NULL OR operation_id > $1) ORDER BY operation_id LIMIT $2`, [after, PAGE_SIZE]);
          for (const row of rows) write({ kind: 'operation', value: fromRow(row) });
          if (rows.length < PAGE_SIZE) break;
          after = rows.at(-1).operation_id;
        }
        let floorAfter = [null, null, null];
        for (;;) {
          const { rows } = await client.query(`SELECT * FROM peer_execution_generation_floors
            WHERE ($1::uuid IS NULL OR (host_instance_id, peer_instance_id, action) > ($1::uuid, $2::uuid, $3::text))
            ORDER BY host_instance_id, peer_instance_id, action LIMIT $4`, [...floorAfter, PAGE_SIZE]);
          for (const row of rows) write({ kind: 'floor', value: { hostInstanceId: row.host_instance_id,
            peerInstanceId: row.peer_instance_id, action: row.action, generation: Number(row.generation) } });
          if (rows.length < PAGE_SIZE) break;
          const last = rows.at(-1);
          floorAfter = [last.host_instance_id, last.peer_instance_id, last.action];
        }
        fs.fsyncSync(fd);
      } finally { fs.closeSync(fd); }
      fs.renameSync(pending, authority.recoveryPath);
      if (process.platform !== 'win32') {
        const directory = fs.openSync(dataDir, 'r');
        try { fs.fsyncSync(directory); } finally { fs.closeSync(directory); }
      }
      return authority.recordSnapshot(id, hash.digest('hex'), count);
  };
  const prepareRestore = id => locked(client => captureRestore(client, id, authority.beginRestore(id)));
  const adoptEmptyRestore = id => locked(async client => {
    uuid.parse(id);
    // Before any file publication/recovery, prove this receiver never had facts
    // in the restored schema. A normal capture can NEVER take this retry path.
    const { rows: [row] } = await client.query('SELECT EXISTS(SELECT 1 FROM peer_execution_operations) OR EXISTS(SELECT 1 FROM peer_execution_generation_floors) AS occupied');
    let current;
    if (!row.occupied) current = authority.recoverEmptyAdoption(id);
    else current = authority.read();
    if (current?.phase === 'ready' && current.settledRecoveryId === id) return current;
    if (current && current.emptyAdoptionId !== id) return current;
    if (row.occupied) throw executionAuthorityError('Legacy empty adoption has unexpected permanent execution facts.');
    return captureRestore(client, id, authority.beginEmptyAdoption(id));
  });
  // Stream twice: authenticate ALL recovery bytes before applying any rows.
  const visitFacts = async visit => {
    const hash = createHash('sha256');
    let buffer = Buffer.alloc(0), count = 0;
    const stream = fs.createReadStream(authority.recoveryPath, { highWaterMark: MAX_FACT_BYTES });
    for await (const chunk of stream) {
      hash.update(chunk); buffer = Buffer.concat([buffer, chunk]);
      let newline;
      while ((newline = buffer.indexOf(10)) >= 0) {
        if (newline >= MAX_FACT_BYTES) throw executionAuthorityError('Oversized execution recovery fact.');
        const fact = factSchema.parse(JSON.parse(buffer.subarray(0, newline).toString('utf8')));
        buffer = buffer.subarray(newline + 1); count++;
        if (visit) await visit(fact);
      }
      if (buffer.length >= MAX_FACT_BYTES) throw executionAuthorityError('Oversized execution recovery fact.');
    }
    if (buffer.length) throw executionAuthorityError('Truncated execution recovery evidence.');
    return { digest: hash.digest('hex'), count };
  };
  const reconcileRestore = async id => {
    const record = authority.read();
    if (record?.phase === 'ready' && record.settledRecoveryId === id) return record;
    if (record?.recovery?.id !== id || record.phase !== 'reconciling')
      throw executionAuthorityError('Complete execution recovery capture before reconciling.');
    const proof = await visitFacts();
    if (proof.digest !== record.recovery.digest || proof.count !== record.recovery.count)
      throw executionAuthorityError('Execution recovery evidence is missing or conflicting.');
    const { createMaintenanceAdmission } = await import('../lib/maintenanceAdmission.js');
    const coordinator = createMaintenanceAdmission(dataDir);
    const journal = coordinator.getExclusive();
    const journalIdentity = journal && { id: journal.id, revision: journal.revision, fingerprint: journal.fingerprint };
    const journalOperation = row => ({ operationId: row.operationId, requestId: row.binding.requestId,
      peerInstanceId: row.binding.peerInstanceId, hostInstanceId: row.binding.hostInstanceId,
      grantId: row.binding.grantId, grantGeneration: row.binding.grantGeneration, scope: row.binding.scope,
      pairBinding: row.binding.pairBinding, intent: row.binding.intent, receiverVersion: row.binding.receiverVersion,
      evidenceDigest: row.binding.evidenceDigest });
    const mergeClaims = claims => {
      const present = claims.filter(Boolean);
      if (present.some(claim => claim.id !== present[0].id || claim.fingerprint !== present[0].fingerprint))
        throw executionAuthorityError('Conflicting execution owner evidence cannot be discarded.');
      return present.reduce((newest, claim) => !newest || claim.revision > newest.revision ? claim : newest, null);
    };
    await locked(async client => {
      const replayed = await visitFacts(async fact => {
      if (fact.kind === 'floor') {
        const floor = fact.value;
        await client.query(`INSERT INTO peer_execution_generation_floors (host_instance_id, peer_instance_id, action, generation)
          VALUES ($1, $2, $3, $4) ON CONFLICT (host_instance_id, peer_instance_id, action)
          DO UPDATE SET generation = GREATEST(peer_execution_generation_floors.generation, EXCLUDED.generation)`,
        [floor.hostInstanceId, floor.peerInstanceId, floor.action, floor.generation]);
        return;
      }
      const saved = fact.value;
      const { rows } = await client.query(`SELECT * FROM peer_execution_operations WHERE operation_id = $1 OR
        (host_instance_id = $2 AND peer_instance_id = $3 AND request_id = $4) FOR UPDATE`,
      [saved.operationId, saved.binding.hostInstanceId, saved.binding.peerInstanceId, saved.binding.requestId]);
      if (rows.some(row => row.operation_id !== saved.operationId || fromRow(row).fingerprint !== saved.fingerprint))
        throw executionAuthorityError('Restored execution consumption conflicts with non-rewound evidence.');
      if (rows.some(row => terminal(row.state) && terminal(saved.state) && JSON.stringify(receiptSchema.parse(row.receipt)) !== JSON.stringify(saved.receipt)))
        throw executionAuthorityError('Restored terminal evidence conflicts with non-rewound evidence.');
      if (rows.some(row => terminal(row.state) && !terminal(saved.state)))
        throw executionAuthorityError('Restored completion contradicts non-rewound unresolved ownership.');
      if (journal?.operation.operationId === saved.operationId
        && JSON.stringify(journal.operation) !== JSON.stringify(journalOperation(saved)))
        throw executionAuthorityError('Preserved journal ownership conflicts with permanent request consumption.');
      const owner = mergeClaims([saved.claim, ...rows.map(row => fromRow(row).claim),
        journal?.operation.operationId === saved.operationId ? journalIdentity : null]);
      // Missing DB claim is not proof of no launch: journal-start/DB-rollback
      // can leave queued/draining storage behind an already-started owner.
      const state = terminal(saved.state) ? saved.state : 'uncertain';
      const receipt = terminal(saved.state) ? saved.receipt : null;
      await client.query(`INSERT INTO peer_execution_operations
        (operation_id, host_instance_id, peer_instance_id, request_id, fingerprint, binding, execution_epoch, state, revision, claim, receipt)
        VALUES ($1,$2,$3,$4,$5,$6::jsonb,$7,$8,$9,$10::jsonb,$11::jsonb)
        ON CONFLICT (operation_id) DO UPDATE SET state = EXCLUDED.state,
        revision = GREATEST(peer_execution_operations.revision, EXCLUDED.revision), claim = EXCLUDED.claim, receipt = EXCLUDED.receipt, updated_at = NOW()`,
      [saved.operationId, saved.binding.hostInstanceId, saved.binding.peerInstanceId, saved.binding.requestId, saved.fingerprint,
        toPgJson(saved.binding), saved.binding.executionEpoch, state, saved.revision + 1, toPgJson(owner), toPgJson(receipt)]);
      });
      if (replayed.digest !== proof.digest || replayed.count !== proof.count)
        throw executionAuthorityError('Execution recovery evidence changed during reconciliation.');
    // An old/different snapshot may contain owners not present in this machine's
    // capture. They also cannot become launchable just because generic restore ended.
      await client.query(`UPDATE peer_execution_operations SET state = 'uncertain', revision = revision + 1,
        receipt = NULL, updated_at = NOW() WHERE state IN ('queued', 'draining', 'in-flight', 'awaiting-reconnect')`);
      if (journal) {
        const { rows: [owner] } = await client.query('SELECT * FROM peer_execution_operations WHERE operation_id = $1', [journal.operation.operationId]);
        if (!owner || JSON.stringify(journal.operation) !== JSON.stringify(journalOperation(fromRow(owner))))
          throw executionAuthorityError('Preserved journal ownership has no matching permanent consumption.');
      }
      const latest = coordinator.getExclusive();
      if (JSON.stringify(latest) !== JSON.stringify(journal))
        throw executionAuthorityError('Preserved journal ownership changed during reconciliation.');
      const current = authority.read();
      if (current?.epoch !== record.epoch || current.recovery?.id !== id)
        throw executionAuthorityError('The execution recovery owner changed during reconciliation.');
    });
    const current = authority.read();
    if (current?.phase === 'ready' && current.settledRecoveryId === id) return current;
    return authority.completeRestore(id);
  };
  return { initialize, consume, retireUnconsumed, read, readRequest, generationFloor, list, listActive, transition, advanceGenerationFloor, prepareRestore, adoptEmptyRestore, reconcileRestore, authority };
}