Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Receiver-owned execution state machine. An accepted request is never a completion. */
import { createHash } from 'node:crypto';
import { canonicalStringify } from '../lib/objects.js';
import { createMutex } from '../lib/asyncMutex.js';
import { peerExecutionError } from './peerExecutionGrants.js';
import { peerExecutionPreflightPayloadSchema } from '../lib/peerAdminValidation.js';
import { peerAdminPairBinding, signPeerAdmin, verifyPeerAdminSignature } from './peerAdministration.js';
const hash = value => createHash('sha256').update(canonicalStringify(value)).digest('hex');
const claimRef = claim => ({ id: claim.id, revision: claim.revision, fingerprint: claim.fingerprint });
const terminal = operation => ['succeeded', 'failed'].includes(operation.state);
const fail = (code, message) => { throw peerExecutionError(code, message); };
const journalOperation = operation => {
const { executionEpoch: _epoch, ...binding } = operation.binding;
return { operationId: operation.operationId, ...binding };
};
export function createPeerExecutionReceiver({ ledger, grants, coordinator, adapters, caller, version,
now = Date.now, changed = () => {}, identityLock = fn => fn(), terminalProofs = new Map(), resume = input => coordinator.resume(input) }) {
const withLock = createMutex();
const locked = fn => identityLock(() => withLock(fn));
const preflights = new Map();
const pending = new Map();
const running = new Set();
let draining = false;
let drainAgain = false;
const emit = () => { try { changed(); } catch { console.error('❌ Peer execution status notification failed.'); } };
const assertCurrent = async (peerId, binding) => {
const pair = await grants.current(peerId, binding.intent.action);
const grant = pair.grant;
ledger.authority.requireReady(binding.executionEpoch);
if (pair.self.instanceId !== binding.hostInstanceId || pair.peer.instanceId !== binding.peerInstanceId
|| peerAdminPairBinding(pair.peer, pair.self) !== binding.pairBinding
|| grant.id !== binding.grantId || grant.generation !== binding.grantGeneration
|| version !== binding.receiverVersion) fail('PEER_EXECUTION_AUTHORITY_CHANGED', 'Pair, grant, receiver or execution authority changed.');
return pair;
};
const receipt = (operation, pair) => {
const payload = { protocolVersion: 1, scope: 'execution-v1', requestId: operation.binding.requestId,
senderInstanceId: pair.peer.instanceId, targetInstanceId: pair.self.instanceId,
operationId: operation.operationId, state: operation.state, revision: operation.revision, code: operation.receipt?.code ?? null };
return { payload, signature: signPeerAdmin(pair.peer, 'execution-receipt', payload) };
};
const findRequest = (pair, requestId) => ledger.readRequest({ hostInstanceId: pair.self.instanceId, peerInstanceId: pair.peer.instanceId, requestId });
const preflight = (req, input) => locked(async () => {
for (const [key, value] of preflights) if (value.payload.expiresAt <= now()) preflights.delete(key);
const pair = await caller(req);
// Default-deny is independent of database availability. Do not read the
// ledger or probe adapters until this exact pair has an execution grant.
const { grant } = await grants.current(pair.peer.id, input.intent.action);
if (await findRequest(pair, input.requestId)) fail('PEER_EXECUTION_CONSUMED', 'Use status for an already-consumed request.');
if (preflights.size >= 128) fail('PEER_EXECUTION_PREFLIGHT_LIMIT', 'Too many outstanding execution preflights.');
const evidence = await adapters.prepare(input.intent);
const binding = { hostInstanceId: pair.self.instanceId, peerInstanceId: pair.peer.instanceId,
requestId: input.requestId, grantId: grant.id, grantGeneration: grant.generation, scope: 'execution-v1',
pairBinding: peerAdminPairBinding(pair.peer, pair.self), intent: input.intent, receiverVersion: version,
evidenceDigest: hash(evidence), executionEpoch: grant.executionEpoch };
await caller(req);
await assertCurrent(pair.peer.id, binding);
const payload = { protocolVersion: 1, scope: 'execution-v1', requestId: input.requestId,
grantId: grant.id, grantGeneration: grant.generation, intent: input.intent, version,
evidenceDigest: binding.evidenceDigest, executionEpoch: binding.executionEpoch,
senderInstanceId: pair.peer.instanceId, targetInstanceId: pair.self.instanceId,
expiresAt: Math.min(now() + 60_000, grant.expiresAt) };
const key = `${pair.peer.instanceId}:${input.requestId}`;
if (preflights.has(key)) fail('PEER_EXECUTION_PREFLIGHT_REPLAY', 'Use a fresh request identity for preflight.');
preflights.set(key, { payload, binding, evidence });
return { payload, signature: signPeerAdmin(pair.peer, 'execution-preflight', payload) };
});
const finish = async (operation, result, claim) => {
const state = result.state;
if (!['succeeded', 'failed'].includes(state) || !/^[A-Z0-9_]{1,100}$/.test(result.code))
fail('PEER_EXECUTION_RECEIPT_INVALID', 'The adapter did not prove a terminal outcome.');
const epoch = ledger.authority.read()?.epoch;
const proof = { outcome: state, code: result.code, evidenceDigest: result.evidenceDigest ?? hash(result), executionEpoch: epoch };
const completed = terminal(operation) ? operation : await ledger.transition(operation.operationId, operation.revision, state, { receipt: proof, authorityEpoch: epoch });
if (claim) ledger.authority.requireReady(completed.receipt.executionEpoch);
if (claim) {
const current = coordinator.getExclusive();
if (!current || current.id !== claim.id || current.fingerprint !== claim.fingerprint
|| hash(current.operation) !== hash(journalOperation(completed))
|| (completed.claim && (completed.claim.id !== current.id || completed.claim.fingerprint !== current.fingerprint)))
fail('PEER_EXECUTION_CLAIM_CHANGED', 'Completion cannot release a different maintenance owner.');
const receiptDigest = hash(completed.receipt);
terminalProofs.set(current.operation.operationId, { fingerprint: current.fingerprint, receiptDigest, outcome: state });
coordinator.settleExclusive(claimRef(current), { outcome: state, receiptDigest });
terminalProofs.delete(current.operation.operationId);
}
pending.delete(operation.operationId);
const status = coordinator.status();
if (status.hold?.owner === `peer-execution:${operation.operationId}` && !coordinator.getExclusive())
await resume({ id: status.hold.id, revision: status.hold.revision });
emit();
return completed;
};
const uncertain = async (operation, claim) => {
const current = coordinator.getExclusive();
if (claim && current?.id === claim.id && current.phase !== 'uncertain')
coordinator.transitionExclusive(claimRef(current), 'uncertain');
if (!terminal(operation) && operation.state !== 'uncertain')
await ledger.transition(operation.operationId, operation.revision, 'uncertain');
pending.delete(operation.operationId);
emit();
};
const drain = async () => {
if (draining) { drainAgain = true; return; }
draining = true;
try {
for (const [id, accepted] of pending) {
let launched;
await locked(async () => {
let operation = await ledger.read(id);
if (!operation || terminal(operation)) { pending.delete(id); return; }
let claim;
try {
await assertCurrent(accepted.peerId, operation.binding);
let status = coordinator.status();
if (status.state === 'normal') status = coordinator.begin({ reason: 'Authorized peer administration; existing work settles naturally.', owner: `peer-execution:${id}` });
if (status.hold?.owner !== `peer-execution:${id}`) return;
if (operation.state === 'queued') operation = await ledger.transition(id, operation.revision, 'draining');
if (status.state !== 'ready') return;
claim = coordinator.claimReady({ id: status.hold.id, revision: status.hold.revision, operation: journalOperation(operation) }, coordinator.observeIdle());
const evidence = await adapters.prepare(operation.binding.intent);
if (hash(evidence) !== operation.binding.evidenceDigest)
fail('PEER_EXECUTION_EVIDENCE_CHANGED', 'Receiver-side requirements changed during drain. Preview again.');
await assertCurrent(accepted.peerId, operation.binding);
// No await between the last authority check and the coordinator's
// atomic idle/revision transition; resume and ordinary admission fence here.
claim = coordinator.transitionExclusive(claimRef(claim), 'in-flight', coordinator.observeIdle());
operation = await ledger.transition(id, operation.revision, 'in-flight', { claim: claimRef(claim) });
await assertCurrent(accepted.peerId, operation.binding);
const capability = coordinator.issueExecutionCapability(claim);
const result = await adapters.run(operation.binding.intent, evidence, { capability, coordinator });
launched = { operation, claim, result };
running.add(id);
pending.delete(id);
emit();
} catch (error) {
const current = coordinator.getExclusive();
if (current?.operation.operationId === id && current.phase !== 'reserved') {
await uncertain(operation, current);
} else {
if (current?.operation.operationId === id) coordinator.settleExclusive(claimRef(current), { outcome: 'cancelled', receiptDigest: null });
await finish(operation, { state: 'failed', code: /^[A-Z0-9_]{1,100}$/.test(error.code) ? error.code : 'PEER_EXECUTION_PREFLIGHT_FAILED' }, null);
}
}
});
if (launched) {
const { operation, claim, result } = launched;
// Completion owns its rejection outside the Express lifecycle. No
// timeout, rejection or reconnect path ever calls run again.
Promise.resolve(result.completion ?? result).then(outcome => locked(async () => {
const current = await ledger.read(operation.operationId);
running.delete(operation.operationId);
if (outcome.state === 'awaiting-reconnect') {
const owner = coordinator.getExclusive();
coordinator.transitionExclusive(claimRef(owner), 'awaiting-reconnect');
await ledger.transition(current.operationId, current.revision, 'awaiting-reconnect');
emit();
} else await finish(current, outcome, claim);
})).catch(async () => {
running.delete(operation.operationId);
try { await locked(async () => uncertain(await ledger.read(operation.operationId), claim)); }
catch { console.error('❌ Peer execution completion is uncertain; maintenance ownership remains held.'); }
});
}
}
} finally {
draining = false;
if (drainAgain) { drainAgain = false; queueMicrotask(kick); }
}
};
const kick = () => { drain().catch(() => console.error('❌ Peer execution drain needs recovery; no retry launch was scheduled.')); };
const dispatch = async (req, input) => {
const response = await locked(async () => {
const pair = await caller(req);
const previous = await findRequest(pair, input.requestId);
if (previous) {
const b = previous.binding;
if (b.grantId !== input.grantId || b.grantGeneration !== input.grantGeneration || b.evidenceDigest !== input.evidenceDigest
|| b.executionEpoch !== input.executionEpoch || b.receiverVersion !== input.version || hash(b.intent) !== hash(input.intent))
fail('PEER_EXECUTION_REPLAY_CONFLICT', 'This request identity already belongs to different input.');
return receipt(previous, pair);
}
const key = `${pair.peer.instanceId}:${input.requestId}`;
const saved = preflights.get(key);
if (!saved || saved.payload.expiresAt <= now()) fail('PEER_EXECUTION_PREFLIGHT_STALE', 'A fresh receiver preflight is required.');
const { scope: _scope, senderInstanceId: _sender, targetInstanceId: _target, expiresAt: _expiry, ...expected } = saved.payload;
if (hash(expected) !== hash(input)) fail('PEER_EXECUTION_PREFLIGHT_STALE', 'Dispatch differs from the signed preflight.');
await assertCurrent(pair.peer.id, saved.binding);
const { operation, isNew } = await ledger.consume(saved.binding);
preflights.delete(key);
if (isNew) pending.set(operation.operationId, { peerId: pair.peer.id });
emit();
return receipt(operation, pair);
});
kick();
return response;
};
const reconcileOperation = async operation => {
const claim = coordinator.getExclusive();
const owns = claim?.operation.operationId === operation.operationId;
if (terminal(operation)) {
await finish(operation, { state: operation.state, code: operation.receipt.code }, owns ? claim : null);
} else if ((owns && claim.phase !== 'reserved') || ['in-flight', 'awaiting-reconnect', 'uncertain'].includes(operation.state)) {
const result = await adapters.reconcile(operation);
if (result && ['succeeded', 'failed'].includes(result.state)) await finish(operation, result, owns ? claim : null);
else await uncertain(operation, owns ? claim : null);
} else {
if (owns) coordinator.settleExclusive(claimRef(claim), { outcome: 'cancelled', receiptDigest: null });
await finish(operation, { state: 'failed', code: 'PEER_EXECUTION_INTERRUPTED_BEFORE_LAUNCH' }, null);
}
};
const status = (req, { requestId, preflight: envelope }) => locked(async () => {
const pair = await caller(req);
let operation = await findRequest(pair, requestId);
if (!operation && envelope) {
const payload = envelope.payload;
if (!peerExecutionPreflightPayloadSchema.safeParse(payload).success
|| !verifyPeerAdminSignature(pair.peer, 'execution-preflight', payload, envelope.signature)
|| payload.requestId !== requestId || payload.senderInstanceId !== pair.peer.instanceId
|| payload.targetInstanceId !== pair.self.instanceId)
fail('PEER_EXECUTION_RECOVERY_UNVERIFIED', 'The original request preview could not be verified.');
operation = await ledger.retireUnconsumed({ hostInstanceId: pair.self.instanceId, peerInstanceId: pair.peer.instanceId,
requestId, grantId: payload.grantId, grantGeneration: payload.grantGeneration, scope: 'execution-v1',
pairBinding: peerAdminPairBinding(pair.peer, pair.self), intent: payload.intent, receiverVersion: payload.version,
evidenceDigest: payload.evidenceDigest, executionEpoch: payload.executionEpoch });
preflights.delete(`${pair.peer.instanceId}:${requestId}`);
}
if (!operation) throw peerExecutionError('PEER_EXECUTION_NOT_FOUND', 'Execution request not found.', 404);
// A deliberate receipt read may observe completion written after server boot.
// It never dispatches work, retries a launch, or cancels live local work.
if (!pending.has(operation.operationId) && !running.has(operation.operationId)) {
await reconcileOperation(operation);
operation = await ledger.read(operation.operationId);
}
return receipt(operation, pair);
});
const recover = () => locked(async () => {
// Reconcile both sides of the terminal-DB -> journal-settle -> resume gap.
const rows = new Map((await ledger.listActive()).map(row => [row.operationId, row]));
const owner = coordinator.getExclusive()?.operation.operationId
?? coordinator.status().hold?.owner?.match(/^peer-execution:([a-f0-9-]{36})$/)?.[1];
if (owner && !rows.has(owner)) {
const operation = await ledger.read(owner);
if (!operation) fail('PEER_EXECUTION_OWNER_MISSING', 'Maintenance ownership has no matching durable consumption.');
rows.set(owner, operation);
}
for (const operation of rows.values()) await reconcileOperation(operation);
});
return { preflight, dispatch, status, recover, kick, drain, notify: emit,
describe: peerId => locked(() => grants.describe(peerId)),
saveGrant: async (input, req) => { const result = await locked(() => grants.save(input, req)); emit(); kick(); return result; } };
}