Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { existsSync, lstatSync, mkdirSync, linkSync, unlinkSync, openSync, closeSync, fsyncSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import { z } from 'zod';
import { PATHS } from './paths.js';
import { assertNotRealDataWrite } from './testDataIsolation.js';
import { createDatabaseAuthority } from './databaseAuthority.js';
export const databaseMaintenanceEndpointSchema = z.object({
mode: z.enum(['native', 'docker']),
host: z.string().min(1).max(255),
port: z.number().int().min(1).max(65535),
database: z.string().min(1).max(63),
user: z.string().min(1).max(63),
}).strict();
const stages = ['accepted', 'quiescing', 'exporting', 'importing', 'committing', 'verifying', 'verified'];
const journalSchema = z.object({
version: z.literal(1),
id: z.string().uuid(),
stage: z.enum(stages),
createdAt: z.string().datetime(),
source: databaseMaintenanceEndpointSchema,
target: databaseMaintenanceEndpointSchema,
}).strict();
const coordinatorSchema = z.object({
id: z.string().uuid(),
token: z.string().uuid(),
}).strict();
const successorSchema = coordinatorSchema.extend({ previousToken: z.string().uuid() });
const producerSnapshotSchema = z.object({
id: z.string().uuid(),
producers: z.array(z.object({
name: z.enum(['portos-cos', 'portos-server']),
pmId: z.number().int().nonnegative(), pid: z.number().int().nonnegative(),
cwd: z.string().min(1), script: z.string().min(1),
status: z.enum(['online', 'stopped']),
}).strict()).length(2),
}).strict();
const sha256Schema = z.string().regex(/^[0-9a-f]{64}$/);
// Transfer evidence. The dump name is derived from the operation, never a path
// chosen by a caller, so a manifest cannot point the import anywhere else.
const transferDumpSchema = z.object({
id: z.string().uuid(),
source: databaseMaintenanceEndpointSchema,
file: z.string().regex(/^portos-maintenance-[0-9a-f-]{36}\.sql$/),
bytes: z.number().int().positive(),
sha256: sha256Schema,
}).strict();
const transferImportSchema = z.object({
id: z.string().uuid(),
target: databaseMaintenanceEndpointSchema,
sha256: sha256Schema,
}).strict();
// A restarted ordinary server proved ITS OWN pool reaches the recorded target.
// Evidence, not authority: the coordinator accepts it only for the PM2 pid it
// reads back itself AND that process's start time (a reused pid cannot inherit
// an earlier process's proof), and every booting process proves its own pool.
// startedAt is null only on Windows, which has no process-table start time.
const targetProofSchema = z.object({
id: z.string().uuid(),
pid: z.number().int().positive(),
startedAt: z.number().int().nullable(),
target: databaseMaintenanceEndpointSchema,
}).strict();
const recoveryIntentSchema = z.object({
id: z.string().uuid(),
previousToken: z.string().uuid(),
recoveryToken: z.string().uuid(),
}).strict();
const workerGroupSchema = coordinatorSchema.extend({ pgid: z.number().int().positive() }).strict();
const decisionSchema = coordinatorSchema.extend({
stage: z.enum(stages),
action: z.discriminatedUnion('kind', [
z.object({ kind: z.literal('transition'), nextStage: z.enum(stages) }).strict(),
z.object({ kind: z.literal('recovery'), recoveryToken: z.string().uuid() }).strict(),
]),
});
const normalizedHost = host => ['localhost', '127.0.0.1', '::1', '[::1]'].includes(host.toLowerCase())
? 'loopback' : host.toLowerCase();
const databaseMaintenanceError = () => Object.assign(
new Error('Persistent database maintenance is active or requires recovery. Inspect scripts/database-maintenance.mjs status.'),
{ status: 503, code: 'DATABASE_MAINTENANCE' },
);
function syncDirectory(path) {
// Windows does not expose directory fsync through Node. The fence still
// exists before journal publication; unreadable/incomplete state fails shut.
if (process.platform === 'win32') return;
const fd = openSync(path, 'r');
try { fsyncSync(fd); } finally { closeSync(fd); }
}
function writeDurableExclusive(path, value) {
const fd = openSync(path, 'wx', 0o600);
try {
writeFileSync(fd, JSON.stringify(value, null, 2) + '\n');
fsyncSync(fd);
} finally {
closeSync(fd);
}
}
/**
* The DIRECTORY is the fence, not a successfully parsed journal. In particular,
* a crash between mkdir and journal publication must never reopen admission.
* Admission linearizes at the synchronous filesystem check, before the first
* pool checkout/await. A call admitted just before publication may acquire its
* connection or spawn afterward. This is NOT a quiescence/snapshot guarantee:
* the coordinator must drain/stop every writer and in-flight spawn before dump.
*/
export function createDatabaseMaintenanceJournal(dataDir = PATHS.data) {
const activeDir = join(dataDir, 'database-maintenance');
const recordPath = join(activeDir, 'operation.json');
const isFenced = () => {
try {
// Unlike existsSync, lstat distinguishes missing from unreadable and
// treats a dangling symlink as a fence, not a missing operation.
lstatSync(activeDir);
return true;
} catch (err) {
return err.code !== 'ENOENT';
}
};
const assertAdmission = () => {
if (isFenced()) throw databaseMaintenanceError();
};
const read = () => {
if (!isFenced()) return null;
try {
const fence = lstatSync(activeDir);
if (!fence.isDirectory() || !lstatSync(recordPath).isFile()) {
throw databaseMaintenanceError();
}
let current = journalSchema.parse(JSON.parse(readFileSync(recordPath, 'utf8')));
// Immutable publications cannot be overwritten by a delayed older writer.
// Legacy journals keep their recorded stage as the history's starting point.
for (const stage of stages.slice(stages.indexOf(current.stage) + 1)) {
const stagePath = join(activeDir, 'published-' + stage + '.json');
try { lstatSync(stagePath); } catch (err) {
if (err.code === 'ENOENT') break;
throw err;
}
if (!lstatSync(stagePath).isFile()) throw databaseMaintenanceError();
const next = journalSchema.parse(JSON.parse(readFileSync(stagePath, 'utf8')));
if (JSON.stringify(next) !== JSON.stringify({ ...current, stage })) throw databaseMaintenanceError();
current = next;
}
// The walk ends at the first missing publication, and a release or
// cancellation that moves the fence mid-walk makes every later one look
// missing. The history is complete only if the SAME fence is still in
// place afterward: a moved fence (ENOENT) reports no operation below; a
// different one at this path fails closed.
const after = lstatSync(activeDir);
if (after.ino !== fence.ino || after.dev !== fence.dev) throw databaseMaintenanceError();
return current;
} catch {
// The fence check and the record reads are separate syscalls. A release
// or cancellation that renames the fence between them is not damage:
// once the directory itself is gone, no operation is active. Anything
// still fenced (partial publication, damaged record) fails closed.
if (!isFenced()) return null;
throw databaseMaintenanceError();
}
};
const begin = ({ source, target }) => {
assertNotRealDataWrite(activeDir, 'database maintenance begin');
const record = journalSchema.parse({
version: 1, id: randomUUID(), stage: 'accepted',
createdAt: new Date().toISOString(), source, target,
});
if (source.mode === target.mode ||
(normalizedHost(source.host) === normalizedHost(target.host) && source.port === target.port)) {
throw new Error('Database maintenance requires distinct source and target backends.');
}
mkdirSync(dataDir, { recursive: true });
// Exclusive mkdir is also the cross-process begin lock. No rollback on a
// failed publication: leaving the fence closed is the recoverable outcome.
mkdirSync(activeDir, { mode: 0o700 });
syncDirectory(dataDir);
const temporary = join(activeDir, 'operation.pending');
writeDurableExclusive(temporary, record);
renameSync(temporary, recordPath);
syncDirectory(activeDir);
return record;
};
// Initial durable owner. Successors require explicit same-operation recovery
// backed by the detached supervisor's exit receipt, never PID/elapsed time.
const acquireCoordinator = (id) => {
assertNotRealDataWrite(activeDir, 'database maintenance ownership');
const current = read();
if (!current || current.id !== id || current.stage !== 'accepted') throw databaseMaintenanceError();
const token = randomUUID();
writeDurableExclusive(join(activeDir, 'cancel-' + id + '.claim'), { id, token });
syncDirectory(activeDir);
const owned = read();
if (!owned || owned.id !== id || owned.stage !== 'accepted') throw databaseMaintenanceError();
return token;
};
// Recovery appends an immutable successor instead of overwriting ownership.
// Each predecessor admits exactly one successor, even across processes. A
// missing/partial worker receipt never authorizes reclaiming an owner.
// The ordered ownership chain, oldest first; empty when never claimed.
const readCoordinatorChain = (id, rejectedPredecessor = null) => {
const current = read();
if (!current || current.id !== id) throw databaseMaintenanceError();
const ownerPath = join(activeDir, 'cancel-' + id + '.claim');
try { lstatSync(ownerPath); } catch (err) {
if (err.code === 'ENOENT') return [];
throw databaseMaintenanceError();
}
if (!lstatSync(ownerPath).isFile()) throw databaseMaintenanceError();
let owner = coordinatorSchema.parse(JSON.parse(readFileSync(ownerPath, 'utf8')));
if (owner.id !== id) throw databaseMaintenanceError();
const chain = [];
while (true) {
if (chain.some(value => value.token === owner.token)) throw databaseMaintenanceError();
chain.push(owner);
const nextPath = join(activeDir, 'coordinator-after-' + owner.token + '.json');
try { lstatSync(nextPath); } catch (err) {
if (err.code === 'ENOENT') return chain;
throw databaseMaintenanceError();
}
if (!lstatSync(nextPath).isFile()) throw databaseMaintenanceError();
const next = successorSchema.parse(JSON.parse(readFileSync(nextPath, 'utf8')));
if (next.id !== id || next.previousToken !== owner.token || owner.token === rejectedPredecessor) throw databaseMaintenanceError();
owner = { id, token: next.token };
}
};
const readCoordinator = (id, rejectedPredecessor = null) => readCoordinatorChain(id, rejectedPredecessor).at(-1) ?? null;
const assertCoordinator = (id, token) => {
const owner = readCoordinator(id);
if (!owner || typeof token !== 'string' || owner.token !== token) throw databaseMaintenanceError();
return owner;
};
const workerDirectory = token => join(activeDir, 'worker-' + token);
// Reserve once, BEFORE spawnDetached. Never reuse this directory: resetting
// its exit sentinel could let an old supervisor acknowledge a later worker.
// The caller must use cleanup:false and launch only the coordinator here.
const reserveCoordinatorWorker = (id, token) => {
assertNotRealDataWrite(activeDir, 'database maintenance worker reservation');
const owner = assertCoordinator(id, token);
const directory = workerDirectory(token);
try { lstatSync(directory); throw databaseMaintenanceError(); } catch (err) {
if (err.code !== 'ENOENT') throw err;
}
const pending = join(activeDir, 'worker-pending-' + randomUUID());
mkdirSync(pending, { mode: 0o700 });
try {
writeDurableExclusive(join(pending, 'owner.json'), owner);
syncDirectory(pending);
// The complete, nonempty reservation is published at once. Another
// launch cannot replace it (rename refuses a nonempty destination).
renameSync(pending, directory);
syncDirectory(activeDir);
} finally {
rmSync(pending, { recursive: true, force: true });
}
return directory;
};
const coordinatorStatusForOwner = (id, owner) => {
if (!owner) return { state: 'unclaimed' };
const directory = workerDirectory(owner.token);
try { lstatSync(directory); } catch (err) {
if (err.code === 'ENOENT') return { state: 'unregistered' };
throw databaseMaintenanceError();
}
if (!lstatSync(directory).isDirectory()) throw databaseMaintenanceError();
const bindingPath = join(directory, 'owner.json');
if (!lstatSync(bindingPath).isFile()) throw databaseMaintenanceError();
const binding = coordinatorSchema.parse(JSON.parse(readFileSync(bindingPath, 'utf8')));
if (binding.id !== id || binding.token !== owner.token) throw databaseMaintenanceError();
const exitPath = join(directory, 'exit');
try { lstatSync(exitPath); } catch (err) {
if (err.code === 'ENOENT') return { state: 'awaiting-exit' };
throw databaseMaintenanceError();
}
if (!lstatSync(exitPath).isFile()) throw databaseMaintenanceError();
const raw = readFileSync(exitPath, 'utf8');
if (!/^-?\d{1,10}\r?\n?$/.test(raw)) throw databaseMaintenanceError();
const exitCode = Number(raw.trim());
if (!Number.isSafeInteger(exitCode) || exitCode < -2147483648 || exitCode > 4294967295) throw databaseMaintenanceError();
return { state: 'exited', exitCode };
};
const coordinatorStatus = id => coordinatorStatusForOwner(id, readCoordinator(id));
// The fixed maintenance worker checks both ownership and the supervisor's
// one-use reservation. This grants no ordinary database/spawn admission.
const assertCoordinatorWorker = (id, token) => {
assertCoordinator(id, token);
if (coordinatorStatus(id).state !== 'awaiting-exit') throw databaseMaintenanceError();
return read();
};
const enterCoordinatorWorker = (id, token) => {
assertNotRealDataWrite(activeDir, 'database maintenance worker entry');
const current = assertCoordinatorWorker(id, token);
// A second invocation with copied arguments cannot become a second worker.
writeDurableExclusive(join(workerDirectory(token), 'started.json'), { id, token });
syncDirectory(workerDirectory(token));
assertCoordinatorWorker(id, token);
return current;
};
// Optional JSON record: null when absent, fenced when present but invalid.
const readRecord = (path, schema) => {
try { lstatSync(path); } catch (err) {
if (err.code === 'ENOENT') return null;
throw databaseMaintenanceError();
}
try {
if (!lstatSync(path).isFile()) throw databaseMaintenanceError();
return schema.parse(JSON.parse(readFileSync(path, 'utf8')));
} catch {
throw databaseMaintenanceError();
}
};
// For stages that run inside the one entered worker process. Entry itself
// stays one-use; this only proves the caller is that entered owner.
const assertEnteredCoordinatorWorker = (id, token) => {
const current = assertCoordinatorWorker(id, token);
const started = readRecord(join(workerDirectory(token), 'started.json'), coordinatorSchema);
if (!started || started.id !== id || started.token !== token) throw databaseMaintenanceError();
return current;
};
// The entered worker's POSIX process group. Every database child it starts
// (dump/import) inherits this group, so a successor can prove none survive.
const recordCoordinatorGroup = (id, token, pgid) => {
assertNotRealDataWrite(activeDir, 'database maintenance worker group');
assertEnteredCoordinatorWorker(id, token);
const value = workerGroupSchema.parse({ id, token, pgid });
writeDurableExclusive(join(workerDirectory(token), 'group.json'), value);
syncDirectory(workerDirectory(token));
assertEnteredCoordinatorWorker(id, token);
return value;
};
// Earlier owners of this operation. Recovery requires each one's supervisor
// exit receipt, but that says nothing about descendants it may have left.
// A worker records its group before starting any database child, so a
// predecessor without a group record (never entered, crashed before that
// point, or an inspection-only legacy worker) started no dump or import.
const readPredecessorWorkers = (id, token) => {
assertEnteredCoordinatorWorker(id, token);
const chain = readCoordinatorChain(id);
if (chain.at(-1)?.token !== token) throw databaseMaintenanceError();
return chain.slice(0, -1).map(({ token: previous }) => {
const directory = workerDirectory(previous);
const started = readRecord(join(directory, 'started.json'), coordinatorSchema);
const group = readRecord(join(directory, 'group.json'), workerGroupSchema);
if ((started && (started.id !== id || started.token !== previous))
|| (group && (!started || group.id !== id || group.token !== previous))) throw databaseMaintenanceError();
return { started: Boolean(started), pgid: group?.pgid ?? null };
});
};
const transferDumpPath = id => join(dataDir, 'db-dumps', 'portos-maintenance-' + z.string().uuid().parse(id) + '.sql');
const transferDumpRecordPath = join(activeDir, 'transfer-dump.json');
const transferImportRecordPath = join(activeDir, 'transfer-import.json');
const readTransferDump = (id) => {
const current = read();
if (!current || current.id !== id) throw databaseMaintenanceError();
const dump = readRecord(transferDumpRecordPath, transferDumpSchema);
if (dump && (dump.id !== id || JSON.stringify(dump.source) !== JSON.stringify(current.source)
|| dump.file !== 'portos-maintenance-' + id + '.sql')) throw databaseMaintenanceError();
return dump;
};
const readTransferImport = (id) => {
const current = read();
if (!current || current.id !== id) throw databaseMaintenanceError();
const receipt = readRecord(transferImportRecordPath, transferImportSchema);
if (!receipt) return null;
const dump = readTransferDump(id);
if (!dump || receipt.id !== id || receipt.sha256 !== dump.sha256
|| JSON.stringify(receipt.target) !== JSON.stringify(current.target)) throw databaseMaintenanceError();
return receipt;
};
// No-replace publication of already-fsynced bytes; the first writer wins.
const publishFirst = (destination, value) => {
const pending = join(activeDir, 'record-' + randomUUID() + '.pending');
writeDurableExclusive(pending, value);
try {
linkSync(pending, destination);
} catch (err) {
if (err.code !== 'EEXIST') throw err;
} finally {
unlinkSync(pending);
}
syncDirectory(activeDir);
};
// Immutable publication: a retry of the identical record is accepted; a
// different record is never substituted.
const publishImmutable = (destination, value) => {
publishFirst(destination, value);
if (readFileSync(destination, 'utf8') !== JSON.stringify(value, null, 2) + '\n') throw databaseMaintenanceError();
};
// Publish the COMPLETE recovery dump's identity. The caller has already
// fsynced the dump and its directory; the import may use only these bytes.
const recordTransferDump = (id, token, dump) => {
assertNotRealDataWrite(activeDir, 'database maintenance dump publication');
const current = assertEnteredCoordinatorWorker(id, token);
const value = transferDumpSchema.parse(dump);
if (current.stage !== 'exporting' || value.id !== id) throw databaseMaintenanceError();
publishImmutable(transferDumpRecordPath, value);
assertEnteredCoordinatorWorker(id, token);
return readTransferDump(id);
};
// A committed single-transaction import into the recorded target.
const recordTransferImport = (id, token, receipt) => {
assertNotRealDataWrite(activeDir, 'database maintenance import receipt');
const current = assertEnteredCoordinatorWorker(id, token);
const value = transferImportSchema.parse(receipt);
if (current.stage !== 'importing' || value.id !== id) throw databaseMaintenanceError();
publishImmutable(transferImportRecordPath, value);
assertEnteredCoordinatorWorker(id, token);
return readTransferImport(id);
};
// Bounded operator status: never paths, digests, endpoints or tokens.
const transferStatus = (id) => ({
dump: readTransferDump(id) ? 'recorded' : 'absent',
import: readTransferImport(id) ? 'committed' : 'pending',
});
// Retain the pre-stop producer identities across same-operation recovery.
// Partial publication is never interpreted as an empty producer set.
const readProducerSnapshot = (id, token) => {
assertCoordinatorWorker(id, token);
const path = join(activeDir, 'producers.json');
try {
if (!lstatSync(path).isFile()) throw databaseMaintenanceError();
const snapshot = producerSnapshotSchema.parse(JSON.parse(readFileSync(path, 'utf8')));
if (snapshot.id !== id || new Set(snapshot.producers.map(p => p.name)).size !== 2
|| new Set(snapshot.producers.map(p => p.pmId)).size !== 2) throw databaseMaintenanceError();
return snapshot.producers;
} catch (err) {
if (err.code === 'ENOENT' && read()?.stage === 'accepted') return null;
throw databaseMaintenanceError();
}
};
const recordProducerSnapshot = (id, token, producers) => {
assertNotRealDataWrite(activeDir, 'database maintenance producer inventory');
assertCoordinatorWorker(id, token);
if (read()?.stage !== 'accepted' || readProducerSnapshot(id, token)) throw databaseMaintenanceError();
const snapshot = producerSnapshotSchema.parse({ id, producers });
writeDurableExclusive(join(activeDir, 'producers.json'), snapshot);
syncDirectory(activeDir);
return readProducerSnapshot(id, token);
};
// A durable CAS for EACH outgoing (owner, stage) state. Transition and
// recovery compete on the SAME file, so a predecessor paused after checking
// its token cannot publish a new stage after losing ownership. Interrupted
// stage decisions can be completed identically before attempting recovery.
const decide = (id, token, stage, action) => {
const value = decisionSchema.parse({ id, token, stage, action });
const destination = join(activeDir, 'decision-' + stage + '-' + token + '.json');
const pending = join(activeDir, 'decision-' + randomUUID() + '.pending');
writeDurableExclusive(pending, value);
try {
linkSync(pending, destination);
} catch (err) {
if (err.code !== 'EEXIST') throw err;
} finally {
unlinkSync(pending);
}
syncDirectory(activeDir);
if (!lstatSync(destination).isFile()) throw databaseMaintenanceError();
const winner = decisionSchema.parse(JSON.parse(readFileSync(destination, 'utf8')));
if (winner.id !== id || winner.token !== token || winner.stage !== stage) throw databaseMaintenanceError();
return winner.action;
};
// This transfers journal ownership only, NEVER database authority. Before
// export/import a successor must re-establish writer/child quiescence: the
// worker's exit says nothing about descendants it might have left behind.
// No PID probe, clock, reverse operation, or source-success inference occurs.
const recoverCoordinator = (id, previousToken, recoveryToken) => {
assertNotRealDataWrite(activeDir, 'database maintenance coordinator recovery');
coordinatorSchema.parse({ id, token: recoveryToken });
coordinatorSchema.parse({ id, token: previousToken });
if (recoveryToken === previousToken) throw databaseMaintenanceError();
const owner = readCoordinator(id, recoveryToken);
// The caller persists its recovery token before attempting publication.
// A crash after link/fsync but before returning must be retryable by that
// same claimant; another claimant must never learn or reuse its token.
if (owner?.token === recoveryToken) {
const published = successorSchema.parse(JSON.parse(readFileSync(
join(activeDir, 'coordinator-after-' + previousToken + '.json'), 'utf8')));
if (published.id !== id || published.previousToken !== previousToken || published.token !== recoveryToken) throw databaseMaintenanceError();
syncDirectory(activeDir);
return recoveryToken;
}
assertCoordinator(id, previousToken);
if (coordinatorStatus(id).state !== 'exited') throw databaseMaintenanceError();
// Every unreleased stage can advance here; each interrupted transition
// completion moves strictly forward, never reverses an import to success
// or a committed mode back to the source.
while (true) {
const current = read();
if (!stages.includes(current?.stage)) throw databaseMaintenanceError();
const decision = decide(id, previousToken, current.stage, { kind: 'recovery', recoveryToken });
if (decision.kind === 'recovery') {
if (decision.recoveryToken !== recoveryToken) throw databaseMaintenanceError();
break;
}
if (read()?.stage === current.stage) transition(id, previousToken, current.stage, decision.nextStage);
}
const directory = workerDirectory(previousToken);
const fd = openSync(join(directory, 'exit'), 'r+');
try { fsyncSync(fd); } finally { closeSync(fd); }
syncDirectory(directory);
const next = { id, token: recoveryToken, previousToken };
const pending = join(activeDir, 'coordinator-' + randomUUID() + '.pending');
writeDurableExclusive(pending, next);
try {
// EEXIST is a lost race unless this is the SAME recorded recovery
// request. Different claimants must never receive the winning token.
linkSync(pending, join(activeDir, 'coordinator-after-' + previousToken + '.json'));
} catch (err) {
if (err.code !== 'EEXIST' || readCoordinator(id)?.token !== recoveryToken) throw err;
} finally {
unlinkSync(pending);
}
syncDirectory(activeDir);
assertCoordinator(id, next.token);
return next.token;
};
const transition = (id, token, expectedStage, nextStage) => {
assertNotRealDataWrite(activeDir, 'database maintenance transition');
const current = read();
if (!current || current.id !== id || !stages.includes(expectedStage)
|| stages.indexOf(nextStage) !== stages.indexOf(expectedStage) + 1) throw databaseMaintenanceError();
assertCoordinator(id, token);
// An acknowledged publication may have crashed before returning to its
// caller. Only that exact owned transition is idempotent, never a reversal.
if (current.stage === nextStage) {
syncDirectory(activeDir);
return current;
}
if (current.stage !== expectedStage) throw databaseMaintenanceError();
// Mode commit is reachable only from a committed import of this operation.
if (nextStage === 'committing' && !readTransferImport(id)) throw databaseMaintenanceError();
// Do not reinterpret interrupted writes from the old publication protocol.
if (existsSync(join(activeDir, 'stage-' + expectedStage + '.claim'))) throw databaseMaintenanceError();
const decision = decide(id, token, expectedStage, { kind: 'transition', nextStage });
if (decision.kind !== 'transition' || decision.nextStage !== nextStage) throw databaseMaintenanceError();
const next = journalSchema.parse({ ...current, stage: nextStage });
const pending = join(activeDir, 'publication-' + randomUUID() + '.pending');
writeDurableExclusive(pending, next);
try {
// link is an atomic, no-replace publication of already-fsynced bytes.
// A competing owner-token retry may win; both describe the same transition.
linkSync(pending, join(activeDir, 'published-' + nextStage + '.json'));
} catch (err) {
if (err.code !== 'EEXIST') throw err;
} finally {
unlinkSync(pending);
}
syncDirectory(activeDir);
const published = read();
if (!published || published.id !== id || stages.indexOf(published.stage) < stages.indexOf(nextStage)) throw databaseMaintenanceError();
return next;
};
const cancel = (id, source) => {
assertNotRealDataWrite(activeDir, 'database maintenance cancel');
// A per-operation exclusive cancellation claim prevents two cancellations
// from renaming a later operation after the first has released this one.
// An interrupted cancellation leaves this marker and fails closed; it must
// never be automatically stolen on a timeout or an unverified PID guess.
const initial = read();
if (!initial || initial.id !== id) throw new Error('Database maintenance operation does not match.');
const parsedSource = databaseMaintenanceEndpointSchema.parse(source);
if (JSON.stringify(initial.source) !== JSON.stringify(parsedSource)) {
throw new Error('Saved database endpoint changed; source-only cancellation refused.');
}
writeDurableExclusive(join(activeDir, 'cancel-' + id + '.claim'), { id });
const current = read();
if (current?.id !== id || current.stage !== 'accepted') throw databaseMaintenanceError();
const archiveDir = join(dataDir, 'database-maintenance-cancelled');
mkdirSync(archiveDir, { recursive: true, mode: 0o700 });
// Archive before reopening admission; retain the operation's direction as
// local recovery evidence. Never remove/overwrite an existing archive.
const destination = join(archiveDir, id);
if (existsSync(destination)) throw databaseMaintenanceError();
renameSync(activeDir, destination);
syncDirectory(archiveDir);
syncDirectory(dataDir);
return { id, stage: 'cancelled' };
};
// Operation-scoped archive for detached-writer records proven quiescent.
const reconciledWritersDirectory = join(activeDir, 'reconciled-writers');
const targetProofPath = (pid, startedAt) => join(activeDir, 'target-proof-' + z.number().int().positive().parse(pid)
+ (startedAt === undefined ? '' : '-' + z.number().int().nullable().parse(startedAt)) + '.json');
// Published by a booting ordinary server AFTER its own pool passed the
// read-only target verification. Only verifying/verified may carry one.
const recordTargetProof = (id, pid, startedAt) => {
assertNotRealDataWrite(activeDir, 'database maintenance target proof');
const current = read();
if (!current || current.id !== id || !['verifying', 'verified'].includes(current.stage)) throw databaseMaintenanceError();
publishImmutable(targetProofPath(pid, startedAt), targetProofSchema.parse({ id, pid, startedAt, target: current.target }));
return readTargetProof(id, pid, startedAt);
};
function readTargetProof(id, pid, startedAt) {
const current = read();
if (!current || current.id !== id) throw databaseMaintenanceError();
// Retain already-written proofs from older installs, but never let a
// reused PID inherit evidence belonging to another process incarnation.
const proof = readRecord(targetProofPath(pid, startedAt), targetProofSchema)
?? readRecord(targetProofPath(pid), targetProofSchema);
if (proof && (proof.id !== id || proof.pid !== pid
|| JSON.stringify(proof.target) !== JSON.stringify(current.target))) throw databaseMaintenanceError();
return proof && startedAt !== undefined && proof.startedAt !== startedAt ? null : proof;
}
/**
* Same-operation recovery, as one retryable step for the operator/API. The
* successor token is persisted BEFORE recoverCoordinator publishes it, so a
* crash after publication is completed by the next call with the SAME token
* instead of stranding an owner nobody can launch. Returns the token that may
* launch a worker, or null while the current owner's worker has no exit
* receipt (running, or interrupted before its supervisor recorded one).
*/
const prepareRecovery = (id) => {
assertNotRealDataWrite(activeDir, 'database maintenance recovery');
const current = read();
if (!current || current.id !== id) throw databaseMaintenanceError();
const owner = readCoordinator(id);
if (!owner) throw databaseMaintenanceError();
// Bind the receipt to the owner we read. A concurrent recovery may publish
// a successor before this read; its unregistered state must never authorize
// launching our stale predecessor token. Its persisted intent is adopted below.
const status = coordinatorStatusForOwner(id, owner);
if (status.state === 'unregistered') return owner.token;
if (status.state !== 'exited') return null;
const intentPath = join(activeDir, 'recovery-after-' + owner.token + '.json');
// Concurrent requests race to publish; every one adopts the winner.
if (!readRecord(intentPath, recoveryIntentSchema)) {
publishFirst(intentPath, recoveryIntentSchema.parse({ id, previousToken: owner.token, recoveryToken: randomUUID() }));
}
const intent = readRecord(intentPath, recoveryIntentSchema);
if (!intent || intent.id !== id || intent.previousToken !== owner.token) throw databaseMaintenanceError();
return recoverCoordinator(id, owner.token, intent.recoveryToken);
};
/**
* Reopen admission for a VERIFIED operation, from its entered worker only.
* The authority record (target in, source retired) is durable BEFORE the
* fence directory moves, so no process can be admitted without the stale-
* backend check seeing it. A crash between the two leaves `verified` fenced
* and the next recovery repeats both steps idempotently.
*/
const releaseAdmission = (id, token) => {
assertNotRealDataWrite(activeDir, 'database maintenance release');
const current = assertEnteredCoordinatorWorker(id, token);
if (current.stage !== 'verified' || !readTransferImport(id)) throw databaseMaintenanceError();
createDatabaseAuthority(dataDir).record({ operationId: id, source: current.source, target: current.target });
const archiveDir = join(dataDir, 'database-maintenance-completed');
mkdirSync(archiveDir, { recursive: true, mode: 0o700 });
const destination = join(archiveDir, id);
if (existsSync(destination)) throw databaseMaintenanceError();
assertEnteredCoordinatorWorker(id, token);
renameSync(activeDir, destination);
syncDirectory(archiveDir);
syncDirectory(dataDir);
return { id, stage: 'released', source: current.source.mode, target: current.target.mode };
};
return { isFenced, assertAdmission, read, begin, cancel, acquireCoordinator, transition, reconciledWritersDirectory,
recordTargetProof, readTargetProof, prepareRecovery, releaseAdmission,
reserveCoordinatorWorker, coordinatorStatus, recoverCoordinator, assertCoordinatorWorker, enterCoordinatorWorker,
readProducerSnapshot, recordProducerSnapshot, assertEnteredCoordinatorWorker, recordCoordinatorGroup,
readPredecessorWorkers, transferDumpPath, readTransferDump, readTransferImport, recordTransferDump,
recordTransferImport, transferStatus };
}
const journal = createDatabaseMaintenanceJournal();
export const assertDatabaseAdmission = journal.assertAdmission;