Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { maintenance } from './maintenanceAdmission.js';
import { backupSharedAdmission } from './backupSharedAdmission.js';
/**
* Cross-process admission for durable file-plus-row publications during backup.
* Callers hold one mutation lease across BOTH stores, not around individual
* filesystem or SQL operations. The backup closes admission before rsync and
* keeps it closed through the database dump and manifest write.
*
* Database maintenance shares the boundary. A cut is refused while a maintenance
* fence is up, and maintenance that must not observe or replace a half-published
* pair takes the same cut around its own destructive step. Publication itself
* waits for a snapshot; unreadable ownership fails closed. Under a database
* fence the database already rejects the row write.
* Every retryable refusal carries BACKUP_SNAPSHOT_BUSY.
*/
import { AsyncLocalStorage } from 'node:async_hooks';
import { assertDatabaseAdmission } from './databaseMaintenanceJournal.js';
const publicationScope = new AsyncLocalStorage();
const DEFAULT_DRAIN_TIMEOUT_MS = 120_000;
let admitted = 0;
let cutRequested = false;
let cutActive = false;
let drainWaiters = [];
let publicationWaiters = [];
const busyError = message => Object.assign(new Error(message), { status: 503, code: 'BACKUP_SNAPSHOT_BUSY' });
function releasePublications() {
const waiters = publicationWaiters;
publicationWaiters = [];
for (const resolve of waiters) resolve();
}
const DEFAULT_WAIT_NOTICE_MS = 60_000;
/**
* Hold one admission across an entire file-plus-row workflow. Nested calls reuse it.
*
* A caller publishing output that already exists (a finished render) passes
* `onWait` with a `timeoutMs` long enough to outlast a whole backup cut, which
* spans rsync and the database dump. `onWait` runs every `waitNoticeMs` while
* admission is still closed, so the caller can keep its job watchdog alive and
* say why it is waiting; only a wait that started no writes is retried.
*/
export async function withBackupAssetPublication(work, { timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS, onWait = null, waitNoticeMs = DEFAULT_WAIT_NOTICE_MS } = {}) {
const deadline = Date.now() + timeoutMs;
for (;;) {
let started = false;
const attemptMs = onWait ? Math.max(0, Math.min(waitNoticeMs, deadline - Date.now())) : timeoutMs;
try {
return await maintenance.continueSettlement(() => withAdmittedBackupAssetPublication(() => {
started = true;
return work();
}, attemptMs), { hasStarted: () => started });
} catch (error) {
if (!onWait || started || error?.code !== 'BACKUP_SNAPSHOT_BUSY' || Date.now() >= deadline) throw error;
onWait(error);
}
}
}
function publicationTimeoutError() {
let owner;
try { owner = backupSharedAdmission.status().cut; }
catch (error) { owner = { unreadable: true, error: error.message }; }
return Object.assign(busyError(`Timed out waiting for backup publication admission${owner?.path ? ` at ${owner.path}` : ''}. No publication writes started. Reconcile the interrupted backup or restore before retiring its exact ownership record; do not remove it by age or PID.`), {
owner, ...(owner?.path ? { recoveryPath: owner.path } : {}),
});
}
async function waitForLocalCut(deadline) {
const remaining = deadline - Date.now();
if (remaining <= 0) throw publicationTimeoutError();
await new Promise((resolve, reject) => {
const clear = () => {
clearTimeout(timer);
publicationWaiters = publicationWaiters.filter(waiter => waiter !== resumed);
};
const resumed = () => { clear(); resolve(); };
const timer = setTimeout(() => { clear(); reject(publicationTimeoutError()); }, remaining);
publicationWaiters.push(resumed);
});
}
function retainUncertainPublication(scope, error) {
if (error?.backupPublicationUncertain !== true) return;
scope.uncertain = true;
// Diagnostic decoration is best effort; a frozen error cannot release authority.
try {
error.backupPublicationOwner = { ...scope.sharedLease.owner, path: scope.sharedLease.path };
error.recoveryPath ??= scope.sharedLease.path;
} catch { /* The durable owner remains the recovery authority. */ }
}
async function withAdmittedBackupAssetPublication(work, timeoutMs) {
const scope = publicationScope.getStore();
if (scope?.active) {
try { return await work(); }
catch (error) {
retainUncertainPublication(scope, error);
throw error;
}
}
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) throw new TypeError('Publication timeoutMs must be a non-negative finite number');
const deadline = Date.now() + timeoutMs;
// Work spawned by a still-admitted lease joins it: the cut already waits for that lease.
const joinsAdmitted = scope?.spawnedBy?.active === true;
while (!joinsAdmitted && (cutRequested || cutActive)) {
await waitForLocalCut(deadline);
}
let sharedLease;
while (!(sharedLease = backupSharedAdmission.tryPublication({ parent: joinsAdmitted ? scope.spawnedBy.sharedLease : null }))) {
const remaining = deadline - Date.now();
if (remaining <= 0) throw publicationTimeoutError();
await new Promise(resolve => setTimeout(resolve, Math.min(25, remaining)));
}
admitted += 1;
const lease = { active: true, sharedLease };
try {
return await publicationScope.run(lease, work);
} catch (error) {
retainUncertainPublication(lease, error);
throw error;
} finally {
lease.active = false;
try {
if (lease.uncertain) {
// The owner was already durable before work started. A diagnostic
// marker failure must never release that existing recovery blocker.
try { sharedLease.markUncertain(); }
catch (error) { console.error(`❌ Backup publication recovery marker failed at ${sharedLease.path}: ${error.message}`); }
} else sharedLease.release();
} finally {
// Settled local callbacks no longer count as running. The retained
// shared owner supplies concrete diagnostics to the next refused cut.
admitted -= 1;
if (admitted === 0) {
const waiters = drainWaiters;
drainWaiters = [];
for (const resolve of waiters) resolve();
}
}
}
}
/**
* Whether the caller runs inside an active admission lease, so no cut can be
* active until its work settles. A listener scope created by
* runOutsideBackupAssetPublication is not a lease of its own.
*/
export function holdsBackupAssetPublication() {
return publicationScope.getStore()?.active === true;
}
/** Read-only ownership evidence for interrupted-publication recovery. */
export function backupPublicationAdmissionStatus() {
return backupSharedAdmission.status();
}
/**
* Whether a backup cut is requested or active. Optional housekeeping that would
* otherwise wait out the whole snapshot can defer to its next pass instead.
*/
export function backupSnapshotCutPending() {
return cutRequested || cutActive || backupSharedAdmission.cutPending();
}
/** Wait for already admitted workflows to settle, or reject when they do not. */
async function awaitDrain(timeoutMs) {
let timer;
let onDrained;
try {
await Promise.race([
new Promise(resolve => {
onDrained = resolve;
drainWaiters.push(resolve);
}),
new Promise((_, reject) => {
timer = setTimeout(() => reject(busyError('Timed out draining asset publications for backup')), timeoutMs);
timer.unref?.();
}),
]);
} finally {
clearTimeout(timer);
drainWaiters = drainWaiters.filter(resolve => resolve !== onDrained);
}
}
/**
* Run a synchronous fan-out (an event emit) so each listener acquires its OWN
* lease while the caller's lease is still held, even if a cut is already
* waiting to drain. A listener reusing the caller's lease would continue
* unadmitted once the caller finished and could land mid-cut; one queued behind
* the cut would leave the caller's publication half drained. Listeners must
* call withBackupAssetPublication before their first await, and the caller must
* not await them.
*/
export function runOutsideBackupAssetPublication(work) {
return publicationScope.run({ active: false, spawnedBy: publicationScope.getStore() }, work);
}
/**
* Close admission and drain already admitted workflows before the caller reads
* or replaces both stores. The returned release function owns only this cut. A
* failed drain (timeout, or a maintenance fence that closed meanwhile) reopens
* admission and rejects: runBackup then marks its incomplete snapshot failed,
* and maintenance acceptance refuses before publishing its fence.
*/
export async function acquireBackupSnapshotCut({ timeoutMs = DEFAULT_DRAIN_TIMEOUT_MS } = {}) {
// A cut requested from inside an admitted workflow would wait on itself for the whole timeout.
if (publicationScope.getStore()?.active) throw new Error('Backup snapshot cut cannot be acquired inside an asset publication');
if (cutRequested || cutActive) throw busyError('Backup snapshot cut already owned');
assertDatabaseAdmission();
const sharedCut = backupSharedAdmission.reserveCut();
cutRequested = true;
const deadline = Date.now() + timeoutMs;
try {
if (admitted > 0) await awaitDrain(timeoutMs);
let blockers;
while ((blockers = backupSharedAdmission.publications()).length > 0) {
if (Date.now() >= deadline) throw Object.assign(busyError('Timed out draining asset publications for backup. Reconcile interrupted publications before retrying; ownership records must not be removed by age or PID.'), { blockers });
await new Promise(resolve => setTimeout(resolve, Math.min(25, Math.max(1, deadline - Date.now()))));
}
assertDatabaseAdmission();
} catch (error) {
sharedCut.release();
cutRequested = false;
releasePublications();
throw error;
}
cutActive = true;
cutRequested = false;
let released = false;
return () => {
if (released) return;
sharedCut.release();
released = true;
cutActive = false;
releasePublications();
};
}