Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { hydratePrivateKeys, persistPrivateKeys } from './privateKeyStore.js';
import { join } from 'path';
import { stat } from 'node:fs/promises';
import { EventEmitter } from 'events';
import { safeJSONParse, PATHS, atomicWrite, tryReadFile, tryReadFileStrict } from '../lib/fileUtils.js';
import { createFileWriteQueue, drainDetachedWrites, trackDetachedWrite } from '../lib/fileWriteQueue.js';
import { canonicalStringify, isPlainObject, POLLUTING_KEYS } from '../lib/objects.js';
// `./userActions.js` is imported lazily at its single call site below, NOT here.
// It reaches the DB schema/query layer, and settings.js is imported by nearly
// every service — a static edge instantiated that subtree in ~200 suites that
// never write a ledger row.
// POLLUTING_KEYS (`__proto__`/`constructor`/`prototype`) is the project-wide
// prototype-pollution denylist (defined in server/lib/objects.js). Without it,
// rebuilding the cleaned object via `cleaned[k] = v` against a payload that
// JSON.parse exposed those names on would invoke the prototype setter and
// mutate Object.prototype. Settings never legitimately uses these keys.
const SETTINGS_FILE = join(PATHS.data, 'settings.json');
// Keys that belong to the MortalLoom iCloud store (MortalLoom.json), NOT to
// PortOS settings (data/settings.json). A historical bug or hand-edit can pollute
// settings.json with these top-level arrays/objects, which then bloats every
// `GET /api/settings` response and rides forward through every
// `saveSettings({ ...current, x: y })` mutation. Strip them on both read and
// write so the corruption auto-heals on the next save and can't propagate.
// Superset of ARRAY_KEYS in mortalLoomStore.js — also includes the non-array
// store objects `profile` and `genomeScanRecord` observed in the actual
// corruption. Keep both sides in sync when MortalLoom adds new store keys.
const MORTALLOOM_STORE_KEYS = new Set([
'alcoholDrinks', 'alcoholPresets', 'bloodTests', 'bodyEntries',
'epigeneticTests', 'eyeExams', 'goals', 'habits', 'healthMetrics',
'nicotineEntries', 'nicotinePresets', 'saunaPresets', 'saunaSessions',
'profile', 'genomeScanRecord'
]);
// Pure rebuild — drops MortalLoom store keys and prototype-pollution keys.
// Non-plain-object inputs (arrays, null, primitives) pass through unchanged;
// rebuilding them as `{}` would silently coerce-then-lose the original value.
// Warning emission is the caller's responsibility (see `save()`) so a single
// updateSettings call produces at most one log line, tied to a successful
// persisted write.
const stripStoreKeys = (settings) => {
if (!isPlainObject(settings)) return settings;
const cleaned = {};
for (const [k, v] of Object.entries(settings)) {
if (POLLUTING_KEYS.has(k)) continue;
if (MORTALLOOM_STORE_KEYS.has(k)) continue;
cleaned[k] = v;
}
return cleaned;
};
// Tiny pub/sub so cache holders (annotationIdentity, etc.) can invalidate on
// writes without each subscribing through socket.io. Listeners receive the
// merged settings object so they can pick fields they care about. Use a
// shared module-level emitter so duplicate imports observe the same bus.
export const settingsEvents = new EventEmitter();
// Cache holders that subscribe per-process can accumulate without bound on
// hot-reload — bump the cap so vitest's per-test re-imports don't trip the
// default-10-listeners warning.
settingsEvents.setMaxListeners(50);
// Reads are always silent — a polluted file would otherwise spam logs on
// every GET /api/settings. `save()` warns based on what it's HANDED, so:
// - `updateSettings(patch)` exposes both disk pollution (via the unstripped
// raw snapshot) AND patch pollution to save(), yielding one consolidated
// warning per successful write.
// - Manual `getSettings() → modify → saveSettings(...)` flows hand save() an
// already-stripped object, so no warning fires — but those flows also
// can't reintroduce store-key pollution, so silence is correct.
// - A direct `saveSettings(badObject)` with store keys warns once after the
// write resolves.
const loadRaw = async () => {
const raw = await tryReadFile(SETTINGS_FILE);
return hydratePrivateKeys(safeJSONParse(raw ?? '{}', {}), PATHS.data);
};
/**
* Security-sensitive STRICT read of settings.json for the auth gate (#2684).
*
* The normal loadRaw()/getSettings() path collapses THREE distinct states to
* `{}`: an ABSENT file (fresh install — legitimately no settings), a file that
* exists but can't be READ (permission / I/O error), and a file that reads but
* won't PARSE (truncated / corrupt JSON, or a non-object root). For feature
* reads that collapse is fine, but for the auth-enabled decision it fails OPEN:
* a corrupt settings.json makes `isAuthEnabled()` compute `false` over `{}` and
* silently disables the password gate (see `server/services/authGate.js`).
*
* This read preserves the distinction so a security caller can fail CLOSED on
* the two failure modes while still treating a genuinely absent file as
* "auth off" (the correct default for a fresh install):
* { present: false, corrupt: false } → absent (ENOENT): auth legitimately off
* { present: true, corrupt: true } → exists but unreadable/unparseable: fail closed
* { present: true, corrupt: false } → parsed cleanly into `settings`
*
* `settings` is always a plain object (`{}` in the two non-clean cases) so
* callers can read it unconditionally; they gate their fail-closed behavior on
* `corrupt`. The default path is the real SETTINGS_FILE; the optional argument
* exists so unit tests can point it at a temp file without touching real data.
*/
export const readSettingsStrict = async (filePath = SETTINGS_FILE) => {
// `tryReadFileStrict` keeps the distinction `tryReadFile` collapses, straight
// off the read's own errno — no second `access()` probe, which could disagree
// with the read it is explaining. `ok: false` = we could NOT confirm absence
// (EACCES on the file or a parent dir, ENOTDIR, EIO) → fail closed.
const { ok, value: raw } = await tryReadFileStrict(filePath);
if (!ok) return { present: true, corrupt: true, settings: {} };
if (raw === null) return { present: false, corrupt: false, settings: {} };
// Present and read. safeJSONParse returns null for malformed/empty input; a
// valid-JSON but non-object root (array, string, number) is likewise not a
// settings document — either way, treat it as corrupt rather than reading
// properties off it.
const parsed = safeJSONParse(raw, null);
if (!isPlainObject(parsed)) return { present: true, corrupt: true, settings: {} };
return { present: true, corrupt: false, settings: parsed };
};
// Read-side cache. `getSettings()` is called 100+ times across the codebase
// (many per request), and each call previously did a fresh filesystem read +
// JSON.parse. Settings is app-wide, rarely-changing data — a textbook memoize.
// Correctness contract:
// - The cache is populated lazily on the first `getSettings()` and refreshed
// from the `settings:updated` event the SAME emitter already fires on every
// `save()` — so any write through the app (saveSettings/updateSettings/
// updateSettingsWith) keeps the cache warm and consistent.
// - Write paths (`save`, `updateSettings`, `updateSettingsWith`) deliberately
// still read `loadRaw()` off disk inside the write queue for their
// read-merge-write, so their base snapshot is always the freshest persisted
// truth, never the cache. The post-write `settings:updated` then syncs the
// cache. This keeps the write-ordering guarantees intact.
// - `null` = not-yet-loaded (distinct from a legitimately empty `{}`), so a
// fresh process reads disk exactly once.
// - The prior `stripStoreKeys(await loadRaw())` handed every caller a fresh deep
// object graph, so callers may mutate nested settings in place. To preserve
// that isolation the cache is never handed out directly: reads deep-clone it,
// and the `settings:updated` listener stores a private clone (save() emits the
// same `cleaned` object it also returns to its caller). structuredClone is the
// right tool for this JSON-shaped data and stays far cheaper than the disk
// read + full-tree JSON.parse it replaces.
// The local review bridge writes from another process. Reviewer-selection
// reads explicitly refresh from disk metadata; ordinary getSettings() calls
// keep this hot cache without paying a stat on every read.
const readSettingsSignature = async () => {
try {
const { mtimeMs, size } = await stat(SETTINGS_FILE);
return { mtimeMs, size };
} catch (error) {
if (error.code === 'ENOENT') return null;
// Metadata is advisory; the strict read below owns unreadable/corrupt
// handling and invalidates consumers without broadcasting auth-off data.
return undefined;
}
};
let settingsSignature;
let settingsCache = null;
settingsEvents.on('settings:updated', (cleaned) => {
settingsCache = structuredClone(cleaned);
});
// Test-only: drop the read cache so a suite that stubs the on-disk file per
// test observes each fresh stub. Production code never calls this — writes keep
// the cache coherent via the `settings:updated` event above.
export const __resetSettingsCache = () => {
settingsCache = null;
settingsSignature = undefined;
};
// Re-sync the read cache with the current on-disk settings.json and notify every
// `settings:updated` listener. Call after a process replaces settings.json
// OUTSIDE the normal save() path — e.g. a backup restore rsyncs it into place —
// since such a write bypasses the save()-emitted event the cache rides on and
// would otherwise leave every settings consumer serving pre-restore values.
const reloadSettingsFromDisk = async (signaturePromise = readSettingsSignature()) => {
const signature = await signaturePromise;
const { corrupt, settings } = await readSettingsStrict();
if (corrupt) {
// Do NOT broadcast a corrupt on-disk settings.json as `{}` (issue #2684). The
// concrete path is a backup restore of a malformed snapshot: backup.js calls
// reloadSettings() after installing it. Emitting `settings:updated` with `{}`
// would prime the read cache AND the auth enabled-cache to an empty/disabled
// state — reopening the gate (FAIL OPEN) and sticking there because the cache
// is no longer null. Invalidate instead: drop the read cache and signal
// consumers (auth) to drop theirs, so the next read goes to disk and fails
// closed / re-reads. Self-heals on the next clean read.
settingsCache = null;
settingsSignature = undefined;
settingsEvents.emit('settings:invalidated');
return {};
}
const cleaned = stripStoreKeys(await hydratePrivateKeys(settings, PATHS.data));
settingsSignature = signature;
settingsEvents.emit('settings:updated', cleaned);
return cleaned;
};
export const reloadSettings = () => reloadSettingsFromDisk();
// Serialize all writes to settings.json on a single tail so an updateSettings
// read-merge-write can't interleave with a concurrent save (two browser tabs,
// a background job racing a user save) and clobber the other's patch. Ordinary
// reads stay off the queue; explicit disk refreshes serialize their cache events.
const queueWrite = createFileWriteQueue();
// Share the write queue so an older disk snapshot cannot emit after a newer
// in-process save. reloadSettings itself stays usable inside the restore lock.
export const refreshSettingsIfChangedOnDisk = () => queueWrite(async () => {
const signature = await readSettingsSignature();
if (settingsCache !== null && settingsSignature !== undefined && signature !== undefined
&& signature?.mtimeMs === settingsSignature?.mtimeMs
&& signature?.size === settingsSignature?.size) return;
await reloadSettingsFromDisk(signature);
});
/**
* Drain admitted settings writes and hold later writes through an out-of-band
* restore and its cache reconciliation. The callback must not re-enter any
* settings write API; reloadSettings reads directly and is safe inside it.
* For a full restore acquire this queue before CoS config, then CoS state.
*/
export const withLiveSettingsRestore = (restore) => queueWrite(restore);
// Longest scalar the ledger keeps verbatim. A settings value longer than this
// is a blob (a prompt template, a key list), and the useful fact for "what did I
// change?" is THAT it changed, not its full text.
const LEDGER_SCALAR_MAX_CHARS = 120;
// `undefined` counts: it is how "the key was not set before" arrives, and a
// first-time set is exactly the case where a before/after pair is most useful.
// It is normalized to null in the recorded value.
const isShortScalar = (value) => (
value === null
|| value === undefined
|| typeof value === 'boolean'
|| typeof value === 'number'
|| (typeof value === 'string' && value.length <= LEDGER_SCALAR_MAX_CHARS)
);
// Shallow top-level diff of the settings object. Top-level is the right grain:
// the settings file is a map of feature slices, and "the `federation` slice
// changed" is the answerable question — a deep diff would produce a per-leaf
// path list nobody reads and would carry slice internals into the log.
//
// Values are only kept for SHORT SCALARS; everything else records the fact of the
// change alone. Secret-shaped keys are NOT filtered here — `recordUserAction`
// redacts by key name and lists the paths under `payload.redactedKeys`, so the
// denylist has exactly one home.
//
// Equality goes through `canonicalStringify` (key-order independent) so a slice
// that was merely REBUILT — same values, different insertion order — is not
// reported as an edit the user did not make.
const diffSettings = (prev, next) => {
const keys = [...new Set([...Object.keys(prev), ...Object.keys(next)])].sort();
const keysChanged = [];
const changes = {};
for (const key of keys) {
const before = prev[key];
const after = next[key];
if (canonicalStringify(before) === canonicalStringify(after)) continue;
keysChanged.push(key);
changes[key] = isShortScalar(before) && isShortScalar(after)
? { from: before ?? null, to: after ?? null }
: { changed: true };
}
return { keysChanged, changes };
};
/**
* Persist settings.
*
* `actor` (#5594) names WHO is saving, and defaults to `'system'` — the honest
* default, because most callers here are schedulers, sync hooks, and internal
* feature writes. Settings HTTP mutations pass `'user'`. `reloadSettings` does
* not call this at all, so a backup restore is excluded by construction.
*/
const save = async (settings, { actor = 'system', skipUserAction = false } = {}, previousSettings) => {
const cleaned = stripStoreKeys(settings);
// Stamp `timezoneUpdatedAt` whenever the effective `timezone` actually
// changes, so timezone-dependent schedulers can gate catch-up/re-evaluation
// logic on "when did the zone last change" (see meatspacePostReminder's
// missed-slot catch-up, #2040). Compare against the previous on-disk value so
// unrelated settings saves NEVER touch the field. The read runs inside the
// same queued write turn (every caller wraps save in queueWrite), so the
// comparison is against the freshest persisted snapshot, and any prior
// `timezoneUpdatedAt` on an unchanged-timezone save rides through untouched
// via the merged object.
const prev = isPlainObject(cleaned) ? previousSettings ?? stripStoreKeys(await loadRaw()) : null;
// Diff BEFORE the timezoneUpdatedAt stamp below, so a timezone change reports
// `keysChanged: ['timezone']` rather than dragging in the derived stamp.
const change = isPlainObject(prev) ? diffSettings(prev, cleaned) : null;
if (isPlainObject(prev) && cleaned.timezone !== prev.timezone) {
cleaned.timezoneUpdatedAt = Date.now();
}
// Stamp `backupConfigUpdatedAt` whenever a registration-affecting backup
// field actually changes (cron expression, enabled, or destination) — the
// same shape as `timezoneUpdatedAt` above — so backupScheduler.js's
// missed-slot catch-up (#8456) can tell "this config produced the missed
// slot" from "the config was edited after the slot was missed" and skip a
// catch-up the current settings never scheduled.
if (isPlainObject(prev)) {
const prevBackup = isPlainObject(prev.backup) ? prev.backup : {};
const nextBackup = isPlainObject(cleaned.backup) ? cleaned.backup : {};
const backupScheduleChanged = prevBackup.cronExpression !== nextBackup.cronExpression
|| prevBackup.enabled !== nextBackup.enabled
|| (prevBackup.destPath || null) !== (nextBackup.destPath || null);
if (backupScheduleChanged) {
cleaned.backupConfigUpdatedAt = Date.now();
}
}
// atomicWrite (temp-file + rename) so a mid-write crash never truncates
// settings.json. Pass a pre-stringified string to preserve the trailing
// newline; atomicWrite's own JSON.stringify omits it.
await atomicWrite(SETTINGS_FILE, JSON.stringify(await persistPrivateKeys(cleaned, PATHS.data, { previousSettings: prev }), null, 2) + '\n');
await hydratePrivateKeys(cleaned, PATHS.data);
// Another process can replace the file between our write and a stat. Never
// label this in-memory save with metadata that might belong to that writer;
// the next reviewer-selection read reconciles the persisted snapshot once.
settingsSignature = undefined;
// Warn AFTER the successful write so a thrown write never produces
// a misleading "stripped" log line for a write that didn't happen.
if (isPlainObject(settings)) {
const polluted = Object.keys(settings).filter((k) => MORTALLOOM_STORE_KEYS.has(k));
if (polluted.length > 0) {
console.warn(`⚠️ settings.json: stripped MortalLoom store keys: ${polluted.join(', ')}`);
}
}
settingsEvents.emit('settings:updated', cleaned);
// A no-op save writes no ledger row: the settings page PUTs the whole object
// on every visit, and a log full of "changed nothing" rows would bury the
// changes that matter.
if (!skipUserAction && change && change.keysChanged.length > 0) {
recordSettingsChange(actor, change);
}
return cleaned;
};
// Record a settings change in the operator-action ledger (#5594) as detached
// bookkeeping: the row is a Postgres write, and awaiting it inside the queued
// save turn made a DB outage stall every later settings write for the pool
// connect timeout and then reject a save whose file was already on disk — which
// in turn skipped setPassword's session revocation (#12018). The ledger is an
// audit trail, not part of the save's success contract, so the chain is not
// awaited by the caller and owns its own rejection. It is tracked so a drain
// point (`flushSettingsLedger`, the test runner's per-test teardown) can wait
// for it: a suite that removes its temp data root under an in-flight row would
// otherwise see the row recreate the directory and leak it.
const recordSettingsChange = (actor, change) => {
const happenedAt = new Date().toISOString();
const changedKeys = change.keysChanged.join(',');
trackDetachedWrite(import('./userActions.js')
.then(({ recordUserAction }) => recordUserAction({
type: 'settings.update',
actor,
summary: `Updated settings: ${change.keysChanged.join(', ')}`,
payload: { keysChanged: change.keysChanged, changes: change.changes },
source: { service: 'settings', fn: 'save' },
happenedAt,
// Timestamp because a save is not an idempotent retry, PLUS the changed
// keys because `toISOString()` only resolves to the millisecond — two
// distinct saves landing in the same one would otherwise silently collapse
// to a single row. Deterministic (not random) so a genuine duplicate still
// dedupes.
dedupeKey: `settings.update:${happenedAt}:${changedKeys.slice(0, 200)}`,
}))
.catch((error) => {
console.error(`❌ Settings ledger write failed for keys ${changedKeys}: ${error.message}`);
}));
};
// Await every ledger write scheduled so far. The writes are detached from the
// save that scheduled them, so a caller that must observe the row (a test, a
// shutdown drain) waits here rather than on the save's promise.
export const flushSettingsLedger = () => drainDetachedWrites();
/**
* `getSettings`, but reporting whether the snapshot came from a CORRUPT read.
*
* `getSettings()` deliberately hands back an empty object when settings.json is
* malformed or unreadable, which makes "corrupt" indistinguishable from "fresh
* install with nothing configured". That is the right default for most readers —
* they want a usable object — but it is wrong for any caller whose decision is
* spend-bearing or privacy-bearing, where guessing "nothing is configured" can
* silently do the opposite of what the user set up (see the federated media
* router, which must fail loudly rather than render locally).
*
* @returns {Promise<{corrupt: boolean, settings: object}>}
*/
export const getSettingsWithStatus = async () => {
if (settingsCache === null) {
// Route the cold read through the STRICT reader so a corrupt/unreadable
// settings.json does NOT poison the cache with `{}` (issue #2684). Caching a
// corrupt-derived empty object would strand every consumer — verifyPassword,
// schedulers, feature reads — on empty settings until a save() or restart,
// defeating the no-restart self-heal the auth fail-closed path promises. This
// covers BOTH failure modes (malformed content and an unreadable-but-present
// file); only a genuinely ABSENT file (fresh install) caches `{}`.
const signature = await readSettingsSignature();
const { corrupt, settings } = await readSettingsStrict();
const loaded = stripStoreKeys(await hydratePrivateKeys(settings, PATHS.data));
// A save()/reloadSettings() may have populated the cache via the
// settings:updated listener while this cold read was awaiting the disk read.
// Prefer that fresher in-memory value over our (older) on-disk snapshot.
if (settingsCache !== null) return { corrupt: false, settings: structuredClone(settingsCache) };
// On a corrupt read, hand back the empty snapshot WITHOUT caching it, so the
// very next call re-reads and picks up a repair immediately.
if (corrupt) return { corrupt: true, settings: structuredClone(loaded) };
settingsCache = loaded;
settingsSignature = signature;
}
// Hand out a private deep copy so a caller mutating nested settings in place
// can't corrupt the shared cache — matching the prior per-call
// `stripStoreKeys(await loadRaw())` deep-copy semantics, minus the I/O.
return { corrupt: false, settings: structuredClone(settingsCache) };
};
export const getSettings = async () => (await getSettingsWithStatus()).settings;
export const saveSettings = (settings, options) => queueWrite(() => save(settings, options));
// The merge base for every partial write, read strictly inside the queued turn.
// loadRaw() reads a present-but-unreadable or malformed file as `{}`, and a
// partial write merged onto that replaced every setting (auth secret included)
// with just its patch. A confirmed-absent file (fresh install) is a valid empty
// base; anything else refuses until the file is repaired or restored.
const loadWriteBase = async () => {
const { corrupt, settings } = await readSettingsStrict();
if (corrupt) {
const err = new Error('Cannot update settings: data/settings.json is unreadable or malformed — repair it or restore it from a backup');
err.code = 'SETTINGS_UNREADABLE';
throw err;
}
return hydratePrivateKeys(settings, PATHS.data);
};
// Merge against the unstripped on-disk snapshot so save() sees every
// MortalLoom store key in one place — guaranteeing exactly one warning
// per updateSettings call, only when the write succeeds. The whole
// read-merge-write runs inside one queued turn so it merges against the
// freshest persisted snapshot, not a stale pre-image.
export const updateSettings = (patch, options) => queueWrite(async () => {
const raw = await loadWriteBase();
const previous = structuredClone(stripStoreKeys(raw));
const incoming = isPlainObject(patch) ? patch : {};
const merged = { ...raw, ...incoming };
return save(merged, options, previous);
});
// Read-modify-write INSIDE the write queue. For callers whose merge semantics
// `updateSettings`' shallow `{ ...raw, ...patch }` can't express — a deep merge
// (`deepMerge(current, …)`), or building the next object by spreading/deleting a
// sub-key. `mutate(current)` receives the freshest stripped settings (same shape
// `getSettings()` returns) and returns the FULL next settings object to persist.
//
// This closes the stale-base window in the old `getSettings() → modify →
// saveSettings(...)` pattern: there, a concurrent `updateSettings` landing
// between the external read and the queued write was clobbered by the caller's
// stale pre-image. Here the read and the write share one queued turn, so the
// mutator always sees the latest persisted snapshot. An unreadable or malformed
// file refuses (loadWriteBase) rather than handing the mutator an empty base.
export const updateSettingsWith = (mutate, options) => queueWrite(async () => {
const current = stripStoreKeys(await loadWriteBase());
const previous = structuredClone(current);
const next = await mutate(current);
// Guard the mutator's return BEFORE persisting: unlike the old
// saveSettings(objectLiteral) callers, a mutator with a missing `return`
// (or one that returns an array/primitive) would otherwise serialize
// garbage like `undefined`/`"foo"` into settings.json. Fail loud instead.
if (!isPlainObject(next)) {
throw new TypeError('updateSettingsWith: mutate() must return a plain settings object');
}
return save(next, options, previous);
});