Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Writers Room — storage backend dispatcher (#1017).
*
* Folders, works (+ decomposed draft versions), and exercises used to live in a
* bespoke file layout (folders.json, exercises.json, per-work manifest.json). As
* of #1017 their METADATA lives in PostgreSQL (writers_room_folders /
* writers_room_works / writers_room_draft_versions / writers_room_exercises).
* The draft PROSE BODIES stay on disk (file-primary) — local.js still owns the
* .md read/write; this dispatcher only routes the metadata.
*
* The public surface (folders/works/exercises CRUD) is identical across both
* backends, so local.js calls `store()` and keeps its API unchanged:
*
* - PostgreSQL (./db.js) for normal installs.
* - File backend only under MEMORY_BACKEND=file or NODE_ENV=test (both
* UNSUPPORTED for production). The file backend reproduces the legacy
* on-disk JSON format so tests boot without a DB exactly as before.
*
* Works FEDERATE across peers as of #1565 via the per-record peer-sync push
* pipeline (not the 60s snapshot loop), so this dispatcher also routes the
* federation primitives (readWork includeDeleted, listWorkIds includeDeleted,
* mergeWorksFromSync, pruneTombstonedWorks) to whichever backend is active —
* both implement the same LWW/tombstone semantics from syncLogic.js. There is
* still NO mutation epoch here: works ride per-record pushes, not the snapshot
* checksum a storage swap would invalidate.
*
* The first PG-backed call runs a one-time, marker-gated import of the legacy
* data/writers-room file store into the tables (migrateWritersRoomToDB).
*/
import { readFile, readdir } from 'fs/promises';
import { atomicWrite, ensureDir, readJSONFile, safeJSONParse } from '../../lib/fileUtils.js';
import { createPgFileFacade, resolvePgBackend } from '../../lib/pgFileFacade.js';
import { ServerError } from '../../lib/errorHandler.js';
import {
maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord, flushBaseHashes,
} from '../../lib/conflictJournal.js';
import {
WRITERS_ROOM_WORK_KIND, mergeWorkRecord,
WRITERS_ROOM_FOLDER_KIND, mergeFolderRecord,
WRITERS_ROOM_EXERCISE_KIND, mergeExerciseRecord,
} from './syncLogic.js';
import {
WORK_ID_RE, wrRoot, wrFoldersFile, wrExercisesFile, wrWorksDir, wrManifestPath,
} from './_shared.js';
// --- File backend (escape hatch / tests): the legacy on-disk JSON format ---
// Reproduces local.js's original storage primitives verbatim so an install on
// the file escape hatch (and every test that boots without a DB) behaves
// identically to the pre-#1017 file layout.
function makeFileBackend() {
const loadFolders = async () => {
await ensureDir(wrRoot());
// Folder metadata is durable user state, even on the test/escape-hatch backend.
const raw = await readJSONFile(wrFoldersFile(), [], { strict: true });
return Array.isArray(raw) ? raw : [];
};
const saveFolders = async (folders) => {
await ensureDir(wrRoot());
await atomicWrite(wrFoldersFile(), folders);
};
const loadExercises = async () => {
await ensureDir(wrRoot());
// Exercise history must not be replaced by an empty fallback after a failed read.
const raw = await readJSONFile(wrExercisesFile(), [], { strict: true });
return Array.isArray(raw) ? raw : [];
};
const saveExercises = async (exercises) => {
await ensureDir(wrRoot());
await atomicWrite(wrExercisesFile(), exercises);
};
const loadManifest = async (workId) => {
const path = wrManifestPath(workId);
const content = await readFile(path, 'utf-8').catch((err) => {
if (err.code === 'ENOENT') return null;
throw err;
});
if (content === null) return null;
// Surface corrupted manifests as a deterministic CORRUPTED_MANIFEST 500 (the
// pre-#1017 file behavior) so listWorks can drop the work without masking
// the issue and direct callers get an actionable error, not a raw SyntaxError.
const parsed = safeJSONParse(content, null, { allowArray: false, logError: true, context: path });
// `allowArray: false` only rejects a root array — a bare JSON scalar
// (corrupted manifest) still parses, so also check the shape here.
if (parsed === null || typeof parsed !== 'object') {
console.warn(`⚠️ wr: corrupted manifest at ${path} (work ${workId})`);
throw new ServerError(`Corrupted writers-room manifest for ${workId}`, {
status: 500, code: 'CORRUPTED_MANIFEST', context: { workId },
});
}
return parsed;
};
const saveManifest = async (workId, manifest) => {
await ensureDir(`${wrWorksDir()}/${workId}/drafts`);
await atomicWrite(wrManifestPath(workId), manifest);
};
const listWorkIds = async () => {
await ensureDir(wrWorksDir());
const entries = await readdir(wrWorksDir(), { withFileTypes: true });
return entries.filter((e) => e.isDirectory() && WORK_ID_RE.test(e.name)).map((e) => e.name);
};
// The live-work set, shared by listWorks + countWorks below so the listing and
// the tally can't disagree about which works exist.
//
// Tolerate a corrupted manifest per work (drop it) so one bad work doesn't 500
// the whole library — the pre-#1017 file behavior. Re-throw anything else
// (permission errors, EIO): masking those would hide a real outage behind a
// plausible-looking short count.
const loadLiveManifests = async () => {
const ids = await listWorkIds();
const manifests = await Promise.all(ids.map((id) => loadManifest(id).catch((err) => {
if (err?.code === 'CORRUPTED_MANIFEST') {
console.warn(`⚠️ wr: dropped work ${id} from listing — corrupted manifest`);
return null;
}
throw err;
})));
return manifests.filter((m) => m != null && m.deleted !== true);
};
// Shared LWW merge for a body-less record array (folders/exercises) — mirrors
// mergeWorksFromSync below minus the per-work manifest file. Loads the array
// once, applies the shared pure decision per remote record, journals the
// about-to-be-overwritten local version when remote wins, and saves once.
const mergeBodylessArray = async (load, save, kind, mergeFn, remoteRecords, source) => {
if (!Array.isArray(remoteRecords)) return { applied: false, count: 0 };
const all = await load();
let changed = 0;
for (const remote of remoteRecords) {
const idx = all.findIndex((r) => r.id === remote?.id);
const local = idx >= 0 ? all[idx] : null;
const { next, inserted, remoteWins, changed: didChange } = mergeFn(local, remote);
if (!next) continue;
if (!inserted && (!remoteWins || !didChange)) continue;
if (!inserted) {
await maybeJournalBeforeOverwrite({ kind, id: next.id, local, remote: next, source });
}
if (idx >= 0) all[idx] = next; else all.push(next);
await setSyncBaseHash(kind, next.id, contentHashForRecord(kind, next));
changed += 1;
}
if (changed > 0) await save(all);
await flushBaseHashes();
return changed === 0 ? { applied: false, count: 0 } : { applied: true, count: changed };
};
// Hard-remove tombstoned records whose `deletedAt` is older than the cutoff;
// returns their ids (the sync.js facade evicts each one's base hash).
const pruneTombstonedArray = async (load, save, olderThanMs) => {
if (!Number.isFinite(olderThanMs)) return { pruned: 0, ids: [] };
const all = await load();
const ids = [];
for (const r of all) {
if (r?.deleted !== true) continue;
const ts = Date.parse(r.deletedAt || '');
if (Number.isFinite(ts) && ts < olderThanMs) ids.push(r.id);
}
if (ids.length > 0) {
const dropped = new Set(ids);
await save(all.filter((r) => !dropped.has(r.id)));
}
return { pruned: ids.length, ids };
};
return {
name: 'file',
// folders. A soft-deleted folder (tombstone) stays in folders.json so its
// deletion federates; live reads filter it out, mirroring the PG backend's
// `WHERE deleted = FALSE`.
listFolders: async () => (await loadFolders()).filter((f) => f.deleted !== true),
readFolder: async (id, { includeDeleted = false } = {}) => {
const folder = (await loadFolders()).find((f) => f.id === id) ?? null;
if (!folder) return null;
if (!includeDeleted && folder.deleted === true) return null;
return folder;
},
listFolderIds: async ({ includeDeleted = false } = {}) => (await loadFolders())
.filter((f) => includeDeleted || f.deleted !== true).map((f) => f.id),
writeFolder: async (folder) => {
const folders = await loadFolders();
const idx = folders.findIndex((f) => f.id === folder.id);
if (idx >= 0) folders[idx] = folder; else folders.push(folder);
await saveFolders(folders);
},
deleteFolder: async (id) => {
// Soft-delete tombstone (#1645) so the deletion federates — mirror the PG
// backend. Idempotent: a missing or already-deleted folder is a no-op.
const folders = await loadFolders();
const idx = folders.findIndex((f) => f.id === id);
if (idx < 0 || folders[idx].deleted === true) return;
const now = new Date().toISOString();
folders[idx] = { ...folders[idx], deleted: true, deletedAt: now, updatedAt: now };
await saveFolders(folders);
},
mergeFoldersFromSync: async (remoteFolders, { source = { via: 'sync', peerId: null } } = {}) =>
mergeBodylessArray(loadFolders, saveFolders, WRITERS_ROOM_FOLDER_KIND, mergeFolderRecord, remoteFolders, source),
pruneTombstonedFolders: async (olderThanMs) => pruneTombstonedArray(loadFolders, saveFolders, olderThanMs),
// exercises (same body-less tombstone-aware contract as folders).
listExercises: async () => (await loadExercises()).filter((e) => e.deleted !== true),
readExercise: async (id, { includeDeleted = false } = {}) => {
const exercise = (await loadExercises()).find((e) => e.id === id) ?? null;
if (!exercise) return null;
if (!includeDeleted && exercise.deleted === true) return null;
return exercise;
},
listExerciseIds: async ({ includeDeleted = false } = {}) => (await loadExercises())
.filter((e) => includeDeleted || e.deleted !== true).map((e) => e.id),
writeExercise: async (exercise) => {
const all = await loadExercises();
const idx = all.findIndex((e) => e.id === exercise.id);
if (idx >= 0) all[idx] = exercise; else all.push(exercise);
await saveExercises(all);
},
mergeExercisesFromSync: async (remoteExercises, { source = { via: 'sync', peerId: null } } = {}) =>
mergeBodylessArray(loadExercises, saveExercises, WRITERS_ROOM_EXERCISE_KIND, mergeExerciseRecord, remoteExercises, source),
pruneTombstonedExercises: async (olderThanMs) => pruneTombstonedArray(loadExercises, saveExercises, olderThanMs),
// works (manifest carries its own drafts[] in the file format). A
// soft-deleted work (tombstone) stays on disk; readWork/listWorks/listWorkIds
// filter it out unless includeDeleted, mirroring the PG backend's
// `WHERE deleted = FALSE`.
readWork: async (id, { includeDeleted = false } = {}) => {
const manifest = await loadManifest(id);
if (!manifest) return null;
if (!includeDeleted && manifest.deleted === true) return null;
return manifest;
},
listWorkIds: async ({ includeDeleted = false } = {}) => {
const ids = await listWorkIds();
if (includeDeleted) return ids;
// Load manifests concurrently (mirrors listWorks below) to filter tombstones.
const manifests = await Promise.all(ids.map((id) => loadManifest(id).catch(() => null)));
return ids.filter((_, i) => manifests[i] && manifests[i].deleted !== true);
},
listWorks: loadLiveManifests,
listOrderedWorkIds: async () => (await loadLiveManifests())
.sort((a, b) => (b.updatedAt || '').localeCompare(a.updatedAt || '') || a.id.localeCompare(b.id))
.map(work => work.id),
readWorksPage: async (ids) => (await Promise.all(ids.map(id => loadManifest(id).catch(() => null))))
.filter(work => work && !work.deleted),
// No cheap count on the file layout — `deleted` lives inside each manifest,
// so the live set can only be known by reading them. Counting the shared
// live-set reader makes agreement with listWorks structural rather than a
// duplicated filter that could drift. The file backend is a dev/test escape
// hatch (never a deployment mode), so its cost here is not worth splitting
// the two readers apart to shave.
countWorks: async () => (await loadLiveManifests()).length,
writeWork: async (manifest) => { await saveManifest(manifest.id, manifest); return manifest; },
deleteWork: async (id) => {
// Soft-delete tombstone (#1565) so the deletion federates — mirror the PG
// backend. Leave the .md bodies on disk until hard-prune. Idempotent.
const manifest = await loadManifest(id).catch(() => null);
if (!manifest || manifest.deleted === true) return;
const now = new Date().toISOString();
await saveManifest(id, { ...manifest, deleted: true, deletedAt: now, updatedAt: now });
},
mergeWorksFromSync: async (remoteWorks, { source = { via: 'sync', peerId: null } } = {}) => {
if (!Array.isArray(remoteWorks)) return { applied: false, count: 0 };
let changed = 0;
for (const remote of remoteWorks) {
const local = await loadManifest(remote?.id).catch(() => null);
const { next, inserted, remoteWins, changed: didChange } = mergeWorkRecord(local, remote);
if (!next) continue;
if (!inserted && (!remoteWins || !didChange)) continue;
if (!inserted) {
await maybeJournalBeforeOverwrite({ kind: WRITERS_ROOM_WORK_KIND, id: next.id, local, remote: next, source });
}
await saveManifest(next.id, next);
await setSyncBaseHash(WRITERS_ROOM_WORK_KIND, next.id, contentHashForRecord(WRITERS_ROOM_WORK_KIND, next));
changed += 1;
}
await flushBaseHashes();
return changed === 0 ? { applied: false, count: 0 } : { applied: true, count: changed };
},
pruneTombstonedWorks: async (olderThanMs) => {
if (!Number.isFinite(olderThanMs)) return { pruned: 0, ids: [] };
const ids = await listWorkIds();
const manifests = await Promise.all(ids.map((id) => loadManifest(id).catch(() => null)));
const out = ids.filter((_, i) => {
const m = manifests[i];
const deletedMs = m?.deleted === true ? Date.parse(m.deletedAt || '') : NaN;
return Number.isFinite(deletedMs) && deletedMs < olderThanMs;
});
// The facade (sync.js) rm's each id's dir (manifest + bodies) and evicts the
// base hash — uniform with the PG backend, whose rows are dropped here.
return { pruned: out.length, ids: out };
},
listDraftVersionsForCoherence: async () => {
const ids = await listWorkIds();
const out = [];
for (const id of ids) {
const m = await loadManifest(id).catch(() => null);
if (!m || m.deleted === true) continue;
for (const d of m?.drafts || []) {
out.push({ id: d.id, workId: id, contentFile: d.contentFile, contentHash: d.contentHash });
}
}
return out;
},
};
}
// --- PostgreSQL backend: pure leaf I/O from ./db.js ---
function makePgBackend(db) {
return { name: 'postgres', ...db };
}
const pgBackend = () => resolvePgBackend({
requirement: 'Writers Room requires PostgreSQL — run `npm run setup:db` (dev/test only: set MEMORY_BACKEND=file for the unsupported file backend)',
migrate: async () => {
const { migrateWritersRoomToDB } = await import('../../scripts/migrateWritersRoomToDB.js');
await migrateWritersRoomToDB();
},
loadDb: () => import('./db.js'),
makePg: (db) => makePgBackend(db),
});
// Memoize the backend-selection PROMISE (not just the result) so two concurrent
// first calls — e.g. the boot warm racing a request — don't both import the PG
// module and run the migration twice.
const facade = createPgFileFacade({
makeFile: () => makeFileBackend(),
makePg: () => pgBackend(),
});
const { getBackend } = facade;
export function _resetWritersRoomStore() {
facade.reset();
}
/** The Writers Room metadata store — backend resolved lazily per call. */
export function writersRoomStore() {
return {
getBackendName: () => facade.getBackendName(),
listFolders: async () => (await getBackend()).listFolders(),
readFolder: async (id, opts) => (await getBackend()).readFolder(id, opts),
listFolderIds: async (opts) => (await getBackend()).listFolderIds(opts),
writeFolder: async (folder) => (await getBackend()).writeFolder(folder),
deleteFolder: async (id) => (await getBackend()).deleteFolder(id),
mergeFoldersFromSync: async (remoteFolders, opts) => (await getBackend()).mergeFoldersFromSync(remoteFolders, opts),
pruneTombstonedFolders: async (olderThanMs) => (await getBackend()).pruneTombstonedFolders(olderThanMs),
listExercises: async () => (await getBackend()).listExercises(),
readExercise: async (id, opts) => (await getBackend()).readExercise(id, opts),
listExerciseIds: async (opts) => (await getBackend()).listExerciseIds(opts),
writeExercise: async (exercise) => (await getBackend()).writeExercise(exercise),
mergeExercisesFromSync: async (remoteExercises, opts) => (await getBackend()).mergeExercisesFromSync(remoteExercises, opts),
pruneTombstonedExercises: async (olderThanMs) => (await getBackend()).pruneTombstonedExercises(olderThanMs),
readWork: async (id, opts) => (await getBackend()).readWork(id, opts),
listWorkIds: async (opts) => (await getBackend()).listWorkIds(opts),
listWorks: async () => (await getBackend()).listWorks(),
listOrderedWorkIds: async () => (await getBackend()).listOrderedWorkIds(),
readWorksPage: async (ids) => (await getBackend()).readWorksPage(ids),
countWorks: async () => (await getBackend()).countWorks(),
writeWork: async (manifest) => (await getBackend()).writeWork(manifest),
deleteWork: async (id) => (await getBackend()).deleteWork(id),
mergeWorksFromSync: async (remoteWorks, opts) => (await getBackend()).mergeWorksFromSync(remoteWorks, opts),
pruneTombstonedWorks: async (olderThanMs) => (await getBackend()).pruneTombstonedWorks(olderThanMs),
listDraftVersionsForCoherence: async () => (await getBackend()).listDraftVersionsForCoherence(),
};
}