Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: serialize Digital Twin metadata writes and write soul documents atomically (#10939)
Adam Eivyec8ee0bc4ece
diff --git a/server/services/digital-twin-documents.js b/server/services/digital-twin-documents.jsindex d50e052821..91ef3c2cce 100644--- a/server/services/digital-twin-documents.js+++ b/server/services/digital-twin-documents.js@@ -1,8 +1,9 @@-import { readFile, writeFile, unlink, stat } from 'fs/promises';+import { readFile, unlink, stat } from 'fs/promises';import { existsSync } from 'fs';import { join } from 'path';import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';-import { loadMeta, saveMeta } from './digital-twin-meta.js';+import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';+import { atomicWrite } from '../lib/fileUtils.js';import { extractVersion } from './digital-twin-meta.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';@@ -41,9 +42,9 @@ export async function getDocumentById(id) {// The document file and the meta row that names it are one workflow, so each// mutation holds the backup lease from its first byte change through the meta// write (#9982).-export const createDocument = (data) => withBackupAssetPublication(() => createDocumentLeased(data));-export const updateDocument = (id, updates) => withBackupAssetPublication(() => updateDocumentLeased(id, updates));-export const deleteDocument = (id) => withBackupAssetPublication(() => deleteDocumentLeased(id));+export const createDocument = (data) => withBackupAssetPublication(() => withMetaLock(() => createDocumentLeased(data)));+export const updateDocument = (id, updates) => withBackupAssetPublication(() => withMetaLock(() => updateDocumentLeased(id, updates)));+export const deleteDocument = (id) => withBackupAssetPublication(() => withMetaLock(() => deleteDocumentLeased(id)));async function createDocumentLeased(data) {await ensureSoulDir();@@ -57,7 +58,7 @@ async function createDocumentLeased(data) {}// Write the file- await writeFile(filePath, data.content);+ await atomicWrite(filePath, data.content);// Re-creating a filename that was previously deleted must clear its tombstone// and stamp a creation time that STRICTLY supersedes the deletion — otherwise@@ -98,7 +99,7 @@ async function updateDocumentLeased(id, updates) {// Update file content if providedif (updates.content) {- await writeFile(filePath, updates.content);+ await atomicWrite(filePath, updates.content);docMeta.version = extractVersion(updates.content);}diff --git a/server/services/digital-twin-meta.concurrency.test.js b/server/services/digital-twin-meta.concurrency.test.jsnew file mode 100644index 0000000000..d88658f09c--- /dev/null+++ b/server/services/digital-twin-meta.concurrency.test.js@@ -0,0 +1,85 @@+/**+ * Concurrent metadata mutations (#10939): documents, personas and settings all+ * load -> mutate -> saveMeta against one meta.json. When the cache is cold (TTL+ * expiry, a peer-sync apply) each overlapping caller reads its own copy from+ * disk, so without a shared lock the last saveMeta silently drops the others'+ * records and tombstones. Real disk paths, PATHS pointed at a temp dir.+ */++import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';+import { readdirSync, readFileSync, rmSync } from 'fs';+import { join } from 'path';+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';++const tempRoot = createTempDataRoot('portos-dt-meta-concurrency-');+const twinDir = join(tempRoot, 'digital-twin');++vi.mock('../lib/fileUtils.js', async () => {+ const actual = await vi.importActual('../lib/fileUtils.js');+ return makePathsProxy(actual, { dataRoot: tempRoot });+});++const { createDocument, updateDocument, deleteDocument } = await import('./digital-twin-documents.js');+const { createPersona, deletePersona } = await import('./digital-twin-personas.js');+const { loadMeta, saveMeta, updateSettings, cache, digitalTwinEvents } = await import('./digital-twin-meta.js');++const coldCache = () => {+ cache.meta.data = null;+ cache.meta.timestamp = 0;+};++beforeEach(async () => {+ rmSync(twinDir, { recursive: true, force: true });+ await saveMeta({ version: '1.0.0', documents: [], deletedDocuments: [], personas: [], deletedPersonas: [], settings: {} });+ coldCache();+ // Every save drops the cache again, so each overlapping caller that is not+ // serialized re-reads its own snapshot from disk.+ digitalTwinEvents.removeAllListeners('meta:changed');+ digitalTwinEvents.on('meta:changed', coldCache);+});++afterAll(() => {+ digitalTwinEvents.removeAllListeners('meta:changed');+ rmSync(tempRoot, { recursive: true, force: true });+});++describe('digital twin meta write serialization (#10939)', () => {+ it('keeps every persona, tombstone and setting from overlapping mutations', async () => {+ const seed = await createPersona({ name: 'Seed', instructions: 'Seed persona.' });+ coldCache();++ const [a, b] = await Promise.all([+ createPersona({ name: 'A', instructions: 'Persona A.' }),+ createPersona({ name: 'B', instructions: 'Persona B.' }),+ deletePersona(seed.id),+ updateSettings({ maxContextTokens: 1234 }),+ ]);++ const meta = JSON.parse(readFileSync(join(twinDir, 'meta.json'), 'utf-8'));+ expect(meta.personas.map((p) => p.id).sort()).toEqual([a.id, b.id].sort());+ expect(meta.deletedPersonas.map((t) => t.id)).toEqual([seed.id]);+ expect(meta.settings.maxContextTokens).toBe(1234);+ });++ it('keeps concurrent document creates and deletes, writing files atomically', async () => {+ const first = await createDocument({ filename: 'FIRST.md', title: 'First', category: 'core', content: '# First\n' });+ coldCache();++ await Promise.all([+ createDocument({ filename: 'SECOND.md', title: 'Second', category: 'core', content: '# Second\n' }),+ createDocument({ filename: 'THIRD.md', title: 'Third', category: 'core', content: '# Third\n' }),+ deleteDocument(first.id),+ ]);++ const meta = JSON.parse(readFileSync(join(twinDir, 'meta.json'), 'utf-8'));+ expect(meta.documents.map((d) => d.filename).sort()).toEqual(['SECOND.md', 'THIRD.md']);+ expect(meta.deletedDocuments.map((t) => t.filename)).toEqual(['FIRST.md']);++ const second = meta.documents.find((d) => d.filename === 'SECOND.md');+ await updateDocument(second.id, { content: '# Second v2\n' });+ expect(readFileSync(join(twinDir, 'SECOND.md'), 'utf-8')).toBe('# Second v2\n');+ // atomicWrite renames a temp file into place; nothing may be left behind.+ expect(readdirSync(twinDir).filter((f) => f.endsWith('.tmp'))).toEqual([]);+ expect((await loadMeta()).documents).toHaveLength(2);+ });+});diff --git a/server/services/digital-twin-meta.js b/server/services/digital-twin-meta.jsindex dd3950ad99..56f4455aa9 100644--- a/server/services/digital-twin-meta.js+++ b/server/services/digital-twin-meta.js@@ -4,6 +4,7 @@ import { join } from 'path';import EventEmitter from 'events';import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';+import { createMutex } from '../lib/asyncMutex.js';import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';export const META_FILE = join(DIGITAL_TWIN_DIR, 'meta.json');@@ -22,6 +23,13 @@ const cache = {};export const CACHE_TTL_MS = 5000;+// One lock for every load -> mutate -> saveMeta workflow. loadMeta hands every+// caller the same cached object, so two overlapping read-modify-write flows+// would otherwise each push onto it and the later saveMeta would persist+// whichever tombstone/record set it happened to hold. NOT re-entrant: never+// call withMetaLock from inside a withMetaLock callback.+export const withMetaLock = createMutex();+// Expose cache for modules that manage test/document caches (testing.js)export { cache };@@ -166,16 +174,16 @@ export async function saveMeta(meta) {digitalTwinEvents.emit('meta:changed', meta);}-export async function updateMeta(updates) {+export const updateMeta = (updates) => withMetaLock(async () => {const meta = await loadMeta();const updated = { ...meta, ...updates };await saveMeta(updated);return updated;-}+});-export async function updateSettings(settings) {+export const updateSettings = (settings) => withMetaLock(async () => {const meta = await loadMeta();meta.settings = { ...meta.settings, ...settings };await saveMeta(meta);return meta.settings;-}+});diff --git a/server/services/digital-twin-personas.js b/server/services/digital-twin-personas.jsindex 780577e3a1..296eed5de0 100644--- a/server/services/digital-twin-personas.js+++ b/server/services/digital-twin-personas.js@@ -13,7 +13,7 @@*/import { generateId, now } from './digital-twin-helpers.js';-import { loadMeta, saveMeta } from './digital-twin-meta.js';+import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';import { recordTombstone } from '../lib/tombstones.js';export async function getPersonas() {@@ -26,7 +26,9 @@ export async function getPersonaById(id) {return personas.find(p => p.id === id) || null;}-export async function createPersona({ name, description, instructions, traitAdjustments }) {+export const createPersona = (...args) => withMetaLock(() => createPersonaLocked(...args));++async function createPersonaLocked({ name, description, instructions, traitAdjustments }) {const meta = await loadMeta();if (!Array.isArray(meta.personas)) meta.personas = [];@@ -47,7 +49,9 @@ export async function createPersona({ name, description, instructions, traitAdjureturn persona;}-export async function updatePersona(id, updates) {+export const updatePersona = (...args) => withMetaLock(() => updatePersonaLocked(...args));++async function updatePersonaLocked(id, updates) {const meta = await loadMeta();const persona = (meta.personas || []).find(p => p.id === id);if (!persona) {@@ -74,7 +78,9 @@ export async function updatePersona(id, updates) {return persona;}-export async function deletePersona(id) {+export const deletePersona = (...args) => withMetaLock(() => deletePersonaLocked(...args));++async function deletePersonaLocked(id) {const meta = await loadMeta();const before = (meta.personas || []).length;meta.personas = (meta.personas || []).filter(p => p.id !== id);@@ -103,7 +109,9 @@ export async function deletePersona(id) {* Set (or clear, with personaId === null) the active persona. Validates the id* exists so the settings pointer can't dangle.*/-export async function setActivePersona(personaId) {+export const setActivePersona = (...args) => withMetaLock(() => setActivePersonaLocked(...args));++async function setActivePersonaLocked(personaId) {const meta = await loadMeta();if (personaId !== null && !(meta.personas || []).some(p => p.id === personaId)) {throw new Error(`Persona ${personaId} not found`);diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.jsindex 09b8e1de89..50755ae094 100644--- a/server/services/digital-twin.test.js+++ b/server/services/digital-twin.test.js@@ -430,9 +430,9 @@ describe('digital-twin.js', () => {expect(result.filename).toBe('NEW_DOC.md');expect(result.id).toBe('test-uuid-1');expect(result.content).toBe('# New Doc\n\nContent here.');- // writeFile for the document file; atomicWrite for the meta save- expect(writeFile).toHaveBeenCalledTimes(1);- expect(atomicWrite).toHaveBeenCalledTimes(1);+ // Document file and meta save both go through atomicWrite (#10939)+ expect(writeFile).not.toHaveBeenCalled();+ expect(atomicWrite).toHaveBeenCalledTimes(2);});it('should throw if document already exists', async () => {@@ -483,8 +483,9 @@ describe('digital-twin.js', () => {});expect(result).not.toBeNull();- // writeFile for content + saveMeta- expect(writeFile).toHaveBeenCalled();+ // content + meta both via atomicWrite, never an in-place truncating write (#10939)+ expect(writeFile).not.toHaveBeenCalled();+ expect(atomicWrite).toHaveBeenCalledTimes(2);});it('should return null for non-existent document', async () => {