Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: serialize Digital Twin metadata writes and write soul documents atomically (#10939)

Adam Eivyec8ee0bc4ece
diff --git a/server/services/digital-twin-documents.js b/server/services/digital-twin-documents.js
index d50e052821..91ef3c2cce 100644
--- a/server/services/digital-twin-documents.js
+++ b/server/services/digital-twin-documents.js
@@ -1,8 +1,9 @@
-import { readFile, writeFile, unlink, stat } from 'fs/promises';
+import { readFile, unlink, stat } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
-import { loadMeta, saveMeta } from './digital-twin-meta.js';
+import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';
+import { atomicWrite } from '../lib/fileUtils.js';
import { extractVersion } from './digital-twin-meta.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';
@@ -41,9 +42,9 @@ export async function getDocumentById(id) {
// The document file and the meta row that names it are one workflow, so each
// mutation holds the backup lease from its first byte change through the meta
// write (#9982).
-export const createDocument = (data) => withBackupAssetPublication(() => createDocumentLeased(data));
-export const updateDocument = (id, updates) => withBackupAssetPublication(() => updateDocumentLeased(id, updates));
-export const deleteDocument = (id) => withBackupAssetPublication(() => deleteDocumentLeased(id));
+export const createDocument = (data) => withBackupAssetPublication(() => withMetaLock(() => createDocumentLeased(data)));
+export const updateDocument = (id, updates) => withBackupAssetPublication(() => withMetaLock(() => updateDocumentLeased(id, updates)));
+export const deleteDocument = (id) => withBackupAssetPublication(() => withMetaLock(() => deleteDocumentLeased(id)));
async function createDocumentLeased(data) {
await ensureSoulDir();
@@ -57,7 +58,7 @@ async function createDocumentLeased(data) {
}
// Write the file
- await writeFile(filePath, data.content);
+ await atomicWrite(filePath, data.content);
// Re-creating a filename that was previously deleted must clear its tombstone
// and stamp a creation time that STRICTLY supersedes the deletion — otherwise
@@ -98,7 +99,7 @@ async function updateDocumentLeased(id, updates) {
// Update file content if provided
if (updates.content) {
- await writeFile(filePath, updates.content);
+ await atomicWrite(filePath, updates.content);
docMeta.version = extractVersion(updates.content);
}
diff --git a/server/services/digital-twin-meta.concurrency.test.js b/server/services/digital-twin-meta.concurrency.test.js
new file mode 100644
index 0000000000..d88658f09c
--- /dev/null
+++ b/server/services/digital-twin-meta.concurrency.test.js
@@ -0,0 +1,85 @@
+/**
+ * Concurrent metadata mutations (#10939): documents, personas and settings all
+ * load -> mutate -> saveMeta against one meta.json. When the cache is cold (TTL
+ * expiry, a peer-sync apply) each overlapping caller reads its own copy from
+ * disk, so without a shared lock the last saveMeta silently drops the others'
+ * records and tombstones. Real disk paths, PATHS pointed at a temp dir.
+ */
+
+import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
+import { readdirSync, readFileSync, rmSync } from 'fs';
+import { join } from 'path';
+import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+const tempRoot = createTempDataRoot('portos-dt-meta-concurrency-');
+const twinDir = join(tempRoot, 'digital-twin');
+
+vi.mock('../lib/fileUtils.js', async () => {
+ const actual = await vi.importActual('../lib/fileUtils.js');
+ return makePathsProxy(actual, { dataRoot: tempRoot });
+});
+
+const { createDocument, updateDocument, deleteDocument } = await import('./digital-twin-documents.js');
+const { createPersona, deletePersona } = await import('./digital-twin-personas.js');
+const { loadMeta, saveMeta, updateSettings, cache, digitalTwinEvents } = await import('./digital-twin-meta.js');
+
+const coldCache = () => {
+ cache.meta.data = null;
+ cache.meta.timestamp = 0;
+};
+
+beforeEach(async () => {
+ rmSync(twinDir, { recursive: true, force: true });
+ await saveMeta({ version: '1.0.0', documents: [], deletedDocuments: [], personas: [], deletedPersonas: [], settings: {} });
+ coldCache();
+ // Every save drops the cache again, so each overlapping caller that is not
+ // serialized re-reads its own snapshot from disk.
+ digitalTwinEvents.removeAllListeners('meta:changed');
+ digitalTwinEvents.on('meta:changed', coldCache);
+});
+
+afterAll(() => {
+ digitalTwinEvents.removeAllListeners('meta:changed');
+ rmSync(tempRoot, { recursive: true, force: true });
+});
+
+describe('digital twin meta write serialization (#10939)', () => {
+ it('keeps every persona, tombstone and setting from overlapping mutations', async () => {
+ const seed = await createPersona({ name: 'Seed', instructions: 'Seed persona.' });
+ coldCache();
+
+ const [a, b] = await Promise.all([
+ createPersona({ name: 'A', instructions: 'Persona A.' }),
+ createPersona({ name: 'B', instructions: 'Persona B.' }),
+ deletePersona(seed.id),
+ updateSettings({ maxContextTokens: 1234 }),
+ ]);
+
+ const meta = JSON.parse(readFileSync(join(twinDir, 'meta.json'), 'utf-8'));
+ expect(meta.personas.map((p) => p.id).sort()).toEqual([a.id, b.id].sort());
+ expect(meta.deletedPersonas.map((t) => t.id)).toEqual([seed.id]);
+ expect(meta.settings.maxContextTokens).toBe(1234);
+ });
+
+ it('keeps concurrent document creates and deletes, writing files atomically', async () => {
+ const first = await createDocument({ filename: 'FIRST.md', title: 'First', category: 'core', content: '# First\n' });
+ coldCache();
+
+ await Promise.all([
+ createDocument({ filename: 'SECOND.md', title: 'Second', category: 'core', content: '# Second\n' }),
+ createDocument({ filename: 'THIRD.md', title: 'Third', category: 'core', content: '# Third\n' }),
+ deleteDocument(first.id),
+ ]);
+
+ const meta = JSON.parse(readFileSync(join(twinDir, 'meta.json'), 'utf-8'));
+ expect(meta.documents.map((d) => d.filename).sort()).toEqual(['SECOND.md', 'THIRD.md']);
+ expect(meta.deletedDocuments.map((t) => t.filename)).toEqual(['FIRST.md']);
+
+ const second = meta.documents.find((d) => d.filename === 'SECOND.md');
+ await updateDocument(second.id, { content: '# Second v2\n' });
+ expect(readFileSync(join(twinDir, 'SECOND.md'), 'utf-8')).toBe('# Second v2\n');
+ // atomicWrite renames a temp file into place; nothing may be left behind.
+ expect(readdirSync(twinDir).filter((f) => f.endsWith('.tmp'))).toEqual([]);
+ expect((await loadMeta()).documents).toHaveLength(2);
+ });
+});
diff --git a/server/services/digital-twin-meta.js b/server/services/digital-twin-meta.js
index dd3950ad99..56f4455aa9 100644
--- a/server/services/digital-twin-meta.js
+++ b/server/services/digital-twin-meta.js
@@ -4,6 +4,7 @@ import { join } from 'path';
import EventEmitter from 'events';
import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';
import { safeJSONParse, atomicWrite } from '../lib/fileUtils.js';
+import { createMutex } from '../lib/asyncMutex.js';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
export const META_FILE = join(DIGITAL_TWIN_DIR, 'meta.json');
@@ -22,6 +23,13 @@ const cache = {
};
export const CACHE_TTL_MS = 5000;
+// One lock for every load -> mutate -> saveMeta workflow. loadMeta hands every
+// caller the same cached object, so two overlapping read-modify-write flows
+// would otherwise each push onto it and the later saveMeta would persist
+// whichever tombstone/record set it happened to hold. NOT re-entrant: never
+// call withMetaLock from inside a withMetaLock callback.
+export const withMetaLock = createMutex();
+
// Expose cache for modules that manage test/document caches (testing.js)
export { cache };
@@ -166,16 +174,16 @@ export async function saveMeta(meta) {
digitalTwinEvents.emit('meta:changed', meta);
}
-export async function updateMeta(updates) {
+export const updateMeta = (updates) => withMetaLock(async () => {
const meta = await loadMeta();
const updated = { ...meta, ...updates };
await saveMeta(updated);
return updated;
-}
+});
-export async function updateSettings(settings) {
+export const updateSettings = (settings) => withMetaLock(async () => {
const meta = await loadMeta();
meta.settings = { ...meta.settings, ...settings };
await saveMeta(meta);
return meta.settings;
-}
+});
diff --git a/server/services/digital-twin-personas.js b/server/services/digital-twin-personas.js
index 780577e3a1..296eed5de0 100644
--- a/server/services/digital-twin-personas.js
+++ b/server/services/digital-twin-personas.js
@@ -13,7 +13,7 @@
*/
import { generateId, now } from './digital-twin-helpers.js';
-import { loadMeta, saveMeta } from './digital-twin-meta.js';
+import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';
import { recordTombstone } from '../lib/tombstones.js';
export async function getPersonas() {
@@ -26,7 +26,9 @@ export async function getPersonaById(id) {
return personas.find(p => p.id === id) || null;
}
-export async function createPersona({ name, description, instructions, traitAdjustments }) {
+export const createPersona = (...args) => withMetaLock(() => createPersonaLocked(...args));
+
+async function createPersonaLocked({ name, description, instructions, traitAdjustments }) {
const meta = await loadMeta();
if (!Array.isArray(meta.personas)) meta.personas = [];
@@ -47,7 +49,9 @@ export async function createPersona({ name, description, instructions, traitAdju
return persona;
}
-export async function updatePersona(id, updates) {
+export const updatePersona = (...args) => withMetaLock(() => updatePersonaLocked(...args));
+
+async function updatePersonaLocked(id, updates) {
const meta = await loadMeta();
const persona = (meta.personas || []).find(p => p.id === id);
if (!persona) {
@@ -74,7 +78,9 @@ export async function updatePersona(id, updates) {
return persona;
}
-export async function deletePersona(id) {
+export const deletePersona = (...args) => withMetaLock(() => deletePersonaLocked(...args));
+
+async function deletePersonaLocked(id) {
const meta = await loadMeta();
const before = (meta.personas || []).length;
meta.personas = (meta.personas || []).filter(p => p.id !== id);
@@ -103,7 +109,9 @@ export async function deletePersona(id) {
* Set (or clear, with personaId === null) the active persona. Validates the id
* exists so the settings pointer can't dangle.
*/
-export async function setActivePersona(personaId) {
+export const setActivePersona = (...args) => withMetaLock(() => setActivePersonaLocked(...args));
+
+async function setActivePersonaLocked(personaId) {
const meta = await loadMeta();
if (personaId !== null && !(meta.personas || []).some(p => p.id === personaId)) {
throw new Error(`Persona ${personaId} not found`);
diff --git a/server/services/digital-twin.test.js b/server/services/digital-twin.test.js
index 09b8e1de89..50755ae094 100644
--- a/server/services/digital-twin.test.js
+++ b/server/services/digital-twin.test.js
@@ -430,9 +430,9 @@ describe('digital-twin.js', () => {
expect(result.filename).toBe('NEW_DOC.md');
expect(result.id).toBe('test-uuid-1');
expect(result.content).toBe('# New Doc\n\nContent here.');
- // writeFile for the document file; atomicWrite for the meta save
- expect(writeFile).toHaveBeenCalledTimes(1);
- expect(atomicWrite).toHaveBeenCalledTimes(1);
+ // Document file and meta save both go through atomicWrite (#10939)
+ expect(writeFile).not.toHaveBeenCalled();
+ expect(atomicWrite).toHaveBeenCalledTimes(2);
});
it('should throw if document already exists', async () => {
@@ -483,8 +483,9 @@ describe('digital-twin.js', () => {
});
expect(result).not.toBeNull();
- // writeFile for content + saveMeta
- expect(writeFile).toHaveBeenCalled();
+ // content + meta both via atomicWrite, never an in-place truncating write (#10939)
+ expect(writeFile).not.toHaveBeenCalled();
+ expect(atomicWrite).toHaveBeenCalledTimes(2);
});
it('should return null for non-existent document', async () => {