Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve peer journal edits during local writes

Adam Eivy769a281406b0
diff --git a/server/services/brainJournal.js b/server/services/brainJournal.js
index d44a04057c..f92de3f7be 100644
--- a/server/services/brainJournal.js
+++ b/server/services/brainJournal.js
@@ -63,16 +63,9 @@ export const updateSettings = settingsStore.update;
// ─── Store ─────────────────────────────────────────────────────────────────
-// Serialize every read→mutate→write of a journal entry so a fire-and-forget
-// Obsidian path-persist can't interleave with an appendJournal() and clobber
-// newer segments. PortOS is single-user, but dictation bursts schedule
-// overlapping async tasks (appendJournal → fire-and-forget syncToObsidian →
-// persistObsidianLocation) that all want to rewrite the same entry. This mutex
-// guarantees at most one read-modify-write runs at a time.
-//
-// Entry storage is delegated to brainStorage (type `journals`): getById reads
-// the live record, upsertWithId(date, …) persists it AND appends to the brain
-// sync log so the edit federates. We never touch journals.json directly here.
+// Coordinate local journal writes with the machine-local Obsidian sidecar.
+// Peer applies do not take this mutex: putEntry also runs its read/transform
+// inside brainStorage's per-record queue, shared with every Brain writer.
const storeMutex = createMutex();
// ── Local Obsidian-location sidecar (NOT synced) ─────────────────────────────
@@ -153,7 +146,10 @@ async function getEntry(date) {
return { ...entry, obsidianPath: loc.obsidianPath, obsidianVaultId: loc.obsidianVaultId };
}
-// Persist a fully-formed entry under its date key. The inner `id` AND the
+// Transform the fresh entry under brainStorage's per-record write lock, shared
+// with peer applies. A local-only mutex cannot protect a snapshot read before
+// this queue: an accepted peer edit could otherwise be overwritten by it.
+// Returning null leaves the entry unchanged. The inner `id` AND the
// machine-local Obsidian fields are stripped before storage: the map key IS the
// id (so an inner `id` would be redundant and pollute the synced record /
// reconcile checksum), and obsidianPath/obsidianVaultId must never federate
@@ -161,10 +157,16 @@ async function getEntry(date) {
// because this module emits its own richer bridge-shaped events (journals:upserted
// with the segment, journals:appended, etc.) right after, and we don't want the
// generic brainStorage `journals:upserted` to double-fire.
-async function putEntry(date, entry) {
- // eslint-disable-next-line no-unused-vars
- const { id: _id, obsidianPath: _op, obsidianVaultId: _ov, ...rest } = entry;
- const saved = await brainStorage.upsertWithId('journals', date, rest, { emitEvent: false });
+async function putEntry(date, transform) {
+ const saved = await brainStorage.upsertWithId('journals', date, async (existing) => {
+ const loc = await getObsidianLocation(date);
+ const entry = await transform(existing ? { ...existing, ...loc } : null);
+ if (!entry) return null;
+ // eslint-disable-next-line no-unused-vars
+ const { id: _id, obsidianPath: _op, obsidianVaultId: _ov, ...rest } = entry;
+ return rest;
+ }, { emitEvent: false });
+ if (!saved) return null;
// Re-attach the local location so the returned entry shape is unchanged for callers.
const loc = await getObsidianLocation(date);
return { ...saved, obsidianPath: loc.obsidianPath, obsidianVaultId: loc.obsidianVaultId };
@@ -287,8 +289,7 @@ function scheduleObsidianSync(entry) {
*/
export async function setJournalContent(date, content, { ifMatchUpdatedAt } = {}) {
if (!isIsoDate(date)) throw new Error(`invalid date: ${date}`);
- const entry = await storeMutex(async () => {
- const existing = await getEntry(date);
+ const entry = await storeMutex(() => putEntry(date, (existing) => {
// Precondition: a full rewrite based on a known updatedAt must not clobber
// a concurrent append (voice dictation is the common case — the client
// autosave and the append share the same day key). When the on-disk entry
@@ -318,8 +319,8 @@ export async function setJournalContent(date, content, { ifMatchUpdatedAt } = {}
// to a single 'edit' segment that represents the rewrite.
segments: clean ? [{ text: clean, at: now(), source: 'edit' }] : [],
};
- return putEntry(date, next);
- });
+ return next;
+ }));
scheduleObsidianSync(entry);
// Per-entry event so downstream syncers (memory bridge) can update the
// single affected day without iterating the whole store.
@@ -345,17 +346,18 @@ export async function appendJournal(date, text, { source = 'text' } = {}) {
const segmentSource = normalizeSource(source);
const { entry, segment } = await storeMutex(async () => {
- const existing = await getEntry(date);
- const base = existing || newEntry(date);
- const segmentLocal = { text: clean, at: now(), source: segmentSource };
- const next = {
- ...base,
- segments: [...(Array.isArray(base.segments) ? base.segments : []), segmentLocal],
- content: base.content
- ? `${base.content.trimEnd()}\n\n${clean}`
- : clean,
- };
- const saved = await putEntry(date, next);
+ let segmentLocal;
+ const saved = await putEntry(date, (existing) => {
+ const base = existing || newEntry(date);
+ segmentLocal = { text: clean, at: now(), source: segmentSource };
+ return {
+ ...base,
+ segments: [...(Array.isArray(base.segments) ? base.segments : []), segmentLocal],
+ content: base.content
+ ? `${base.content.trimEnd()}\n\n${clean}`
+ : clean,
+ };
+ });
return { entry: saved, segment: segmentLocal };
});
scheduleObsidianSync(entry);
@@ -382,18 +384,19 @@ export async function appendJournal(date, text, { source = 'text' } = {}) {
*/
export async function upsertAutoSection(date, body, markers) {
if (!isIsoDate(date)) throw new Error(`invalid date: ${date}`);
- const entry = await storeMutex(async () => {
- const existing = await getEntry(date);
+ let changed = false;
+ const entry = await storeMutex(() => putEntry(date, (existing) => {
const base = existing || newEntry(date);
const nextContent = replaceMarkedSection(base.content || '', body, markers);
// Idempotent re-run against an unchanged day: skip the write, sync-log
// append, and Obsidian round-trip entirely.
- if (existing && nextContent === (base.content || '')) return existing;
+ if (existing && nextContent === (base.content || '')) return null;
// Nothing to persist: no prior entry and the draft produced no section.
if (!existing && !nextContent.trim()) return null;
- return putEntry(date, { ...base, content: nextContent });
- });
- if (!entry) return null;
+ changed = true;
+ return { ...base, content: nextContent };
+ }));
+ if (!changed) return entry;
scheduleObsidianSync(entry);
brainEvents.emit('journals:upserted', { entry });
return entry;
diff --git a/server/services/brainJournal.storage.test.js b/server/services/brainJournal.storage.test.js
new file mode 100644
index 0000000000..c287312bd2
--- /dev/null
+++ b/server/services/brainJournal.storage.test.js
@@ -0,0 +1,126 @@
+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
+
+vi.mock('../lib/fileUtils.js', async (importOriginal) => makePathsProxy(await importOriginal(), {
+ dataRoot: () => lazyTempDataRoot('journal-storage-'),
+}));
+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: async () => 'local-instance' }));
+vi.mock('./obsidian.js', () => ({ upsertNote: vi.fn(), deleteNote: vi.fn() }));
+vi.mock('./brainSyncLog.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, appendLocalChanges: vi.fn(actual.appendLocalChanges) };
+});
+
+import * as journal from './brainJournal.js';
+import * as storage from './brainStorage.js';
+import { appendLocalChanges } from './brainSyncLog.js';
+import { PATHS } from '../lib/fileUtils.js';
+
+const store = storage.brainCollectionStores().find(item => item.type === 'journals');
+const markers = { start: '<!-- test:start -->', end: '<!-- test:end -->' };
+const peerTimestamp = '2026-04-17T12:00:01.000Z';
+
+beforeEach(async () => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(new Date('2026-04-17T12:00:00.000Z'));
+ vi.clearAllMocks();
+ journal._clearObsidianLocationsCacheForTest();
+ await journal.updateSettings({ autoSync: false });
+});
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.useRealTimers();
+});
+afterAll(cleanupTempDataRoots);
+
+// Place a real peer write immediately before the local operation enters the
+// storage queue. The old journal code had already read its stale snapshot here;
+// the callback form must instead read the peer's accepted state under the lock.
+function applyPeerBeforeAdmission(date, base) {
+ const queue = store.queueRecordWrite;
+ const peer = {
+ ...base,
+ content: 'Initial text\n\nPeer contribution',
+ segments: [{ text: 'Peer contribution', at: peerTimestamp, source: 'text' }],
+ updatedAt: peerTimestamp,
+ };
+ delete peer.id;
+ delete peer.obsidianPath;
+ delete peer.obsidianVaultId;
+ vi.spyOn(store, 'queueRecordWrite').mockImplementationOnce(async (id, work) => {
+ expect(await storage.applyRemoteRecord('journals', date, peer, 'update')).toEqual({ applied: true });
+ return queue(id, work);
+ });
+ vi.setSystemTime(new Date('2026-04-17T12:00:02.000Z'));
+ return peer;
+}
+
+describe('journal writes sharing the Brain record queue', () => {
+ it('appends to accepted peer content and persists both contributions', async () => {
+ const date = '2026-04-17';
+ const base = await journal.setJournalContent(date, 'Initial text');
+ applyPeerBeforeAdmission(date, base);
+
+ const entry = await journal.appendJournal(date, 'Local contribution', { source: 'voice' });
+
+ expect(entry.content).toBe('Initial text\n\nPeer contribution\n\nLocal contribution');
+ expect(entry.segments.map(segment => segment.text)).toEqual(['Peer contribution', 'Local contribution']);
+ expect(JSON.parse(readFileSync(store.recordPath(date), 'utf8')).content).toBe(entry.content);
+ });
+
+ it('rejects a stale full replacement after peer admission and returns the fresh local mirror context', async () => {
+ const date = '2026-04-18';
+ const base = await journal.setJournalContent(date, 'Initial text');
+ writeFileSync(join(PATHS.brain, 'journal-obsidian-locations.json'), JSON.stringify({
+ [date]: { obsidianPath: 'Daily Log/2026-04-18.md', obsidianVaultId: 'local-vault' },
+ }));
+ journal._clearObsidianLocationsCacheForTest();
+ const peer = applyPeerBeforeAdmission(date, base);
+ appendLocalChanges.mockClear();
+
+ await expect(journal.setJournalContent(date, 'Stale rewrite', { ifMatchUpdatedAt: base.updatedAt }))
+ .rejects.toMatchObject({
+ status: 409,
+ code: 'STALE_JOURNAL',
+ context: { entry: { content: peer.content, updatedAt: peerTimestamp,
+ obsidianPath: 'Daily Log/2026-04-18.md', obsidianVaultId: 'local-vault' } },
+ });
+
+ expect(await storage.getById('journals', date)).toMatchObject(peer);
+ expect(appendLocalChanges).not.toHaveBeenCalled();
+ });
+
+ it('splices an automatic section into fresh peer text without changing its segments', async () => {
+ const date = '2026-04-19';
+ const base = await journal.setJournalContent(date, 'Initial text');
+ const peer = applyPeerBeforeAdmission(date, base);
+
+ const entry = await journal.upsertAutoSection(date, 'Activity summary', markers);
+
+ expect(entry.content).toBe(`${peer.content}\n\n${markers.start}\nActivity summary\n${markers.end}`);
+ expect(entry.segments).toEqual(peer.segments);
+ expect(JSON.parse(readFileSync(store.recordPath(date), 'utf8')).content).toBe(entry.content);
+ });
+
+ it('leaves storage, federation and events untouched for unchanged or empty auto sections', async () => {
+ const date = '2026-04-20';
+ const entry = await journal.upsertAutoSection(date, 'Activity summary', markers);
+ const bytes = readFileSync(store.recordPath(date), 'utf8');
+ const emitted = vi.fn();
+ storage.brainEvents.on('journals:upserted', emitted);
+ appendLocalChanges.mockClear();
+ try {
+ vi.setSystemTime(new Date('2026-04-17T12:00:03.000Z'));
+ expect(await journal.upsertAutoSection(date, 'Activity summary', markers)).toEqual(entry);
+ expect(await journal.upsertAutoSection('2026-04-21', '', markers)).toBeNull();
+ expect(readFileSync(store.recordPath(date), 'utf8')).toBe(bytes);
+ expect(await storage.getById('journals', '2026-04-21')).toBeNull();
+ expect(appendLocalChanges).not.toHaveBeenCalled();
+ expect(emitted).not.toHaveBeenCalled();
+ } finally {
+ storage.brainEvents.off('journals:upserted', emitted);
+ }
+ });
+});
diff --git a/server/services/brainJournal.test.js b/server/services/brainJournal.test.js
index 28c8c36113..d0ee30cd72 100644
--- a/server/services/brainJournal.test.js
+++ b/server/services/brainJournal.test.js
@@ -62,6 +62,11 @@ vi.mock('./brainStorage.js', () => ({
.map(([id, rec]) => ({ id, ...rec }))),
upsertWithId: vi.fn(async (_type, id, record) => {
const existing = journalRecords.get(id);
+ const live = existing && !existing._deleted ? { id, ...existing } : null;
+ if (typeof record === 'function') {
+ record = await record(live);
+ if (record == null) return live;
+ }
clock.n += 1;
const stamp = `2026-04-17T12:00:${String(clock.n).padStart(2, '0')}.000Z`;
const stored = {
@@ -84,7 +89,7 @@ vi.mock('./brainStorage.js', () => ({
}));
import * as journal from './brainJournal.js';
-import { brainEvents, getAll, getById } from './brainStorage.js';
+import { brainEvents, getAll } from './brainStorage.js';
import { atomicWrite } from '../lib/fileUtils.js';
import * as obsidian from './obsidian.js';
@@ -299,8 +304,7 @@ describe('brainJournal', () => {
it('accepts a rewrite when the on-disk entry has no updatedAt to compare', async () => {
// Legacy / hand-edited stores may lack a clock. Rejecting forever would
// trap the day; accept the write when there is nothing to precondition on.
- getById.mockResolvedValueOnce({
- id: '2026-04-17',
+ journalRecords.set('2026-04-17', {
date: '2026-04-17',
content: 'legacy body',
segments: [{ text: 'legacy body', at: '2026-04-17T10:00:00.000Z', source: 'edit' }],
diff --git a/server/services/brainStorage.js b/server/services/brainStorage.js
index 3655f5cca1..d323df2fd2 100644
--- a/server/services/brainStorage.js
+++ b/server/services/brainStorage.js
@@ -21,9 +21,9 @@
* read a stale snapshot and drop the other write (the AGENTS.md-sanctioned
* "serialize two write paths that mutate the same record" case). Writes to
* different records no longer serialize against each other. brainJournal layers
- * its own storeMutex ON TOP for the read→mutate→write of a single journal entry;
- * that only ever nests storeMutex → this per-id queue (one direction), so no
- * deadlock.
+ * its own storeMutex ON TOP for local sidecar coordination; journal transforms
+ * run inside this per-id queue too, so peer applies cannot invalidate their
+ * snapshot. Locks only nest storeMutex → this queue (one direction).
*
* FEDERATION is unaffected by the layout. Brain federates strictly per-record
* through this module's API seams (`getRawRecords` / `applyRemoteRecord`) plus
@@ -558,6 +558,11 @@ export async function updateWith(type, id, fn) {
* pass the FULL desired record (this does not merge unknown fields the way
* `update()` does). `emitEvent:false` lets a caller (e.g. brainJournal) emit its
* own richer, bridge-shaped event instead of the generic one.
+ * A function receives the fresh live `{ id, ...record }` (or null) INSIDE the
+ * per-id queue and returns the full replacement. Returning null/undefined
+ * leaves storage, events and the sync log untouched and returns the live record.
+ * Use this form for derived content and preconditions that must serialize with
+ * peer applies as well as other local writes.
* createOnly:true returns null for any existing record, including tombstones.
*/
export async function upsertWithId(type, id, recordData, { emitEvent = true, createOnly = false } = {}) {
@@ -574,11 +579,15 @@ export async function upsertWithId(type, id, recordData, { emitEvent = true, cre
// Test presence inside the queue so a peer or local write cannot race.
if (createOnly && existing) return null;
const live = existing && !isTombstone(existing) ? existing : null;
+ const input = typeof recordData === 'function'
+ ? await recordData(live ? { id, ...live } : null)
+ : recordData;
+ if (typeof recordData === 'function' && input == null) return live ? { id, ...live } : null;
const timestamp = now();
const originInstanceId = live?.originInstanceId ?? fallbackOrigin;
const record = {
- ...recordData,
+ ...input,
originInstanceId,
createdAt: live?.createdAt ?? timestamp,
updatedAt: timestamp