Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve peer journal edits during local writes
Adam Eivy769a281406b0
diff --git a/server/services/brainJournal.js b/server/services/brainJournal.jsindex d44a04057c..f92de3f7be 100644--- a/server/services/brainJournal.js+++ b/server/services/brainJournal.js@@ -63,16 +63,9 @@ export const updateSettings = settingsStore.update;// ─── Store ─────────────────────────────────────────────────────────────────-// Serialize every read→mutate→write of a journal entry so a fire-and-forget-// Obsidian path-persist can't interleave with an appendJournal() and clobber-// newer segments. PortOS is single-user, but dictation bursts schedule-// overlapping async tasks (appendJournal → fire-and-forget syncToObsidian →-// persistObsidianLocation) that all want to rewrite the same entry. This mutex-// guarantees at most one read-modify-write runs at a time.-//-// Entry storage is delegated to brainStorage (type `journals`): getById reads-// the live record, upsertWithId(date, …) persists it AND appends to the brain-// sync log so the edit federates. We never touch journals.json directly here.+// Coordinate local journal writes with the machine-local Obsidian sidecar.+// Peer applies do not take this mutex: putEntry also runs its read/transform+// inside brainStorage's per-record queue, shared with every Brain writer.const storeMutex = createMutex();// ── Local Obsidian-location sidecar (NOT synced) ─────────────────────────────@@ -153,7 +146,10 @@ async function getEntry(date) {return { ...entry, obsidianPath: loc.obsidianPath, obsidianVaultId: loc.obsidianVaultId };}-// Persist a fully-formed entry under its date key. The inner `id` AND the+// Transform the fresh entry under brainStorage's per-record write lock, shared+// with peer applies. A local-only mutex cannot protect a snapshot read before+// this queue: an accepted peer edit could otherwise be overwritten by it.+// Returning null leaves the entry unchanged. The inner `id` AND the// machine-local Obsidian fields are stripped before storage: the map key IS the// id (so an inner `id` would be redundant and pollute the synced record /// reconcile checksum), and obsidianPath/obsidianVaultId must never federate@@ -161,10 +157,16 @@ async function getEntry(date) {// because this module emits its own richer bridge-shaped events (journals:upserted// with the segment, journals:appended, etc.) right after, and we don't want the// generic brainStorage `journals:upserted` to double-fire.-async function putEntry(date, entry) {- // eslint-disable-next-line no-unused-vars- const { id: _id, obsidianPath: _op, obsidianVaultId: _ov, ...rest } = entry;- const saved = await brainStorage.upsertWithId('journals', date, rest, { emitEvent: false });+async function putEntry(date, transform) {+ const saved = await brainStorage.upsertWithId('journals', date, async (existing) => {+ const loc = await getObsidianLocation(date);+ const entry = await transform(existing ? { ...existing, ...loc } : null);+ if (!entry) return null;+ // eslint-disable-next-line no-unused-vars+ const { id: _id, obsidianPath: _op, obsidianVaultId: _ov, ...rest } = entry;+ return rest;+ }, { emitEvent: false });+ if (!saved) return null;// Re-attach the local location so the returned entry shape is unchanged for callers.const loc = await getObsidianLocation(date);return { ...saved, obsidianPath: loc.obsidianPath, obsidianVaultId: loc.obsidianVaultId };@@ -287,8 +289,7 @@ function scheduleObsidianSync(entry) {*/export async function setJournalContent(date, content, { ifMatchUpdatedAt } = {}) {if (!isIsoDate(date)) throw new Error(`invalid date: ${date}`);- const entry = await storeMutex(async () => {- const existing = await getEntry(date);+ const entry = await storeMutex(() => putEntry(date, (existing) => {// Precondition: a full rewrite based on a known updatedAt must not clobber// a concurrent append (voice dictation is the common case — the client// autosave and the append share the same day key). When the on-disk entry@@ -318,8 +319,8 @@ export async function setJournalContent(date, content, { ifMatchUpdatedAt } = {}// to a single 'edit' segment that represents the rewrite.segments: clean ? [{ text: clean, at: now(), source: 'edit' }] : [],};- return putEntry(date, next);- });+ return next;+ }));scheduleObsidianSync(entry);// Per-entry event so downstream syncers (memory bridge) can update the// single affected day without iterating the whole store.@@ -345,17 +346,18 @@ export async function appendJournal(date, text, { source = 'text' } = {}) {const segmentSource = normalizeSource(source);const { entry, segment } = await storeMutex(async () => {- const existing = await getEntry(date);- const base = existing || newEntry(date);- const segmentLocal = { text: clean, at: now(), source: segmentSource };- const next = {- ...base,- segments: [...(Array.isArray(base.segments) ? base.segments : []), segmentLocal],- content: base.content- ? `${base.content.trimEnd()}\n\n${clean}`- : clean,- };- const saved = await putEntry(date, next);+ let segmentLocal;+ const saved = await putEntry(date, (existing) => {+ const base = existing || newEntry(date);+ segmentLocal = { text: clean, at: now(), source: segmentSource };+ return {+ ...base,+ segments: [...(Array.isArray(base.segments) ? base.segments : []), segmentLocal],+ content: base.content+ ? `${base.content.trimEnd()}\n\n${clean}`+ : clean,+ };+ });return { entry: saved, segment: segmentLocal };});scheduleObsidianSync(entry);@@ -382,18 +384,19 @@ export async function appendJournal(date, text, { source = 'text' } = {}) {*/export async function upsertAutoSection(date, body, markers) {if (!isIsoDate(date)) throw new Error(`invalid date: ${date}`);- const entry = await storeMutex(async () => {- const existing = await getEntry(date);+ let changed = false;+ const entry = await storeMutex(() => putEntry(date, (existing) => {const base = existing || newEntry(date);const nextContent = replaceMarkedSection(base.content || '', body, markers);// Idempotent re-run against an unchanged day: skip the write, sync-log// append, and Obsidian round-trip entirely.- if (existing && nextContent === (base.content || '')) return existing;+ if (existing && nextContent === (base.content || '')) return null;// Nothing to persist: no prior entry and the draft produced no section.if (!existing && !nextContent.trim()) return null;- return putEntry(date, { ...base, content: nextContent });- });- if (!entry) return null;+ changed = true;+ return { ...base, content: nextContent };+ }));+ if (!changed) return entry;scheduleObsidianSync(entry);brainEvents.emit('journals:upserted', { entry });return entry;diff --git a/server/services/brainJournal.storage.test.js b/server/services/brainJournal.storage.test.jsnew file mode 100644index 0000000000..c287312bd2--- /dev/null+++ b/server/services/brainJournal.storage.test.js@@ -0,0 +1,126 @@+import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { readFileSync, writeFileSync } from 'fs';+import { join } from 'path';+import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';++vi.mock('../lib/fileUtils.js', async (importOriginal) => makePathsProxy(await importOriginal(), {+ dataRoot: () => lazyTempDataRoot('journal-storage-'),+}));+vi.mock('./instanceIdentity.js', () => ({ getInstanceId: async () => 'local-instance' }));+vi.mock('./obsidian.js', () => ({ upsertNote: vi.fn(), deleteNote: vi.fn() }));+vi.mock('./brainSyncLog.js', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, appendLocalChanges: vi.fn(actual.appendLocalChanges) };+});++import * as journal from './brainJournal.js';+import * as storage from './brainStorage.js';+import { appendLocalChanges } from './brainSyncLog.js';+import { PATHS } from '../lib/fileUtils.js';++const store = storage.brainCollectionStores().find(item => item.type === 'journals');+const markers = { start: '<!-- test:start -->', end: '<!-- test:end -->' };+const peerTimestamp = '2026-04-17T12:00:01.000Z';++beforeEach(async () => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(new Date('2026-04-17T12:00:00.000Z'));+ vi.clearAllMocks();+ journal._clearObsidianLocationsCacheForTest();+ await journal.updateSettings({ autoSync: false });+});+afterEach(() => {+ vi.restoreAllMocks();+ vi.useRealTimers();+});+afterAll(cleanupTempDataRoots);++// Place a real peer write immediately before the local operation enters the+// storage queue. The old journal code had already read its stale snapshot here;+// the callback form must instead read the peer's accepted state under the lock.+function applyPeerBeforeAdmission(date, base) {+ const queue = store.queueRecordWrite;+ const peer = {+ ...base,+ content: 'Initial text\n\nPeer contribution',+ segments: [{ text: 'Peer contribution', at: peerTimestamp, source: 'text' }],+ updatedAt: peerTimestamp,+ };+ delete peer.id;+ delete peer.obsidianPath;+ delete peer.obsidianVaultId;+ vi.spyOn(store, 'queueRecordWrite').mockImplementationOnce(async (id, work) => {+ expect(await storage.applyRemoteRecord('journals', date, peer, 'update')).toEqual({ applied: true });+ return queue(id, work);+ });+ vi.setSystemTime(new Date('2026-04-17T12:00:02.000Z'));+ return peer;+}++describe('journal writes sharing the Brain record queue', () => {+ it('appends to accepted peer content and persists both contributions', async () => {+ const date = '2026-04-17';+ const base = await journal.setJournalContent(date, 'Initial text');+ applyPeerBeforeAdmission(date, base);++ const entry = await journal.appendJournal(date, 'Local contribution', { source: 'voice' });++ expect(entry.content).toBe('Initial text\n\nPeer contribution\n\nLocal contribution');+ expect(entry.segments.map(segment => segment.text)).toEqual(['Peer contribution', 'Local contribution']);+ expect(JSON.parse(readFileSync(store.recordPath(date), 'utf8')).content).toBe(entry.content);+ });++ it('rejects a stale full replacement after peer admission and returns the fresh local mirror context', async () => {+ const date = '2026-04-18';+ const base = await journal.setJournalContent(date, 'Initial text');+ writeFileSync(join(PATHS.brain, 'journal-obsidian-locations.json'), JSON.stringify({+ [date]: { obsidianPath: 'Daily Log/2026-04-18.md', obsidianVaultId: 'local-vault' },+ }));+ journal._clearObsidianLocationsCacheForTest();+ const peer = applyPeerBeforeAdmission(date, base);+ appendLocalChanges.mockClear();++ await expect(journal.setJournalContent(date, 'Stale rewrite', { ifMatchUpdatedAt: base.updatedAt }))+ .rejects.toMatchObject({+ status: 409,+ code: 'STALE_JOURNAL',+ context: { entry: { content: peer.content, updatedAt: peerTimestamp,+ obsidianPath: 'Daily Log/2026-04-18.md', obsidianVaultId: 'local-vault' } },+ });++ expect(await storage.getById('journals', date)).toMatchObject(peer);+ expect(appendLocalChanges).not.toHaveBeenCalled();+ });++ it('splices an automatic section into fresh peer text without changing its segments', async () => {+ const date = '2026-04-19';+ const base = await journal.setJournalContent(date, 'Initial text');+ const peer = applyPeerBeforeAdmission(date, base);++ const entry = await journal.upsertAutoSection(date, 'Activity summary', markers);++ expect(entry.content).toBe(`${peer.content}\n\n${markers.start}\nActivity summary\n${markers.end}`);+ expect(entry.segments).toEqual(peer.segments);+ expect(JSON.parse(readFileSync(store.recordPath(date), 'utf8')).content).toBe(entry.content);+ });++ it('leaves storage, federation and events untouched for unchanged or empty auto sections', async () => {+ const date = '2026-04-20';+ const entry = await journal.upsertAutoSection(date, 'Activity summary', markers);+ const bytes = readFileSync(store.recordPath(date), 'utf8');+ const emitted = vi.fn();+ storage.brainEvents.on('journals:upserted', emitted);+ appendLocalChanges.mockClear();+ try {+ vi.setSystemTime(new Date('2026-04-17T12:00:03.000Z'));+ expect(await journal.upsertAutoSection(date, 'Activity summary', markers)).toEqual(entry);+ expect(await journal.upsertAutoSection('2026-04-21', '', markers)).toBeNull();+ expect(readFileSync(store.recordPath(date), 'utf8')).toBe(bytes);+ expect(await storage.getById('journals', '2026-04-21')).toBeNull();+ expect(appendLocalChanges).not.toHaveBeenCalled();+ expect(emitted).not.toHaveBeenCalled();+ } finally {+ storage.brainEvents.off('journals:upserted', emitted);+ }+ });+});diff --git a/server/services/brainJournal.test.js b/server/services/brainJournal.test.jsindex 28c8c36113..d0ee30cd72 100644--- a/server/services/brainJournal.test.js+++ b/server/services/brainJournal.test.js@@ -62,6 +62,11 @@ vi.mock('./brainStorage.js', () => ({.map(([id, rec]) => ({ id, ...rec }))),upsertWithId: vi.fn(async (_type, id, record) => {const existing = journalRecords.get(id);+ const live = existing && !existing._deleted ? { id, ...existing } : null;+ if (typeof record === 'function') {+ record = await record(live);+ if (record == null) return live;+ }clock.n += 1;const stamp = `2026-04-17T12:00:${String(clock.n).padStart(2, '0')}.000Z`;const stored = {@@ -84,7 +89,7 @@ vi.mock('./brainStorage.js', () => ({}));import * as journal from './brainJournal.js';-import { brainEvents, getAll, getById } from './brainStorage.js';+import { brainEvents, getAll } from './brainStorage.js';import { atomicWrite } from '../lib/fileUtils.js';import * as obsidian from './obsidian.js';@@ -299,8 +304,7 @@ describe('brainJournal', () => {it('accepts a rewrite when the on-disk entry has no updatedAt to compare', async () => {// Legacy / hand-edited stores may lack a clock. Rejecting forever would// trap the day; accept the write when there is nothing to precondition on.- getById.mockResolvedValueOnce({- id: '2026-04-17',+ journalRecords.set('2026-04-17', {date: '2026-04-17',content: 'legacy body',segments: [{ text: 'legacy body', at: '2026-04-17T10:00:00.000Z', source: 'edit' }],diff --git a/server/services/brainStorage.js b/server/services/brainStorage.jsindex 3655f5cca1..d323df2fd2 100644--- a/server/services/brainStorage.js+++ b/server/services/brainStorage.js@@ -21,9 +21,9 @@* read a stale snapshot and drop the other write (the AGENTS.md-sanctioned* "serialize two write paths that mutate the same record" case). Writes to* different records no longer serialize against each other. brainJournal layers- * its own storeMutex ON TOP for the read→mutate→write of a single journal entry;- * that only ever nests storeMutex → this per-id queue (one direction), so no- * deadlock.+ * its own storeMutex ON TOP for local sidecar coordination; journal transforms+ * run inside this per-id queue too, so peer applies cannot invalidate their+ * snapshot. Locks only nest storeMutex → this queue (one direction).** FEDERATION is unaffected by the layout. Brain federates strictly per-record* through this module's API seams (`getRawRecords` / `applyRemoteRecord`) plus@@ -558,6 +558,11 @@ export async function updateWith(type, id, fn) {* pass the FULL desired record (this does not merge unknown fields the way* `update()` does). `emitEvent:false` lets a caller (e.g. brainJournal) emit its* own richer, bridge-shaped event instead of the generic one.+ * A function receives the fresh live `{ id, ...record }` (or null) INSIDE the+ * per-id queue and returns the full replacement. Returning null/undefined+ * leaves storage, events and the sync log untouched and returns the live record.+ * Use this form for derived content and preconditions that must serialize with+ * peer applies as well as other local writes.* createOnly:true returns null for any existing record, including tombstones.*/export async function upsertWithId(type, id, recordData, { emitEvent = true, createOnly = false } = {}) {@@ -574,11 +579,15 @@ export async function upsertWithId(type, id, recordData, { emitEvent = true, cre// Test presence inside the queue so a peer or local write cannot race.if (createOnly && existing) return null;const live = existing && !isTombstone(existing) ? existing : null;+ const input = typeof recordData === 'function'+ ? await recordData(live ? { id, ...live } : null)+ : recordData;+ if (typeof recordData === 'function' && input == null) return live ? { id, ...live } : null;const timestamp = now();const originInstanceId = live?.originInstanceId ?? fallbackOrigin;const record = {- ...recordData,+ ...input,originInstanceId,createdAt: live?.createdAt ?? timestamp,updatedAt: timestamp