Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: route twin feedback sync through feedbackLoop and union entries by id (#12441)
[._.]/ Adam Eivyc9ebde1280dd
diff --git a/server/services/digital-twin-sync.js b/server/services/digital-twin-sync.jsindex a3c67f77f3..085d38db7a 100644--- a/server/services/digital-twin-sync.js+++ b/server/services/digital-twin-sync.js@@ -15,7 +15,7 @@ import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';* - identity.json — LWW on updatedAt* - chronotype.json — deep union (derived markers, derivedAt tiebreak)* - longevity.json — deep union (derived markers, derivedAt tiebreak)- * - feedback.json — LWW on updatedAt+ * - feedback.json — union of entries by id (stats/weights LWW)* - taste-profile.json — per-section union of responses (never lose answers)* - meta.json — union of documents/histories/personas, deep-union* enrichment, fill-missing settings, and the analyzed@@ -247,6 +247,53 @@ function pickStatus(a, b) {return (TASTE_STATUS_RANK[b] ?? -1) > (TASTE_STATUS_RANK[a] ?? -1) ? b : a;}+/**+ * Merge behavioral feedback. Entries are immutable records, so union by id+ * (newest createdAt wins a duplicate id; local wins ties) and keep the latest+ * 500 like the local cap. Feedback recorded on both machines between syncs+ * survives on both. Counters never drop below what either side recorded; the+ * weight adjustments follow whichever side recalculated last.+ */+export function mergeFeedback(local, remote) {+ if (!isPlainObject(remote) || !Array.isArray(remote.entries)) return { merged: local, changed: false };+ if (!isPlainObject(local)) return { merged: remote, changed: true };++ const localEntries = Array.isArray(local.entries) ? local.entries : [];+ const byId = new Map();+ for (const e of localEntries) if (isPlainObject(e) && e.id) byId.set(e.id, e);+ let changed = false;+ for (const e of remote.entries) {+ if (!isPlainObject(e) || !e.id) continue;+ const cur = byId.get(e.id);+ if (!cur || (e.createdAt || '') > (cur.createdAt || '')) {+ byId.set(e.id, e);+ changed = true;+ }+ }+ if (!changed) return { merged: local, changed: false };++ const entries = [...byId.values()]+ .sort((a, b) => (a.createdAt || '').localeCompare(b.createdAt || ''))+ .slice(-500);+ const count = (v) => entries.filter((e) => e.validation === v).length;+ const stat = (k, n) => Math.max(local.stats?.[k] || 0, remote.stats?.[k] || 0, n);+ const remoteRecalcNewer = (remote.lastCalculatedAt || '') > (local.lastCalculatedAt || '');+ const weightSource = remoteRecalcNewer ? remote : local;+ const merged = {+ ...local,+ entries,+ stats: {+ totalFeedback: stat('totalFeedback', entries.length),+ soundsLikeMe: stat('soundsLikeMe', count('sounds_like_me')),+ doesntSoundLikeMe: stat('doesntSoundLikeMe', count('doesnt_sound_like_me')),+ },+ documentWeightAdjustments: weightSource.documentWeightAdjustments ?? {},+ lastCalculatedAt: weightSource.lastCalculatedAt ?? null,+ updatedAt: [local.updatedAt, remote.updatedAt].filter(Boolean).sort().pop() ?? null,+ };+ return { merged, changed: true };+}+/*** Merge taste profiles. Within each section, responses union by questionId* (LWW on updatedAt||answeredAt) so answers given on either machine survive;@@ -887,6 +934,14 @@ async function applyTaste(remoteTaste) {return (await applyTasteProfileMerge((local) => mergeTaste(local, remoteTaste))) ? 1 : 0;}+// feedbackLoop keeps a never-invalidated cache and a write queue; merging through+// it keeps a later local submit from overwriting synced entries.+async function applyFeedback(remoteFeedback) {+ if (!isPlainObject(remoteFeedback)) return 0;+ const { applyFeedbackMerge } = await import('./feedbackLoop.js');+ return (await applyFeedbackMerge((local) => mergeFeedback(local, remoteFeedback))) ? 1 : 0;+}+async function applyMeta(remoteMeta) {if (!isPlainObject(remoteMeta)) return 0;const { loadMeta, saveMeta } = await import('./digital-twin-meta.js');@@ -919,7 +974,7 @@ export async function applyDigitalTwinRemote(remoteData) {count += await applyMerge(IDENTITY_FILE, remoteData.identity, (l, r) => mergeObjectLWW(l, r, 'updatedAt'));count += await applyMerge(CHRONOTYPE_FILE, remoteData.chronotype, (l, r) => mergeDeepUnion(l, r, 'derivedAt'));count += await applyMerge(LONGEVITY_FILE, remoteData.longevity, (l, r) => mergeDeepUnion(l, r, 'derivedAt'));- count += await applyMerge(FEEDBACK_FILE, remoteData.feedback, (l, r) => mergeObjectLWW(l, r, 'updatedAt'));+ count += await applyFeedback(remoteData.feedback);count += await applyTaste(remoteData.taste);// Observed evidence (Phase 7): regenerated derived records — body is LWW on// derivedAt; taste also preserves the newest user-triggered AI interpretation.diff --git a/server/services/feedbackLoop.js b/server/services/feedbackLoop.jsindex 0d516f7cd4..c4dade8f91 100644--- a/server/services/feedbackLoop.js+++ b/server/services/feedbackLoop.js@@ -6,11 +6,10 @@* Tracks feedback patterns and adjusts document weights for confidence scoring.*/-import { readFile } from 'fs/promises';-import { existsSync } from 'fs';import { join } from 'path';import { v4 as uuidv4 } from '../lib/uuid.js';-import { atomicWrite, ensureDir, safeJSONParse, PATHS } from '../lib/fileUtils.js';+import { atomicWrite, ensureDir, readJSONFile, PATHS } from '../lib/fileUtils.js';+import { createFileWriteQueue } from '../lib/fileWriteQueue.js';import { digitalTwinEvents } from './digital-twin.js';const DIGITAL_TWIN_DIR = PATHS.digitalTwin;@@ -26,36 +25,57 @@ function now() {let feedbackCache = null;-async function loadFeedback() {- if (feedbackCache) return feedbackCache;+// Peer sync (digital-twin-sync) and local mutators share feedback.json; every+// read-modify-write runs in this queue against a fresh read so neither side+// overwrites the other from a stale cache.+const queueFeedbackWrite = createFileWriteQueue();- const defaultData = {+function defaultFeedback() {+ return {version: '1.0.0',entries: [],stats: { totalFeedback: 0, soundsLikeMe: 0, doesntSoundLikeMe: 0 },documentWeightAdjustments: {},lastCalculatedAt: null};+}- if (!existsSync(FEEDBACK_FILE)) {- await saveFeedback(defaultData);- return defaultData;- }+// Strict: ENOENT yields the default, but an unreadable or malformed file throws+// so a mutator never replaces user feedback it failed to parse.+async function readFeedbackFresh() {+ return (await readJSONFile(FEEDBACK_FILE, null, { strict: true })) ?? defaultFeedback();+}- const raw = await readFile(FEEDBACK_FILE, 'utf-8');- feedbackCache = safeJSONParse(raw, defaultData);+async function loadFeedback() {+ if (feedbackCache) return feedbackCache;+ feedbackCache = await readFeedbackFresh();return feedbackCache;}async function saveFeedback(data) {- if (!existsSync(DIGITAL_TWIN_DIR)) {- await ensureDir(DIGITAL_TWIN_DIR);- }+ await ensureDir(DIGITAL_TWIN_DIR);data.updatedAt = now();await atomicWrite(FEEDBACK_FILE, data);feedbackCache = data;}+/**+ * Apply a peer-sync merge to feedback.json inside the write queue. `mergeFn(local)`+ * returns `{ merged, changed }`; the cache is replaced with what was written.+ * Returns whether anything was written.+ */+export function applyFeedbackMerge(mergeFn) {+ return queueFeedbackWrite(async () => {+ const local = await readJSONFile(FEEDBACK_FILE, null, { strict: true });+ const { merged, changed } = mergeFn(local);+ if (!changed) return false;+ await ensureDir(DIGITAL_TWIN_DIR);+ await atomicWrite(FEEDBACK_FILE, merged);+ feedbackCache = merged;+ return true;+ });+}+// =============================================================================// PUBLIC API// =============================================================================@@ -73,8 +93,6 @@ async function saveFeedback(data) {* @param {string[]} [params.documentsUsed] - Which twin documents influenced the response*/export async function submitFeedback({ contentType, validation, contentSnippet, context, providerId, model, documentsUsed }) {- const data = await loadFeedback();-const entry = {id: uuidv4(),contentType,@@ -87,19 +105,22 @@ export async function submitFeedback({ contentType, validation, contentSnippet,createdAt: now()};- data.entries.push(entry);+ const data = await queueFeedbackWrite(async () => {+ const fresh = await readFeedbackFresh();+ fresh.entries.push(entry);- // Update running stats- data.stats.totalFeedback++;- if (validation === 'sounds_like_me') data.stats.soundsLikeMe++;- else if (validation === 'doesnt_sound_like_me') data.stats.doesntSoundLikeMe++;+ fresh.stats.totalFeedback++;+ if (validation === 'sounds_like_me') fresh.stats.soundsLikeMe++;+ else if (validation === 'doesnt_sound_like_me') fresh.stats.doesntSoundLikeMe++;- // Keep last 500 entries to prevent unbounded growth- if (data.entries.length > 500) {- data.entries = data.entries.slice(-500);- }+ // Keep last 500 entries to prevent unbounded growth+ if (fresh.entries.length > 500) {+ fresh.entries = fresh.entries.slice(-500);+ }- await saveFeedback(data);+ await saveFeedback(fresh);+ return fresh;+ });digitalTwinEvents.emit('feedback:submitted', entry);console.log(`🔄 Feedback submitted: ${validation} on ${contentType} (${data.stats.totalFeedback} total)`);@@ -202,8 +223,12 @@ export async function getFeedbackStats() {* Documents that appear in "doesn't sound like me" responses get reduced.* Returns the new weight adjustments (deltas from base weight).*/-export async function recalculateWeights() {- const data = await loadFeedback();+export function recalculateWeights() {+ return queueFeedbackWrite(recalculateWeightsLocked);+}++async function recalculateWeightsLocked() {+ const data = await readFeedbackFresh();const entries = data.entries.filter(e => e.documentsUsed?.length > 0);if (entries.length < 5) {diff --git a/server/services/feedbackLoop.test.js b/server/services/feedbackLoop.test.jsnew file mode 100644index 0000000000..96ba650be5--- /dev/null+++ b/server/services/feedbackLoop.test.js@@ -0,0 +1,79 @@+import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdir, readFile, rm, writeFile } from 'fs/promises';+import { join } from 'path';+import { mockPathsDataRoot } from '../lib/mockPathsDataRoot.js';++const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-feedback-' });++vi.mock('../lib/fileUtils.js', async () => {+ const actual = await vi.importActual('../lib/fileUtils.js');+ return makeProxy(actual);+});++vi.mock('./digital-twin.js', () => ({ digitalTwinEvents: { emit: vi.fn() } }));++const FILE = join(tempRoot, 'digital-twin', 'feedback.json');+const entry = (id, createdAt, validation = 'sounds_like_me') => ({ id, validation, createdAt, contentType: 'enrichment', documentsUsed: [] });+const doc = (entries, updatedAt) => ({+ version: '1.0.0', entries, updatedAt,+ stats: { totalFeedback: entries.length, soundsLikeMe: entries.length, doesntSoundLikeMe: 0 },+ documentWeightAdjustments: {}, lastCalculatedAt: null,+});+const readDisk = async () => JSON.parse(await readFile(FILE, 'utf-8'));++let feedback;+let sync;+beforeEach(async () => {+ vi.resetModules();+ await rm(join(tempRoot, 'digital-twin'), { recursive: true, force: true });+ await mkdir(join(tempRoot, 'digital-twin'), { recursive: true });+ feedback = await import('./feedbackLoop.js');+ sync = await import('./digital-twin-sync.js');+});+afterAll(cleanup);++const merge = (remote) => feedback.applyFeedbackMerge((local) => sync.mergeFeedback(local, remote));++describe('feedback peer merge vs local submit', () => {+ it('keeps synced entries when a local submit follows a peer merge (stale cache)', async () => {+ await feedback.submitFeedback({ contentType: 'enrichment', validation: 'sounds_like_me', contentSnippet: 'a' }); // primes cache+ expect(await merge(doc([entry('remote-1', '2026-01-01T00:00:00.000Z')], '2026-01-01T00:00:00.000Z'))).toBe(true);++ await feedback.submitFeedback({ contentType: 'enrichment', validation: 'not_quite', contentSnippet: 'b' });++ const ids = (await readDisk()).entries.map((e) => e.id);+ expect(ids).toContain('remote-1');+ expect(ids).toHaveLength(3);+ expect((await feedback.getRecentFeedback()).length).toBe(3);+ });++ it('converges to the union when both machines recorded feedback between syncs', async () => {+ const a = doc([entry('shared', '2026-01-01T00:00:00.000Z'), entry('a-only', '2026-01-02T00:00:00.000Z')], '2026-01-03T00:00:00.000Z');+ const b = doc([entry('shared', '2026-01-01T00:00:00.000Z'), entry('b-only', '2026-01-02T12:00:00.000Z')], '2026-01-02T12:00:00.000Z');+ await writeFile(FILE, JSON.stringify(a));++ await merge(b);+ const onA = (await readDisk()).entries.map((e) => e.id).sort();+ const onB = sync.mergeFeedback(b, a).merged.entries.map((e) => e.id).sort();+ expect(onA).toEqual(['a-only', 'b-only', 'shared']);+ expect(onB).toEqual(onA);+ });++ it('does not rewrite when the remote adds nothing', async () => {+ const a = doc([entry('x', '2026-01-01T00:00:00.000Z')], '2026-01-03T00:00:00.000Z');+ await writeFile(FILE, JSON.stringify(a));+ expect(await merge(doc([entry('x', '2026-01-01T00:00:00.000Z')], '2026-01-04T00:00:00.000Z'))).toBe(false);+ });++ it('refuses to overwrite a malformed feedback.json on merge or submit', async () => {+ await writeFile(FILE, '{ not json');+ await expect(merge(doc([entry('r', '2026-01-01T00:00:00.000Z')], 'x'))).rejects.toThrow();+ await expect(feedback.submitFeedback({ contentType: 'enrichment', validation: 'sounds_like_me', contentSnippet: 'c' })).rejects.toThrow();+ expect(await readFile(FILE, 'utf-8')).toBe('{ not json');+ });++ it('initializes a missing file on first submit', async () => {+ await feedback.submitFeedback({ contentType: 'enrichment', validation: 'sounds_like_me', contentSnippet: 'c' });+ expect((await readDisk()).stats.totalFeedback).toBe(1);+ });+});