Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest';
import { mkdir, readFile, writeFile, rm } from 'fs/promises';
import { dirname, join } from 'path';
import { mockPathsDataRoot, mockNoPeers, mockNoPeerSync, mockTestIdentity } from '../lib/mockPathsDataRoot.js';
// Keep the real parser/writer; inject an OS fault only for the targeted file.
const faults = vi.hoisted(() => new Map());
vi.mock('fs/promises', async (original) => {
const actual = await original();
return { ...actual, readFile: (...args) => faults.has(args[0])
? Promise.reject(Object.assign(new Error('injected read fault'), { code: faults.get(args[0]) }))
: actual.readFile(...args) };
});
const { tempRoot, makeProxy, cleanup } = mockPathsDataRoot({ prefix: 'portos-sharing-durability-' });
vi.mock('../lib/fileUtils.js', async (original) => new Proxy(makeProxy(await original()), {
get: (target, key) => key === 'dataPath' ? (...parts) => join(tempRoot, ...parts) : target[key],
}));
// syncWithPeer logs the peer through instances.js's pure `peerLogLabel`; pull
// the real one so a log helper cannot turn an injected store fault into a
// missing-export error. Everything else stays doubled (no live peers).
vi.mock('./instances.js', async (importOriginal) => mockNoPeers({}, {
peerLogLabel: (await importOriginal()).peerLogLabel,
resolveEffectiveCategories: () => ({}),
updatePeer: async () => {},
}));
vi.mock('./instanceIdentity.js', () => mockTestIdentity());
vi.mock('./sharing/peerSync.js', () => mockNoPeerSync({}, {
getOutboundCoverageForPeer: async () => ({ universe: new Set(), pipeline: new Set(), mediaCollections: new Set() }),
}));
// The queue owns data/media-jobs.json (its unreadable-snapshot latch is pinned in mediaJobQueue/index.test.js).
vi.mock('./mediaJobQueue/index.js', () => ({ getJob: () => null, importArchivedJobs: async () => 0 }));
vi.mock('./sharing/annotationIdentity.js', () => ({ resolveBucketSourceName: async () => 'Test' }));
vi.mock('../lib/peerHttpClient.js', () => ({ peerFetch: vi.fn(() => { throw new Error('Unexpected network request'); }) }));
const backup = await import('./backup.js');
const dataSync = await import('./dataSync.js');
const peerUsage = await import('./peerUsage.js');
const quotas = await import('./providerQuotaShare.js');
const sync = await import('./syncOrchestrator.js');
const buckets = await import('./sharing/buckets.js');
const subscriptions = await import('./sharing/subscriptions.js');
const manifests = await import('./sharing/manifest.js');
const importer = await import('./sharing/importer.js');
const annotations = await import('./sharing/annotationsSync.js');
const alcohol = await import('./meatspaceAlcohol.js');
const nicotine = await import('./meatspaceNicotine.js');
const { PATHS } = await import('../lib/fileUtils.js');
const { PORTOS_SCHEMA_VERSIONS } = await import('../lib/schemaVersions.js');
const bucketPath = join(tempRoot, 'bucket');
let bucket;
const seed = async (path, value) => {
await mkdir(dirname(path), { recursive: true });
await writeFile(path, typeof value === 'string' ? value : JSON.stringify(value));
};
const card = (family) => ({ family, limits: [{ key: 'weekly', percentUsed: 20 }], fetchedAt: '2026-09-01T00:00:00Z' });
const incoming = { id: 'incoming', senderInstanceId: 'remote', kind: 'media', recordIds: ['job-new'], assetRefs: [] };
beforeEach(async () => {
faults.clear();
annotations.__resetBucketAssetKeysCache();
subscriptions.__resetForTests();
await rm(tempRoot, { recursive: true, force: true });
await mkdir(bucketPath, { recursive: true });
bucket = await buckets.createBucket({ name: 'Bucket', path: bucketPath, mode: 'inbox' });
await seed(join(bucketPath, 'manifests', 'incoming.json'), incoming);
await seed(join(bucketPath, 'records', 'media', 'job-new.json'), { id: 'job-new', kind: 'image', status: 'completed' });
await seed(join(bucketPath, 'assets', 'images', 'note.png'), 'image');
await seed(join(bucketPath, 'assets', 'images', 'new.png'), 'image');
});
afterAll(cleanup);
// Each row pins a separate durable mutation boundary; rejection must preserve
// bytes, and a repair must unblock the same queue/operation without a restart.
const cases = [
['backup state', () => join(tempRoot, 'backup/state.json'), { lastRun: 'old' },
() => backup.saveState({ status: 'ok' }), saved => expect(saved).toMatchObject({ lastRun: 'old', status: 'ok' })],
['goals', () => join(PATHS.digitalTwin, 'goals.json'), { goals: [{ id: 'old' }] },
() => dataSync.applyRemote('goals', { goals: [{ id: 'new', updatedAt: '2026-09-02' }] }),
saved => expect(saved.goals.map(g => g.id)).toEqual(['old', 'new'])],
['character', () => join(tempRoot, 'character.json'), { events: [{ id: 'old', timestamp: '2026-09-01' }], syncedTaskIds: ['old'] },
() => dataSync.applyRemote('character', { events: [{ id: 'new', timestamp: '2026-09-02' }], updatedAt: '2026-09-02', level: 99 }),
saved => { expect(saved.events.map(e => e.id)).toEqual(['old', 'new']); expect(saved.syncedTaskIds).toEqual(['old']); expect(saved).not.toHaveProperty('level'); }],
['personal log', () => join(tempRoot, 'meatspace/daily-log.json'), { entries: [{ date: '2026-09-01' }], custom: true },
() => dataSync.applyRemote('meatspace', { 'daily-log.json': { entries: [{ date: '2026-09-02' }] } }),
saved => { expect(saved.entries).toHaveLength(2); expect(saved.custom).toBe(true); }],
['personal config', () => join(tempRoot, 'meatspace/config.json'), { updatedAt: '2026-09-03', custom: true },
() => dataSync.applyRemote('meatspace', { 'config.json': { updatedAt: '2026-09-02', custom: false } }), saved => expect(saved.custom).toBe(true)],
['peer retirements', () => peerUsage.PEER_USAGE_FILE, { instances: { retained: { capturedAt: '2026-09-01' } }, tombstones: [] },
() => peerUsage.forgetInstanceUsage('retired'), saved => { expect(saved.instances).toHaveProperty('retained'); expect(saved.tombstones).toEqual(expect.arrayContaining([expect.objectContaining({ instanceId: 'retired' })])); }],
['quota families', () => quotas.PROVIDER_QUOTAS_FILE, { quotas: [card('old')] },
() => quotas.recordLocalQuotaCards([card('new')]), saved => expect(saved.quotas.map(c => c.family)).toEqual(['old', 'new'])],
['bucket registry', () => join(tempRoot, 'sharing/buckets.json'), () => ({ buckets: [bucket] }),
async () => { const other = join(tempRoot, 'other'); await mkdir(other, { recursive: true }); return buckets.createBucket({ name: 'Other', path: other }); },
saved => expect(saved.buckets.map(b => b.name)).toEqual(['Bucket', 'Other'])],
['bucket identity', () => join(bucketPath, 'bucket.json'), { id: 'original', schemaVersion: 1, custom: true },
() => buckets.ensureBucketLayout(bucket), saved => expect(saved).toEqual({ id: 'original', schemaVersion: 1, custom: true })],
['subscriptions', () => join(tempRoot, 'sharing/subscriptions.json'), { subscriptions: [{ id: 'old', recordKind: 'series', recordId: 'old' }] },
() => subscriptions.adoptImportedSubscription({ bucketId: bucket.id, recordKind: 'series', recordId: 'new' }),
saved => expect(saved.subscriptions.map(s => s.recordId)).toEqual(['old', 'new'])],
['sharing cursors', () => join(tempRoot, 'sharing/cursors', `${bucket.id}.json`), { processedById: { old: 'old-id' }, processed: ['legacy'] },
() => manifests.markProcessed(bucket.id, 'new', 'new-id'), saved => { expect(saved.processedById).toEqual({ old: 'old-id', new: 'new-id' }); expect(saved.processed).toEqual(['legacy']); }],
['import inbox', () => join(tempRoot, 'sharing/inbox', `${bucket.id}.json`), { items: [{ manifestId: 'old' }] },
() => importer.processManifest(bucket.id, 'incoming.json'), saved => expect(saved.items.map(i => i.manifestId)).toEqual(['old', 'incoming'])],
['annotation tombstones', () => join(bucketPath, 'records/media-annotations/test-instance.json'), { annotations: { 'image:note.png': { note: 'old' } } },
() => annotations.exportAnnotationsToBucket({ ...bucket, mode: 'auto-merge' }, { 'image:new.png': { note: 'new' } }, 'test-instance'),
saved => { expect(saved.annotations['image:note.png'].note).toBe(''); expect(saved.annotations['image:new.png'].note).toBe('new'); }],
['sync cursors', () => join(tempRoot, 'instances_sync_cursors.json'), { old: { brainSeq: 42 } },
() => sync.syncWithPeer({ instanceId: 'new', name: 'Test' }), saved => { expect(saved.old).toEqual({ brainSeq: 42 }); expect(saved.new.lastSyncAt).toBeTruthy(); }],
];
describe.each(cases)('%s preserves unreadable data', (_name, getPath, fixture, mutate, retained) => {
it('preserves corrupt/empty bytes and read faults, then retries after repair', async () => {
const path = getPath();
for (const bytes of ['{"truncated":', 'not json', '']) {
await seed(path, bytes);
await expect(mutate()).rejects.toMatchObject({ code: 'UNREADABLE_STORE' });
expect(await readFile(path, 'utf8')).toBe(bytes);
}
await seed(path, typeof fixture === 'function' ? fixture() : fixture);
const original = await readFile(path, 'utf8');
faults.set(path, 'EACCES');
await expect(mutate()).rejects.toMatchObject({ code: 'UNREADABLE_STORE' });
faults.delete(path);
expect(await readFile(path, 'utf8')).toBe(original);
await mutate();
retained(JSON.parse(await readFile(path, 'utf8')));
});
it('initializes only when the file is absent', async () => {
const path = getPath();
await rm(path, { force: true });
await mutate();
expect(JSON.parse(await readFile(path, 'utf8'))).toBeTruthy();
});
});
it('merges same-date daily-log tenants without duplicating them on replay', async () => {
const path = join(PATHS.meatspace, 'daily-log.json');
const date = '2026-01-02';
await seed(path, {
lastEntryDate: date,
entries: [{
date,
body: { weightLbs: 170 },
alcohol: { drinks: [{ name: 'Example beer', oz: 12, abv: 5, count: 1 }] },
}],
custom: true,
});
const remote = {
'daily-log.json': {
entries: [{
date,
body: { fatPct: 20 },
alcohol: { drinks: [{ name: 'Example wine', oz: 5, abv: 12, count: 1 }] },
nicotine: { items: [{ product: 'Example gum', mgPerUnit: 2, count: 2 }] },
}],
},
};
const first = await dataSync.applyRemote('meatspace', remote);
const saved = JSON.parse(await readFile(path, 'utf8'));
const entry = saved.entries[0];
expect(first).toEqual({ applied: true, count: 1 });
expect(entry.body).toEqual({ weightLbs: 170, fatPct: 20 });
expect(entry.alcohol.drinks).toHaveLength(2);
expect(entry.alcohol.standardDrinks).toBe(2);
expect(entry.nicotine.items).toHaveLength(1);
expect(entry.nicotine.totalMg).toBe(4);
expect(saved.lastEntryDate).toBe(date);
expect(saved.custom).toBe(true);
const replay = await dataSync.applyRemote('meatspace', remote);
const replayed = JSON.parse(await readFile(path, 'utf8'));
expect(replay).toEqual({ applied: false, count: 0 });
expect(replayed.entries[0]).toEqual(entry);
});
// Identified alcohol/nicotine events (#8143): identical rows logged on two peers
// are two events, a replay adds nothing, and legacy/v0 payloads stay safe.
describe('daily-log event identity (#8143)', () => {
const date = '2026-01-03';
const path = () => join(PATHS.meatspace, 'daily-log.json');
const stamp = '2026-01-03T20:00:00.000Z';
const lager = (id, extra = {}) => ({ id, name: 'Example Lager', oz: 12, abv: 5, count: 1, createdAt: stamp, updatedAt: stamp, ...extra });
const day = (drinks) => ({ date, alcohol: { drinks, standardDrinks: 0 } });
const snapshot = (...entries) => ({ 'daily-log.json': { entries } });
afterEach(() => vi.useRealTimers());
const savedDay = async () => JSON.parse(await readFile(path(), 'utf8')).entries.find((e) => e.date === date);
// Each "peer" logs through the real services onto an empty log at the same
// instant, so the rows are identical apart from their event identity.
const logOnFreshPeer = async () => {
await rm(path(), { force: true });
vi.useFakeTimers({ toFake: ['Date'], now: new Date(stamp) });
await alcohol.logDrink({ name: 'Example Lager', oz: 12, abv: 5, count: 1, date });
await nicotine.logNicotine({ product: 'Example Pouch', mgPerUnit: 3, count: 1, date });
return JSON.parse(await readFile(path(), 'utf8'));
};
it('keeps identical independent events from both peers in either sync order', async () => {
const peers = [await logOnFreshPeer(), await logOnFreshPeer()];
vi.useRealTimers();
for (const [local, remote] of [peers, [...peers].reverse()]) {
await seed(path(), local);
const peer = { 'daily-log.json': remote };
expect(await dataSync.applyRemote('meatspace', peer)).toEqual({ applied: true, count: 1 });
const merged = await savedDay();
expect(merged.alcohol.drinks).toHaveLength(2);
expect(merged.alcohol.standardDrinks).toBe(2);
expect(merged.nicotine.items).toHaveLength(2);
expect(merged.nicotine.totalMg).toBe(6);
expect(await dataSync.applyRemote('meatspace', peer)).toEqual({ applied: false, count: 0 });
expect(await savedDay()).toEqual(merged);
}
});
it('resolves copies of one event the same way whichever side holds which', async () => {
const legacy = { name: 'Example Stout', oz: 12, abv: 5, count: 1 };
const newerEdit = lager('drink-x', { count: 1, abv: 10, updatedAt: '2026-01-04T08:00:00.000Z' });
// A pre-#8143 peer folds a second log into the row in place, without restamping.
const bumpedInPlace = lager('drink-x', { count: 2 });
const cases = [
// Equal stamps: the in-place increment is a real drink, so the larger count wins.
[[lager('drink-x'), legacy], [legacy, lager('drink-x'), bumpedInPlace], 2, 3],
[[bumpedInPlace, legacy], [legacy, lager('drink-x')], 2, 3],
// A restamped edit wins over a stale copy regardless of its count.
[[bumpedInPlace, legacy], [newerEdit, legacy], 1, 3],
[[newerEdit, legacy], [bumpedInPlace, legacy], 1, 3],
];
for (const [localDrinks, remoteDrinks, count, total] of cases) {
await seed(path(), { entries: [day(localDrinks)] });
await dataSync.applyRemote('meatspace', snapshot(day(remoteDrinks)));
const merged = await savedDay();
// The legacy row keeps content dedupe; the identified event is one row.
expect(merged.alcohol.drinks).toHaveLength(2);
expect(merged.alcohol.drinks.find((d) => d.id === 'drink-x').count).toBe(count);
expect(merged.alcohol.standardDrinks).toBe(total);
}
});
it('does not merge a legacy row back in after one peer edits it', async () => {
const legacyLog = { entries: [day([{ name: 'Example Stout', oz: 12, abv: 5, count: 1 }])] };
await seed(path(), legacyLog);
await alcohol.updateDrink(date, 0, { count: 2 });
const edited = JSON.parse(await readFile(path(), 'utf8'));
// Pulling the unedited peer's copy leaves the edit as the only row...
await dataSync.applyRemote('meatspace', { 'daily-log.json': legacyLog });
expect((await savedDay()).alcohol.drinks).toEqual(edited.entries[0].alcohol.drinks);
// ...and the unedited peer drops its legacy copy when it pulls the edit.
await seed(path(), legacyLog);
await dataSync.applyRemote('meatspace', { 'daily-log.json': edited });
const merged = await savedDay();
expect(merged.alcohol.drinks).toEqual(edited.entries[0].alcohol.drinks);
expect(merged.alcohol.standardDrinks).toBe(2);
});
it('refuses a daily log from a peer on a newer meatspace schema', async () => {
await seed(path(), { entries: [day([lager('drink-a')])] });
const before = await readFile(path(), 'utf8');
const result = await dataSync.applyRemote('meatspace', snapshot(day([lager('drink-b')])), {
portosMeta: { portosVersion: '99.0.0', schemaVersions: { meatspace: PORTOS_SCHEMA_VERSIONS.meatspace + 1 } },
});
expect(result.applied).toBe(false);
expect(result.blockedBySchema.ahead).toEqual([
{ category: 'meatspace', senderV: PORTOS_SCHEMA_VERSIONS.meatspace + 1, receiverV: PORTOS_SCHEMA_VERSIONS.meatspace },
]);
expect(await readFile(path(), 'utf8')).toBe(before);
});
});
// Deletes and date moves of identified events converge across peers (#8154).
// Peer A acts through the real services on a log both peers share; peer B
// still holds the original rows, as an offline (or pre-#8154) peer would.
describe('daily-log deletes and date moves (#8154)', () => {
const date = '2026-01-03';
const nextDate = '2026-01-04';
const stamp = '2026-01-03T20:00:00.000Z';
const path = () => join(PATHS.meatspace, 'daily-log.json');
const lager = (id, extra = {}) => ({ id, name: 'Example Lager', oz: 12, abv: 5, count: 1, createdAt: stamp, updatedAt: stamp, ...extra });
const pouch = { id: 'pouch-1', product: 'Example Pouch', mgPerUnit: 3, count: 1, createdAt: stamp, updatedAt: stamp };
const shared = () => ({
entries: [{
date,
alcohol: { drinks: [lager('drink-1'), lager('drink-2', { oz: 24 })], standardDrinks: 3 },
nicotine: { items: [{ ...pouch }], totalMg: 3 },
}],
});
const read = async () => JSON.parse(await readFile(path(), 'utf8'));
const drinkIdsByDate = (log) => Object.fromEntries(
log.entries.filter((e) => e.alcohol).map((e) => [e.date, e.alcohol.drinks.map((d) => d.id)]),
);
afterEach(() => vi.useRealTimers());
const actOnPeerA = async (action, at = '2026-01-05T09:00:00.000Z') => {
await seed(path(), shared());
vi.useFakeTimers({ toFake: ['Date'], now: new Date(at) });
await action();
vi.useRealTimers();
return read();
};
// Row order within a day follows whichever side is local; identity does not.
const byId = (a, b) => (a.id < b.id ? -1 : 1);
const unordered = (log) => log.entries.map((e) => ({
...e,
...(e.alcohol && { alcohol: { ...e.alcohol, drinks: [...e.alcohol.drinks].sort(byId) } }),
...(e.nicotine && { nicotine: { ...e.nicotine, items: [...e.nicotine.items].sort(byId) } }),
}));
// Merge in both directions; each side must reach the same log, and a replay
// of the same snapshot must be a no-op.
const syncBothWays = async (peerA, peerB) => {
const results = [];
for (const [local, remote] of [[peerA, peerB], [peerB, peerA]]) {
await seed(path(), local);
await dataSync.applyRemote('meatspace', { 'daily-log.json': remote });
const merged = await read();
expect(await dataSync.applyRemote('meatspace', { 'daily-log.json': remote })).toEqual({ applied: false, count: 0 });
results.push(merged);
}
expect(unordered(results[1])).toEqual(unordered(results[0]));
expect(results[1].eventTombstones).toEqual(results[0].eventTombstones);
return results[0];
};
it('keeps a deleted drink and nicotine item deleted on both peers', async () => {
const peerA = await actOnPeerA(async () => {
await alcohol.removeDrink(date, 0);
await nicotine.removeNicotine(date, 0);
});
const merged = await syncBothWays(peerA, shared());
const day = merged.entries.find((e) => e.date === date);
expect(day.alcohol.drinks.map((d) => d.id)).toEqual(['drink-2']);
expect(day.alcohol.standardDrinks).toBe(2);
expect(day.nicotine).toBeUndefined();
// The receiving peer keeps the tombstones so it can pass the delete on.
expect(merged.eventTombstones.map((t) => t.id).sort()).toEqual(['drink-1', 'pouch-1']);
});
it('leaves exactly one copy of a moved drink, on the new date', async () => {
const peerA = await actOnPeerA(() => alcohol.updateDrink(date, 0, { date: nextDate }));
const merged = await syncBothWays(peerA, shared());
expect(drinkIdsByDate(merged)).toEqual({ [date]: ['drink-2'], [nextDate]: ['drink-1'] });
expect(merged.entries.find((e) => e.date === date).alcohol.standardDrinks).toBe(2);
expect(merged.entries.find((e) => e.date === nextDate).alcohol.standardDrinks).toBe(1);
});
it('does not suppress an identical drink logged again after the delete', async () => {
const peerA = await actOnPeerA(async () => {
await alcohol.removeDrink(date, 0);
await alcohol.logDrink({ name: 'Example Lager', oz: 12, abv: 5, count: 1, date });
});
const merged = await syncBothWays(peerA, shared());
const drinks = merged.entries.find((e) => e.date === date).alcohol.drinks;
expect(drinks.map((d) => d.id)).not.toContain('drink-1');
expect(drinks).toHaveLength(2);
expect(merged.entries.find((e) => e.date === date).alcohol.standardDrinks).toBe(3);
});
it('keeps a drink a peer edited after the delete, and retires the tombstone', async () => {
const peerA = await actOnPeerA(() => alcohol.removeDrink(date, 0));
const peerB = shared();
peerB.entries[0].alcohol.drinks[0].count = 2;
peerB.entries[0].alcohol.drinks[0].updatedAt = '2026-01-06T09:00:00.000Z';
const merged = await syncBothWays(peerA, peerB);
const drinks = merged.entries.find((e) => e.date === date).alcohol.drinks;
expect(drinks.find((d) => d.id === 'drink-1').count).toBe(2);
expect(merged.eventTombstones).toBeUndefined();
});
});
// An unreadable shared identity must not leave a new local registry entry.
it('does not register a bucket whose existing identity cannot be read', async () => {
const other = join(tempRoot, 'other');
const identityPath = join(other, 'bucket.json');
const registryPath = join(tempRoot, 'sharing/buckets.json');
await seed(identityPath, '{"id":');
const before = await readFile(registryPath, 'utf8');
await expect(buckets.createBucket({ name: 'Other', path: other })).rejects.toMatchObject({ code: 'UNREADABLE_STORE' });
expect(await readFile(registryPath, 'utf8')).toBe(before);
expect(await readFile(identityPath, 'utf8')).toBe('{"id":');
});
it('merges federated goals without dropping a queued local edit', async () => {
const { mutateGoals } = await import('./identity/store.js');
const goalsPath = join(PATHS.digitalTwin, 'goals.json');
await seed(goalsPath, { goals: [{ id: 'local', progress: 0 }] });
await Promise.all([
dataSync.applyRemote('goals', { goals: [{ id: 'remote', updatedAt: '2026-01-01' }] }),
mutateGoals(doc => {
doc.goals.find(goal => goal.id === 'local').progress = 50;
return doc;
})
]);
expect(JSON.parse(await readFile(goalsPath, 'utf8')).goals).toEqual([
{ id: 'local', progress: 50 }, { id: 'remote', updatedAt: '2026-01-01' }
]);
});