Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { mkdir, rm, writeFile, readdir } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
import { tmpdir } from 'os';
import { createHash } from 'crypto';
// Completed-agent CoS history federation (#1650). We exercise peerSync.js's
// cos-history functions against a tmpdir-redirected PATHS.cos, stubbing the
// network (peerFetch) and the peer registry (instances). cosAgentIndex.js is mocked
// so the receiver's index merge is a spy — it must NEVER touch the real
// data/cos/agents index. The heavy record-graph modules are mocked exactly as
// peerSync.test.js does, purely so importing peerSync.js stays cheap + offline.
import { PATHS } from '../../lib/fileUtils.js';
vi.mock('../instances.js', () => ({
DEFAULT_SYNC_CATEGORIES: {},
getPeers: vi.fn().mockResolvedValue([]),
enqueueReciprocalSync: vi.fn().mockResolvedValue({ ok: true }),
}));
vi.mock('../instanceIdentity.js', () => ({
UNKNOWN_INSTANCE_ID: 'unknown',
getInstanceId: vi.fn().mockResolvedValue('test-instance'),
}));
vi.mock('../universeBuilder.js', async () => ({ getUniverse: vi.fn(), mergeUniversesFromSync: vi.fn(), listUniverses: vi.fn() }));
vi.mock('../pipeline/series.js', async () => ({ getSeries: vi.fn(), mergeSeriesFromSync: vi.fn(), listSeries: vi.fn() }));
vi.mock('../pipeline/issues.js', async () => ({ listIssues: vi.fn(), mergeIssuesFromSync: vi.fn() }));
vi.mock('../pipeline/manuscriptReview.js', async () => ({ getReview: vi.fn(), mergeReviewFromSync: vi.fn() }));
vi.mock('../pipeline/reverseOutline.js', async () => ({ getStoredOutline: vi.fn(), mergeOutlineFromSync: vi.fn() }));
vi.mock('../mediaCollections.js', async () => ({
getCollection: vi.fn(), listCollections: vi.fn(), findCollectionByUniverseId: vi.fn(),
findCollectionBySeriesId: vi.fn(), mergeMediaCollectionsFromSync: vi.fn(),
}));
vi.mock('../mediaAssetIndex/index.js', () => ({ reconcileMediaAssets: vi.fn().mockResolvedValue(undefined) }));
vi.mock('../../lib/peerHttpClient.js', async (importOriginal) => ({
peerFetch: vi.fn(),
readPeerBody: (await importOriginal()).readPeerBody,
}));
// cosAgentIndex is dynamic-imported by the receiver to merge the agentId→date index.
// Spy it so the merge is observable AND can't write the real data/cos index.
vi.mock('../cosAgentIndex.js', () => ({ addAgentArchivesToIndex: vi.fn().mockResolvedValue(0) }));
import {
buildCosHistoryManifest,
diffCosHistoryManifestAgainstLocal,
syncCosHistoryFromPeer,
} from './peerSync.js';
import { getPeers } from '../instances.js';
import { peerFetch } from '../../lib/peerHttpClient.js';
import { addAgentArchivesToIndex } from '../cosAgentIndex.js';
// A chunked body with no Content-Length that never ends; only the streaming
// cap in readPeerBody can stop it.
const endlessChunkedRes = () => {
const chunk = new Uint8Array(8 * 1024 * 1024);
return new Response(new ReadableStream({ pull(controller) { controller.enqueue(chunk); } }));
};
const sha = (s) => createHash('sha256').update(s).digest('hex');
function byteRes(content) {
const buf = Buffer.from(content);
return {
ok: true,
headers: new Headers({ 'content-length': String(buf.length) }),
arrayBuffer: async () => buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength),
};
}
function manifestRes(obj) {
return { ok: true, headers: { get: () => null }, json: async () => obj };
}
const PEER = { instanceId: 'peer-1', name: 'P1', address: '10.0.0.2', port: 5555, fullSync: true, enabled: true };
let tmp;
let originalCos;
beforeEach(async () => {
originalCos = PATHS.cos;
tmp = join(tmpdir(), `portos-cos-history-${process.pid}-${Math.random().toString(36).slice(2)}`);
PATHS.cos = tmp;
await mkdir(join(tmp, 'agents'), { recursive: true });
vi.mocked(peerFetch).mockReset();
vi.mocked(getPeers).mockResolvedValue([PEER]);
vi.mocked(addAgentArchivesToIndex).mockClear();
});
afterEach(async () => {
PATHS.cos = originalCos;
await rm(tmp, { recursive: true, force: true });
});
async function seedArchive(date, agentId, files) {
const dir = join(tmp, 'agents', date, agentId);
await mkdir(dir, { recursive: true });
for (const [name, content] of Object.entries(files)) {
await writeFile(join(dir, name), content);
}
}
describe('buildCosHistoryManifest', () => {
it.each([undefined, '{', 'null', '[]', '"legacy"', '42'])(
'excludes archive files when metadata cannot classify privacy (%s)', async (metadata) => {
await seedArchive('2026-06-20', 'agent-unclassified', {
...(metadata === undefined ? {} : { 'metadata.json': metadata }),
'output.txt': 'unclassified finding', 'prompt.txt': 'unclassified source',
});
expect((await buildCosHistoryManifest()).entries).toEqual([]);
}
);
it.each([{ taskAnalysisType: 'private-security-assessment' }, { machineLocal: true }, { machineLocal: 'true' }])('excludes every machine-local archive file (%j)', async (metadata) => {
await seedArchive('2026-06-20', 'agent-private', {
'metadata.json': JSON.stringify({ metadata }),
'output.txt': 'private finding', 'prompt.txt': 'private source',
});
expect((await buildCosHistoryManifest()).entries).toEqual([]);
});
it('hashes each archive file, skips index.json + flat running dirs, is deterministic', async () => {
await seedArchive('2026-06-20', 'agent-abc', {
'metadata.json': '{"id":"agent-abc"}',
'output.txt': 'line1\nline2\n',
'prompt.txt': 'do the thing',
});
// Noise that must be excluded: the type-level index, and a flat (running) dir.
await writeFile(join(tmp, 'agents', 'index.json'), '{"agent-abc":"2026-06-20"}');
await mkdir(join(tmp, 'agents', 'agent-running'), { recursive: true });
await writeFile(join(tmp, 'agents', 'agent-running', 'metadata.json'), '{}');
const m = await buildCosHistoryManifest();
expect(m.schemaVersion).toBe(1);
expect(m.manifestHash).toMatch(/^[a-f0-9]{64}$/);
expect(m.entries).toHaveLength(3);
const files = m.entries.map((e) => e.file).sort();
expect(files).toEqual(['metadata.json', 'output.txt', 'prompt.txt']);
expect(m.entries.every((e) => e.date === '2026-06-20' && e.agentId === 'agent-abc')).toBe(true);
expect(m.entries.find((e) => e.file === 'output.txt').sha256).toBe(sha('line1\nline2\n'));
// Deterministic: a second build over the same disk yields the same hash.
const again = await buildCosHistoryManifest();
expect(again.manifestHash).toBe(m.manifestHash);
});
it('returns an empty manifest when no archives exist', async () => {
const m = await buildCosHistoryManifest();
expect(m.entries).toHaveLength(0);
expect(m.schemaVersion).toBe(1);
});
});
describe('diffCosHistoryManifestAgainstLocal', () => {
it('flags absent + hash-mismatched files, leaves present-matching alone', async () => {
await seedArchive('2026-06-20', 'agent-abc', { 'metadata.json': 'GOOD' });
const entries = [
{ date: '2026-06-20', agentId: 'agent-abc', file: 'metadata.json', sha256: sha('GOOD') }, // present+match
{ date: '2026-06-20', agentId: 'agent-abc', file: 'output.txt', sha256: sha('B') }, // absent
];
const missing = await diffCosHistoryManifestAgainstLocal(entries);
expect(missing.map((e) => e.file)).toEqual(['output.txt']);
// A locally-present file whose hash differs is also missing.
const mismatch = await diffCosHistoryManifestAgainstLocal([
{ date: '2026-06-20', agentId: 'agent-abc', file: 'metadata.json', sha256: sha('STALE') },
]);
expect(mismatch).toHaveLength(1);
});
it('silently rejects path-traversal segments before any FS op', async () => {
const evil = [
{ date: '../../etc', agentId: 'agent-abc', file: 'metadata.json', sha256: sha('x') },
{ date: '2026-06-20', agentId: '../../secrets', file: 'metadata.json', sha256: sha('x') },
{ date: '2026-06-20', agentId: 'agent-abc', file: '../../../etc/passwd', sha256: sha('x') },
];
expect(await diffCosHistoryManifestAgainstLocal(evil)).toEqual([]);
});
});
describe('syncCosHistoryFromPeer', () => {
it('skips a non-full-sync peer (no network)', async () => {
const r = await syncCosHistoryFromPeer({ ...PEER, fullSync: false });
expect(r).toEqual({ pulled: 0, skipped: 'not-fullsync' });
expect(peerFetch).not.toHaveBeenCalled();
});
it('skips a chunked manifest that streams past the byte cap', async () => {
vi.mocked(peerFetch).mockResolvedValue(endlessChunkedRes());
expect(await syncCosHistoryFromPeer(PEER)).toEqual({ pulled: 0, skipped: 'too-large' });
expect(peerFetch).toHaveBeenCalledOnce();
});
it('gently skips a sender whose manifest schema is ahead', async () => {
vi.mocked(peerFetch).mockResolvedValue(manifestRes({ schemaVersion: 999, manifestHash: sha('x'), entries: [] }));
const r = await syncCosHistoryFromPeer(PEER);
expect(r).toEqual({ pulled: 0, skipped: 'schema-ahead' });
});
it('pulls missing archive files, writes them, and merges the index', async () => {
const meta = '{"id":"agent-abc"}';
const out = 'transcript\n';
const prompt = 'sys prompt';
const entries = [
{ date: '2026-06-20', agentId: 'agent-abc', file: 'metadata.json', sha256: sha(meta) },
{ date: '2026-06-20', agentId: 'agent-abc', file: 'output.txt', sha256: sha(out) },
{ date: '2026-06-20', agentId: 'agent-abc', file: 'prompt.txt', sha256: sha(prompt) },
];
const byFile = { 'metadata.json': meta, 'output.txt': out, 'prompt.txt': prompt };
vi.mocked(peerFetch).mockImplementation(async (url) => {
if (url.includes('cos-history-manifest')) {
return manifestRes({ schemaVersion: 1, manifestHash: sha('m'), entries });
}
const file = decodeURIComponent(url.match(/file=([^&]+)/)[1]);
return byteRes(byFile[file]);
});
const r = await syncCosHistoryFromPeer(PEER);
expect(r).toEqual({ pulled: 3, missing: 0 });
const dir = join(tmp, 'agents', '2026-06-20', 'agent-abc');
expect(existsSync(join(dir, 'metadata.json'))).toBe(true);
expect(existsSync(join(dir, 'output.txt'))).toBe(true);
expect(existsSync(join(dir, 'prompt.txt'))).toBe(true);
// Index merge fired with the landed (date, agentId) pairs.
expect(addAgentArchivesToIndex).toHaveBeenCalledTimes(1);
const pairs = vi.mocked(addAgentArchivesToIndex).mock.calls[0][0];
expect(pairs).toContainEqual({ date: '2026-06-20', agentId: 'agent-abc' });
});
it('reconciles the index from the manifest even when every file is already present (crash-recovery)', async () => {
// Simulate a prior sweep that landed the bytes but never persisted the index.
const meta = '{"id":"agent-abc"}';
await seedArchive('2026-06-20', 'agent-abc', { 'metadata.json': meta });
const entries = [{ date: '2026-06-20', agentId: 'agent-abc', file: 'metadata.json', sha256: sha(meta) }];
vi.mocked(peerFetch).mockResolvedValue(manifestRes({ schemaVersion: 1, manifestHash: sha('present'), entries }));
const r = await syncCosHistoryFromPeer(PEER);
expect(r).toEqual({ pulled: 0 }); // nothing to pull — but the index must still be repaired
expect(addAgentArchivesToIndex).toHaveBeenCalledTimes(1);
expect(vi.mocked(addAgentArchivesToIndex).mock.calls[0][0]).toContainEqual({ date: '2026-06-20', agentId: 'agent-abc' });
// No byte route should have been hit — everything was already on disk.
expect(vi.mocked(peerFetch).mock.calls.every(([u]) => u.includes('cos-history-manifest'))).toBe(true);
});
it('withholds the manifest hash on a partial pull (corrupt byte) so it retries', async () => {
const meta = 'META';
const out = 'OUT';
const entries = [
{ date: '2026-06-20', agentId: 'agent-abc', file: 'metadata.json', sha256: sha(meta) },
{ date: '2026-06-20', agentId: 'agent-abc', file: 'output.txt', sha256: sha(out) },
];
vi.mocked(peerFetch).mockImplementation(async (url) => {
if (url.includes('cos-history-manifest')) {
return manifestRes({ schemaVersion: 1, manifestHash: sha('m2'), entries });
}
const file = decodeURIComponent(url.match(/file=([^&]+)/)[1]);
// output.txt arrives corrupted → hash mismatch → discarded → stays missing.
return byteRes(file === 'output.txt' ? 'CORRUPT' : meta);
});
const first = await syncCosHistoryFromPeer(PEER);
expect(first).toEqual({ pulled: 1, missing: 1 });
expect(existsSync(join(tmp, 'agents', '2026-06-20', 'agent-abc', 'metadata.json'))).toBe(true);
expect(existsSync(join(tmp, 'agents', '2026-06-20', 'agent-abc', 'output.txt'))).toBe(false);
// Hash was NOT recorded, so a second identical sweep re-fetches the manifest
// (not short-circuited as 'unchanged') and re-attempts the still-missing file.
vi.mocked(peerFetch).mockClear();
await syncCosHistoryFromPeer(PEER);
const fetchedManifest = vi.mocked(peerFetch).mock.calls.some(([u]) => u.includes('cos-history-manifest'));
expect(fetchedManifest).toBe(true);
});
});