Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* End-to-end cover for the deleted-document tombstone (#3530): a document the
* user deletes must stay deleted across a peer sync with a machine that still
* has it, the delete must PROPAGATE to that machine, and a document re-created
* after a delete must not be permanently suppressed.
*
* Exercises the real disk paths (meta.json + the .md files) with PATHS pointed
* at a temp dir, so the merge, the file reap, and the add-only write all run.
*/
import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
import { existsSync, rmSync, mkdirSync, writeFileSync, readFileSync } from 'fs';
import { join } from 'path';
import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';
// `digital-twin-sync.js` and `digital-twin-helpers.js` both capture
// PATHS.digitalTwin at module load, so the root is fixed for the whole file and
// per-test isolation comes from wiping the dir + the meta cache in beforeEach.
const tempRoot = createTempDataRoot('portos-dt-tombstone-');
const twinDir = join(tempRoot, 'digital-twin');
vi.mock('../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../lib/fileUtils.js');
return makePathsProxy(actual, { dataRoot: tempRoot });
});
const { createDocument, updateDocument, deleteDocument } = await import('./digital-twin-documents.js');
const { ensureDocumentInMeta } = await import('./digital-twin-helpers.js');
const { loadMeta, saveMeta, cache, digitalTwinEvents } = await import('./digital-twin-meta.js');
const { applyDigitalTwinRemote, getDigitalTwinSnapshot } = await import('./digital-twin-sync.js');
const FILENAME = 'CUSTOM_ROUTINE.md';
const CONTENT = '# Custom Routine\n\nExample content.\n';
const docPath = join(twinDir, FILENAME);
const newDoc = () => createDocument({ filename: FILENAME, title: 'Custom Routine', category: 'lifestyle', content: CONTENT });
/** The snapshot a peer that still holds the document would ship. */
const peerSnapshotWithDoc = (meta) => ({
meta: { ...meta, deletedDocuments: [] },
documents: { [FILENAME]: CONTENT },
});
beforeEach(async () => {
rmSync(twinDir, { recursive: true, force: true });
cache.meta.data = null;
cache.meta.timestamp = 0;
await saveMeta({ version: '1.0.0', documents: [], deletedDocuments: [] });
});
afterAll(() => rmSync(tempRoot, { recursive: true, force: true }));
describe('digital twin document tombstones (#3530)', () => {
it('records a tombstone on delete and keeps the document deleted through a peer sync', async () => {
const created = await newDoc();
const peer = peerSnapshotWithDoc(await loadMeta());
expect(existsSync(docPath)).toBe(true);
expect(await deleteDocument(created.id)).toBe(true);
const afterDelete = await loadMeta();
expect(afterDelete.documents.map((d) => d.filename)).not.toContain(FILENAME);
expect(afterDelete.deletedDocuments.map((t) => t.filename)).toEqual([FILENAME]);
expect(existsSync(docPath)).toBe(false);
// The peer still has the document and ships it back — the pre-#3530 bug.
await applyDigitalTwinRemote(peer);
const afterSync = await loadMeta();
expect(afterSync.documents.map((d) => d.filename)).not.toContain(FILENAME);
expect(existsSync(docPath)).toBe(false);
// …and it stays deleted on every subsequent cycle, not just the first.
await applyDigitalTwinRemote(peer);
expect((await loadMeta()).documents.map((d) => d.filename)).not.toContain(FILENAME);
expect(existsSync(docPath)).toBe(false);
});
it('ships tombstones in the snapshot so peers can see the delete', async () => {
const created = await newDoc();
await deleteDocument(created.id);
const { data } = await getDigitalTwinSnapshot();
expect(data.meta.deletedDocuments.map((t) => t.filename)).toEqual([FILENAME]);
expect(Object.keys(data.documents)).not.toContain(FILENAME);
});
it('propagates a peer\'s delete: removes the local metadata entry AND the .md file', async () => {
await newDoc();
expect(existsSync(docPath)).toBe(true);
// The peer deleted it after our copy was created.
const { applied } = await applyDigitalTwinRemote({
meta: { documents: [], deletedDocuments: [{ filename: FILENAME, deletedAt: new Date(Date.now() + 60_000).toISOString() }] },
documents: {},
});
expect(applied).toBe(true);
const meta = await loadMeta();
expect(meta.documents.map((d) => d.filename)).not.toContain(FILENAME);
expect(meta.deletedDocuments.map((t) => t.filename)).toEqual([FILENAME]);
expect(existsSync(docPath)).toBe(false);
});
it('does not suppress a document re-created after the delete, and drops the stale tombstone', async () => {
const created = await newDoc();
await deleteDocument(created.id);
const staleTombstone = (await loadMeta()).deletedDocuments;
expect(staleTombstone).toHaveLength(1);
const recreated = await newDoc();
expect(recreated.createdAt > staleTombstone[0].deletedAt).toBe(true);
expect((await loadMeta()).deletedDocuments).toEqual([]);
// A peer that has not seen the re-create still ships the old tombstone.
await applyDigitalTwinRemote({ meta: { documents: [], deletedDocuments: staleTombstone }, documents: {} });
const meta = await loadMeta();
expect(meta.documents.map((d) => d.filename)).toContain(FILENAME);
expect(meta.deletedDocuments).toEqual([]);
expect(existsSync(docPath)).toBe(true);
});
it('keeps a document edited here after another machine deleted it', async () => {
const created = await newDoc();
await updateDocument(created.id, { content: `${CONTENT}\nEdited.\n` });
const edited = (await loadMeta()).documents.find((d) => d.filename === FILENAME);
expect(edited.updatedAt).toBeTruthy();
// The peer's delete predates our edit, so the edit is the user's last word.
const deletedAt = new Date(Date.parse(edited.updatedAt) - 1_000).toISOString();
await applyDigitalTwinRemote({ meta: { documents: [], deletedDocuments: [{ filename: FILENAME, deletedAt }] }, documents: {} });
const meta = await loadMeta();
expect(meta.documents.map((d) => d.filename)).toContain(FILENAME);
expect(meta.deletedDocuments).toEqual([]);
expect(existsSync(docPath)).toBe(true);
});
it('re-registering an enrichment target clears a tombstone the peer merged in', async () => {
const meta = await loadMeta();
meta.documents.push({ id: 'x', filename: 'MEMORIES.md', title: 'Memories', category: 'enrichment', enabled: true, priority: 30 });
meta.deletedDocuments = [{ filename: 'MEMORIES.md', deletedAt: new Date().toISOString() }];
await saveMeta(meta);
ensureDocumentInMeta(meta, 'MEMORIES.md', 'Memories', 'enrichment');
expect(meta.deletedDocuments).toEqual([]);
expect(meta.documents.find((d) => d.filename === 'MEMORIES.md').createdAt).toBeTruthy();
});
it('still accepts a document the peer has that was never deleted here', async () => {
await applyDigitalTwinRemote({
meta: { documents: [{ id: 'peer-1', filename: FILENAME, title: 'Custom Routine', category: 'lifestyle' }], deletedDocuments: [] },
documents: { [FILENAME]: CONTENT },
});
expect((await loadMeta()).documents.map((d) => d.filename)).toContain(FILENAME);
expect(existsSync(docPath)).toBe(true);
});
});
it('invalidates only after synchronized document files are available', async () => {
const observed = [];
const listener = () => observed.push(existsSync(docPath));
digitalTwinEvents.on('sync:completed', listener);
try {
await applyDigitalTwinRemote({
meta: { ...(await loadMeta()), documents: [{ id: 'remote-example', filename: FILENAME, title: 'Example', enabled: true }] },
documents: { [FILENAME]: CONTENT },
});
expect(observed).toEqual([true]);
await applyDigitalTwinRemote({ documents: { [FILENAME]: CONTENT } });
expect(observed).toEqual([true]);
} finally {
digitalTwinEvents.off('sync:completed', listener);
}
});
describe('digital twin document create failure paths (#11681)', () => {
const metaPath = join(twinDir, 'meta.json');
it.each([
{ name: 'new file', orphan: false },
{ name: 'pre-existing orphan', orphan: true },
])('retries a $name after a failed metadata write without leaking cached changes', async ({ orphan }) => {
// Keep cache expiry from hiding an immediate retry's stale-state regression.
const clock = vi.spyOn(Date, 'now').mockReturnValue(Date.now());
try {
await saveMeta({ ...(await loadMeta()), deletedDocuments: [
{ filename: FILENAME, deletedAt: new Date(Date.now() - 60_000).toISOString() },
] });
const before = structuredClone(await loadMeta());
const persisted = readFileSync(metaPath, 'utf-8');
if (orphan) writeFileSync(docPath, 'orphan');
// A directory at the meta path makes the real metadata write fail.
rmSync(metaPath, { force: true });
mkdirSync(metaPath);
await expect(newDoc()).rejects.toThrow();
expect(existsSync(docPath)).toBe(orphan);
expect(await loadMeta()).toEqual(before);
// Repair I/O without resetting the cache, as an immediate user retry would.
rmSync(metaPath, { recursive: true, force: true });
writeFileSync(metaPath, persisted);
const created = await newDoc();
const saved = JSON.parse(readFileSync(metaPath, 'utf-8'));
expect(saved.documents).toEqual([expect.objectContaining({ id: created.id, filename: FILENAME })]);
expect(saved.deletedDocuments).toEqual([]);
expect(readFileSync(docPath, 'utf-8')).toBe(CONTENT);
} finally {
clock.mockRestore();
}
});
it('adopts an orphan file that has no meta entry', async () => {
writeFileSync(docPath, 'orphan');
const created = await newDoc();
expect(readFileSync(docPath, 'utf-8')).toBe(CONTENT);
expect((await loadMeta()).documents.map(d => d.id)).toContain(created.id);
});
it('still rejects a filename that is already in meta', async () => {
await newDoc();
await expect(newDoc()).rejects.toThrow(/already exists/);
});
});