Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { readFile, readdir } from 'fs/promises';
import { join } from 'path';
import EventEmitter from 'events';
import { AsyncLocalStorage } from 'node:async_hooks';
import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';
import { safeJSONParse, atomicWrite, tryReadFileStrict, unreadableStoreError } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { createMutex } from '../lib/asyncMutex.js';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
export const META_FILE = join(DIGITAL_TWIN_DIR, 'meta.json');
// Event emitter for digital twin data changes
export const digitalTwinEvents = new EventEmitter();
// In-memory cache
const cache = {
meta: { data: null, timestamp: 0 },
documents: { data: null, timestamp: 0 },
tests: { data: null, timestamp: 0 },
valuesTests: { data: null, timestamp: 0 },
adversarialTests: { data: null, timestamp: 0 },
multiTurnTests: { data: null, timestamp: 0 }
};
export const CACHE_TTL_MS = 5000;
// One lock for every load -> mutate -> saveMeta workflow. loadMeta hands every
// caller the same cached object, so two overlapping read-modify-write flows
// would otherwise each push onto it and the later saveMeta would persist
// whichever tombstone/record set it happened to hold. NOT re-entrant: never
// call withMetaLock from inside a withMetaLock callback.
const metaMutex = createMutex();
// Marks code running inside withMetaLock: loadMeta refuses an untrusted file
// there instead of serving defaults, so a transient read fault cannot hand a
// mutation defaults that a later, healthy read in saveMeta would let through.
const metaLockContext = new AsyncLocalStorage();
export const withMetaLock = (fn) => metaMutex(() => metaLockContext.run(true, fn));
// Expose cache for modules that manage test/document caches (testing.js)
export { cache };
// Default meta structure
export const DEFAULT_META = {
version: '1.0.0',
documents: [],
// Tombstones for documents the user deleted (#3530) — peer sync is otherwise
// add-only and would resurrect them from a machine that still has the file.
deletedDocuments: [],
testHistory: [],
valuesTestHistory: [],
adversarialTestHistory: [],
multiTurnTestHistory: [],
personas: [],
// Tombstones for personas the user deleted (#3533) — the persona union is
// add-only, so a peer that still has the persona would otherwise re-add it.
deletedPersonas: [],
enrichment: { completedCategories: [], lastSession: null },
settings: { autoInjectToCoS: true, maxContextTokens: 4000, includePrivacyContext: false }
};
export async function loadMeta() {
if (cache.meta.data && (Date.now() - cache.meta.timestamp) < CACHE_TTL_MS) {
return cache.meta.data;
}
await ensureSoulDir();
const { ok, value: content } = await tryReadFileStrict(META_FILE);
if (ok && content === null) {
// Only ENOENT proves the file was never written: scan existing documents
// and build initial meta
const meta = await buildInitialMeta();
await saveMeta(meta);
return meta;
}
const parsed = ok ? readMetaObject(content) : null;
if (!parsed) {
// Unreadable or malformed is not "no meta". Mutations (inside withMetaLock)
// refuse outright; readers get uncached defaults, and saveMeta still refuses
// to persist over the file until it is repaired, so the registry, histories
// and tombstones are never replaced.
console.error(`❌ Digital twin meta.json is unreadable or malformed: ${META_FILE}`);
if (metaLockContext.getStore()) throw unreadableStoreError(META_FILE);
return structuredClone(DEFAULT_META);
}
const validated = digitalTwinMetaSchema.safeParse(parsed);
cache.meta.data = validated.success ? validated.data : { ...DEFAULT_META, ...parsed };
cache.meta.timestamp = Date.now();
return cache.meta.data;
}
async function buildInitialMeta() {
// Deep clone — a shallow spread would share DEFAULT_META's arrays, so the
// `documents.push` below would mutate the module-level default in place.
const meta = structuredClone(DEFAULT_META);
const files = await readdir(DIGITAL_TWIN_DIR).catch(() => []);
const mdFiles = files.filter(f => f.endsWith('.md'));
for (const file of mdFiles) {
const content = await readFile(join(DIGITAL_TWIN_DIR, file), 'utf-8').catch(() => '');
const title = extractTitle(content) || file.replace('.md', '');
const category = inferCategory(file);
const version = extractVersion(content);
meta.documents.push({
id: generateId(),
filename: file,
title,
category,
version,
enabled: true,
priority: getPriorityForFile(file),
weight: 5 // Default weight
});
}
meta.documents.sort((a, b) => a.priority - b.priority);
return meta;
}
function extractTitle(content) {
const match = content.match(/^#\s+(.+)/m);
return match ? match[1].trim() : null;
}
export { extractTitle };
function extractVersion(content) {
const match = content.match(/\*\*Version:\*\*\s*([\d.]+)/);
return match ? match[1] : null;
}
export { extractVersion };
function inferCategory(filename) {
const upper = filename.toUpperCase();
// Audio/Music
if (upper.startsWith('AUDIO') || upper.includes('MUSIC')) return 'audio';
// Behavioral tests
if (upper.includes('BEHAVIORAL') || upper.includes('TEST_SUITE')) return 'behavioral';
// Entertainment (movies, books, TV, games)
if (upper.includes('MOVIE') || upper.includes('FILM') || upper.includes('BOOK') ||
upper.includes('TV') || upper.includes('GAME') || upper.includes('ENTERTAINMENT')) return 'entertainment';
// Professional
if (upper.includes('CAREER') || upper.includes('SKILL') || upper.includes('WORK') ||
upper.includes('PROFESSIONAL')) return 'professional';
// Lifestyle
if (upper.includes('ROUTINE') || upper.includes('HABIT') || upper.includes('HEALTH') ||
upper.includes('LIFESTYLE') || upper.includes('DAILY')) return 'lifestyle';
// Social
if (upper.includes('SOCIAL') || upper.includes('COMMUNICATION') ||
upper.includes('RELATIONSHIP')) return 'social';
// Creative
if (upper.includes('AESTHETIC') || upper.includes('CREATIVE') || upper.includes('ART') ||
upper.includes('DESIGN')) return 'creative';
// Enrichment (generic enrichment outputs)
if (['MEMORIES.md', 'FAVORITES.md', 'PREFERENCES.md'].includes(filename)) return 'enrichment';
// Default to core identity
return 'core';
}
function getPriorityForFile(filename) {
const priorities = {
'SOUL.md': 1,
'Expanded.md': 2,
'BEHAVIORAL_TEST_SUITE.md': 100
};
return priorities[filename] || 50;
}
const readMetaObject = (content) => {
const parsed = safeJSONParse(content, null, { allowArray: false });
return isPlainObject(parsed) ? parsed : null;
};
// The on-disk file, not the caller's snapshot, decides whether a write is safe:
// loadMeta serves defaults for an untrusted file, and any mutation built on
// them would otherwise overwrite the real document registry and tombstones.
async function assertMetaWritable() {
const { ok, value } = await tryReadFileStrict(META_FILE);
if (!ok || (value !== null && !readMetaObject(value))) throw unreadableStoreError(META_FILE);
}
export async function saveMeta(meta) {
await ensureSoulDir();
await assertMetaWritable();
await atomicWrite(META_FILE, meta);
cache.meta.data = meta;
cache.meta.timestamp = Date.now();
digitalTwinEvents.emit('meta:changed', meta);
}
export const updateMeta = (updates) => withMetaLock(async () => {
const meta = await loadMeta();
const updated = { ...meta, ...updates };
await saveMeta(updated);
return updated;
});
export const updateSettings = (settings) => withMetaLock(async () => {
const meta = await loadMeta();
meta.settings = { ...meta.settings, ...settings };
await saveMeta(meta);
return meta.settings;
});