Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

7.8 KB
Raw Blame
import { readFile, readdir } from 'fs/promises';
import { join } from 'path';
import EventEmitter from 'events';
import { AsyncLocalStorage } from 'node:async_hooks';
import { digitalTwinMetaSchema } from '../lib/digitalTwinValidation.js';
import { safeJSONParse, atomicWrite, tryReadFileStrict, unreadableStoreError } from '../lib/fileUtils.js';
import { isPlainObject } from '../lib/objects.js';
import { createMutex } from '../lib/asyncMutex.js';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';

export const META_FILE = join(DIGITAL_TWIN_DIR, 'meta.json');

// Event emitter for digital twin data changes
export const digitalTwinEvents = new EventEmitter();

// In-memory cache
const cache = {
  meta: { data: null, timestamp: 0 },
  documents: { data: null, timestamp: 0 },
  tests: { data: null, timestamp: 0 },
  valuesTests: { data: null, timestamp: 0 },
  adversarialTests: { data: null, timestamp: 0 },
  multiTurnTests: { data: null, timestamp: 0 }
};
export const CACHE_TTL_MS = 5000;

// One lock for every load -> mutate -> saveMeta workflow. loadMeta hands every
// caller the same cached object, so two overlapping read-modify-write flows
// would otherwise each push onto it and the later saveMeta would persist
// whichever tombstone/record set it happened to hold. NOT re-entrant: never
// call withMetaLock from inside a withMetaLock callback.
const metaMutex = createMutex();
// Marks code running inside withMetaLock: loadMeta refuses an untrusted file
// there instead of serving defaults, so a transient read fault cannot hand a
// mutation defaults that a later, healthy read in saveMeta would let through.
const metaLockContext = new AsyncLocalStorage();
export const withMetaLock = (fn) => metaMutex(() => metaLockContext.run(true, fn));

// Expose cache for modules that manage test/document caches (testing.js)
export { cache };

// Default meta structure
export const DEFAULT_META = {
  version: '1.0.0',
  documents: [],
  // Tombstones for documents the user deleted (#3530) — peer sync is otherwise
  // add-only and would resurrect them from a machine that still has the file.
  deletedDocuments: [],
  testHistory: [],
  valuesTestHistory: [],
  adversarialTestHistory: [],
  multiTurnTestHistory: [],
  personas: [],
  // Tombstones for personas the user deleted (#3533) — the persona union is
  // add-only, so a peer that still has the persona would otherwise re-add it.
  deletedPersonas: [],
  enrichment: { completedCategories: [], lastSession: null },
  settings: { autoInjectToCoS: true, maxContextTokens: 4000, includePrivacyContext: false }
};

export async function loadMeta() {
  if (cache.meta.data && (Date.now() - cache.meta.timestamp) < CACHE_TTL_MS) {
    return cache.meta.data;
  }

  await ensureSoulDir();

  const { ok, value: content } = await tryReadFileStrict(META_FILE);
  if (ok && content === null) {
    // Only ENOENT proves the file was never written: scan existing documents
    // and build initial meta
    const meta = await buildInitialMeta();
    await saveMeta(meta);
    return meta;
  }

  const parsed = ok ? readMetaObject(content) : null;
  if (!parsed) {
    // Unreadable or malformed is not "no meta". Mutations (inside withMetaLock)
    // refuse outright; readers get uncached defaults, and saveMeta still refuses
    // to persist over the file until it is repaired, so the registry, histories
    // and tombstones are never replaced.
    console.error(`❌ Digital twin meta.json is unreadable or malformed: ${META_FILE}`);
    if (metaLockContext.getStore()) throw unreadableStoreError(META_FILE);
    return structuredClone(DEFAULT_META);
  }
  const validated = digitalTwinMetaSchema.safeParse(parsed);

  cache.meta.data = validated.success ? validated.data : { ...DEFAULT_META, ...parsed };
  cache.meta.timestamp = Date.now();
  return cache.meta.data;
}

async function buildInitialMeta() {
  // Deep clone — a shallow spread would share DEFAULT_META's arrays, so the
  // `documents.push` below would mutate the module-level default in place.
  const meta = structuredClone(DEFAULT_META);

  const files = await readdir(DIGITAL_TWIN_DIR).catch(() => []);
  const mdFiles = files.filter(f => f.endsWith('.md'));

  for (const file of mdFiles) {
    const content = await readFile(join(DIGITAL_TWIN_DIR, file), 'utf-8').catch(() => '');
    const title = extractTitle(content) || file.replace('.md', '');
    const category = inferCategory(file);
    const version = extractVersion(content);

    meta.documents.push({
      id: generateId(),
      filename: file,
      title,
      category,
      version,
      enabled: true,
      priority: getPriorityForFile(file),
      weight: 5 // Default weight
    });
  }

  meta.documents.sort((a, b) => a.priority - b.priority);

  return meta;
}

function extractTitle(content) {
  const match = content.match(/^#\s+(.+)/m);
  return match ? match[1].trim() : null;
}

export { extractTitle };

function extractVersion(content) {
  const match = content.match(/\*\*Version:\*\*\s*([\d.]+)/);
  return match ? match[1] : null;
}

export { extractVersion };

function inferCategory(filename) {
  const upper = filename.toUpperCase();

  // Audio/Music
  if (upper.startsWith('AUDIO') || upper.includes('MUSIC')) return 'audio';

  // Behavioral tests
  if (upper.includes('BEHAVIORAL') || upper.includes('TEST_SUITE')) return 'behavioral';

  // Entertainment (movies, books, TV, games)
  if (upper.includes('MOVIE') || upper.includes('FILM') || upper.includes('BOOK') ||
      upper.includes('TV') || upper.includes('GAME') || upper.includes('ENTERTAINMENT')) return 'entertainment';

  // Professional
  if (upper.includes('CAREER') || upper.includes('SKILL') || upper.includes('WORK') ||
      upper.includes('PROFESSIONAL')) return 'professional';

  // Lifestyle
  if (upper.includes('ROUTINE') || upper.includes('HABIT') || upper.includes('HEALTH') ||
      upper.includes('LIFESTYLE') || upper.includes('DAILY')) return 'lifestyle';

  // Social
  if (upper.includes('SOCIAL') || upper.includes('COMMUNICATION') ||
      upper.includes('RELATIONSHIP')) return 'social';

  // Creative
  if (upper.includes('AESTHETIC') || upper.includes('CREATIVE') || upper.includes('ART') ||
      upper.includes('DESIGN')) return 'creative';

  // Enrichment (generic enrichment outputs)
  if (['MEMORIES.md', 'FAVORITES.md', 'PREFERENCES.md'].includes(filename)) return 'enrichment';

  // Default to core identity
  return 'core';
}

function getPriorityForFile(filename) {
  const priorities = {
    'SOUL.md': 1,
    'Expanded.md': 2,
    'BEHAVIORAL_TEST_SUITE.md': 100
  };
  return priorities[filename] || 50;
}

const readMetaObject = (content) => {
  const parsed = safeJSONParse(content, null, { allowArray: false });
  return isPlainObject(parsed) ? parsed : null;
};

// The on-disk file, not the caller's snapshot, decides whether a write is safe:
// loadMeta serves defaults for an untrusted file, and any mutation built on
// them would otherwise overwrite the real document registry and tombstones.
async function assertMetaWritable() {
  const { ok, value } = await tryReadFileStrict(META_FILE);
  if (!ok || (value !== null && !readMetaObject(value))) throw unreadableStoreError(META_FILE);
}

export async function saveMeta(meta) {
  await ensureSoulDir();
  await assertMetaWritable();
  await atomicWrite(META_FILE, meta);
  cache.meta.data = meta;
  cache.meta.timestamp = Date.now();
  digitalTwinEvents.emit('meta:changed', meta);
}

export const updateMeta = (updates) => withMetaLock(async () => {
  const meta = await loadMeta();
  const updated = { ...meta, ...updates };
  await saveMeta(updated);
  return updated;
});

export const updateSettings = (settings) => withMetaLock(async () => {
  const meta = await loadMeta();
  meta.settings = { ...meta.settings, ...settings };
  await saveMeta(meta);
  return meta.settings;
});