Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

6.8 KB
Raw Blame
import { readFile, unlink, stat } from 'fs/promises';
import { existsSync } from 'fs';
import { join, resolve } from 'path';
import { DIGITAL_TWIN_DIR, generateId, ensureSoulDir } from './digital-twin-helpers.js';
import { loadMeta, saveMeta, withMetaLock } from './digital-twin-meta.js';
import { atomicWrite } from '../lib/fileUtils.js';
import { documentFilenameSchema } from '../lib/digitalTwinValidation.js';
import { isPathInsideDir } from '../lib/pathSafety.js';
import { ServerError } from '../lib/errorHandler.js';
import { extractVersion } from './digital-twin-meta.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { recordTombstone, clearTombstone, tombstoneTimestamp, supersedingTimestamp } from '../lib/tombstones.js';

// Legacy metadata may bypass schema parsing; reject individual bad records
// before any filename reaches the filesystem, without hiding valid neighbors.
const usableDocument = doc => doc && documentFilenameSchema.safeParse(doc.filename).success;
const documentRecords = meta => Array.isArray(meta.documents) ? meta.documents : [];

export async function getDocuments() {
  const meta = await loadMeta();
  const existing = documentRecords(meta).filter(usableDocument).filter(doc => existsSync(join(DIGITAL_TWIN_DIR, doc.filename)));
  const stats = await Promise.all(existing.map(doc => stat(join(DIGITAL_TWIN_DIR, doc.filename))));
  return existing.map((doc, i) => ({
    ...doc,
    lastModified: stats[i].mtime.toISOString(),
    size: stats[i].size
  }));
}

export async function getDocumentById(id) {
  const meta = await loadMeta();
  const docMeta = documentRecords(meta).find(d => usableDocument(d) && d.id === id);

  if (!docMeta) return null;

  const filePath = join(DIGITAL_TWIN_DIR, docMeta.filename);
  if (!existsSync(filePath)) return null;

  const content = await readFile(filePath, 'utf-8');
  const stats = await stat(filePath);

  return {
    ...docMeta,
    content,
    lastModified: stats.mtime.toISOString(),
    size: stats.size
  };
}

// The document file and the meta row that names it are one workflow, so each
// mutation holds the backup lease from its first byte change through the meta
// write (#9982).
export const createDocument = (data) => withBackupAssetPublication(() => withMetaLock(() => createDocumentLeased(data)));
export const updateDocument = (id, updates) => withBackupAssetPublication(() => withMetaLock(() => updateDocumentLeased(id, updates)));
export const deleteDocument = (id) => withBackupAssetPublication(() => withMetaLock(() => deleteDocumentLeased(id)));

function documentWritePath(filename) {
  documentFilenameSchema.parse(filename);
  const filePath = resolve(DIGITAL_TWIN_DIR, filename);
  if (!isPathInsideDir(DIGITAL_TWIN_DIR, filePath)) {
    throw new ServerError('Invalid document destination', { status: 400, code: 'VALIDATION_ERROR' });
  }
  return filePath;
}

async function createDocumentLeased(data) {
  // Validate before even creating the document directory or initializing metadata.
  const filePath = documentWritePath(data.filename);
  await ensureSoulDir();

  // loadMeta shares its cached object; publish these changes only after saveMeta succeeds.
  const meta = structuredClone(await loadMeta());

  // A file with no meta entry is an orphan from a failed earlier create; adopt it.
  const fileExisted = existsSync(filePath);
  if (fileExisted && documentRecords(meta).some(d => d?.filename === data.filename)) {
    throw new Error(`Document ${data.filename} already exists`);
  }

  await atomicWrite(filePath, data.content);

  // Re-creating a filename that was previously deleted must clear its tombstone
  // and stamp a creation time that STRICTLY supersedes the deletion — otherwise
  // a peer that still holds the old tombstone would reap the new document on the
  // next sync (#3530).
  const deletedAt = tombstoneTimestamp(meta.deletedDocuments, data.filename, 'filename');

  // Add to meta
  const docMeta = {
    id: generateId(),
    filename: data.filename,
    title: data.title,
    category: data.category,
    version: extractVersion(data.content),
    enabled: data.enabled !== false,
    priority: data.priority || 50,
    weight: data.weight || 5,
    createdAt: supersedingTimestamp(deletedAt)
  };

  meta.documents.push(docMeta);
  meta.documents.sort((a, b) => a.priority - b.priority);
  meta.deletedDocuments = clearTombstone(meta.deletedDocuments, data.filename, 'filename');
  // Remove only the file this attempt created; preserve a pre-existing orphan.
  await saveMeta(meta).catch(async (err) => {
    if (!fileExisted) await unlink(filePath).catch(() => {});
    throw err;
  });

  console.log(`🧬 Created soul document: ${data.filename}`);
  return { ...docMeta, content: data.content };
}

async function updateDocumentLeased(id, updates) {
  const meta = await loadMeta();
  const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);

  if (docIndex === -1) return null;

  const docMeta = meta.documents[docIndex];
  const filePath = documentWritePath(docMeta.filename);

  // Update file content if provided
  if (updates.content) {
    await atomicWrite(filePath, updates.content);
    docMeta.version = extractVersion(updates.content);
  }

  // Update metadata
  if (updates.title) docMeta.title = updates.title;
  if (updates.enabled !== undefined) docMeta.enabled = updates.enabled;
  if (updates.priority !== undefined) {
    docMeta.priority = updates.priority;
    meta.documents.sort((a, b) => a.priority - b.priority);
  }
  if (updates.weight !== undefined) docMeta.weight = updates.weight;

  // Stamp the edit so a delete performed on ANOTHER machine before this edit
  // reached it can't reap the document and destroy the edit with it: the sync
  // merge keeps whichever of create/edit/delete happened last (#3530).
  docMeta.updatedAt = new Date().toISOString();

  meta.documents[docIndex] = docMeta;
  await saveMeta(meta);

  console.log(`🧬 Updated soul document: ${docMeta.filename}`);
  return await getDocumentById(id);
}

async function deleteDocumentLeased(id) {
  const meta = await loadMeta();
  const docIndex = documentRecords(meta).findIndex(d => usableDocument(d) && d.id === id);

  if (docIndex === -1) return false;

  const docMeta = meta.documents[docIndex];
  const filePath = join(DIGITAL_TWIN_DIR, docMeta.filename);

  // Delete file
  if (existsSync(filePath)) {
    await unlink(filePath);
  }

  // Remove from meta and tombstone the filename so peer sync — which unions
  // documents add-only — can't resurrect it from a machine that still has the
  // file (#3530). The tombstone also propagates the delete to those peers.
  meta.documents.splice(docIndex, 1);
  meta.deletedDocuments = recordTombstone(meta.deletedDocuments, docMeta.filename, { keyField: 'filename' });
  await saveMeta(meta);

  console.log(`🧬 Deleted soul document: ${docMeta.filename}`);
  return true;
}