Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

23.4 KB
Raw Blame
import { selectPersistentMindContextEvents } from '../lib/persistentMindContextEvents.js';
/**
 * Persistent-mind context rollups and explicit Brain promotion.
 *
 * Raw events remain in the bounded machine-local run-event ledger. Older
 * sealed ranges are represented by provenance-tagged rollups in a separate
 * machine-local cache, so assembling one turn never requires an unbounded JSONL
 * read. A corrupt/unreadable cache fails closed rather than becoming `[]` and
 * overwriting the only summaries of history that has left raw retention.
 */

import { join } from 'path';
import { isRuntimeBudgetError } from '../lib/aiToolkit/errorDetection.js';
import {
  PATHS,
  atomicWrite,
  readJSONFileStrict,
  sha256Text,
} from '../lib/fileUtils.js';
import { createFileWriteQueue } from '../lib/fileWriteQueue.js';
import {
  PERSISTENT_MIND_ID,
  PERSISTENT_MIND_ROLLUP_PROMPT_VERSION,
  PERSISTENT_MIND_TRAJECTORY_LIMITS,
  assemblePersistentMindContext,
  buildPersistentMindRollup,
  isPersistentMindCallDenial,
  isStoredPersistentMindRollup,
} from '../lib/persistentMindTrajectory.js';
import {
  appendMindEvent,
  readPersistentMindHistory,
} from './agentRunEventLog.js';
import {
  extractPersistentMindJournal,
  readPersistentMindJournal,
} from './persistentMindJournal.js';
import { persistentMindJournalDigest } from '../lib/persistentMindJournal.js';
import { PERSISTENT_MIND_CHOSEN_NAME_TAG, persistentMindChooseNameSchema, resolvePersistentMindChosenName } from '../lib/persistentMindChosenName.js';
import * as memoryBackend from './memoryBackend.js';
import {
  PERSISTENT_MIND_MEMORY_PROTECTION_TAGS,
  comparePersistentMindMemories,
  persistentMindMemoryProtection,
  persistentMindMemoryTags,
  persistentMindProtectMemorySchema,
  projectPersistentMindMemory,
} from '../lib/persistentMindMemory.js';

const ROLLUP_PATH = join(PATHS.cos, 'persistent-mind-rollups.json');
const ROLLUP_STORE_SCHEMA_VERSION = 1;
const queueRollupWrite = createFileWriteQueue();
const queueMemoryWrite = createFileWriteQueue();
const promotionRuns = new Map();
const memoryCreationRuns = new Map();

const emptyStore = () => ({ schemaVersion: ROLLUP_STORE_SCHEMA_VERSION, rollups: [] });

async function loadRollupStore() {
  const { ok, value } = await readJSONFileStrict(ROLLUP_PATH, emptyStore());
  if (!ok) throw new Error('Persistent mind rollup cache is unreadable');
  if (!value || typeof value !== 'object' || Array.isArray(value)
      || value.schemaVersion !== ROLLUP_STORE_SCHEMA_VERSION || !Array.isArray(value.rollups)
      || value.rollups.some((rollup) => !isStoredPersistentMindRollup(rollup))) {
    throw new Error('Persistent mind rollup cache has an invalid shape');
  }
  return value;
}

export async function readPersistentMindRollups(mindId = PERSISTENT_MIND_ID) {
  const store = await loadRollupStore();
  return store.rollups
    .filter((rollup) => rollup.mindId === mindId)
    .sort((a, b) => a.source.fromSequence - b.source.fromSequence);
}

/** Drop derived summaries for one mind so the next wake rebuilds from retained history. */
export function clearPersistentMindRollups(mindId = PERSISTENT_MIND_ID) {
  return queueRollupWrite(async () => {
    const store = await loadRollupStore();
    const rollups = store.rollups.filter((rollup) => rollup.mindId !== mindId);
    const cleared = store.rollups.length - rollups.length;
    await atomicWrite(ROLLUP_PATH, { schemaVersion: ROLLUP_STORE_SCHEMA_VERSION, rollups });
    return { cleared };
  });
}

/** Protected memories take precedence over ordinary importance-ranked context. */
export async function readPersistentMindMemories(mindId = PERSISTENT_MIND_ID) {
  const options = { status: 'active', sourceAgentId: mindId, sortBy: 'importance', sortOrder: 'desc', limit: 100 };
  // Query each protection tier separately so a low-score identity cannot fall
  // outside the first page of ordinary memories before we get to sort it.
  const pages = await Promise.all([
    memoryBackend.getMemories(options),
    memoryBackend.getMemories({ ...options, tags: [PERSISTENT_MIND_CHOSEN_NAME_TAG] }),
    ...Object.values(PERSISTENT_MIND_MEMORY_PROTECTION_TAGS).map((tag) => memoryBackend.getMemories({ ...options, tags: [tag] })),
  ]);
  const candidates = [...new Map(pages.flatMap((page) => page.memories || []).map((memory) => [memory.id, memory])).values()];
  const details = await Promise.all(candidates.map((memory) => memoryBackend.peekMemory(memory.id)));
  return details.filter((memory) => memory?.status === 'active' && memory.sourceAgentId === mindId)
    .sort(comparePersistentMindMemories).slice(0, 100).map(projectPersistentMindMemory);
}

/** Named installs need only one record for frequent page/turn identity refreshes. */
export async function readPersistentMindName(mindId = PERSISTENT_MIND_ID) {
  const { memories = [] } = await memoryBackend.getMemories({
    status: 'active', sourceAgentId: mindId, tags: [PERSISTENT_MIND_CHOSEN_NAME_TAG],
    sortBy: 'updatedAt', sortOrder: 'desc', limit: 1,
  });
  const memory = memories[0] ? await memoryBackend.peekMemory(memories[0].id) : null;
  if (memory?.sourceAgentId === mindId && memory.status === 'active') {
    const name = resolvePersistentMindChosenName([memory]);
    if (name) return name;
  }
  return resolvePersistentMindChosenName(await readPersistentMindMemories(mindId));
}

/** Bulk cleanup never archives protected memories, regardless of its caller. */
export function archivePersistentMindMemories(mindId = PERSISTENT_MIND_ID) {
  return queueMemoryWrite(async () => {
    let archived = 0;
    let preserved = 0;
    let offset = 0;
    while (true) {
      const result = await memoryBackend.getMemories({
        status: 'active', sourceAgentId: mindId,
        sortBy: 'createdAt', sortOrder: 'asc', limit: 100, offset,
      });
      const candidates = result.memories || [];
      if (candidates.length === 0) break;
      let batchArchived = 0;
      for (const candidate of candidates) {
        const memory = await memoryBackend.peekMemory(candidate.id);
        if (memory?.status !== 'active' || memory.sourceAgentId !== mindId) continue;
        if (persistentMindMemoryProtection(memory) !== 'standard') {
          preserved += 1;
          continue;
        }
        await memoryBackend.archiveMemory(memory.id);
        archived += 1;
        batchArchived += 1;
      }
      // Archived rows leave the active result set; protected rows remain. Move
      // past those retained rows even when an entire page is protected.
      offset += candidates.length - batchArchived;
    }
    return { archived, preserved };
  });
}

export function recordPersistentMindRollup(input) {
  const rollup = buildPersistentMindRollup(input);
  return queueRollupWrite(async () => {
    const store = await loadRollupStore();
    const ordered = [...store.rollups.filter((item) => item.id !== rollup.id), rollup]
      .sort((a, b) => a.source.fromSequence - b.source.fromSequence || a.source.toSequence - b.source.toSequence);
    const latestReadyByMind = new Map();
    for (const item of ordered.filter((candidate) => candidate.status === 'ready')) {
      const previous = latestReadyByMind.get(item.mindId);
      if (!previous || item.source.toSequence > previous.source.toSequence) {
        latestReadyByMind.set(item.mindId, item);
      }
    }
    const latestReadyIds = new Set([...latestReadyByMind.values()].map((item) => item.id));
    const tail = ordered
      .filter((item) => !latestReadyIds.has(item.id))
      .slice(-Math.max(0, PERSISTENT_MIND_TRAJECTORY_LIMITS.maxStoredRollups - latestReadyIds.size));
    const keptIds = new Set([...latestReadyIds, ...tail.map((item) => item.id)]);
    const rollups = ordered.filter((item) => keptIds.has(item.id));
    await atomicWrite(ROLLUP_PATH, { schemaVersion: ROLLUP_STORE_SCHEMA_VERSION, rollups });
    return rollup;
  });
}

const latestReadyRollup = (rollups, promptVersion) => rollups
  .filter((rollup) => rollup.status === 'ready' && rollup.provenance.promptVersion === promptVersion)
  .sort((a, b) => a.source.toSequence - b.source.toSequence)
  .at(-1) || null;

// `attempted: false` means the summarizer never reached a provider — the
// per-call boundary refused it (budget, authorization, lifecycle). Sealing a
// FAILED rollup for that range would be a lie AND permanent: the range id is
// deterministic, so `alreadyAttempted` would stop every later turn from
// retrying, and a transient refusal would silently cost the mind that stretch
// of its life forever.
const summaryOutcome = (summarize, input) => Promise.resolve()
  .then(() => summarize(input))
  .then(
    (summary) => typeof summary === 'string' && summary.trim()
      ? { ok: true, attempted: true, summary }
      : { ok: false, attempted: true, error: 'Persistent mind summarizer returned no summary text' },
    (error) => {
      if (isRuntimeBudgetError(error)) throw error;
      return {
        ok: false,
        attempted: !isPersistentMindCallDenial(error),
        error: String(error?.message || error || 'Persistent mind summary failed').slice(0, 500),
      };
    }
  );

/**
 * Assemble context and, when a summarizer is available, incrementally seal the
 * older range that just fell outside the recent verbatim window.
 */
export async function preparePersistentMindContext({
  mindId = PERSISTENT_MIND_ID,
  identity = '',
  instructions = '',
  memories = [],
  maxChars,
  recentEventLimit = PERSISTENT_MIND_TRAJECTORY_LIMITS.recentContextEvents,
  promptVersion = PERSISTENT_MIND_ROLLUP_PROMPT_VERSION,
  providerId = null,
  model = null,
  summarize = null,
  extractJournal = null,
  forceSummary = false,
} = {}) {
  const [history, initialRollups, initialJournal] = await Promise.all([
    readPersistentMindHistory(mindId),
    readPersistentMindRollups(mindId),
    readPersistentMindJournal(mindId),
  ]);
  let rollups = initialRollups;
  let journal = initialJournal;
  const recent = selectPersistentMindContextEvents(history, mindId).slice(-Math.max(1, recentEventLimit));
  const older = recent.length ? history.filter(event => event.sequence < recent[0].sequence) : [];
  let coverageGap = null;

  if (older.length > 0) {
    const previous = latestReadyRollup(rollups, promptVersion);
    const coveredThrough = previous?.source.toSequence ?? -1;
    const rangeEvents = older.filter((event) => event.sequence > coveredThrough);
    if (rangeEvents.length > 0) {
      const expectedPredecessor = previous?.source.toSequence ?? null;
      const brokenLinkIndex = rangeEvents.findIndex((event, index) => {
        const expected = index === 0 ? expectedPredecessor : rangeEvents[index - 1].sequence;
        return event.data?.previousSequence !== expected;
      });
      if (brokenLinkIndex >= 0) {
        const brokenEvent = rangeEvents[brokenLinkIndex];
        const expected = brokenLinkIndex === 0
          ? expectedPredecessor
          : rangeEvents[brokenLinkIndex - 1].sequence;
        const actualPredecessor = brokenEvent.data?.previousSequence;
        coverageGap = {
          expectedAfterSequence: expected,
          retainedFromSequence: brokenEvent.sequence,
          recordedPredecessorSequence: Number.isSafeInteger(actualPredecessor) ? actualPredecessor : null,
        };
      }
      const source = {
        fromSequence: previous?.source.fromSequence ?? rangeEvents[0].sequence,
        toSequence: rangeEvents.at(-1).sequence,
        fromEventId: previous?.source.fromEventId ?? rangeEvents[0].eventId,
        toEventId: rangeEvents.at(-1).eventId,
      };
      const rollupId = `${mindId}:${source.fromSequence}-${source.toSequence}:v${promptVersion}`;
      const alreadyAttempted = rollups.some((rollup) => rollup.id === rollupId);
      // Validate coverage against the complete ledger, then project prompt evidence.
      // Filtering never edits retained history or its provenance chain.
      const contextEvents = selectPersistentMindContextEvents(history, mindId)
        .filter(event => event.sequence > coveredThrough && event.sequence <= source.toSequence);
      // The journal is extracted BEFORE the range is sealed, so the summary
      // compacts from typed events — what is still decided, owed, open and
      // risky — instead of re-compressing the transcript. A failed extraction
      // never blocks the seal: the rollup is still strictly better than losing
      // the range, and a later turn can re-extract.
      if (!coverageGap && typeof extractJournal === 'function' && (forceSummary || !alreadyAttempted)) {
        const extraction = await extractPersistentMindJournal({
          mindId,
          events: contextEvents,
          range: { fromSequence: rangeEvents[0].sequence, toSequence: rangeEvents.at(-1).sequence },
          extract: extractJournal,
          providerId,
          model,
          isCallDenial: isPersistentMindCallDenial,
        });
        if (extraction.attempted) journal = await readPersistentMindJournal(mindId);
        if (extraction.error) console.error(`❌ Persistent mind journal extraction failed: ${extraction.error}`);
      }
      if (!coverageGap && typeof summarize === 'function' && (forceSummary || !alreadyAttempted)) {
        const outcome = await summaryOutcome(summarize, {
          mindId,
          source,
          events: contextEvents,
          journal,
          previousSummary: previous?.summary ?? null,
          previousProvenance: previous?.provenance ?? null,
          promptVersion,
        });
        // A refused call never reached a provider, so the range stays
        // unattempted and a later turn can still seal it.
        if (outcome.attempted) {
          const rollup = await recordPersistentMindRollup({
            id: rollupId,
            mindId,
            status: outcome.ok ? 'ready' : 'failed',
            summary: outcome.ok ? outcome.summary : null,
            error: outcome.ok ? null : outcome.error,
            source,
            providerId,
            model,
            promptVersion,
          });
          await appendMindEvent({
            kind: 'mind.summary',
            mindId,
            // Keyed on the rollup's own createdAt (unique per attempt), not just
            // rollup.id: a forceSummary retry reuses the same rollup id, and the
            // shared ledger dedupes mind events by eventId regardless of age — an
            // id derived from rollup.id alone would make a successful retry's
            // event silently drop, leaving the replayed trajectory stuck showing
            // the earlier failed attempt forever.
            eventId: `mind-summary-${sha256Text(`${rollup.id}:${rollup.provenance.createdAt}`).slice(0, 32)}`,
            data: {
              rollupId: rollup.id,
              status: rollup.status,
              fromSequence: source.fromSequence,
              toSequence: source.toSequence,
              providerId,
              model,
              promptVersion,
              summaryText: rollup.summary,
              error: rollup.error,
            },
          });
          rollups = await readPersistentMindRollups(mindId);
        }
      }
    }
  }

  const chosenName = resolvePersistentMindChosenName(memories);
  return assemblePersistentMindContext({
    mindId,
    identity: [chosenName
      ? `Current chosen display name: ${JSON.stringify(chosenName)}. This supersedes older name memories; mindId and trajectory remain unchanged.`
      : 'No chosen display name yet.', identity].filter(Boolean).join('\n'),
    instructions,
    memories,
    events: history,
    rollups,
    journalDigest: persistentMindJournalDigest(journal, mindId),
    maxChars,
    recentEventLimit,
    promptVersion,
    coverageGap,
  });
}

/** One protected record is the current name; older identity prose remains history. */
export function choosePersistentMindName(input, mindId = PERSISTENT_MIND_ID) {
  const { name } = persistentMindChooseNameSchema.parse(input);
  return queueMemoryWrite(async () => {
    const memories = await readPersistentMindMemories(mindId);
    const previousName = resolvePersistentMindChosenName(memories);
    const existing = memories.find((memory) => memory.tags?.includes(PERSISTENT_MIND_CHOSEN_NAME_TAG));
    const fields = {
      content: name,
      summary: `Current chosen display name: ${name}`,
      tags: persistentMindMemoryTags([PERSISTENT_MIND_CHOSEN_NAME_TAG, 'name'], 'core-identity'),
      importance: 1,
    };
    const memory = existing
      ? await memoryBackend.updateMemory(existing.id, fields)
      : await memoryBackend.createMemory({ ...fields, type: 'fact', category: 'preferences', sourceAgentId: mindId, status: 'active' });
    if (!memory) throw new Error('Chosen name could not be saved');
    return { ok: true, success: true, name, previousName, mindId, memoryId: memory.id };
  });
}

export function createPersistentMindMemory({ mindId = PERSISTENT_MIND_ID, protection = 'standard', ...input } = {}) {
  return queueMemoryWrite(async () => projectPersistentMindMemory(await memoryBackend.createMemory({
    ...input,
    tags: persistentMindMemoryTags(input.tags, protection),
    sourceAgentId: mindId,
    status: 'active',
  })));
}

async function findExistingAutomaticMemory({ memoryApi, mindId, turnId, content }) {
  const result = await memoryApi.getMemories({
    status: 'active',
    sourceAgentId: mindId,
    limit: 1000,
  });
  const candidates = result.memories || [];
  for (const candidate of candidates) {
    const memory = await memoryApi.peekMemory(candidate.id);
    if (memory?.sourceTaskId === turnId && memory.content === content) return memory;
  }
  return null;
}

async function performAutomaticMemoryCreation({
  candidateId,
  mindId = PERSISTENT_MIND_ID,
  turnId = null,
  content,
  summary,
  type = 'observation',
  category = 'other',
  tags = [],
  protection = 'standard',
  memoryApi = memoryBackend,
} = {}) {
  if (typeof candidateId !== 'string' || !candidateId.trim()) {
    throw new Error('Persistent mind memory candidate id is required');
  }
  if (typeof content !== 'string' || !content.trim()) {
    throw new Error('Persistent mind memory content is required');
  }
  const id = candidateId.trim();
  const normalizedContent = content.trim().slice(0, 10_240);
  const history = await readPersistentMindHistory(mindId);
  const previous = history.find((event) => (
    event.kind === 'mind.memory.created' && event.data?.candidateId === id
  ));
  if (previous?.data?.memoryId) {
    return { success: true, duplicate: true, memory: { id: previous.data.memoryId } };
  }

  const existing = await findExistingAutomaticMemory({ memoryApi, mindId, turnId, content: normalizedContent });
  const memory = existing || await memoryApi.createMemory({
    type,
    content: normalizedContent,
    summary: typeof summary === 'string' ? summary.trim().slice(0, 500) : undefined,
    category,
    tags: persistentMindMemoryTags(tags, protection),
    sourceTaskId: turnId,
    sourceAgentId: mindId,
    status: 'active',
  });
  return { success: true, duplicate: Boolean(existing), memory };
}

/** Persist a structured memory emitted by the mind without a human approval step. */
export function createPersistentMindMemoryFromCandidate(input = {}) {
  const key = `${input.mindId || PERSISTENT_MIND_ID}:${input.candidateId || ''}`;
  if (memoryCreationRuns.has(key)) return memoryCreationRuns.get(key);
  const run = queueMemoryWrite(() => performAutomaticMemoryCreation(input)).finally(() => memoryCreationRuns.delete(key));
  memoryCreationRuns.set(key, run);
  return run;
}

export function updatePersistentMindMemory(memoryId, updates, mindId = PERSISTENT_MIND_ID) {
  return queueMemoryWrite(async () => {
    const existing = await memoryBackend.peekMemory(memoryId);
    if (!existing || existing.sourceAgentId !== mindId) return null;
    const { protection, ...fields } = updates;
    // Older clients send tags without a protection field. Keep the protection
    // unless the user explicitly selects a different level.
    const tags = persistentMindMemoryTags(fields.tags ?? existing.tags, protection ?? persistentMindMemoryProtection(existing));
    return projectPersistentMindMemory(await memoryBackend.updateMemory(memoryId, { ...fields, tags }));
  });
}

/** The mind may protect its own active records, never remove their protection. */
export function protectPersistentMindMemory(input, mindId = PERSISTENT_MIND_ID) {
  const { memoryId, protection } = persistentMindProtectMemorySchema.parse(input);
  return queueMemoryWrite(async () => {
    const existing = await memoryBackend.peekMemory(memoryId);
    if (!existing || existing.sourceAgentId !== mindId || existing.status !== 'active') {
      return { ok: false, success: false, error: 'Active mind-owned memory not found' };
    }
    const level = persistentMindMemoryProtection(existing) === 'core-identity' ? 'core-identity' : protection;
    await memoryBackend.updateMemory(memoryId, { tags: persistentMindMemoryTags(existing.tags, level) });
    return { ok: true, success: true, memoryId, protection: level };
  });
}

/** Add an attributable comment/idea to the trajectory. */
export function appendPersistentMindAnnotation({
  id,
  mindId = PERSISTENT_MIND_ID,
  turnId = null,
  targetEventId = null,
  text,
  at,
} = {}) {
  if (typeof id !== 'string' || !id.trim() || typeof text !== 'string' || !text.trim()) {
    return Promise.resolve({ appended: false, error: 'Annotation id and text are required' });
  }
  return appendMindEvent({
    kind: 'mind.annotation.accepted',
    mindId,
    turnId,
    at,
    eventId: `mind-annotation:${id.trim()}`,
    data: {
      annotationId: id.trim(),
      targetEventId: typeof targetEventId === 'string' ? targetEventId : null,
      displayText: text.trim(),
      textChars: text.trim().length,
    },
  });
}

/**
 * Promote one user-approved fact into the existing Brain backend. Absence of an
 * explicit `approved: true` is a refusal, not a pending or empty memory.
 */
async function performPersistentMindPromotion({
  id,
  approved,
  mindId = PERSISTENT_MIND_ID,
  turnId = null,
  sourceEventId = null,
  content,
  summary,
  type = 'fact',
  category = 'other',
  tags = [],
  memoryApi = memoryBackend,
} = {}) {
  if (approved !== true) return { success: false, error: 'Explicit user approval is required' };
  if (typeof id !== 'string' || !id.trim()) return { success: false, error: 'Promotion id is required' };
  if (typeof content !== 'string' || !content.trim()) return { success: false, error: 'Memory content is required' };
  const promotionId = id.trim();
  const previous = (await readPersistentMindHistory(mindId)).find(
    (event) => event.kind === 'mind.memory.promoted' && event.data?.promotionId === promotionId
  );
  if (previous) {
    return { success: true, duplicate: true, memory: { id: previous.data.memoryId } };
  }
  const memory = await memoryApi.createMemory({
    type,
    content: content.trim().slice(0, 10_240),
    summary: typeof summary === 'string' ? summary.trim().slice(0, 500) : undefined,
    category,
    tags: [...new Set(Array.isArray(tags) ? tags.filter((tag) => typeof tag === 'string' && tag) : [])].slice(0, 20),
    sourceTaskId: turnId,
    sourceAgentId: mindId,
    status: 'active',
  });
  await appendMindEvent({
    kind: 'mind.memory.promoted',
    mindId,
    turnId,
    eventId: `mind-memory:${memory.id}`,
    data: {
      promotionId,
      memoryId: memory.id,
      sourceEventId,
      type,
      category,
      approved: true,
    },
  });
  return { success: true, memory };
}

export function promotePersistentMindMemory(input = {}) {
  const key = typeof input.id === 'string' ? input.id.trim() : '';
  if (key && promotionRuns.has(key)) return promotionRuns.get(key);
  const run = performPersistentMindPromotion(input).finally(() => promotionRuns.delete(key));
  if (key) promotionRuns.set(key, run);
  return run;
}