Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

29.8 KB
Raw Blame
/**
 * Memory Service
 *
 * Core CRUD and search operations for the CoS memory system.
 * Stores facts, learnings, observations, decisions, preferences, and context.
 *
 * Orchestration only: persistence + caches + the write mutex live in
 * `memoryStore.js`; pure index projection/filter/sort/RRF helpers live in
 * `../lib/memoryQuery.js`. This module wires those primitives into the public
 * CRUD/search API and emits cosEvents. Public exports are unchanged so all
 * import sites (and `memoryBackend.js`'s dynamic import) keep working.
 */

import { v4 as uuidv4 } from '../lib/uuid.js';
import { persistentMindMemoryProtection } from '../lib/persistentMindMemory.js';
import { DEFAULT_MEMORY_LINK_TYPE, isSymmetricMemoryLinkType } from '../lib/memoryLinkTypes.js';
import { cosEvents } from './cosEvents.js';
import { findTopK, findAboveThreshold, clusterBySimilarity } from '../lib/vectorMath.js';
import * as notifications from './notifications.js';
import * as memoryBM25 from './memoryBM25.js';
import { DEFAULT_MEMORY_CONFIG, generateSummary, decrementAgentPendingApproval } from './memoryConfig.js';
import {
  withMemoryLock,
  loadIndex,
  saveIndex,
  loadEmbeddings,
  saveEmbeddings,
  loadMemory,
  saveMemory,
  deleteMemoryFiles,
  invalidateCaches
} from './memoryStore.js';
import {
  projectIndexMeta,
  filterMemoryIndex,
  compareMemoryEntries,
  passesSearchMetaFilters,
  passesHybridMetaFilters,
  fuseRankingsRRF
} from '../lib/memoryQuery.js';

export { DEFAULT_MEMORY_CONFIG };

// Re-export the cache invalidator so existing import sites keep working.
export { invalidateCaches };

/**
 * Create a new memory
 */
export async function createMemory(data, embedding = null) {
  return withMemoryLock(async () => {
    const index = await loadIndex();
    const embeddings = await loadEmbeddings();

    const now = new Date().toISOString();
    const id = uuidv4();

    const memory = {
      id,
      type: data.type,
      content: data.content,
      summary: data.summary || generateSummary(data.content),
      category: data.category || 'other',
      tags: data.tags || [],
      relatedMemories: data.relatedMemories || [],
      sourceTaskId: data.sourceTaskId || null,
      sourceAgentId: data.sourceAgentId || null,
      sourceAppId: data.sourceAppId || null,
      embedding: embedding || null,
      embeddingModel: embedding ? DEFAULT_MEMORY_CONFIG.embeddingModel : null,
      confidence: data.confidence ?? 0.8,
      importance: data.importance ?? 0.5,
      accessCount: 0,
      lastAccessed: null,
      createdAt: now,
      updatedAt: now,
      expiresAt: data.expiresAt || null,
      status: data.status || 'active'
    };

    // Save full memory
    await saveMemory(memory);

    // Add to index (lightweight metadata only)
    index.memories.push(projectIndexMeta(memory));
    index.count = index.memories.length;
    await saveIndex(index);

    // Store embedding separately
    if (embedding) {
      embeddings.vectors[id] = embedding;
      embeddings.model = DEFAULT_MEMORY_CONFIG.embeddingModel;
      embeddings.dimension = embedding.length;
      await saveEmbeddings(embeddings);
    }

    // Index in BM25 for text search (async, non-blocking)
    memoryBM25.indexMemory({
      id: memory.id,
      content: memory.content,
      type: memory.type,
      tags: memory.tags,
      source: memory.sourceAppId
    }).catch(err => console.error(`⚠️ BM25 index error: ${err.message}`));

    console.log(`🧠 Memory created: ${memory.type} - ${memory.summary.substring(0, 50)}...`);
    cosEvents.emit('memory:created', { id, type: memory.type, summary: memory.summary });

    return memory;
  });
}

/**
 * Read a memory by ID without updating access stats
 */
export async function peekMemory(id) {
  return withMemoryLock(() => loadMemory(id));
}

/**
 * Get a memory by ID (updates access stats)
 */
export async function getMemory(id) {
  return withMemoryLock(async () => {
    const memory = await loadMemory(id);
    if (!memory) return null;

    // Update access stats
    memory.accessCount += 1;
    memory.lastAccessed = new Date().toISOString();
    await saveMemory(memory);

    return memory;
  });
}

export async function countMemories(options = {}) {
  const index = await loadIndex();
  return filterMemoryIndex(index.memories, options).length;
}

/**
 * Get memories with filters
 */
export async function getMemories(options = {}) {
  const index = await loadIndex();
  let memories = filterMemoryIndex(index.memories, options);

  // Sort
  const sortBy = options.sortBy || 'createdAt';
  const sortOrder = options.sortOrder || 'desc';
  memories.sort(compareMemoryEntries(sortBy, sortOrder));

  // Paginate
  const offset = options.offset || 0;
  const limit = options.limit || 50;
  const total = memories.length;
  memories = memories.slice(offset, offset + limit);

  return { total, memories };
}

/**
 * Update a memory
 */
export async function updateMemory(id, updates) {
  return withMemoryLock(async () => {
    const memory = await loadMemory(id);
    if (!memory) return null;

    // Apply updates
    const updatableFields = ['content', 'summary', 'type', 'category', 'tags', 'confidence', 'importance', 'relatedMemories', 'status', 'expiresAt', 'sourceAppId'];
    for (const field of updatableFields) {
      if (updates[field] !== undefined) {
        memory[field] = updates[field];
      }
    }

    // Update summary if content changed. Gate on "is a string", not truthiness:
    // clearing content to "" is an intentional edit that must regenerate (clear)
    // the summary, not leave the stale one behind (absent-vs-cleared, AGENTS.md).
    // The typeof check keeps null/undefined/absent out of generateSummary (which
    // would throw on .length) while still treating "" as a present-but-empty clear.
    if (typeof updates.content === 'string' && updates.summary === undefined) {
      memory.summary = generateSummary(updates.content);
    }

    memory.updatedAt = new Date().toISOString();
    await saveMemory(memory);

    // Update index
    const index = await loadIndex();
    const idx = index.memories.findIndex(m => m.id === id);
    if (idx !== -1) {
      index.memories[idx] = projectIndexMeta(memory);
      await saveIndex(index);
    }

    // Update BM25 index if content or tags changed. Gate on shape, not truthiness:
    // clearing content to "" (a string) must reindex so the now-empty document
    // stops matching its old terms, and clearing tags to [] (Array.isArray) must
    // reindex too (absent-vs-cleared, AGENTS.md). null/undefined/absent are skipped.
    if (typeof updates.content === 'string' || Array.isArray(updates.tags)) {
      memoryBM25.indexMemory({
        id: memory.id,
        content: memory.content,
        type: memory.type,
        tags: memory.tags,
        source: memory.sourceAppId
      }).catch(err => console.error(`⚠️ BM25 index error: ${err.message}`));
    }

    console.log(`🧠 Memory updated: ${id}`);
    cosEvents.emit('memory:updated', { id, updates });

    return memory;
  });
}

/**
 * Update a memory's embedding
 */
export async function updateMemoryEmbedding(id, embedding) {
  return withMemoryLock(async () => {
    const memory = await loadMemory(id);
    if (!memory) return null;

    // Update memory with new embedding
    memory.embedding = embedding;
    memory.embeddingModel = DEFAULT_MEMORY_CONFIG.embeddingModel;
    memory.updatedAt = new Date().toISOString();
    await saveMemory(memory);

    // Update embeddings file
    const embeddings = await loadEmbeddings();
    embeddings.vectors[id] = embedding;
    embeddings.model = DEFAULT_MEMORY_CONFIG.embeddingModel;
    embeddings.dimension = embedding.length;
    await saveEmbeddings(embeddings);

    console.log(`🧠 Memory embedding updated: ${id}`);
    return memory;
  });
}

/**
 * Archive a memory: mark it archived; the row and embedding stay, so it is restorable.
 */
export async function archiveMemory(id) {
  return withMemoryLock(async () => {
    // Note: We can't call updateMemory here as it would cause deadlock (both use withMemoryLock)
    // Instead, we handle the archive logic directly within this lock
    const memory = await loadMemory(id);
    if (memory) {
      memory.status = 'archived';
      memory.updatedAt = new Date().toISOString();
      await saveMemory(memory);

      // Update index
      const index = await loadIndex();
      const idx = index.memories.findIndex(m => m.id === id);
      if (idx !== -1) {
        index.memories[idx].status = 'archived';
        await saveIndex(index);
      }
    }

    console.log(`🧠 Memory archived: ${id}`);
    cosEvents.emit('memory:deleted', { id, hard: false });

    return { success: true, id };
  });
}

/**
 * Purge a memory: destroy its files, index entry, embedding and BM25 entry. Not restorable.
 */
export async function purgeMemory(id) {
  return withMemoryLock(async () => {
    await deleteMemoryFiles(id);

    // Remove from index
    const index = await loadIndex();
    index.memories = index.memories.filter(m => m.id !== id);
    index.count = index.memories.length;
    await saveIndex(index);

    // Remove embedding
    const embeddings = await loadEmbeddings();
    delete embeddings.vectors[id];
    await saveEmbeddings(embeddings);

    // Remove from BM25 index
    memoryBM25.removeMemoryFromIndex(id)
      .catch(err => console.error(`⚠️ BM25 remove error: ${err.message}`));

    console.log(`🧠 Memory purged: ${id}`);
    cosEvents.emit('memory:deleted', { id, hard: true });

    return { success: true, id };
  });
}

/**
 * Approve a pending memory (changes status from pending_approval to active)
 */
export async function approveMemory(id) {
  return withMemoryLock(async () => {
    const memory = await loadMemory(id);
    if (!memory) return { success: false, error: 'Memory not found' };
    if (memory.status !== 'pending_approval') {
      return { success: false, error: 'Memory is not pending approval' };
    }

    memory.status = 'active';
    memory.updatedAt = new Date().toISOString();
    await saveMemory(memory);

    // Update index
    const index = await loadIndex();
    const idx = index.memories.findIndex(m => m.id === id);
    if (idx !== -1) {
      index.memories[idx].status = 'active';
      await saveIndex(index);
    }

    console.log(`🧠 Memory approved: ${id}`);
    cosEvents.emit('memory:approved', { id, memory });

    // Remove associated notification
    await notifications.removeByMetadata('memoryId', id);

    // Decrement agent's pendingApproval count
    await decrementAgentPendingApproval(memory.sourceAgentId);

    return { success: true, memory };
  });
}

/**
 * Reject a pending memory (hard deletes it)
 */
export async function rejectMemory(id) {
  return withMemoryLock(async () => {
    const memory = await loadMemory(id);
    if (!memory) return { success: false, error: 'Memory not found' };
    if (memory.status !== 'pending_approval') {
      return { success: false, error: 'Memory is not pending approval' };
    }

    // Store sourceAgentId before deletion
    const sourceAgentId = memory.sourceAgentId;

    // Hard delete - remove files
    await deleteMemoryFiles(id);

    // Remove from index
    const index = await loadIndex();
    index.memories = index.memories.filter(m => m.id !== id);
    index.count = index.memories.length;
    await saveIndex(index);

    // Remove embedding
    const embeddings = await loadEmbeddings();
    delete embeddings.vectors[id];
    await saveEmbeddings(embeddings);

    // Remove from BM25 index
    memoryBM25.removeMemoryFromIndex(id)
      .catch(err => console.error(`⚠️ BM25 remove error: ${err.message}`));

    console.log(`🧠 Memory rejected: ${id}`);
    cosEvents.emit('memory:rejected', { id });

    // Remove associated notification
    await notifications.removeByMetadata('memoryId', id);

    // Decrement agent's pendingApproval count
    await decrementAgentPendingApproval(sourceAgentId);

    return { success: true, id };
  });
}

/**
 * Search memories semantically
 */
export async function searchMemories(queryEmbedding, options = {}) {
  const embeddings = await loadEmbeddings();
  const index = await loadIndex();

  if (!queryEmbedding || Object.keys(embeddings.vectors).length === 0) {
    return { total: 0, memories: [] };
  }

  // Find similar vectors
  const minRelevance = options.minRelevance || 0.7;
  const limit = options.limit || 20;

  const similar = findAboveThreshold(queryEmbedding, embeddings.vectors, minRelevance);

  // Filter by additional options
  let results = similar.slice(0, limit);

  // Get memory metadata from index
  const indexMap = new Map(index.memories.map(m => [m.id, m]));

  results = results
    .map(r => {
      const meta = indexMap.get(r.id);
      if (!passesSearchMetaFilters(meta, options)) return null;
      return { ...meta, similarity: r.similarity };
    })
    .filter(Boolean);

  return { total: results.length, memories: results };
}

/**
 * Hybrid search combining BM25 text matching and vector similarity
 * Uses Reciprocal Rank Fusion (RRF) to merge rankings
 *
 * @param {string} query - Text query for BM25
 * @param {number[]} queryEmbedding - Vector embedding for semantic search
 * @param {Object} options - Search options
 * @returns {Promise<{total: number, memories: Array}>}
 */
export async function hybridSearchMemories(query, queryEmbedding, options = {}) {
  const { limit = 20, minRelevance = 0.5, ftsWeight = options.bm25Weight ?? 0.4, vectorWeight = 0.6 } = options

  const index = await loadIndex()
  const embeddings = await loadEmbeddings()
  const indexMap = new Map(index.memories.map(m => [m.id, m]))

  // Get BM25 results
  const bm25Results = query
    ? await memoryBM25.searchBM25(query, { limit: limit * 2, threshold: 0.05 })
    : []

  // Get vector results
  let vectorResults = []
  if (queryEmbedding && Object.keys(embeddings.vectors).length > 0) {
    const similar = findAboveThreshold(queryEmbedding, embeddings.vectors, minRelevance * 0.5)
    vectorResults = similar.slice(0, limit * 2).map(r => ({
      id: r.id,
      score: r.similarity
    }))
  }

  // Apply Reciprocal Rank Fusion (RRF) to merge BM25 + vector rankings
  const rrfScores = fuseRankingsRRF(bm25Results, vectorResults, { ftsWeight, vectorWeight })

  // Filter and sort by RRF score
  let results = Array.from(rrfScores.entries())
    .map(([id, data]) => {
      const meta = indexMap.get(id)
      if (!passesHybridMetaFilters(meta, options)) return null

      return {
        ...meta,
        rrfScore: data.rrfScore,
        bm25Rank: data.bm25Rank,
        vectorRank: data.vectorRank,
        searchMethod: data.bm25Rank && data.vectorRank ? 'hybrid' :
                     data.bm25Rank ? 'bm25' : 'vector'
      }
    })
    .filter(Boolean)
    .sort((a, b) => b.rrfScore - a.rrfScore)
    .slice(0, limit)

  return { total: results.length, memories: results }
}

/**
 * Rebuild the BM25 index from all memories
 * Call this after bulk imports or to fix index inconsistencies
 */
export async function rebuildBM25Index() {
  const index = await loadIndex()
  const activeMemories = index.memories.filter(m => m.status === 'active')

  // Load full content for each memory
  const documents = []
  for (const meta of activeMemories) {
    const memory = await loadMemory(meta.id)
    if (memory) {
      documents.push({
        id: memory.id,
        content: memory.content,
        type: memory.type,
        tags: memory.tags,
        source: memory.sourceAppId
      })
    }
  }

  return memoryBM25.rebuildIndex(documents)
}

/**
 * Get BM25 index statistics
 */
export async function getBM25Stats() {
  return memoryBM25.getStats()
}

/**
 * Get timeline data (memories grouped by date)
 */
export async function getTimeline(options = {}) {
  const index = await loadIndex();
  let memories = index.memories.filter(m => m.status === 'active');

  // Filter by date range. Compare instants, not strings — `createdAt` is always
  // UTC (`…Z`) while a boundary may carry an offset (`…+02:00`) or be a bare
  // `YYYY-MM-DD`, and those sort lexicographically against each other wrong.
  const startMs = options.startDate ? Date.parse(options.startDate) : NaN;
  const endMs = options.endDate ? Date.parse(options.endDate) : NaN;
  if (!Number.isNaN(startMs)) {
    memories = memories.filter(m => Date.parse(m.createdAt) >= startMs);
  }
  if (!Number.isNaN(endMs)) {
    memories = memories.filter(m => Date.parse(m.createdAt) <= endMs);
  }

  // Filter by types
  if (options.types && options.types.length > 0) {
    memories = memories.filter(m => options.types.includes(m.type));
  }

  // Filter by app
  if (options.appId === '__not_brain') {
    memories = memories.filter(m => m.sourceAppId !== 'brain');
  } else if (options.appId) {
    memories = memories.filter(m => m.sourceAppId === options.appId);
  }

  // Sort by date descending
  memories.sort((a, b) => (b.createdAt > a.createdAt ? 1 : -1));

  // Limit
  const limit = options.limit || 100;
  memories = memories.slice(0, limit);

  // Group by date
  const timeline = {};
  for (const memory of memories) {
    const date = memory.createdAt.split('T')[0];
    if (!timeline[date]) timeline[date] = [];
    timeline[date].push(memory);
  }

  return timeline;
}

/**
 * Get all unique categories
 */
export async function getCategories() {
  const index = await loadIndex();
  const categories = new Map();

  for (const memory of index.memories) {
    if (memory.status !== 'active') continue;
    const count = categories.get(memory.category) || 0;
    categories.set(memory.category, count + 1);
  }

  return Array.from(categories.entries()).map(([name, count]) => ({ name, count }));
}

/**
 * Get all unique tags
 */
export async function getTags() {
  const index = await loadIndex();
  const tags = new Map();

  for (const memory of index.memories) {
    if (memory.status !== 'active') continue;
    for (const tag of memory.tags) {
      const count = tags.get(tag) || 0;
      tags.set(tag, count + 1);
    }
  }

  return Array.from(tags.entries())
    .map(([name, count]) => ({ name, count }))
    .sort((a, b) => b.count - a.count);
}

/**
 * Get related memories (by ID links + embedding similarity)
 */
export async function getRelatedMemories(id, limit = 10) {
  const memory = await loadMemory(id);
  if (!memory) return [];

  const embeddings = await loadEmbeddings();
  const index = await loadIndex();
  const indexMap = new Map(index.memories.map(m => [m.id, m]));

  const related = [];

  // Add explicitly linked memories
  for (const relId of memory.relatedMemories) {
    const meta = indexMap.get(relId);
    if (meta && meta.status === 'active') {
      related.push({ ...meta, relationship: 'linked', similarity: 1.0, linkType: DEFAULT_MEMORY_LINK_TYPE, direction: 'outgoing' });
    }
  }
  for (const link of memory.typedLinks || []) {
    const meta = indexMap.get(link.targetId);
    if (meta && meta.status === 'active') {
      related.push({
        ...meta, relationship: 'linked', similarity: 1.0, linkId: link.id, linkType: link.linkType,
        direction: 'outgoing', note: link.note, createdBy: link.createdBy
      });
    }
  }

  // Add similar by embedding
  if (memory.embedding && embeddings.vectors[id]) {
    const similar = findTopK(memory.embedding, embeddings.vectors, limit + related.length);
    for (const item of similar) {
      if (item.id === id) continue;
      if (related.some(r => r.id === item.id)) continue;

      const meta = indexMap.get(item.id);
      if (meta && meta.status === 'active') {
        related.push({ ...meta, relationship: 'similar', similarity: item.similarity });
      }
    }
  }

  return related.slice(0, limit);
}

/**
 * Get graph data for visualization.
 *
 * `sourceIds` mirrors the PostgreSQL backend (#10953): absent (undefined/null)
 * returns the whole graph; an array — even an empty one — returns only the
 * nodes in it and the edges whose BOTH endpoints are in it. Similarity is
 * still ranked against every stored vector, so an out-of-scope nearest
 * neighbour keeps its top-three slot.
 */
export async function getGraphData({ sourceIds } = {}) {
  if (sourceIds !== undefined && sourceIds !== null && !Array.isArray(sourceIds)) {
    throw new TypeError('getGraphData: sourceIds must be an array');
  }
  const scope = Array.isArray(sourceIds) ? new Set(sourceIds) : null;
  if (scope && scope.size === 0) return { nodes: [], edges: [] };
  const inScope = (id) => !scope || scope.has(id);

  const index = await loadIndex();
  const embeddings = await loadEmbeddings();

  const activeMemories = index.memories.filter(m => m.status === 'active' && inScope(m.id));

  // Build nodes
  const nodes = activeMemories.map(m => ({
    id: m.id,
    type: m.type,
    category: m.category,
    summary: m.summary,
    importance: m.importance
  }));

  // Build edges from explicit links and high similarity
  const edges = [];
  const seenEdges = new Set();

  // Batch load all memories to avoid N+1 query pattern
  // Load all memories in parallel instead of sequential loadMemory calls
  const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
  const loadedMemories = await Promise.all(memoryLoadPromises);
  const memoriesById = new Map();
  loadedMemories.forEach((mem, idx) => {
    if (mem) memoriesById.set(activeMemories[idx].id, mem);
  });

  // Build explicit link edges using pre-loaded memories
  for (const memory of activeMemories) {
    const full = memoriesById.get(memory.id);
    if (!full) continue;

    // Explicit links
    for (const targetId of full.relatedMemories) {
      if (!inScope(targetId)) continue;
      const edgeKey = [memory.id, targetId].sort().join('-');
      if (!seenEdges.has(edgeKey)) {
        seenEdges.add(edgeKey);
        edges.push({ source: memory.id, target: targetId, type: 'linked', linkType: DEFAULT_MEMORY_LINK_TYPE, weight: 1.0 });
      }
    }
    for (const link of full.typedLinks || []) {
      if (!memoriesById.has(link.targetId)) continue;
      edges.push({ source: memory.id, target: link.targetId, type: 'linked', linkType: link.linkType, weight: 1.0 });
    }
  }

  // Add similarity edges (top 3 per node, > 0.8 similarity)
  for (const memory of activeMemories) {
    if (!embeddings.vectors[memory.id]) continue;

    const similar = findTopK(embeddings.vectors[memory.id], embeddings.vectors, 4);
    for (const item of similar) {
      if (item.id === memory.id) continue;
      if (item.similarity < 0.8) continue;
      // Filtered only after the global top-K: an out-of-scope neighbour still
      // consumes its slot rather than letting a lower-ranked one replace it.
      if (!inScope(item.id)) continue;

      const edgeKey = [memory.id, item.id].sort().join('-');
      if (!seenEdges.has(edgeKey)) {
        seenEdges.add(edgeKey);
        edges.push({ source: memory.id, target: item.id, type: 'similar', weight: item.similarity });
      }
    }
  }

  return { nodes, edges };
}

/**
 * Ids of active memories that have no embedding vector yet — the backfill
 * target for "embed missing" (a record whose embedding generation failed or
 * never ran). Mirrors the memoryDB.js implementation against the file store.
 */
export async function getMemoryIdsMissingEmbedding() {
  const index = await loadIndex();
  const embeddings = await loadEmbeddings();
  const missing = new Set();
  for (const m of index.memories) {
    if (m.status !== 'active') continue;
    if (!embeddings.vectors[m.id]) missing.add(m.id);
  }
  return missing;
}

/**
 * Link two memories
 */
export async function linkMemories(sourceId, targetId, { linkType = DEFAULT_MEMORY_LINK_TYPE, note = null, createdBy = 'user' } = {}) {
  return withMemoryLock(async () => {
    const source = await loadMemory(sourceId);
    const target = await loadMemory(targetId);

    if (!source || !target) return { success: false, error: 'Memory not found' };

    // Directed types live on the source record only (dev/test file backend).
    if (!isSymmetricMemoryLinkType(linkType)) {
      source.typedLinks = source.typedLinks || [];
      let link = source.typedLinks.find(l => l.targetId === targetId && l.linkType === linkType);
      if (!link) {
        link = { id: uuidv4(), targetId, linkType, note, createdBy, createdAt: new Date().toISOString() };
        source.typedLinks.push(link);
        source.updatedAt = link.createdAt;
        await saveMemory(source);
      }
      return { success: true, sourceId, targetId, linkType, linkId: link.id };
    }

    // Add bidirectional links
    if (!source.relatedMemories.includes(targetId)) {
      source.relatedMemories.push(targetId);
      source.updatedAt = new Date().toISOString();
      await saveMemory(source);
    }

    if (!target.relatedMemories.includes(sourceId)) {
      target.relatedMemories.push(sourceId);
      target.updatedAt = new Date().toISOString();
      await saveMemory(target);
    }

    return { success: true, sourceId, targetId, linkType, linkId: null };
  });
}

/**
 * Consolidate similar memories (merge duplicates)
 */
export async function consolidateMemories(threshold = 0.9, dryRun = false) {
  const index = await loadIndex();
  const embeddings = await loadEmbeddings();

  const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');

  // Get memories with embeddings
  const memoriesWithEmbeddings = [];
  for (const meta of activeMemories) {
    if (embeddings.vectors[meta.id]) {
      memoriesWithEmbeddings.push({
        id: meta.id,
        embedding: embeddings.vectors[meta.id],
        ...meta
      });
    }
  }

  // Cluster by similarity
  const clusters = clusterBySimilarity(memoriesWithEmbeddings, threshold);
  const duplicateClusters = clusters.filter(c => c.length > 1);

  if (dryRun) {
    return {
      dryRun: true,
      clustersFound: duplicateClusters.length,
      memoriesAffected: duplicateClusters.reduce((sum, c) => sum + c.length, 0),
      clusters: duplicateClusters.map(c => c.map(m => ({ id: m.id, summary: m.summary })))
    };
  }

  // Merge each cluster
  let merged = 0;
  for (const cluster of duplicateClusters) {
    // Sort by importance, keep highest
    cluster.sort((a, b) => (b.importance || 0) - (a.importance || 0));
    const primary = cluster[0];

    for (let i = 1; i < cluster.length; i++) {
      await updateMemory(cluster[i].id, {
        status: 'archived',
        mergedInto: primary.id
      });
      merged++;
    }
  }

  console.log(`🧠 Consolidated ${merged} duplicate memories into ${duplicateClusters.length} clusters`);
  return { merged, clusters: duplicateClusters.length };
}

/**
 * Apply importance decay to old memories
 */
export async function applyDecay(decayRate = 0.01) {
  const index = await loadIndex();
  const now = Date.now();
  let updated = 0;

  // Filter active memories first
  const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');

  // Batch load all active memories to avoid N+1 query pattern
  const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
  const loadedMemories = await Promise.all(memoryLoadPromises);

  // Process each loaded memory
  for (let i = 0; i < activeMemories.length; i++) {
    const meta = activeMemories[i];
    const memory = loadedMemories[i];
    if (!memory || persistentMindMemoryProtection(memory) !== 'standard') continue;

    const ageInDays = (now - new Date(memory.createdAt).getTime()) / (1000 * 60 * 60 * 24);
    const accessRecency = memory.lastAccessed
      ? (now - new Date(memory.lastAccessed).getTime()) / (1000 * 60 * 60 * 24)
      : ageInDays;

    // Decay formula: importance = baseImportance * (1 - decayRate * sqrt(age)) + accessBoost
    const accessBoost = Math.max(0, 0.1 - accessRecency * 0.001);
    const newImportance = Math.max(0.1, memory.importance * (1 - decayRate * Math.sqrt(ageInDays)) + accessBoost);

    // Archive if importance falls below threshold and old enough
    if (newImportance < 0.15 && ageInDays > 30) {
      await updateMemory(meta.id, { status: 'archived', archivedReason: 'decay' });
      updated++;
    } else if (Math.abs(newImportance - memory.importance) > 0.01) {
      await updateMemory(meta.id, { importance: newImportance });
      updated++;
    }
  }

  console.log(`🧠 Decay applied to ${updated} memories`);
  return { updated };
}

/**
 * Clear expired memories
 */
export async function clearExpired() {
  const index = await loadIndex();
  const now = new Date().toISOString();
  let cleared = 0;

  // Filter active memories first
  const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');

  // Batch load all active memories to avoid N+1 query pattern
  const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
  const loadedMemories = await Promise.all(memoryLoadPromises);

  // Process each loaded memory
  for (let i = 0; i < activeMemories.length; i++) {
    const meta = activeMemories[i];
    const memory = loadedMemories[i];
    if (!memory || persistentMindMemoryProtection(memory) !== 'standard') continue;

    if (memory.expiresAt && memory.expiresAt < now) {
      await updateMemory(meta.id, { status: 'expired' });
      cleared++;
    }
  }

  console.log(`🧠 Cleared ${cleared} expired memories`);
  return { cleared };
}

/**
 * Get memory stats
 */
export async function getStats() {
  const index = await loadIndex();
  const embeddings = await loadEmbeddings();

  const byType = {};
  const byCategory = {};
  const byStatus = {};

  for (const memory of index.memories) {
    byType[memory.type] = (byType[memory.type] || 0) + 1;
    byCategory[memory.category] = (byCategory[memory.category] || 0) + 1;
    byStatus[memory.status] = (byStatus[memory.status] || 0) + 1;
  }

  return {
    total: index.count,
    active: byStatus.active || 0,
    archived: byStatus.archived || 0,
    expired: byStatus.expired || 0,
    pendingApproval: byStatus.pending_approval || 0,
    withEmbeddings: Object.keys(embeddings.vectors).length,
    byType,
    byCategory,
    lastUpdated: index.lastUpdated
  };
}

/**
 * Flush BM25 index to disk
 */
export async function flushBM25Index() {
  return memoryBM25.flush();
}