Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Memory Service
*
* Core CRUD and search operations for the CoS memory system.
* Stores facts, learnings, observations, decisions, preferences, and context.
*
* Orchestration only: persistence + caches + the write mutex live in
* `memoryStore.js`; pure index projection/filter/sort/RRF helpers live in
* `../lib/memoryQuery.js`. This module wires those primitives into the public
* CRUD/search API and emits cosEvents. Public exports are unchanged so all
* import sites (and `memoryBackend.js`'s dynamic import) keep working.
*/
import { v4 as uuidv4 } from '../lib/uuid.js';
import { persistentMindMemoryProtection } from '../lib/persistentMindMemory.js';
import { DEFAULT_MEMORY_LINK_TYPE, isSymmetricMemoryLinkType } from '../lib/memoryLinkTypes.js';
import { cosEvents } from './cosEvents.js';
import { findTopK, findAboveThreshold, clusterBySimilarity } from '../lib/vectorMath.js';
import * as notifications from './notifications.js';
import * as memoryBM25 from './memoryBM25.js';
import { DEFAULT_MEMORY_CONFIG, generateSummary, decrementAgentPendingApproval } from './memoryConfig.js';
import {
withMemoryLock,
loadIndex,
saveIndex,
loadEmbeddings,
saveEmbeddings,
loadMemory,
saveMemory,
deleteMemoryFiles,
invalidateCaches
} from './memoryStore.js';
import {
projectIndexMeta,
filterMemoryIndex,
compareMemoryEntries,
passesSearchMetaFilters,
passesHybridMetaFilters,
fuseRankingsRRF
} from '../lib/memoryQuery.js';
export { DEFAULT_MEMORY_CONFIG };
// Re-export the cache invalidator so existing import sites keep working.
export { invalidateCaches };
/**
* Create a new memory
*/
export async function createMemory(data, embedding = null) {
return withMemoryLock(async () => {
const index = await loadIndex();
const embeddings = await loadEmbeddings();
const now = new Date().toISOString();
const id = uuidv4();
const memory = {
id,
type: data.type,
content: data.content,
summary: data.summary || generateSummary(data.content),
category: data.category || 'other',
tags: data.tags || [],
relatedMemories: data.relatedMemories || [],
sourceTaskId: data.sourceTaskId || null,
sourceAgentId: data.sourceAgentId || null,
sourceAppId: data.sourceAppId || null,
embedding: embedding || null,
embeddingModel: embedding ? DEFAULT_MEMORY_CONFIG.embeddingModel : null,
confidence: data.confidence ?? 0.8,
importance: data.importance ?? 0.5,
accessCount: 0,
lastAccessed: null,
createdAt: now,
updatedAt: now,
expiresAt: data.expiresAt || null,
status: data.status || 'active'
};
// Save full memory
await saveMemory(memory);
// Add to index (lightweight metadata only)
index.memories.push(projectIndexMeta(memory));
index.count = index.memories.length;
await saveIndex(index);
// Store embedding separately
if (embedding) {
embeddings.vectors[id] = embedding;
embeddings.model = DEFAULT_MEMORY_CONFIG.embeddingModel;
embeddings.dimension = embedding.length;
await saveEmbeddings(embeddings);
}
// Index in BM25 for text search (async, non-blocking)
memoryBM25.indexMemory({
id: memory.id,
content: memory.content,
type: memory.type,
tags: memory.tags,
source: memory.sourceAppId
}).catch(err => console.error(`⚠️ BM25 index error: ${err.message}`));
console.log(`🧠 Memory created: ${memory.type} - ${memory.summary.substring(0, 50)}...`);
cosEvents.emit('memory:created', { id, type: memory.type, summary: memory.summary });
return memory;
});
}
/**
* Read a memory by ID without updating access stats
*/
export async function peekMemory(id) {
return withMemoryLock(() => loadMemory(id));
}
/**
* Get a memory by ID (updates access stats)
*/
export async function getMemory(id) {
return withMemoryLock(async () => {
const memory = await loadMemory(id);
if (!memory) return null;
// Update access stats
memory.accessCount += 1;
memory.lastAccessed = new Date().toISOString();
await saveMemory(memory);
return memory;
});
}
export async function countMemories(options = {}) {
const index = await loadIndex();
return filterMemoryIndex(index.memories, options).length;
}
/**
* Get memories with filters
*/
export async function getMemories(options = {}) {
const index = await loadIndex();
let memories = filterMemoryIndex(index.memories, options);
// Sort
const sortBy = options.sortBy || 'createdAt';
const sortOrder = options.sortOrder || 'desc';
memories.sort(compareMemoryEntries(sortBy, sortOrder));
// Paginate
const offset = options.offset || 0;
const limit = options.limit || 50;
const total = memories.length;
memories = memories.slice(offset, offset + limit);
return { total, memories };
}
/**
* Update a memory
*/
export async function updateMemory(id, updates) {
return withMemoryLock(async () => {
const memory = await loadMemory(id);
if (!memory) return null;
// Apply updates
const updatableFields = ['content', 'summary', 'type', 'category', 'tags', 'confidence', 'importance', 'relatedMemories', 'status', 'expiresAt', 'sourceAppId'];
for (const field of updatableFields) {
if (updates[field] !== undefined) {
memory[field] = updates[field];
}
}
// Update summary if content changed. Gate on "is a string", not truthiness:
// clearing content to "" is an intentional edit that must regenerate (clear)
// the summary, not leave the stale one behind (absent-vs-cleared, AGENTS.md).
// The typeof check keeps null/undefined/absent out of generateSummary (which
// would throw on .length) while still treating "" as a present-but-empty clear.
if (typeof updates.content === 'string' && updates.summary === undefined) {
memory.summary = generateSummary(updates.content);
}
memory.updatedAt = new Date().toISOString();
await saveMemory(memory);
// Update index
const index = await loadIndex();
const idx = index.memories.findIndex(m => m.id === id);
if (idx !== -1) {
index.memories[idx] = projectIndexMeta(memory);
await saveIndex(index);
}
// Update BM25 index if content or tags changed. Gate on shape, not truthiness:
// clearing content to "" (a string) must reindex so the now-empty document
// stops matching its old terms, and clearing tags to [] (Array.isArray) must
// reindex too (absent-vs-cleared, AGENTS.md). null/undefined/absent are skipped.
if (typeof updates.content === 'string' || Array.isArray(updates.tags)) {
memoryBM25.indexMemory({
id: memory.id,
content: memory.content,
type: memory.type,
tags: memory.tags,
source: memory.sourceAppId
}).catch(err => console.error(`⚠️ BM25 index error: ${err.message}`));
}
console.log(`🧠 Memory updated: ${id}`);
cosEvents.emit('memory:updated', { id, updates });
return memory;
});
}
/**
* Update a memory's embedding
*/
export async function updateMemoryEmbedding(id, embedding) {
return withMemoryLock(async () => {
const memory = await loadMemory(id);
if (!memory) return null;
// Update memory with new embedding
memory.embedding = embedding;
memory.embeddingModel = DEFAULT_MEMORY_CONFIG.embeddingModel;
memory.updatedAt = new Date().toISOString();
await saveMemory(memory);
// Update embeddings file
const embeddings = await loadEmbeddings();
embeddings.vectors[id] = embedding;
embeddings.model = DEFAULT_MEMORY_CONFIG.embeddingModel;
embeddings.dimension = embedding.length;
await saveEmbeddings(embeddings);
console.log(`🧠 Memory embedding updated: ${id}`);
return memory;
});
}
/**
* Archive a memory: mark it archived; the row and embedding stay, so it is restorable.
*/
export async function archiveMemory(id) {
return withMemoryLock(async () => {
// Note: We can't call updateMemory here as it would cause deadlock (both use withMemoryLock)
// Instead, we handle the archive logic directly within this lock
const memory = await loadMemory(id);
if (memory) {
memory.status = 'archived';
memory.updatedAt = new Date().toISOString();
await saveMemory(memory);
// Update index
const index = await loadIndex();
const idx = index.memories.findIndex(m => m.id === id);
if (idx !== -1) {
index.memories[idx].status = 'archived';
await saveIndex(index);
}
}
console.log(`🧠 Memory archived: ${id}`);
cosEvents.emit('memory:deleted', { id, hard: false });
return { success: true, id };
});
}
/**
* Purge a memory: destroy its files, index entry, embedding and BM25 entry. Not restorable.
*/
export async function purgeMemory(id) {
return withMemoryLock(async () => {
await deleteMemoryFiles(id);
// Remove from index
const index = await loadIndex();
index.memories = index.memories.filter(m => m.id !== id);
index.count = index.memories.length;
await saveIndex(index);
// Remove embedding
const embeddings = await loadEmbeddings();
delete embeddings.vectors[id];
await saveEmbeddings(embeddings);
// Remove from BM25 index
memoryBM25.removeMemoryFromIndex(id)
.catch(err => console.error(`⚠️ BM25 remove error: ${err.message}`));
console.log(`🧠 Memory purged: ${id}`);
cosEvents.emit('memory:deleted', { id, hard: true });
return { success: true, id };
});
}
/**
* Approve a pending memory (changes status from pending_approval to active)
*/
export async function approveMemory(id) {
return withMemoryLock(async () => {
const memory = await loadMemory(id);
if (!memory) return { success: false, error: 'Memory not found' };
if (memory.status !== 'pending_approval') {
return { success: false, error: 'Memory is not pending approval' };
}
memory.status = 'active';
memory.updatedAt = new Date().toISOString();
await saveMemory(memory);
// Update index
const index = await loadIndex();
const idx = index.memories.findIndex(m => m.id === id);
if (idx !== -1) {
index.memories[idx].status = 'active';
await saveIndex(index);
}
console.log(`🧠 Memory approved: ${id}`);
cosEvents.emit('memory:approved', { id, memory });
// Remove associated notification
await notifications.removeByMetadata('memoryId', id);
// Decrement agent's pendingApproval count
await decrementAgentPendingApproval(memory.sourceAgentId);
return { success: true, memory };
});
}
/**
* Reject a pending memory (hard deletes it)
*/
export async function rejectMemory(id) {
return withMemoryLock(async () => {
const memory = await loadMemory(id);
if (!memory) return { success: false, error: 'Memory not found' };
if (memory.status !== 'pending_approval') {
return { success: false, error: 'Memory is not pending approval' };
}
// Store sourceAgentId before deletion
const sourceAgentId = memory.sourceAgentId;
// Hard delete - remove files
await deleteMemoryFiles(id);
// Remove from index
const index = await loadIndex();
index.memories = index.memories.filter(m => m.id !== id);
index.count = index.memories.length;
await saveIndex(index);
// Remove embedding
const embeddings = await loadEmbeddings();
delete embeddings.vectors[id];
await saveEmbeddings(embeddings);
// Remove from BM25 index
memoryBM25.removeMemoryFromIndex(id)
.catch(err => console.error(`⚠️ BM25 remove error: ${err.message}`));
console.log(`🧠 Memory rejected: ${id}`);
cosEvents.emit('memory:rejected', { id });
// Remove associated notification
await notifications.removeByMetadata('memoryId', id);
// Decrement agent's pendingApproval count
await decrementAgentPendingApproval(sourceAgentId);
return { success: true, id };
});
}
/**
* Search memories semantically
*/
export async function searchMemories(queryEmbedding, options = {}) {
const embeddings = await loadEmbeddings();
const index = await loadIndex();
if (!queryEmbedding || Object.keys(embeddings.vectors).length === 0) {
return { total: 0, memories: [] };
}
// Find similar vectors
const minRelevance = options.minRelevance || 0.7;
const limit = options.limit || 20;
const similar = findAboveThreshold(queryEmbedding, embeddings.vectors, minRelevance);
// Filter by additional options
let results = similar.slice(0, limit);
// Get memory metadata from index
const indexMap = new Map(index.memories.map(m => [m.id, m]));
results = results
.map(r => {
const meta = indexMap.get(r.id);
if (!passesSearchMetaFilters(meta, options)) return null;
return { ...meta, similarity: r.similarity };
})
.filter(Boolean);
return { total: results.length, memories: results };
}
/**
* Hybrid search combining BM25 text matching and vector similarity
* Uses Reciprocal Rank Fusion (RRF) to merge rankings
*
* @param {string} query - Text query for BM25
* @param {number[]} queryEmbedding - Vector embedding for semantic search
* @param {Object} options - Search options
* @returns {Promise<{total: number, memories: Array}>}
*/
export async function hybridSearchMemories(query, queryEmbedding, options = {}) {
const { limit = 20, minRelevance = 0.5, ftsWeight = options.bm25Weight ?? 0.4, vectorWeight = 0.6 } = options
const index = await loadIndex()
const embeddings = await loadEmbeddings()
const indexMap = new Map(index.memories.map(m => [m.id, m]))
// Get BM25 results
const bm25Results = query
? await memoryBM25.searchBM25(query, { limit: limit * 2, threshold: 0.05 })
: []
// Get vector results
let vectorResults = []
if (queryEmbedding && Object.keys(embeddings.vectors).length > 0) {
const similar = findAboveThreshold(queryEmbedding, embeddings.vectors, minRelevance * 0.5)
vectorResults = similar.slice(0, limit * 2).map(r => ({
id: r.id,
score: r.similarity
}))
}
// Apply Reciprocal Rank Fusion (RRF) to merge BM25 + vector rankings
const rrfScores = fuseRankingsRRF(bm25Results, vectorResults, { ftsWeight, vectorWeight })
// Filter and sort by RRF score
let results = Array.from(rrfScores.entries())
.map(([id, data]) => {
const meta = indexMap.get(id)
if (!passesHybridMetaFilters(meta, options)) return null
return {
...meta,
rrfScore: data.rrfScore,
bm25Rank: data.bm25Rank,
vectorRank: data.vectorRank,
searchMethod: data.bm25Rank && data.vectorRank ? 'hybrid' :
data.bm25Rank ? 'bm25' : 'vector'
}
})
.filter(Boolean)
.sort((a, b) => b.rrfScore - a.rrfScore)
.slice(0, limit)
return { total: results.length, memories: results }
}
/**
* Rebuild the BM25 index from all memories
* Call this after bulk imports or to fix index inconsistencies
*/
export async function rebuildBM25Index() {
const index = await loadIndex()
const activeMemories = index.memories.filter(m => m.status === 'active')
// Load full content for each memory
const documents = []
for (const meta of activeMemories) {
const memory = await loadMemory(meta.id)
if (memory) {
documents.push({
id: memory.id,
content: memory.content,
type: memory.type,
tags: memory.tags,
source: memory.sourceAppId
})
}
}
return memoryBM25.rebuildIndex(documents)
}
/**
* Get BM25 index statistics
*/
export async function getBM25Stats() {
return memoryBM25.getStats()
}
/**
* Get timeline data (memories grouped by date)
*/
export async function getTimeline(options = {}) {
const index = await loadIndex();
let memories = index.memories.filter(m => m.status === 'active');
// Filter by date range. Compare instants, not strings — `createdAt` is always
// UTC (`…Z`) while a boundary may carry an offset (`…+02:00`) or be a bare
// `YYYY-MM-DD`, and those sort lexicographically against each other wrong.
const startMs = options.startDate ? Date.parse(options.startDate) : NaN;
const endMs = options.endDate ? Date.parse(options.endDate) : NaN;
if (!Number.isNaN(startMs)) {
memories = memories.filter(m => Date.parse(m.createdAt) >= startMs);
}
if (!Number.isNaN(endMs)) {
memories = memories.filter(m => Date.parse(m.createdAt) <= endMs);
}
// Filter by types
if (options.types && options.types.length > 0) {
memories = memories.filter(m => options.types.includes(m.type));
}
// Filter by app
if (options.appId === '__not_brain') {
memories = memories.filter(m => m.sourceAppId !== 'brain');
} else if (options.appId) {
memories = memories.filter(m => m.sourceAppId === options.appId);
}
// Sort by date descending
memories.sort((a, b) => (b.createdAt > a.createdAt ? 1 : -1));
// Limit
const limit = options.limit || 100;
memories = memories.slice(0, limit);
// Group by date
const timeline = {};
for (const memory of memories) {
const date = memory.createdAt.split('T')[0];
if (!timeline[date]) timeline[date] = [];
timeline[date].push(memory);
}
return timeline;
}
/**
* Get all unique categories
*/
export async function getCategories() {
const index = await loadIndex();
const categories = new Map();
for (const memory of index.memories) {
if (memory.status !== 'active') continue;
const count = categories.get(memory.category) || 0;
categories.set(memory.category, count + 1);
}
return Array.from(categories.entries()).map(([name, count]) => ({ name, count }));
}
/**
* Get all unique tags
*/
export async function getTags() {
const index = await loadIndex();
const tags = new Map();
for (const memory of index.memories) {
if (memory.status !== 'active') continue;
for (const tag of memory.tags) {
const count = tags.get(tag) || 0;
tags.set(tag, count + 1);
}
}
return Array.from(tags.entries())
.map(([name, count]) => ({ name, count }))
.sort((a, b) => b.count - a.count);
}
/**
* Get related memories (by ID links + embedding similarity)
*/
export async function getRelatedMemories(id, limit = 10) {
const memory = await loadMemory(id);
if (!memory) return [];
const embeddings = await loadEmbeddings();
const index = await loadIndex();
const indexMap = new Map(index.memories.map(m => [m.id, m]));
const related = [];
// Add explicitly linked memories
for (const relId of memory.relatedMemories) {
const meta = indexMap.get(relId);
if (meta && meta.status === 'active') {
related.push({ ...meta, relationship: 'linked', similarity: 1.0, linkType: DEFAULT_MEMORY_LINK_TYPE, direction: 'outgoing' });
}
}
for (const link of memory.typedLinks || []) {
const meta = indexMap.get(link.targetId);
if (meta && meta.status === 'active') {
related.push({
...meta, relationship: 'linked', similarity: 1.0, linkId: link.id, linkType: link.linkType,
direction: 'outgoing', note: link.note, createdBy: link.createdBy
});
}
}
// Add similar by embedding
if (memory.embedding && embeddings.vectors[id]) {
const similar = findTopK(memory.embedding, embeddings.vectors, limit + related.length);
for (const item of similar) {
if (item.id === id) continue;
if (related.some(r => r.id === item.id)) continue;
const meta = indexMap.get(item.id);
if (meta && meta.status === 'active') {
related.push({ ...meta, relationship: 'similar', similarity: item.similarity });
}
}
}
return related.slice(0, limit);
}
/**
* Get graph data for visualization.
*
* `sourceIds` mirrors the PostgreSQL backend (#10953): absent (undefined/null)
* returns the whole graph; an array — even an empty one — returns only the
* nodes in it and the edges whose BOTH endpoints are in it. Similarity is
* still ranked against every stored vector, so an out-of-scope nearest
* neighbour keeps its top-three slot.
*/
export async function getGraphData({ sourceIds } = {}) {
if (sourceIds !== undefined && sourceIds !== null && !Array.isArray(sourceIds)) {
throw new TypeError('getGraphData: sourceIds must be an array');
}
const scope = Array.isArray(sourceIds) ? new Set(sourceIds) : null;
if (scope && scope.size === 0) return { nodes: [], edges: [] };
const inScope = (id) => !scope || scope.has(id);
const index = await loadIndex();
const embeddings = await loadEmbeddings();
const activeMemories = index.memories.filter(m => m.status === 'active' && inScope(m.id));
// Build nodes
const nodes = activeMemories.map(m => ({
id: m.id,
type: m.type,
category: m.category,
summary: m.summary,
importance: m.importance
}));
// Build edges from explicit links and high similarity
const edges = [];
const seenEdges = new Set();
// Batch load all memories to avoid N+1 query pattern
// Load all memories in parallel instead of sequential loadMemory calls
const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
const loadedMemories = await Promise.all(memoryLoadPromises);
const memoriesById = new Map();
loadedMemories.forEach((mem, idx) => {
if (mem) memoriesById.set(activeMemories[idx].id, mem);
});
// Build explicit link edges using pre-loaded memories
for (const memory of activeMemories) {
const full = memoriesById.get(memory.id);
if (!full) continue;
// Explicit links
for (const targetId of full.relatedMemories) {
if (!inScope(targetId)) continue;
const edgeKey = [memory.id, targetId].sort().join('-');
if (!seenEdges.has(edgeKey)) {
seenEdges.add(edgeKey);
edges.push({ source: memory.id, target: targetId, type: 'linked', linkType: DEFAULT_MEMORY_LINK_TYPE, weight: 1.0 });
}
}
for (const link of full.typedLinks || []) {
if (!memoriesById.has(link.targetId)) continue;
edges.push({ source: memory.id, target: link.targetId, type: 'linked', linkType: link.linkType, weight: 1.0 });
}
}
// Add similarity edges (top 3 per node, > 0.8 similarity)
for (const memory of activeMemories) {
if (!embeddings.vectors[memory.id]) continue;
const similar = findTopK(embeddings.vectors[memory.id], embeddings.vectors, 4);
for (const item of similar) {
if (item.id === memory.id) continue;
if (item.similarity < 0.8) continue;
// Filtered only after the global top-K: an out-of-scope neighbour still
// consumes its slot rather than letting a lower-ranked one replace it.
if (!inScope(item.id)) continue;
const edgeKey = [memory.id, item.id].sort().join('-');
if (!seenEdges.has(edgeKey)) {
seenEdges.add(edgeKey);
edges.push({ source: memory.id, target: item.id, type: 'similar', weight: item.similarity });
}
}
}
return { nodes, edges };
}
/**
* Ids of active memories that have no embedding vector yet — the backfill
* target for "embed missing" (a record whose embedding generation failed or
* never ran). Mirrors the memoryDB.js implementation against the file store.
*/
export async function getMemoryIdsMissingEmbedding() {
const index = await loadIndex();
const embeddings = await loadEmbeddings();
const missing = new Set();
for (const m of index.memories) {
if (m.status !== 'active') continue;
if (!embeddings.vectors[m.id]) missing.add(m.id);
}
return missing;
}
/**
* Link two memories
*/
export async function linkMemories(sourceId, targetId, { linkType = DEFAULT_MEMORY_LINK_TYPE, note = null, createdBy = 'user' } = {}) {
return withMemoryLock(async () => {
const source = await loadMemory(sourceId);
const target = await loadMemory(targetId);
if (!source || !target) return { success: false, error: 'Memory not found' };
// Directed types live on the source record only (dev/test file backend).
if (!isSymmetricMemoryLinkType(linkType)) {
source.typedLinks = source.typedLinks || [];
let link = source.typedLinks.find(l => l.targetId === targetId && l.linkType === linkType);
if (!link) {
link = { id: uuidv4(), targetId, linkType, note, createdBy, createdAt: new Date().toISOString() };
source.typedLinks.push(link);
source.updatedAt = link.createdAt;
await saveMemory(source);
}
return { success: true, sourceId, targetId, linkType, linkId: link.id };
}
// Add bidirectional links
if (!source.relatedMemories.includes(targetId)) {
source.relatedMemories.push(targetId);
source.updatedAt = new Date().toISOString();
await saveMemory(source);
}
if (!target.relatedMemories.includes(sourceId)) {
target.relatedMemories.push(sourceId);
target.updatedAt = new Date().toISOString();
await saveMemory(target);
}
return { success: true, sourceId, targetId, linkType, linkId: null };
});
}
/**
* Consolidate similar memories (merge duplicates)
*/
export async function consolidateMemories(threshold = 0.9, dryRun = false) {
const index = await loadIndex();
const embeddings = await loadEmbeddings();
const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');
// Get memories with embeddings
const memoriesWithEmbeddings = [];
for (const meta of activeMemories) {
if (embeddings.vectors[meta.id]) {
memoriesWithEmbeddings.push({
id: meta.id,
embedding: embeddings.vectors[meta.id],
...meta
});
}
}
// Cluster by similarity
const clusters = clusterBySimilarity(memoriesWithEmbeddings, threshold);
const duplicateClusters = clusters.filter(c => c.length > 1);
if (dryRun) {
return {
dryRun: true,
clustersFound: duplicateClusters.length,
memoriesAffected: duplicateClusters.reduce((sum, c) => sum + c.length, 0),
clusters: duplicateClusters.map(c => c.map(m => ({ id: m.id, summary: m.summary })))
};
}
// Merge each cluster
let merged = 0;
for (const cluster of duplicateClusters) {
// Sort by importance, keep highest
cluster.sort((a, b) => (b.importance || 0) - (a.importance || 0));
const primary = cluster[0];
for (let i = 1; i < cluster.length; i++) {
await updateMemory(cluster[i].id, {
status: 'archived',
mergedInto: primary.id
});
merged++;
}
}
console.log(`🧠 Consolidated ${merged} duplicate memories into ${duplicateClusters.length} clusters`);
return { merged, clusters: duplicateClusters.length };
}
/**
* Apply importance decay to old memories
*/
export async function applyDecay(decayRate = 0.01) {
const index = await loadIndex();
const now = Date.now();
let updated = 0;
// Filter active memories first
const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');
// Batch load all active memories to avoid N+1 query pattern
const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
const loadedMemories = await Promise.all(memoryLoadPromises);
// Process each loaded memory
for (let i = 0; i < activeMemories.length; i++) {
const meta = activeMemories[i];
const memory = loadedMemories[i];
if (!memory || persistentMindMemoryProtection(memory) !== 'standard') continue;
const ageInDays = (now - new Date(memory.createdAt).getTime()) / (1000 * 60 * 60 * 24);
const accessRecency = memory.lastAccessed
? (now - new Date(memory.lastAccessed).getTime()) / (1000 * 60 * 60 * 24)
: ageInDays;
// Decay formula: importance = baseImportance * (1 - decayRate * sqrt(age)) + accessBoost
const accessBoost = Math.max(0, 0.1 - accessRecency * 0.001);
const newImportance = Math.max(0.1, memory.importance * (1 - decayRate * Math.sqrt(ageInDays)) + accessBoost);
// Archive if importance falls below threshold and old enough
if (newImportance < 0.15 && ageInDays > 30) {
await updateMemory(meta.id, { status: 'archived', archivedReason: 'decay' });
updated++;
} else if (Math.abs(newImportance - memory.importance) > 0.01) {
await updateMemory(meta.id, { importance: newImportance });
updated++;
}
}
console.log(`🧠 Decay applied to ${updated} memories`);
return { updated };
}
/**
* Clear expired memories
*/
export async function clearExpired() {
const index = await loadIndex();
const now = new Date().toISOString();
let cleared = 0;
// Filter active memories first
const activeMemories = index.memories.filter(m => m.status === 'active' && persistentMindMemoryProtection(m) === 'standard');
// Batch load all active memories to avoid N+1 query pattern
const memoryLoadPromises = activeMemories.map(m => loadMemory(m.id));
const loadedMemories = await Promise.all(memoryLoadPromises);
// Process each loaded memory
for (let i = 0; i < activeMemories.length; i++) {
const meta = activeMemories[i];
const memory = loadedMemories[i];
if (!memory || persistentMindMemoryProtection(memory) !== 'standard') continue;
if (memory.expiresAt && memory.expiresAt < now) {
await updateMemory(meta.id, { status: 'expired' });
cleared++;
}
}
console.log(`🧠 Cleared ${cleared} expired memories`);
return { cleared };
}
/**
* Get memory stats
*/
export async function getStats() {
const index = await loadIndex();
const embeddings = await loadEmbeddings();
const byType = {};
const byCategory = {};
const byStatus = {};
for (const memory of index.memories) {
byType[memory.type] = (byType[memory.type] || 0) + 1;
byCategory[memory.category] = (byCategory[memory.category] || 0) + 1;
byStatus[memory.status] = (byStatus[memory.status] || 0) + 1;
}
return {
total: index.count,
active: byStatus.active || 0,
archived: byStatus.archived || 0,
expired: byStatus.expired || 0,
pendingApproval: byStatus.pending_approval || 0,
withEmbeddings: Object.keys(embeddings.vectors).length,
byType,
byCategory,
lastUpdated: index.lastUpdated
};
}
/**
* Flush BM25 index to disk
*/
export async function flushBM25Index() {
return memoryBM25.flush();
}