Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

6.0 KB
Raw Blame
/**
 * BM25 Index Manager for Memory System
 *
 * Manages the BM25 inverted index for text-based memory search.
 * Complements the vector-based semantic search with keyword matching.
 */

import { promises as fs } from 'fs'
import path from 'path'
import {
  buildInvertedIndex,
  addDocument,
  removeDocument,
  search,
  createEmptyIndex,
  serializeIndex,
  deserializeIndex,
  getIndexStats
} from '../lib/bm25.js'
import { atomicWrite, ensureDir, PATHS } from '../lib/fileUtils.js'

const DATA_DIR = PATHS.memory
const INDEX_FILE = path.join(DATA_DIR, 'bm25-index.json')

// In-memory cache of the index
let indexCache = null
let isDirty = false

/**
 * Load the BM25 index from disk
 * @returns {Promise<Object>} - The BM25 index
 */
async function loadIndex() {
  if (indexCache) return indexCache

  await ensureDir(DATA_DIR)

  const exists = await fs.access(INDEX_FILE).then(() => true).catch(() => false)

  if (!exists) {
    indexCache = createEmptyIndex()
    return indexCache
  }

  const data = await fs.readFile(INDEX_FILE, 'utf-8')
  // Handle empty or malformed index file
  if (!data || !data.trim()) {
    console.log('⚠️ BM25 index file empty, creating fresh index')
    indexCache = createEmptyIndex()
    return indexCache
  }
  // Validate JSON structure before parsing
  const trimmed = data.trim()
  if (!trimmed.startsWith('{') || !trimmed.endsWith('}')) {
    console.log('⚠️ BM25 index file malformed, creating fresh index')
    indexCache = createEmptyIndex()
    return indexCache
  }
  let parsed
  try {
    parsed = JSON.parse(data)
  } catch {
    console.log('⚠️ BM25 index file has invalid JSON, creating fresh index')
    indexCache = createEmptyIndex()
    return indexCache
  }
  indexCache = deserializeIndex(parsed)
  return indexCache
}

/**
 * Save the BM25 index to disk
 * @returns {Promise<void>}
 */
async function saveIndex() {
  if (!indexCache || !isDirty) return

  await ensureDir(DATA_DIR)
  const serialized = serializeIndex(indexCache)
  await atomicWrite(INDEX_FILE, serialized)
  isDirty = false

  console.log(`💾 BM25 index saved: ${indexCache.totalDocs} docs, ${Object.keys(indexCache.terms).length} terms`)
}

/**
 * Build or rebuild the entire index from memories
 * @param {Array<{id: string, content: string, type: string, tags: string[]}>} memories
 * @returns {Promise<Object>} - Index statistics
 */
async function rebuildIndex(memories) {
  await ensureDir(DATA_DIR)

  // Convert memories to documents for indexing
  const documents = memories.map(m => ({
    id: m.id,
    text: buildIndexableText(m)
  }))

  indexCache = buildInvertedIndex(documents)
  isDirty = true

  await saveIndex()

  console.log(`🔄 BM25 index rebuilt: ${indexCache.totalDocs} memories indexed`)
  return getIndexStats(indexCache)
}

/**
 * Build indexable text from a memory object
 * Combines content, type, and tags for better matching
 *
 * @param {Object} memory - Memory object
 * @returns {string} - Text for indexing
 */
function buildIndexableText(memory) {
  const parts = []

  if (memory.content) {
    parts.push(memory.content)
  }

  if (memory.type) {
    parts.push(memory.type)
  }

  if (memory.tags && Array.isArray(memory.tags)) {
    parts.push(memory.tags.join(' '))
  }

  if (memory.source) {
    parts.push(memory.source)
  }

  return parts.join(' ')
}

/**
 * Add or update a memory in the index
 * @param {Object} memory - Memory object with id, content, type, tags
 * @returns {Promise<void>}
 */
async function indexMemory(memory) {
  const index = await loadIndex()

  const text = buildIndexableText(memory)
  addDocument(index, memory.id, text)
  isDirty = true

  // Save periodically (every 10 changes) or on explicit flush
  if (isDirty && index.totalDocs % 10 === 0) {
    await saveIndex()
  }
}

/**
 * Remove a memory from the index
 * @param {string} memoryId - Memory ID to remove
 * @returns {Promise<void>}
 */
async function removeMemoryFromIndex(memoryId) {
  const index = await loadIndex()
  removeDocument(index, memoryId)
  isDirty = true
}

/**
 * Search memories using BM25 text matching
 *
 * @param {string} query - Search query
 * @param {Object} options - Search options
 * @param {number} options.limit - Maximum results (default 20)
 * @param {number} options.threshold - Minimum score threshold (default 0.1)
 * @returns {Promise<Array<{id: string, score: number}>>} - Ranked memory IDs with scores
 */
async function searchBM25(query, options = {}) {
  const { limit = 20, threshold = 0.1 } = options

  const index = await loadIndex()
  const results = search(query, index, { limit, threshold })

  return results.map(r => ({
    id: r.docId,
    score: r.score
  }))
}

/**
 * Get index statistics
 * @returns {Promise<Object>} - Index statistics
 */
async function getStats() {
  const index = await loadIndex()
  return {
    ...getIndexStats(index),
    isDirty,
    indexFile: INDEX_FILE
  }
}

/**
 * Flush pending changes to disk
 * @returns {Promise<void>}
 */
async function flush() {
  await saveIndex()
}

/**
 * Clear the entire index
 * @returns {Promise<void>}
 */
async function clearIndex() {
  indexCache = createEmptyIndex()
  isDirty = true
  await saveIndex()
  console.log('🗑️ BM25 index cleared')
}

/**
 * Check if a memory exists in the index
 * @param {string} memoryId - Memory ID
 * @returns {Promise<boolean>}
 */
async function hasMemory(memoryId) {
  const index = await loadIndex()
  return index.docIds.has(memoryId)
}

/**
 * Batch index multiple memories efficiently
 * @param {Array<Object>} memories - Memories to index
 * @returns {Promise<number>} - Number of memories indexed
 */
async function batchIndex(memories) {
  const index = await loadIndex()

  for (const memory of memories) {
    const text = buildIndexableText(memory)
    addDocument(index, memory.id, text)
  }

  isDirty = true
  await saveIndex()

  return memories.length
}

export {
  loadIndex,
  saveIndex,
  rebuildIndex,
  indexMemory,
  removeMemoryFromIndex,
  searchBM25,
  getStats,
  flush,
  clearIndex,
  hasMemory,
  batchIndex,
  buildIndexableText
}