Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.1 KB
Raw Blame
/**
 * Write one brain store record into the index, or take one out (design
 * D5): the `records` row, and for a searchable stream the document and
 * chunks its text is searched through.
 *
 * Idempotent by record id. Records are immutable (their id names one
 * envelope and its checksum), so a record already indexed with its body
 * only has its shard attribution refreshed on a re-read; it is re-chunked
 * only when its body was a pending blob that has since arrived.
 */

import { posix } from "node:path";

import { REMOTE_DENY_VISIBILITY_TOKEN } from "../../graph/visibility.ts";
import { sha256Hex } from "../../integrity/digest.ts";
import { STORE_FORMAT_VERSION } from "../../store/constants.ts";
import type { StoreRecord } from "../../store/envelope.ts";
import { chunkMarkdown } from "../chunker.ts";
import { expandTextForCjkFts } from "../cjk-tokenizer.ts";
import { extractEntities } from "../entities.ts";
import type { Store } from "../store.ts";
import {
  clearStoreTables,
  deleteRecordRow,
  getRecord,
  holdsStoreRows,
  storeRecordDocumentPaths,
  upsertRecord,
  type RecordRow,
  type StoredRecord,
} from "../store/brain-records.ts";
import type { ChunkInput } from "../store/chunks.ts";
import type { ResolvedSearchConfig } from "../types.ts";
import { SEARCHABLE_STORE_STREAMS, STORE_RECORD_PATH_PREFIX } from "./constants.ts";
import { renderRecordText } from "./record-text.ts";
import type { ShardStat } from "./shard-scan.ts";

/** The `documents.path` a store record's text is indexed under. */
export function storeRecordPath(partition: string, id: string): string {
  return posix.join(STORE_RECORD_PATH_PREFIX, partition, id);
}

function authorityOf(record: StoreRecord): string | null {
  const value = record.body["authority"];
  return typeof value === "string" && value.trim() !== "" ? value : null;
}

function physicalSeconds(hlc: string): number {
  const ms = Number(hlc.split(".")[0]);
  return Number.isFinite(ms) ? Math.floor(ms / 1000) : 0;
}

/** Where a record was read from: the parts of a {@link ShardStat} ingest needs. */
export type RecordPlace = Pick<ShardStat, "key" | "partition" | "stream" | "ledger">;

/**
 * What {@link ingestRecord} did. `refused`: the id is already indexed
 * from the other partition. Records never move between partitions, so the
 * second copy is not indexed (it could otherwise relabel or shadow the
 * first); doctor reports such ids.
 */
export type IngestOutcome = "added" | "refreshed" | "unchanged" | "refused";

function indexText(
  store: Store,
  config: ResolvedSearchConfig,
  record: StoreRecord,
  shard: RecordPlace,
  pending: boolean,
): number {
  const { title, content } = renderRecordText(record, shard.stream, pending);
  const docId = store.upsertDocument({
    path: storeRecordPath(shard.partition, record.id),
    title,
    contentHash: sha256Hex(content),
    mtime: physicalSeconds(record.hlc),
    size: Buffer.byteLength(content),
    pageType: null,
    eventAnchor: null,
    // A private-partition record is reserved exactly like a page that
    // declares `visibility: private`.
    visibility: shard.partition === "private" ? [REMOTE_DENY_VISIBILITY_TOKEN] : [],
    pinned: false,
  });
  const chunked = chunkMarkdown(content, title, {
    maxTokens: config.chunkSize,
    minTokens: config.chunkMinSize,
    overlapTokens: config.chunkOverlap,
  });
  const inputs: ChunkInput[] = chunked.chunks.map((c) => ({
    chunkIndex: c.chunkIndex,
    content: c.content,
    ftsContent: expandTextForCjkFts(c.content),
    contentHash: sha256Hex(c.content),
    startLine: c.startLine,
    endLine: c.endLine,
    tokenCount: c.tokenCount,
    headingPath: c.headingPath,
  }));
  const replaced = store.replaceDocumentChunks(docId, inputs);
  replaced.chunkIds.forEach((cid, i) => {
    if (!replaced.keptChunkIds.has(cid)) {
      store.replaceEntities(cid, extractEntities(chunked.chunks[i]!.content));
    }
  });
  return docId;
}

/** Ingest one verified record read from `shard`. */
export function ingestRecord(
  store: Store,
  config: ResolvedSearchConfig,
  record: StoreRecord,
  shard: RecordPlace,
  pending: boolean,
): IngestOutcome {
  const db = store.brainRecordsConnection();
  const existing = getRecord(db, record.id);
  if (existing !== null && existing.partition !== shard.partition) return "refused";
  const row: RecordRow = {
    id: record.id,
    partition: shard.partition,
    stream: shard.stream,
    ledger: shard.ledger,
    kind: record.kind,
    scope: record.scope,
    topic: record.topic,
    hlc: record.hlc,
    device: record.device,
    agent: record.agent,
    authority: pending ? (existing?.authority ?? null) : authorityOf(record),
    sum: record.sum,
    shard: shard.key,
    bodyPending: pending,
  };
  const searchable = SEARCHABLE_STORE_STREAMS.has(shard.stream);
  if (existing !== null && !existing.bodyPending && (existing.documentId !== null || !searchable)) {
    if (existing.shard !== shard.key)
      upsertRecord(db, { ...row, bodyPending: false }, existing.documentId);
    return "unchanged";
  }
  if (existing !== null && existing.bodyPending && pending) {
    if (existing.shard !== shard.key) upsertRecord(db, row, existing.documentId);
    return "unchanged";
  }
  const docId = searchable ? indexText(store, config, record, shard, pending) : null;
  upsertRecord(db, row, docId);
  return existing === null ? "added" : "refreshed";
}

/**
 * Index the body of a record that was ingested while its blob was still
 * pending, now that the blob has arrived.
 */
export function completePendingRecord(
  store: Store,
  config: ResolvedSearchConfig,
  stored: StoredRecord,
  body: StoreRecord["body"],
): void {
  const record: StoreRecord = {
    v: STORE_FORMAT_VERSION,
    id: stored.id,
    hlc: stored.hlc,
    device: stored.device,
    agent: stored.agent,
    kind: stored.kind,
    scope: stored.scope,
    topic: stored.topic,
    sum: stored.sum,
    body,
  };
  const place: RecordPlace = {
    key: stored.shard,
    partition: stored.partition as RecordPlace["partition"],
    stream: stored.stream as RecordPlace["stream"],
    ledger: stored.ledger,
  };
  ingestRecord(store, config, record, place, false);
}

/** Remove one record and its document from the index. */
export function removeRecord(store: Store, record: StoredRecord): void {
  if (record.documentId !== null)
    store.deleteDocument(storeRecordPath(record.partition, record.id));
  deleteRecordRow(store.brainRecordsConnection(), record.id);
}

/**
 * Take every brain store record out of an index whose vault no longer
 * reads a store (the store was turned off, removed, or no longer
 * resolves). Only an index file shared by both modes (an explicit
 * `search_db_path`) can be in that state; the vault sweep never removes
 * these documents, so this is their cleanup. Returns the number of
 * documents removed, or null when the index held nothing of a store.
 */
export function purgeStoreRecords(store: Store): number | null {
  const db = store.brainRecordsConnection();
  const paths = storeRecordDocumentPaths(db, `${STORE_RECORD_PATH_PREFIX}/`);
  if (paths.length === 0 && !holdsStoreRows(db)) return null;
  for (const path of paths) store.deleteDocument(path);
  clearStoreTables(db);
  return paths.length;
}