Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
21.1 KB
Raw Blame
/**
 * The `documents` table: one row per indexed vault file, its identity
 * (path, basename, title), its stat fingerprint, and the frontmatter
 * facets later phases join on without re-reading the file.
 */

import { Database } from "bun:sqlite";

import { isEventAnchorSource, type EventAnchor } from "../event-anchor.ts";
import {
  DOCUMENT_ATTRIBUTES_TABLE,
  DOCUMENT_VISIBILITY_COLUMN,
  EVENT_TIME_MAX_COLUMN,
  EVENT_TIME_MIN_COLUMN,
  PINNED_COLUMN,
  type DocumentAttribute,
  documentBasename,
  encodeDocumentVisibility,
} from "../schema.ts";
import { SearchError } from "../types.ts";
import { STORE_RECORD_PATH_PREFIX } from "../brain-store/constants.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import { nowIso } from "./sql.ts";
import { purgeVecRowsForDocument } from "./vectors.ts";

/**
 * `documents.event_anchor_examined`: whether an anchor-aware binary has
 * ever resolved this document's event anchor. SQLite has no boolean, so
 * the two values are named once here rather than written as bare 0/1 at
 * each of the four sites that read or write them.
 */
const EXAMINED_YES = 1;
const EXAMINED_NO = 0;

/**
 * `documents.pinned`: the per-note `pinned` frontmatter flag (v13,
 * schema half of t_f7bef96a), named once for the same reason - SQLite
 * has no boolean, and a bare 0/1 at each write site invites a third
 * value nobody registered.
 */
const PINNED_YES = 1;
const PINNED_NO = 0;

export interface DocumentInput {
  readonly path: string; // vault-relative POSIX
  readonly title: string | null;
  readonly contentHash: string;
  readonly mtime: number; // unix seconds
  readonly size: number;
  /**
   * The page's declared frontmatter `type` (normalized token), or null
   * when undeclared. Persisted so link-constraint enforcement can join
   * endpoint types without re-reading files (v6).
   */
  readonly pageType?: string | null;
  /**
   * Transcript turn instant this note was authored at, in unix seconds
   * (conversation chronology, S1 / t_347e8224). Derived from the
   * `authored_at` frontmatter field. Absent / null for a note with no
   * turn instant, which then ranks byte-identically to pre-feature.
   */
  readonly authoredAt?: number | null;
  /**
   * The interval this note is ABOUT, resolved from its own frontmatter or
   * body by `resolveEventAnchor` (provenance at the boundary, v11), with
   * the registered token naming which rung produced it.
   *
   * The three states are distinct and all three are recorded:
   *
   *   - an `EventAnchor` - the caller resolved one;
   *   - `null` - the caller resolved the anchor and the note declares no
   *     readable date, so it keeps ranking on storage mtime exactly as
   *     before the anchor existed;
   *   - ABSENT - the caller did not resolve an anchor at all. The row is
   *     then marked unexamined and `o2b search event-anchor-backfill`
   *     finds it. Omitting the field is not the same statement as
   *     passing `null`, and the column that separates them exists
   *     because a hard time filter now depends on the difference.
   */
  readonly eventAnchor?: EventAnchor | null;
  /**
   * The document's RESOLVED event-time window bounds in unix ms (v13),
   * resolved by `resolveDocumentEventTimeWindow` under the shared rung
   * order (frontmatter validity window > event anchor - admitted rungs
   * only). At least one side is non-null for a declared window; a null
   * side is a declared open one.
   *
   * Absent or null on both keeps the columns NULL - the unmeasured
   * state the query side judges by storage mtime, exactly as before
   * these columns existed.
   */
  readonly eventTimeMinMs?: number | null;
  readonly eventTimeMaxMs?: number | null;
  /**
   * The page's per-note `pinned` frontmatter flag (schema half of
   * t_f7bef96a), persisted so ranking can consume it without re-reading
   * files.
   *
   * ABSENT means the caller did not look, and the column is left NULL -
   * "not pinned" and "nobody looked" are different statements, and only
   * the first is something the page said.
   */
  readonly pinned?: boolean;
  /**
   * The page's normalised `visibility:` tokens, as {@link pageVisibility}
   * reads them - an empty array for a page that declares none (v12).
   *
   * ABSENT means the caller did not measure them, and the column is left
   * NULL. That is a third state and not a synonym for the empty array:
   * "this page declares no visibility" and "nobody looked" are different
   * statements, and only the first is something the page said.
   *
   * The column records what the INDEX knows. It is not the read boundary
   * - that is the live frontmatter check at the three read roots
   * (`isPathReadableAtReach`), which reads the file rather than a
   * snapshot of it and is therefore never stale by a whole index run.
   * Those roots additionally honour this column when they serve indexed
   * text, so a stale private chunk stays private (see
   * {@link indexedVisibilityByPaths}).
   */
  readonly visibility?: ReadonlyArray<string>;
}

export interface DocumentSummary {
  readonly id: number;
  readonly contentHash: string;
  readonly mtime: number;
  readonly size: number;
}

/**
 * Every indexed VAULT document, keyed by path; never a brain store record
 * (paths under {@link STORE_RECORD_PATH_PREFIX}). The indexers compare
 * this map with a walk of the vault, where a store record never appears,
 * so listing one would read as a deleted note and the sweep would remove
 * it. The other callers (the link graph, communities, bridge discovery,
 * belief clustering, index-root coverage) are vault views too and get
 * vault documents only by the same rule. A caller that needs the store
 * records reads them through `brain-records.ts`.
 */
export function listDocuments(db: Database): Map<string, DocumentSummary> {
  const storePrefix = `${STORE_RECORD_PATH_PREFIX}/`;
  const rows = db
    .query<
      {
        id: number;
        path: string;
        content_hash: string;
        mtime: number;
        size: number;
      },
      [number, string]
    >("SELECT id, path, content_hash, mtime, size FROM documents WHERE substr(path, 1, ?) != ?")
    .all(storePrefix.length, storePrefix);
  const map = new Map<string, DocumentSummary>();
  for (const r of rows) {
    map.set(r.path, {
      id: r.id,
      contentHash: r.content_hash,
      mtime: r.mtime,
      size: r.size,
    });
  }
  return map;
}

/**
 * The visibility tokens the index MEASURED for each of `paths`, keyed by
 * path. A path with no row, a NULL (unmeasured) column, or a value that
 * does not decode to a string array is absent from the map: only a
 * measurement is a statement the page made.
 *
 * Read by the reach boundary because a search result serves the INDEXED
 * bytes, not the file's: when a page reserved at index time has since
 * been deleted or moved and a public page written at its path, the file
 * says "public" while the snippet is the private body. See
 * `isPathReadableAtReach`.
 */
export function indexedVisibilityByPaths(
  db: Database,
  paths: ReadonlyArray<string>,
): Map<string, ReadonlyArray<string>> {
  const out = new Map<string, ReadonlyArray<string>>();
  if (paths.length === 0) return out;
  const query = db.query<{ visibility: string | null }, [string]>(
    `SELECT ${DOCUMENT_VISIBILITY_COLUMN} AS visibility FROM documents WHERE path = ?`,
  );
  for (const path of new Set(paths)) {
    const raw = query.get(path)?.visibility ?? null;
    if (raw === null) continue;
    let decoded: unknown;
    try {
      decoded = JSON.parse(raw);
    } catch {
      continue;
    }
    if (Array.isArray(decoded) && decoded.every((t) => typeof t === "string")) {
      out.set(path, Object.freeze([...(decoded as string[])]));
    }
  }
  return out;
}

export function getDocumentIdByPath(db: Database, path: string): number | null {
  const row = db
    .query<{ id: number }, [string]>("SELECT id FROM documents WHERE path = ?")
    .get(path);
  return row?.id ?? null;
}

export function upsertDocument(db: Database, doc: DocumentInput): number {
  const now = nowIso();
  // SQLite RETURNING on INSERT...ON CONFLICT works in 3.35+; bun:sqlite ships modern SQLite.
  const row = db
    .query<
      { id: number },
      [
        string,
        string,
        string | null,
        string,
        number,
        number,
        string | null,
        number | null,
        number | null,
        number | null,
        string | null,
        number,
        string | null,
        number | null,
        number | null,
        number | null,
        string,
        string,
        string,
      ]
    >(
      "INSERT INTO documents(path, basename, title, content_hash, mtime, size, page_type, authored_at, " +
        "  event_anchor_start_ms, event_anchor_end_ms, event_anchor_source, event_anchor_examined, " +
        `  ${DOCUMENT_VISIBILITY_COLUMN}, ` +
        `  ${EVENT_TIME_MIN_COLUMN}, ${EVENT_TIME_MAX_COLUMN}, ${PINNED_COLUMN}, ` +
        "  created_at, updated_at, indexed_at) " +
        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " +
        "ON CONFLICT(path) DO UPDATE SET " +
        "  basename = excluded.basename, " +
        "  title = excluded.title, " +
        "  content_hash = excluded.content_hash, " +
        "  mtime = excluded.mtime, " +
        "  size = excluded.size, " +
        "  page_type = excluded.page_type, " +
        "  authored_at = excluded.authored_at, " +
        "  event_anchor_start_ms = excluded.event_anchor_start_ms, " +
        "  event_anchor_end_ms = excluded.event_anchor_end_ms, " +
        "  event_anchor_source = excluded.event_anchor_source, " +
        "  event_anchor_examined = excluded.event_anchor_examined, " +
        `  ${DOCUMENT_VISIBILITY_COLUMN} = excluded.${DOCUMENT_VISIBILITY_COLUMN}, ` +
        `  ${EVENT_TIME_MIN_COLUMN} = excluded.${EVENT_TIME_MIN_COLUMN}, ` +
        `  ${EVENT_TIME_MAX_COLUMN} = excluded.${EVENT_TIME_MAX_COLUMN}, ` +
        `  ${PINNED_COLUMN} = excluded.${PINNED_COLUMN}, ` +
        "  updated_at = excluded.updated_at, " +
        "  indexed_at = excluded.indexed_at " +
        "RETURNING id",
    )
    .get(
      doc.path,
      documentBasename(doc.path),
      doc.title,
      doc.contentHash,
      doc.mtime,
      doc.size,
      doc.pageType ?? null,
      doc.authoredAt ?? null,
      doc.eventAnchor?.startMs ?? null,
      doc.eventAnchor?.endMs ?? null,
      doc.eventAnchor?.source ?? null,
      // Examined iff the caller stated an answer. `null` IS an answer
      // ("this note declares nothing"); leaving the field out is not.
      doc.eventAnchor === undefined ? EXAMINED_NO : EXAMINED_YES,
      // Absent means unmeasured, which is NULL - see `DocumentInput`.
      doc.visibility === undefined ? null : encodeDocumentVisibility(doc.visibility),
      doc.eventTimeMinMs ?? null,
      doc.eventTimeMaxMs ?? null,
      doc.pinned === undefined ? null : doc.pinned ? PINNED_YES : PINNED_NO,
      now,
      now,
      now,
    );
  if (!row) {
    throw new SearchError("INDEX_UNREADABLE", `upsertDocument returned no id for '${doc.path}'`);
  }
  return row.id;
}

/**
 * Replace a document's stored frontmatter attributes (v14) wholesale: the
 * previous set is deleted and the new one inserted in one transaction, so
 * an interrupted write cannot leave a half-updated filter surface.
 *
 * The caller derives the rows with `documentAttributesFromFrontmatter`
 * (schema.ts) from the SAME parsed frontmatter that feeds the degradation
 * notices - the extractor
 * owns the promoted-key skips and the bounds, so this function is a pure
 * writer. The FK cascade removes the rows when the document goes.
 */
export function replaceDocumentAttributes(
  db: Database,
  documentId: number,
  attrs: ReadonlyArray<DocumentAttribute>,
): void {
  db.exec("BEGIN");
  try {
    db.run(`DELETE FROM ${DOCUMENT_ATTRIBUTES_TABLE} WHERE document_id = ?`, [documentId]);
    if (attrs.length > 0) {
      const insert = db.prepare<undefined, [number, string, string]>(
        `INSERT OR IGNORE INTO ${DOCUMENT_ATTRIBUTES_TABLE}(document_id, key, value) VALUES (?, ?, ?)`,
      );
      for (const attr of attrs) insert.run(documentId, attr.key, attr.value);
    }
    db.exec("COMMIT");
  } catch (e) {
    db.exec("ROLLBACK");
    throw e;
  }
}

/**
 * The documents among `documentIds` the index MEASURED as pinned
 * (`documents.pinned = 1`, v13; t_f7bef96a).
 *
 * Scoped to the caller's candidates and to pinned rows only, so a search
 * reads the handful of rows that can earn the boost rather than every
 * measured row of the corpus. A measured-unpinned row and a row the v13
 * migration carried over unmeasured both answer "not in the set": neither
 * earns a boost, and neither is reported, so an unpinned result's shape
 * is unchanged by the feature.
 */
export function pinnedDocumentIds(db: Database, documentIds: ReadonlyArray<number>): Set<number> {
  const out = new Set<number>();
  if (documentIds.length === 0) return out;
  const rows = db
    .query<{ id: number }, [number, string]>(
      `SELECT id FROM documents WHERE ${PINNED_COLUMN} = ? ` +
        `AND id IN (SELECT value FROM json_each(?))`,
    )
    .all(PINNED_YES, JSON.stringify([...new Set(documentIds)]));
  for (const r of rows) out.add(r.id);
  return out;
}

/**
 * The stored verdict of the promoted filter columns for one document: the
 * `page_type` token (v6) and the measured `pinned` flag (v13). Either side
 * is null when the column cannot speak for the row - `page_type` is written
 * only for scalar-string `type` frontmatter, and `pinned` is NULL on every
 * row an in-place upgrade carried over without re-examining.
 */
export interface PromotedColumnState {
  readonly pageType: string | null;
  readonly pinned: boolean | null;
}

/**
 * The promoted-column verdicts of a candidate pool, keyed by document id.
 *
 * The post-rank promoted-key gate reads this to arbitrate exactly the rows
 * the lanes' SQL admitted through a NULL arm (`column IS NULL`): the arm
 * that admitted such a row proves nothing about the filter, so a NULL
 * column is judged by the live filter and never trusted from SQL, while a
 * non-NULL column keeps its as-indexed SQL verdict. One bounded query per
 * filtered search - the pool is the post-rank window, not the corpus.
 */
export function promotedColumnStates(
  db: Database,
  documentIds: ReadonlyArray<number>,
): Map<number, PromotedColumnState> {
  const out = new Map<number, PromotedColumnState>();
  if (documentIds.length === 0) return out;
  const rows = db
    .query<{ id: number; page_type: string | null; pinned: number | null }, [string]>(
      `SELECT id, page_type, ${PINNED_COLUMN} AS pinned FROM documents ` +
        `WHERE id IN (SELECT value FROM json_each(?))`,
    )
    .all(JSON.stringify([...new Set(documentIds)]));
  for (const r of rows) {
    out.set(r.id, {
      pageType: r.page_type,
      pinned: r.pinned === null ? null : r.pinned === PINNED_YES,
    });
  }
  return out;
}

/**
 * The materialised event anchor of one path, or null when the document is
 * absent from the index or declared no readable date (v11).
 *
 * A row carrying an anchor whose source token is not in the registered
 * vocabulary, or whose bounds are both NULL beside a non-NULL source, is
 * an index this binary cannot interpret - it raises rather than returning
 * null, because "no anchor" and "an anchor I cannot read" are different
 * answers and the caller must not receive the first when the truth is the
 * second.
 *
 * A row that was never EXAMINED also returns null, and deliberately does
 * not raise: it holds exactly the answer the previous binary held, so
 * the query degrades to storage mtime precisely as it did before this
 * feature, rather than failing every query on an index that predates it.
 * That state is not left silent either - it is counted by
 * {@link countUnexaminedEventAnchors}, reported by every index run, and
 * closed by `o2b search event-anchor-backfill --apply`.
 */
export function eventAnchorForPath(db: Database, path: string): EventAnchor | null {
  const row = db
    .query<
      {
        event_anchor_start_ms: number | null;
        event_anchor_end_ms: number | null;
        event_anchor_source: string | null;
      },
      [string]
    >(
      "SELECT event_anchor_start_ms, event_anchor_end_ms, event_anchor_source " +
        "FROM documents WHERE path = ?",
    )
    .get(path);
  if (!row || row.event_anchor_source === null) return null;
  if (!isEventAnchorSource(row.event_anchor_source)) {
    throw new SearchError(
      "INDEX_UNREADABLE",
      `documents.event_anchor_source for '${path}' is not a registered anchor source: '${row.event_anchor_source}'`,
    );
  }
  if (row.event_anchor_start_ms === null && row.event_anchor_end_ms === null) {
    throw new SearchError(
      "INDEX_UNREADABLE",
      `documents row for '${path}' names an anchor source with no bounds on either side`,
    );
  }
  return Object.freeze({
    startMs: row.event_anchor_start_ms,
    endMs: row.event_anchor_end_ms,
    source: row.event_anchor_source,
  });
}

/**
 * Vault-relative paths of the documents no anchor-aware binary has ever
 * examined (v11), in stable path order.
 *
 * These are the rows a v10 index carried across the migration: the
 * columns exist, the fastpaths correctly decline to recompute anything
 * for content that did not change, and so nothing ever populates them.
 * They are NOT documents that declare no date - that is the state this
 * list exists to be distinguishable from.
 */
export function unexaminedEventAnchorPaths(db: Database): string[] {
  return db
    .query<{ path: string }, [number]>(
      "SELECT path FROM documents WHERE event_anchor_examined = ? ORDER BY path",
    )
    .all(EXAMINED_NO)
    .map((r) => r.path);
}

/** How many documents no anchor-aware binary has ever examined (v11). */
export function countUnexaminedEventAnchors(db: Database): number {
  return (
    db
      .query<{ c: number }, [number]>(
        "SELECT count(*) AS c FROM documents WHERE event_anchor_examined = ?",
      )
      .get(EXAMINED_NO)?.c ?? 0
  );
}

/**
 * Record the event anchor of an already-indexed document and mark it
 * examined, touching nothing else on the row - no chunks, no links, no
 * stat fingerprint. This is the backfill's only write.
 *
 * `null` stores the verdict "this note declares no readable date", which
 * is a result and not an absence of one.
 */
export function setEventAnchor(db: Database, path: string, anchor: EventAnchor | null): void {
  db.run(
    "UPDATE documents SET event_anchor_start_ms = ?, event_anchor_end_ms = ?, " +
      "event_anchor_source = ?, event_anchor_examined = ?, updated_at = ? WHERE path = ?",
    [
      anchor?.startMs ?? null,
      anchor?.endMs ?? null,
      anchor?.source ?? null,
      EXAMINED_YES,
      nowIso(),
      path,
    ],
  );
}

/**
 * Delete a document and everything that hangs off it. The vec rows
 * are removed first because the FK cascade does not reach the
 * `chunk_vec` virtual table.
 */
export function deleteDocument(db: Database, vecLoaded: boolean, path: string): void {
  const id = getDocumentIdByPath(db, path);
  if (id === null) return;
  purgeVecRowsForDocument(db, vecLoaded, id);
  db.run("DELETE FROM documents WHERE id = ?", [id]);
}

/**
 * Touch a document's mtime and size without changing its title,
 * hash, or triggering chunk replacement. Used by the indexer's
 * mtime-fastpath fallback to re-arm the stat cache after a
 * same-content touch.
 */
export function touchDocument(db: Database, path: string, mtime: number, size: number): void {
  const now = nowIso();
  db.run(
    "UPDATE documents SET mtime = ?, size = ?, updated_at = ?, indexed_at = ? WHERE path = ?",
    [mtime, size, now, now, path],
  );
}

/**
 * Document id -> { path, title } for every indexed document, reading
 * ONLY the `documents` table (never chunk bodies). The graph query
 * pre-pass uses this for its index-only short-circuit so it can rank and
 * answer from index metadata with zero note bodies hydrated.
 */
export function documentTitles(
  db: Database,
): Map<number, { readonly path: string; readonly title: string | null }> {
  const out = new Map<number, { path: string; title: string | null }>();
  const rows = db
    .query<{ id: number; path: string; title: string | null }, []>(
      "SELECT id, path, title FROM documents",
    )
    .all();
  for (const r of rows) out.set(r.id, { path: r.path, title: r.title });
  return out;
}

export function countDocuments(db: Database): number {
  return db.query<{ c: number }, []>("SELECT count(*) AS c FROM documents").get()?.c ?? 0;
}

/**
 * How many documents sit under one raw path prefix, matched with the same
 * sargable range form the pending-vector census and the two query lanes
 * use (`path-prefix.ts`), so every prefix reader agrees on one row set. A
 * null bound (the prefix's last code point cannot increment) runs the
 * census lower-bound-only.
 */
export function countDocumentsUnderPrefix(db: Database, prefix: string): number {
  const { lo, hi } = pathPrefixRange(prefix);
  if (hi === null) {
    return (
      db
        .query<{ c: number }, [string]>("SELECT count(*) AS c FROM documents WHERE path >= ?")
        .get(lo)?.c ?? 0
    );
  }
  return (
    db
      .query<{ c: number }, [string, string]>(
        "SELECT count(*) AS c FROM documents WHERE path >= ? AND path < ?",
      )
      .get(lo, hi)?.c ?? 0
  );
}