Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
14.8 KB
Raw Blame
/**
 * The index census: one row-count per table the operator surfaces care
 * about, taken together so `o2b search status` reports a single
 * consistent picture rather than four independent probes.
 */

import { Database } from "bun:sqlite";

import { SearchError } from "../search-error.ts";
import { EVENT_TIME_MAX_COLUMN, EVENT_TIME_MIN_COLUMN } from "../schema.ts";
import { countChunks, countChunksWithoutEmbeddings } from "./chunks.ts";
import type { ChunkHoldExclusion } from "./chunks.ts";
import { countDocuments } from "./documents.ts";
import {
  EMBEDDING_DIMENSION_STATE_KEY,
  getState,
  peekReadonlyIndex,
  type IndexPeek,
} from "./state.ts";
import { countEmbeddings, staleEmbeddings } from "./vectors.ts";
import {
  charLengthOverTokenBudget,
  MAX_CODE_POINTS_PER_TOKEN,
  NON_ASCII_CEILING_COEFFICIENT,
  textExtent,
  utf8ByteFloorUnderTokenBudget,
  LOCAL_EMBEDDING_MODEL,
} from "../embeddings/signature.ts";
import { LOCAL_DEFAULT_DIMENSION } from "../embeddings/local-provider.ts";
import type { ResolvedEmbeddingConfig } from "../types.ts";

export interface StoreCounts {
  readonly documents: number;
  readonly chunks: number;
  readonly embeddings: number;
  /** Embeddings whose `model`/`dimension` no longer match the current config. */
  readonly staleEmbeddings: number;
}

/** The embedding pair stored vectors are compared against. */
export interface StaleBaseline {
  readonly model: string | null;
  readonly dimension: number | null;
}

/**
 * Resolve the pair {@link staleEmbeddings} compares against, the way the
 * indexer resolves the active signature: the local embedder is named by
 * its built-in model and default dimension; otherwise the configured
 * dimension wins and the stored one fills in when none is configured.
 * `embedding_dimension` is null unless configured, so reading the
 * configuration alone left the stale count at zero on a default setup.
 */
export function staleBaseline(
  db: Database,
  semantic: Pick<ResolvedEmbeddingConfig, "provider" | "model" | "dimension">,
): StaleBaseline {
  const local = semantic.provider === "local";
  const model = local ? LOCAL_EMBEDDING_MODEL : (semantic.model ?? null);
  if (semantic.dimension !== null && semantic.dimension !== undefined) {
    return { model, dimension: semantic.dimension };
  }
  if (local) return { model, dimension: LOCAL_DEFAULT_DIMENSION };
  const raw = getState(db, EMBEDDING_DIMENSION_STATE_KEY);
  const stored = raw === null ? null : Number(raw);
  return {
    model,
    dimension: stored !== null && Number.isFinite(stored) && stored > 0 ? stored : null,
  };
}

export function counts(
  db: Database,
  configuredModel: string | null,
  configuredDimension: number | null,
): StoreCounts {
  return Object.freeze({
    documents: countDocuments(db),
    chunks: countChunks(db),
    embeddings: countEmbeddings(db),
    staleEmbeddings: staleEmbeddings(db, configuredModel, configuredDimension),
  });
}

/**
 * The event-time window census (t_9e1a4b3f): how many documents fall in
 * a `[sinceMs, untilMs]` window, answered from the persisted
 * `documents.event_time_min` / `event_time_max` bounds as ONE SQL
 * aggregate instead of a full hydration pass. Deliberately a function
 * beside {@link StoreCounts}, not a field on it: `StoreCounts` has no
 * window to be relative to, and its consumers keep their shape.
 */
export interface EventTimeWindowCensus {
  /** Every indexed document - the denominator of the other buckets. */
  readonly documents: number;
  /** Documents whose resolved event-time window is persisted (at least one bound non-null). */
  readonly declared: number;
  /**
   * Declared documents whose window intersects the query window, under
   * exactly the rule `eventTimeInRange` applies at query time: a window
   * is out only when a non-null bound proves it (`event_time_min >
   * until`, or `event_time_max < since`), with open window sides and
   * open query edges handled symmetrically.
   */
  readonly intersecting: number;
  /**
   * Documents with no persisted window - the unmeasured bucket: rows
   * that predate the lazy backfill and rows that declare nothing the
   * resolver can use. The query side judges these by storage mtime,
   * which is what the bucket is named for. They are deliberately NOT
   * counted by their mtime here - that would re-import storage time
   * into an event-time census - so `declared + mtimeFallback` always
   * equals {@link documents}.
   */
  readonly mtimeFallback: number;
  /**
   * The earliest instant any persisted window names (its lower bound, or
   * the upper one for an open-start window), unix ms. Null when no
   * document has a persisted window.
   */
  readonly earliestMs: number | null;
  /** The latest instant any persisted window names, unix ms; null as above. */
  readonly latestMs: number | null;
}

/**
 * Take the census. `sinceMs` / `untilMs` are unix ms and mirror
 * `ResolvedTimeRange`: a null edge is an OPEN one, never zero.
 */
export function eventTimeWindowCensus(
  db: Database,
  sinceMs: number | null,
  untilMs: number | null,
): EventTimeWindowCensus {
  const row = db
    .query<
      {
        documents: number;
        declared: number;
        intersecting: number;
        mtime_fallback: number;
        earliest: number | null;
        latest: number | null;
      },
      { $since: number | null; $until: number | null }
    >(
      `SELECT
         COUNT(*) AS documents,
         COUNT(*) FILTER (
           WHERE ${EVENT_TIME_MIN_COLUMN} IS NOT NULL OR ${EVENT_TIME_MAX_COLUMN} IS NOT NULL
         ) AS declared,
         COUNT(*) FILTER (
           WHERE (${EVENT_TIME_MIN_COLUMN} IS NOT NULL OR ${EVENT_TIME_MAX_COLUMN} IS NOT NULL)
             AND (${EVENT_TIME_MIN_COLUMN} IS NULL OR $until IS NULL
                  OR ${EVENT_TIME_MIN_COLUMN} <= $until)
             AND (${EVENT_TIME_MAX_COLUMN} IS NULL OR $since IS NULL
                  OR ${EVENT_TIME_MAX_COLUMN} >= $since)
         ) AS intersecting,
         COUNT(*) FILTER (
           WHERE ${EVENT_TIME_MIN_COLUMN} IS NULL AND ${EVENT_TIME_MAX_COLUMN} IS NULL
         ) AS mtime_fallback,
         MIN(COALESCE(${EVENT_TIME_MIN_COLUMN}, ${EVENT_TIME_MAX_COLUMN})) AS earliest,
         MAX(COALESCE(${EVENT_TIME_MAX_COLUMN}, ${EVENT_TIME_MIN_COLUMN})) AS latest
       FROM documents`,
    )
    .get({ $since: sinceMs, $until: untilMs });
  if (row === null) {
    // Unreachable while the query is an unfiltered aggregate, and
    // therefore exactly the shape that must not degrade to a zero count:
    // a zero here would be a clean census nobody took.
    throw new SearchError(
      "INDEX_UNREADABLE",
      "event-time window census: the aggregate over `documents` returned no row, so no census " +
        "was taken. Run: o2b search check",
    );
  }
  return Object.freeze({
    documents: row.documents,
    declared: row.declared,
    intersecting: row.intersecting,
    mtimeFallback: row.mtime_fallback,
    earliestMs: row.earliest,
    latestMs: row.latest,
  });
}

/** Chunks that carry no vector, against the chunks there are. */
export interface PendingVectorTally {
  /** Chunks with no row in `embeddings`, held chunks excluded when asked. */
  readonly pending: number;
  /** Chunks in the index at all, so the pending count has a denominator. */
  readonly chunks: number;
}

/**
 * Both counts from one connection, so they describe one instant.
 *
 * `hold` is the config's chunk-hold exclusion key: a chunk held out of
 * embedding under that key is not work the index owes a vector for, so a
 * tally that counted it as pending would recommend a backfill that finds
 * nothing to do and never clears. Callers that have no configuration in
 * hand (the key-free diagnostic reads) omit it and see the raw count.
 */
export function pendingVectorTally(db: Database, hold?: ChunkHoldExclusion): PendingVectorTally {
  return Object.freeze({
    pending: countChunksWithoutEmbeddings(db, undefined, hold),
    chunks: countChunks(db),
  });
}

/**
 * {@link pendingVectorTally} taken WITHOUT opening a `Store`, for
 * `indexCheck` - which deliberately probes an in-memory database and
 * never touches the real index, so anything it reports about stored
 * state has to be read explicitly. `hold` threads the config's chunk-hold
 * exclusion through, exactly as {@link pendingVectorTally} takes it.
 *
 * The peek keeps its three outcomes rather than collapsing to a number:
 * "no index here" and "an index that will not open" are not a pending
 * count of zero, and zero is exactly what a caller would read as a
 * fully-embedded vault.
 */
export function peekPendingVectorsSync(
  dbPath: string,
  hold?: ChunkHoldExclusion,
): IndexPeek<PendingVectorTally> {
  return peekReadonlyIndex(dbPath, (_read, db) => pendingVectorTally(db, hold));
}

/** Split of the chunk table by what the census can decide about it. */
export interface ChunkWindowTally {
  /** Chunks over the window even at the conservative hold bound. */
  readonly overWindow: number;
  /** Chunks the hold bound does not prove over, whose estimates still read over. */
  readonly undecided: number;
}

/** The prefix value that means "this backend prepends nothing". */
export const NO_REQUEST_PREFIX = "";

/**
 * One SQL pass, two aggregates: chunks the census can prove are over
 * `windowTokens`, and chunks it cannot place on either side of it.
 *
 * `requestPrefix` is the text the CONFIGURED BACKEND prepends to every
 * indexed chunk before the provider counts a token - `passage: ` for the
 * recommended e5 default, empty for a backend that implements no
 * instruction-prefix contract. It is measured into both thresholds
 * because the provider counts it: a census that ignored it would report
 * zero for chunks the request path puts over the window by exactly the
 * width of the prefix. It is a required argument for the same reason -
 * an optional one is the drift this parameter exists to close.
 *
 * The unit measured is a token ESTIMATE, never the `chunks.token_count`
 * column: that column is the chunker's own count (whitespace words, one
 * per character for unspaced scripts such as Han), a
 * different unit from a model's token window, and comparing the two as
 * though they matched is the arithmetic this census exists to stop.
 *
 * ## Why two aggregates, and where the boundary sits
 *
 * `overWindow` is the HOLD population, so it uses the hold's own bound:
 * code points charged at no more than one token per
 * {@link MAX_CODE_POINTS_PER_TOKEN}. Only that conservative bound proves a
 * chunk over the window - the chars-over-four average is not a bound, and
 * the phase that embeds must never hold on it.
 *
 * The undecided band is everything the proof cannot settle: chunks whose
 * conservative bound still fits the window but whose HIGH estimate - the
 * average rate for Latin text, a near-character-level charge for Han and
 * supplementary-plane text - reads over it. Three populations land there:
 *
 *   - Latin chunks between the average estimate and the sound bound. The
 *     average reads HIGH on real prose (real tokenizers absorb more than
 *     four code points per token), so acting on it would hold work on a
 *     guess - exactly the defect that moved the hold to the conservative
 *     bound;
 *   - Han text, where a BERT-family tokenizer is roughly
 *     character-level. `BAAI/bge-small-zh-v1.5` declares a 512-token
 *     window, and a Chinese chunk that estimates at 512 costs about four
 *     times that;
 *   - supplementary-plane text, where SQLite's `length()` counts one per
 *     character and `String.length` counts two, so the database reads
 *     half the estimated size.
 *
 * None of the three is fixable by a tighter constant on its own, and all
 * are observable in SQL through the same two measurements: `length()`
 * (code points) and `octet_length()` (UTF-8 bytes), whose difference
 * bounds the non-ASCII code points exactly. Charging those at
 * `MAX_TOKENS_PER_NON_ASCII_CODE_POINT` gives the high estimate, and the
 * gap between the two arms is reported as undecided rather than passed.
 *
 * ## Cost
 *
 * Cheaper than the single aggregate it replaced, not dearer. `length()`
 * walks the UTF-8 body and dominates the query; `octet_length()` reads a
 * stored byte count. The `WHERE` clause therefore tests the free
 * measurement first ({@link utf8ByteFloorUnderTokenBudget}) and only the
 * survivors are walked - which on a wide-window model is no rows at all.
 * Measured on a 21 000-chunk / 137 MB fixture, warm: 307 ms before
 * against 316 ms after at a 512-token window, and 288 ms before against
 * 71 ms after at the 8192-token window `BAAI/bge-m3` declares.
 */
export function chunkWindowTally(
  db: Database,
  windowTokens: number,
  requestPrefix: string,
): ChunkWindowTally {
  const prefix = textExtent(requestPrefix);
  const row = db
    .query<
      { over_window: number; undecided: number },
      {
        $budget: number;
        $holdBudget: number;
        $byteFloor: number;
        $coefficient: number;
        $prefixCodePoints: number;
        $prefixUtf8Bytes: number;
      }
    >(
      // MATERIALIZED, not a plain subquery: `code_points` is read three
      // times below, and without it SQLite re-evaluates `length()` -
      // a full walk of the chunk body - once per reference.
      `WITH extents AS MATERIALIZED (
         SELECT length(content) + $prefixCodePoints AS code_points,
                octet_length(content) + $prefixUtf8Bytes AS utf8_bytes
         FROM chunks
         WHERE octet_length(content) + $prefixUtf8Bytes > $byteFloor
       )
       SELECT
         COUNT(*) FILTER (WHERE code_points > $holdBudget) AS over_window,
         COUNT(*) FILTER (
           WHERE code_points <= $holdBudget
             AND code_points + $coefficient * min(code_points, utf8_bytes - code_points) > $budget
         ) AS undecided
       FROM extents`,
    )
    .get({
      $budget: charLengthOverTokenBudget(windowTokens),
      $holdBudget: windowTokens * MAX_CODE_POINTS_PER_TOKEN,
      $byteFloor: utf8ByteFloorUnderTokenBudget(windowTokens),
      $coefficient: NON_ASCII_CEILING_COEFFICIENT,
      $prefixCodePoints: prefix.codePoints,
      $prefixUtf8Bytes: prefix.utf8Bytes,
    });
  if (row === null) {
    // Unreachable while the query is an unfiltered aggregate, and
    // therefore exactly the shape that must not degrade to a zero count:
    // a zero here would be a clean census nobody took.
    throw new SearchError(
      "INDEX_UNREADABLE",
      "chunk-window census: the aggregate over `chunks` returned no row, so no census was " +
        "taken. Run: o2b search check",
    );
  }
  return Object.freeze({ overWindow: row.over_window, undecided: row.undecided });
}

/**
 * The provable half of {@link chunkWindowTally} on its own, for a caller
 * that already knows the backend sends no prefix. Kept as a named
 * projection rather than a second query so the two cannot drift.
 */
export function chunksOverTokenWindow(db: Database, windowTokens: number): number {
  return chunkWindowTally(db, windowTokens, NO_REQUEST_PREFIX).overWindow;
}