Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
29.1 KB
Raw Blame
/**
 * The `chunks` table: the units search actually ranks. Owns their
 * atomic replacement per document (the FTS5 shadow follows via the
 * chunks_ai/ad/au triggers) and the hydration reads that turn ids back
 * into rendered text.
 */

import { Database } from "bun:sqlite";

import { SearchError } from "../types.ts";
import { CHUNK_HOLDS_TABLE } from "../schema.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import { nowIso, sqlPlaceholders } from "./sql.ts";
import {
  matchCarriedVectors,
  readCarryCandidates,
  restoreCarriedVectors,
} from "./vector-carry-over.ts";
import { purgeVecRowsByChunkIds } from "./vectors.ts";

export interface ChunkInput {
  readonly chunkIndex: number;
  readonly content: string;
  readonly ftsContent?: string;
  readonly contentHash: string;
  readonly startLine: number;
  readonly endLine: number;
  readonly tokenCount: number;
  /**
   * Heading breadcrumb in effect at the chunk (v0.13.0). Indexed in the
   * dedicated FTS column; defaults to "" so callers that do not supply
   * it (and pre-v0.13.0 fixtures) index an empty heading column.
   */
  readonly headingPath?: string;
}

export interface ChunkRow {
  readonly id: number;
  readonly documentId: number;
  readonly chunkIndex: number;
  readonly content: string;
  readonly contentHash: string;
  readonly startLine: number;
  readonly endLine: number;
  readonly tokenCount: number;
}

export interface HydratedChunk {
  readonly chunkId: number;
  readonly documentId: number;
  readonly path: string;
  readonly title: string | null;
  readonly content: string;
  readonly startLine: number;
  readonly endLine: number;
  readonly mtime: number;
  /**
   * Transcript turn instant (unix seconds) or null (conversation
   * chronology, S1). It is the ranker's freshness anchor - age is
   * measured from it, falling back to {@link HydratedChunk.mtime} when it
   * is null - it separates an exact score tie, and it is surfaced on the
   * search result. Null for notes with no turn instant, which keeps their
   * ranking and their ordering byte-identical.
   *
   * BOTH reads project it (D1). It was once absent from the
   * representative-chunk read, which left that row unable to distinguish
   * "this document declares no instant" from "this read did not ask" -
   * and a consumer that routed such a row through the ranker would have
   * silently got the storage clock back. Optional in the type only
   * because a caller may construct a row without one.
   */
  readonly authoredAt?: number | null;
  /**
   * sha256 of the chunk content, written on every index run since schema
   * v1 (`chunks.content_hash`). Projected by {@link hydrateChunks} so the
   * assembly stage can fold byte-identical passages into one result row.
   * ABSENT - not null - on reads that do not project it (the
   * representative-chunk read behind traversal expansion), keeping those
   * rows byte-identical to what they were before the projection existed.
   * That absence is load-bearing: {@link HydratedChunk.authoredAt} is the
   * one field that has since left this rule, and it left it deliberately,
   * for the reason stated above.
   */
  readonly contentHash?: string;
  /**
   * sha256 of the WHOLE source file of this chunk's document, frontmatter
   * included (`documents.content_hash`, also schema v1). The chunk hash
   * alone cannot tell a copied file from two distinct records that happen
   * to share a body: frontmatter is stripped before chunking, so two notes
   * with the same prose but different `authored_at`, `freshness_trend` or
   * declared dates hash their chunks identically while ranking - correctly -
   * differently. Pairing the two hashes makes the duplicate merge fire only
   * on a genuine copy. Absent under the same rule as `contentHash`.
   */
  readonly documentHash?: string;
}

export function getChunksByDocument(db: Database, documentId: number): ChunkRow[] {
  const rows = db
    .query<
      {
        id: number;
        document_id: number;
        chunk_index: number;
        content: string;
        content_hash: string;
        start_line: number;
        end_line: number;
        token_count: number;
      },
      [number]
    >(
      "SELECT id, document_id, chunk_index, content, content_hash, start_line, end_line, token_count " +
        "FROM chunks WHERE document_id = ? ORDER BY chunk_index",
    )
    .all(documentId);
  return rows.map((r) => ({
    id: r.id,
    documentId: r.document_id,
    chunkIndex: r.chunk_index,
    content: r.content,
    contentHash: r.content_hash,
    startLine: r.start_line,
    endLine: r.end_line,
    tokenCount: r.token_count,
  }));
}

/** What one document's chunk replacement wrote. */
export interface ChunkReplacement {
  /** The chunk ids, in `chunkIndex` order: kept rows keep theirs. */
  readonly chunkIds: number[];
  /**
   * Chunks that kept a stored vector: kept in place with a vector of the
   * recorded identity, or new rows that carried the vector of an old chunk
   * with the same `content_hash` (vector carry-over). The embedding phase
   * will not pay for them again.
   */
  readonly embeddingsReused: number;
  /**
   * Ids of the chunks kept in place: same position, same content, same
   * lines and heading. Nothing derived from their content (full-text row,
   * entities, vector) needs to be written again.
   */
  readonly keptChunkIds: ReadonlySet<number>;
}

interface StoredChunk {
  id: number;
  chunk_index: number;
  content_hash: string;
  start_line: number;
  end_line: number;
  token_count: number;
  heading_path: string;
  fts_content: string;
}

/**
 * Replace a document's chunks, rewriting only the ones that changed.
 *
 * A chunk at the same `chunkIndex` with the same content hash, lines,
 * token count, heading and full-text form is KEPT: its row, id, full-text row, entities
 * and vector stay as they are (a kept vector of another embedding
 * identity is dropped, as a replaced one would be). Every other old chunk
 * is deleted and every other new chunk inserted, carrying the stored
 * vector of an old chunk with the same content where the identity allows
 * (the rule and its guard live in `vector-carry-over.ts`). A daily log
 * that grows all day therefore costs its appended tail per run, not the
 * whole file. The old `chunk_vec_map` and `embeddings` rows of deleted
 * chunks go with them through the FK cascade; FTS5 stays in sync through
 * the chunks_ai/ad/au triggers.
 */
export function replaceDocumentChunks(
  db: Database,
  vecLoaded: boolean,
  documentId: number,
  chunks: ReadonlyArray<ChunkInput>,
): ChunkReplacement {
  const ids: number[] = [];
  let embeddingsReused = 0;
  const kept = new Set<number>();
  db.exec("BEGIN");
  try {
    const stored = db
      .query<StoredChunk, [number]>(
        "SELECT id, chunk_index, content_hash, start_line, end_line, token_count, heading_path, " +
          "fts_content FROM chunks WHERE document_id = ?",
      )
      .all(documentId);
    const byIndex = new Map(stored.map((r) => [r.chunk_index, r]));
    const keepAt = chunks.map((c) => {
      const old = byIndex.get(c.chunkIndex);
      const same =
        old !== undefined &&
        old.content_hash === c.contentHash &&
        old.start_line === c.startLine &&
        old.end_line === c.endLine &&
        old.token_count === c.tokenCount &&
        old.heading_path === (c.headingPath ?? "") &&
        // The stored full-text form is derived from the content; a release
        // that derives it differently must not leave kept rows behind.
        old.fts_content === (c.ftsContent ?? c.content);
      if (same) kept.add(old.id);
      return same ? old.id : null;
    });

    const candidates = readCarryCandidates(db, vecLoaded, documentId);
    // A kept chunk whose vector is not of the recorded identity loses it,
    // exactly as a replaced chunk's vector would not be carried.
    const validKept = new Set(candidates.filter((c) => kept.has(c.chunkId)).map((c) => c.chunkId));
    const keptStale = [...kept].filter((id) => !validKept.has(id));
    if (keptStale.length > 0) {
      purgeVecRowsByChunkIds(db, vecLoaded, keptStale);
      const placeholders = sqlPlaceholders(keptStale);
      db.run(`DELETE FROM chunk_vec_map WHERE chunk_id IN (${placeholders})`, keptStale);
      db.run(`DELETE FROM embeddings WHERE chunk_id IN (${placeholders})`, keptStale);
    }

    const newPositions: number[] = [];
    keepAt.forEach((id, position) => {
      if (id === null) newPositions.push(position);
    });
    const carried = matchCarriedVectors(
      candidates.filter((c) => !kept.has(c.chunkId)),
      newPositions.map((p) => chunks[p]!.contentHash),
    ).map((m) => ({ position: newPositions[m.position]!, candidate: m.candidate }));
    const carriedOld = new Set(carried.map((m) => m.candidate.chunkId));
    const removed = stored.map((r) => r.id).filter((id) => !kept.has(id));
    purgeVecRowsByChunkIds(
      db,
      vecLoaded,
      removed.filter((id) => !carriedOld.has(id)),
    );
    if (removed.length > 0) {
      db.run(`DELETE FROM chunks WHERE id IN (${sqlPlaceholders(removed)})`, removed);
    }

    // The page-head rule (schema v14's backfill twin): chunk 0 carries the
    // frontmatter block when it starts with the `---` fence, and the head is
    // then the first content chunk - unless the document has no content
    // chunk at all, in which case chunk 0 is all the page there is. A
    // document without frontmatter starts its content at chunk 0.
    const headIndex =
      chunks[0] !== undefined && chunks[0].content.startsWith("---")
        ? Math.min(1, chunks.length - 1)
        : 0;
    const insert = db.prepare<
      { id: number },
      [
        number,
        number,
        string,
        string,
        string,
        number,
        number,
        number,
        string,
        string,
        string,
        string,
      ]
    >(
      "INSERT INTO chunks(document_id, chunk_index, content, fts_content, content_hash, start_line, end_line, token_count, heading_path, created_at, updated_at, chunk_kind) " +
        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",
    );
    const now = nowIso();
    chunks.forEach((c, position) => {
      const keptId = keepAt[position];
      if (keptId !== null && keptId !== undefined) {
        ids.push(keptId);
        return;
      }
      const row = insert.get(
        documentId,
        c.chunkIndex,
        c.content,
        c.ftsContent ?? c.content,
        c.contentHash,
        c.startLine,
        c.endLine,
        c.tokenCount,
        c.headingPath ?? "",
        now,
        now,
        position === headIndex ? "page" : "section",
      );
      if (!row) throw new SearchError("INDEX_UNREADABLE", "chunk insert returned no id");
      ids.push(row.id);
    });
    restoreCarriedVectors(db, ids, carried);
    embeddingsReused = validKept.size + carried.length;
    db.exec("COMMIT");
  } catch (e) {
    db.exec("ROLLBACK");
    throw e;
  }
  return { chunkIds: ids, embeddingsReused, keptChunkIds: kept };
}

/**
 * {@link replaceDocumentChunks} for callers that need only the new ids
 * (in `chunkIndex` order). The carry-over applies all the same.
 */
export function replaceChunks(
  db: Database,
  vecLoaded: boolean,
  documentId: number,
  chunks: ReadonlyArray<ChunkInput>,
): number[] {
  return replaceDocumentChunks(db, vecLoaded, documentId, chunks).chunkIds;
}

/**
 * Delete a set of chunks by id. Vec rows removed first.
 */
export function deleteChunks(
  db: Database,
  vecLoaded: boolean,
  chunkIds: ReadonlyArray<number>,
): void {
  if (chunkIds.length === 0) return;
  db.exec("BEGIN");
  try {
    purgeVecRowsByChunkIds(db, vecLoaded, chunkIds);
    const placeholders = sqlPlaceholders(chunkIds);
    db.run(`DELETE FROM chunks WHERE id IN (${placeholders})`, chunkIds as number[]);
    db.exec("COMMIT");
  } catch (e) {
    db.exec("ROLLBACK");
    throw e;
  }
}

/** Ordered chunk ids for one document (surprisal, t_fddfe64a). */
export function chunksForDocument(
  db: Database,
  documentId: number,
): ReadonlyArray<{ id: number; chunkIndex: number }> {
  return db
    .query<{ id: number; chunk_index: number }, [number]>(
      "SELECT id, chunk_index FROM chunks WHERE document_id = ? ORDER BY chunk_index ASC",
    )
    .all(documentId)
    .map((r) => ({ id: r.id, chunkIndex: r.chunk_index }));
}

/** Total indexed chunk count - used to judge trigram-prefilter selectivity. */
export function countChunks(db: Database): number {
  return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM chunks").get()?.n ?? 0;
}

/**
 * Optional path scope of the pending-vector census: only chunks whose
 * document path starts with one of the prefixes. An absent or empty list
 * means vault-wide, and then the census SQL is exactly the unscoped form.
 */
export interface PendingVectorScope {
  readonly pathPrefixes?: ReadonlyArray<string>;
}

const PENDING_ANTI_JOIN =
  "FROM chunks c LEFT JOIN embeddings e ON e.chunk_id = c.id WHERE e.chunk_id IS NULL";

/**
 * The identity a persistent chunk hold is matched under (chunk
 * skip-and-hold, F2): the (model, window, signature) triple whose proof
 * put the chunk in `chunk_embedding_holds`. The pending anti-join
 * excludes a held chunk ONLY while this triple still describes the
 * active configuration, so a window or model change releases the hold
 * back into pending without any delete pass.
 */
export interface ChunkHoldExclusion {
  readonly model: string;
  readonly windowTokens: number;
  readonly embeddingSignature: string;
}

/** One recorded hold, as read back for reporting. */
export interface ChunkHoldRow {
  readonly chunkId: number;
  readonly model: string;
  readonly windowTokens: number;
  readonly embeddingSignature: string;
  readonly estimatedTokens: number;
  readonly heldAt: string;
}

/**
 * The anti-join's exclusion of holds made under `hold`: a chunk with a
 * row matching the whole key is owed no vector by this configuration.
 * Bound after the scope's bindings, matching clause order.
 */
const HOLD_NOT_EXISTS =
  ` AND NOT EXISTS (SELECT 1 FROM ${CHUNK_HOLDS_TABLE} h ` +
  "WHERE h.chunk_id = c.id AND h.model = ? AND h.window_tokens = ? " +
  "AND h.embedding_signature = ?)";

/**
 * The anti-join's FROM/WHERE text and bindings for a scope. A scoped
 * census joins `documents` and matches each prefix with the sargable
 * range form every other prefix reader uses (`path-prefix.ts`), which
 * needs no LIKE wildcard escaping and lets `idx_documents_path` answer
 * the match. A prefix whose last code point cannot increment has a null
 * bound and contributes its lower-bound-only clause.
 */
function pendingCensusClause(
  scope: PendingVectorScope | undefined,
  hold: ChunkHoldExclusion | undefined,
): { sql: string; bindings: (string | number)[] } {
  const prefixes = scope?.pathPrefixes ?? [];
  const holdBindings: (string | number)[] = hold
    ? [hold.model, hold.windowTokens, hold.embeddingSignature]
    : [];
  const holdClause = hold ? HOLD_NOT_EXISTS : "";
  if (prefixes.length === 0) {
    return { sql: PENDING_ANTI_JOIN + holdClause, bindings: holdBindings };
  }
  const ranges = prefixes.map((prefix) => pathPrefixRange(prefix));
  const matches = ranges
    .map((range) => (range.hi !== null ? "(d.path >= ? AND d.path < ?)" : "(d.path >= ?)"))
    .join(" OR ");
  return {
    sql:
      "FROM chunks c JOIN documents d ON d.id = c.document_id " +
      "LEFT JOIN embeddings e ON e.chunk_id = c.id " +
      `WHERE e.chunk_id IS NULL AND (${matches})${holdClause}`,
    bindings: [
      ...ranges.flatMap((range) => (range.hi !== null ? [range.lo, range.hi] : [range.lo])),
      ...holdBindings,
    ],
  };
}

/**
 * Chunks that have no row in `embeddings`. Used by the indexer to
 * populate vectors after a fresh index or after the model-change drop,
 * optionally scoped to path prefixes and optionally excluding the chunks
 * held back under `hold`'s key (chunk skip-and-hold, F2).
 */
export function findChunksWithoutEmbeddings(
  db: Database,
  scope?: PendingVectorScope,
  hold?: ChunkHoldExclusion,
): Array<{ chunkId: number; content: string }> {
  const { sql, bindings } = pendingCensusClause(scope, hold);
  const rows = db
    .query<{ id: number; content: string }, (string | number)[]>(
      `SELECT c.id AS id, c.content AS content ${sql} ORDER BY c.id`,
    )
    .all(...bindings);
  return rows.map((r) => ({ chunkId: r.id, content: r.content }));
}

/**
 * How MANY chunks have no row in `embeddings`, without materialising a
 * single one of their bodies.
 *
 * The same anti-join {@link findChunksWithoutEmbeddings} walks, under the
 * same scope and hold exclusion, as a `COUNT(*)`. The row-returning form
 * is the indexer's work queue and loads every pending chunk's full
 * `content` into JS memory to hand it to a provider; a diagnostic that
 * only wants the number must not pay that, which is the whole reason this
 * second query exists rather than a `.length` on the first.
 */
export function countChunksWithoutEmbeddings(
  db: Database,
  scope?: PendingVectorScope,
  hold?: ChunkHoldExclusion,
): number {
  const { sql, bindings } = pendingCensusClause(scope, hold);
  return (
    db.query<{ n: number }, (string | number)[]>(`SELECT COUNT(*) AS n ${sql}`).get(...bindings)
      ?.n ?? 0
  );
}

/** One hold decision to record: the chunk and the proof's numbers. */
export interface ChunkHoldEntry {
  readonly chunkId: number;
  readonly estimatedTokens: number;
  readonly windowTokens: number;
}

/** The (model, signature) pair every hold of one phase run shares. */
export interface ChunkHoldStamp {
  readonly model: string;
  readonly embeddingSignature: string;
}

/**
 * Record the embedding phase's hold decisions (chunk skip-and-hold, F2).
 *
 * `INSERT OR REPLACE` on the (chunk, key) primary key: re-deciding the
 * same hold refreshes its estimate and timestamp rather than stacking a
 * second row, which is what keeps a hand-in-plan re-run of the phase
 * free of duplicate state.
 */
export function recordChunkHolds(
  db: Database,
  entries: ReadonlyArray<ChunkHoldEntry>,
  stamp: ChunkHoldStamp,
): void {
  if (entries.length === 0) return;
  const heldAt = nowIso();
  const insert = db.query<never, [number, string, number, string, number, string]>(
    `INSERT OR REPLACE INTO ${CHUNK_HOLDS_TABLE}` +
      "(chunk_id, model, window_tokens, embedding_signature, estimated_tokens, held_at) " +
      "VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
  );
  // One transaction, the way the sibling bulk writers commit: after a
  // model or window change a large held set is hundreds of rows, and one
  // autocommit (WAL fsync) per row prices the run in syscalls while a
  // failure halfway leaves a partial record beside an in-memory tally
  // that already counts every hold.
  db.transaction(() => {
    for (const entry of entries) {
      insert.run(
        entry.chunkId,
        stamp.model,
        entry.windowTokens,
        stamp.embeddingSignature,
        entry.estimatedTokens,
        heldAt,
      );
    }
  })();
}

/** The hold rows recorded for one chunk, whatever key made them. */
export function chunkHoldRowsFor(db: Database, chunkId: number): ChunkHoldRow[] {
  return chunkHoldRowsForMany(db, [chunkId]);
}

/** Hold rows for many chunks in one read, keyed by chunk id. */
export function chunkHoldRowsForMany(
  db: Database,
  chunkIds: ReadonlyArray<number>,
): ChunkHoldRow[] {
  if (chunkIds.length === 0) return [];
  const placeholders = sqlPlaceholders(chunkIds);
  return db
    .query<
      {
        chunk_id: number;
        model: string;
        window_tokens: number;
        embedding_signature: string;
        estimated_tokens: number;
        held_at: string;
      },
      number[]
    >(
      `SELECT chunk_id, model, window_tokens, embedding_signature, estimated_tokens, held_at ` +
        `FROM ${CHUNK_HOLDS_TABLE} WHERE chunk_id IN (${placeholders}) ORDER BY chunk_id, model`,
    )
    .all(...chunkIds)
    .map((r) => ({
      chunkId: r.chunk_id,
      model: r.model,
      windowTokens: r.window_tokens,
      embeddingSignature: r.embedding_signature,
      estimatedTokens: r.estimated_tokens,
      heldAt: r.held_at,
    }));
}

/** One held chunk with the position that names it outside the index. */
export interface HeldChunkCensusRow {
  readonly path: string;
  readonly chunkIndex: number;
  readonly estimatedTokens: number;
  readonly windowTokens: number;
}

/**
 * The held chunks recorded under one key, with the UNCAPPED total. The
 * rows come back position-ordered and at most `limit` of them, so the
 * caller caps without loading a vault's whole hold list.
 */
export function heldChunkCensus(
  db: Database,
  hold: ChunkHoldExclusion,
  limit: number,
): { total: number; rows: HeldChunkCensusRow[] } {
  // A hold row whose chunk has a vector is not a hold any more (window
  // up, chunk embedded, window back): the report answers to the same fact
  // the pending anti-join answers to, so the operator is never warned
  // about work that is already done.
  const total =
    db
      .query<{ n: number }, (string | number)[]>(
        `SELECT COUNT(*) AS n FROM ${CHUNK_HOLDS_TABLE} h ` +
          "WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
          "AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id)",
      )
      .get(hold.model, hold.windowTokens, hold.embeddingSignature)?.n ?? 0;
  if (total === 0) return { total: 0, rows: [] };
  const rows = db
    .query<
      { path: string; chunk_index: number; estimated_tokens: number; window_tokens: number },
      (string | number)[]
    >(
      "SELECT d.path AS path, c.chunk_index AS chunk_index, " +
        "h.estimated_tokens AS estimated_tokens, h.window_tokens AS window_tokens " +
        `FROM ${CHUNK_HOLDS_TABLE} h ` +
        "JOIN chunks c ON c.id = h.chunk_id " +
        "JOIN documents d ON d.id = c.document_id " +
        "WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
        "AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id) " +
        "ORDER BY d.path, c.chunk_index LIMIT ?",
    )
    .all(hold.model, hold.windowTokens, hold.embeddingSignature, limit)
    .map((r) => ({
      path: r.path,
      chunkIndex: r.chunk_index,
      estimatedTokens: r.estimated_tokens,
      windowTokens: r.window_tokens,
    }));
  return { total, rows };
}

/** Held chunks aggregated to the source document that carries them. */
export interface HeldChunkDocRow {
  readonly path: string;
  readonly heldChunks: number;
  readonly maxEstimatedTokens: number;
}

/**
 * Per-document counts of the holds recorded under one key, document-
 * ordered. The doctor's held-sources check speaks per document, and the
 * aggregation belongs to the SQL rather than to a caller that would
 * have to load every held row to do it.
 */
export function heldChunkDocCensus(db: Database, hold: ChunkHoldExclusion): HeldChunkDocRow[] {
  return db
    .query<{ path: string; held: number; max_est: number }, (string | number)[]>(
      "SELECT d.path AS path, COUNT(*) AS held, MAX(h.estimated_tokens) AS max_est " +
        `FROM ${CHUNK_HOLDS_TABLE} h ` +
        "JOIN chunks c ON c.id = h.chunk_id " +
        "JOIN documents d ON d.id = c.document_id " +
        "WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
        "AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id) " +
        "GROUP BY d.path ORDER BY d.path",
    )
    .all(hold.model, hold.windowTokens, hold.embeddingSignature)
    .map((r) => ({ path: r.path, heldChunks: r.held, maxEstimatedTokens: r.max_est }));
}

export function hydrateChunks(
  db: Database,
  chunkIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
  const out = new Map<number, HydratedChunk>();
  if (chunkIds.length === 0) return out;
  const placeholders = sqlPlaceholders(chunkIds);
  const rows = db
    .query<
      {
        chunk_id: number;
        document_id: number;
        path: string;
        title: string | null;
        content: string;
        start_line: number;
        end_line: number;
        mtime: number;
        authored_at: number | null;
        content_hash: string;
        document_hash: string;
      },
      number[]
    >(
      "SELECT c.id AS chunk_id, c.document_id, d.path AS path, d.title AS title, " +
        "       c.content AS content, c.start_line AS start_line, c.end_line AS end_line, d.mtime AS mtime, " +
        "       d.authored_at AS authored_at, c.content_hash AS content_hash, " +
        "       d.content_hash AS document_hash " +
        "FROM chunks c JOIN documents d ON d.id = c.document_id " +
        `WHERE c.id IN (${placeholders})`,
    )
    .all(...(chunkIds as number[]));
  for (const r of rows) {
    out.set(r.chunk_id, {
      chunkId: r.chunk_id,
      documentId: r.document_id,
      path: r.path,
      title: r.title,
      content: r.content,
      startLine: r.start_line,
      endLine: r.end_line,
      mtime: r.mtime,
      authoredAt: r.authored_at,
      contentHash: r.content_hash,
      documentHash: r.document_hash,
    });
  }
  return out;
}

/**
 * The METADATA-ONLY pool hydration (recall-quality, C5): the same
 * projection as {@link hydrateChunks} minus `c.content` - the one column
 * the candidate pool never consumes before ranking (scoring reads ids,
 * paths, hashes, mtimes and authored instants), and by far the most
 * expensive one to pull for every candidate.
 *
 * The rows type as {@link HydratedChunk} with `content: ""` so the
 * downstream signatures do not fork; the assembly's content tier replaces
 * the placeholder for the ranked rows before the first content consumer
 * (MMR) reads it. A consumer reading `content` off the pool map directly
 * would get the placeholder - which is why the pool map flows only to the
 * scoring stages.
 */
export function hydrateChunkRefs(
  db: Database,
  chunkIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
  const out = new Map<number, HydratedChunk>();
  if (chunkIds.length === 0) return out;
  const placeholders = sqlPlaceholders(chunkIds);
  const rows = db
    .query<
      {
        chunk_id: number;
        document_id: number;
        path: string;
        title: string | null;
        start_line: number;
        end_line: number;
        mtime: number;
        authored_at: number | null;
        content_hash: string;
        document_hash: string;
      },
      number[]
    >(
      "SELECT c.id AS chunk_id, c.document_id, d.path AS path, d.title AS title, " +
        "       c.start_line AS start_line, c.end_line AS end_line, d.mtime AS mtime, " +
        "       d.authored_at AS authored_at, c.content_hash AS content_hash, " +
        "       d.content_hash AS document_hash " +
        "FROM chunks c JOIN documents d ON d.id = c.document_id " +
        `WHERE c.id IN (${placeholders})`,
    )
    .all(...(chunkIds as number[]));
  for (const r of rows) {
    out.set(r.chunk_id, {
      chunkId: r.chunk_id,
      documentId: r.document_id,
      path: r.path,
      title: r.title,
      content: "",
      startLine: r.start_line,
      endLine: r.end_line,
      mtime: r.mtime,
      authoredAt: r.authored_at,
      contentHash: r.content_hash,
      documentHash: r.document_hash,
    });
  }
  return out;
}

/**
 * One representative chunk per document - the lowest `chunk_index`,
 * which for markdown is the document head (title / opening section).
 * The traversal layer surfaces this when a linked document is not
 * already a relevance hit.
 *
 * Projects `authored_at` (D1) on the join that already fetches `mtime`,
 * so both producers of a {@link HydratedChunk} report the same freshness
 * anchor. Today's consumers - traversal expansion, the graph pre-pass and
 * the relational arm - read `path`, `content` and the ids, so this
 * changes no current ranking; what it removes is the trap that a row from
 * here silently answers the storage clock for a question about content
 * age. The chunk and document hashes stay unprojected, for the reason
 * given on {@link HydratedChunk.contentHash}.
 */
export function representativeChunks(
  db: Database,
  documentIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
  const out = new Map<number, HydratedChunk>();
  if (documentIds.length === 0) return out;
  const placeholders = sqlPlaceholders(documentIds);
  const rows = db
    .query<
      {
        chunk_id: number;
        document_id: number;
        path: string;
        title: string | null;
        content: string;
        start_line: number;
        end_line: number;
        mtime: number;
        authored_at: number | null;
      },
      number[]
    >(
      "SELECT c.id AS chunk_id, c.document_id AS document_id, d.path AS path, " +
        "d.title AS title, c.content AS content, c.start_line AS start_line, " +
        "c.end_line AS end_line, d.mtime AS mtime, d.authored_at AS authored_at " +
        "FROM chunks c JOIN documents d ON d.id = c.document_id " +
        `WHERE c.document_id IN (${placeholders}) ` +
        "ORDER BY c.document_id, c.chunk_index ASC",
    )
    .all(...(documentIds as number[]));
  for (const r of rows) {
    if (out.has(r.document_id)) continue; // first row per doc = lowest chunk_index
    out.set(
      r.document_id,
      Object.freeze({
        chunkId: r.chunk_id,
        documentId: r.document_id,
        path: r.path,
        title: r.title,
        content: r.content,
        startLine: r.start_line,
        endLine: r.end_line,
        mtime: r.mtime,
        authoredAt: r.authored_at,
      }),
    );
  }
  return out;
}