Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The `chunks` table: the units search actually ranks. Owns their
* atomic replacement per document (the FTS5 shadow follows via the
* chunks_ai/ad/au triggers) and the hydration reads that turn ids back
* into rendered text.
*/
import { Database } from "bun:sqlite";
import { SearchError } from "../types.ts";
import { CHUNK_HOLDS_TABLE } from "../schema.ts";
import { pathPrefixRange } from "./path-prefix.ts";
import { nowIso, sqlPlaceholders } from "./sql.ts";
import {
matchCarriedVectors,
readCarryCandidates,
restoreCarriedVectors,
} from "./vector-carry-over.ts";
import { purgeVecRowsByChunkIds } from "./vectors.ts";
export interface ChunkInput {
readonly chunkIndex: number;
readonly content: string;
readonly ftsContent?: string;
readonly contentHash: string;
readonly startLine: number;
readonly endLine: number;
readonly tokenCount: number;
/**
* Heading breadcrumb in effect at the chunk (v0.13.0). Indexed in the
* dedicated FTS column; defaults to "" so callers that do not supply
* it (and pre-v0.13.0 fixtures) index an empty heading column.
*/
readonly headingPath?: string;
}
export interface ChunkRow {
readonly id: number;
readonly documentId: number;
readonly chunkIndex: number;
readonly content: string;
readonly contentHash: string;
readonly startLine: number;
readonly endLine: number;
readonly tokenCount: number;
}
export interface HydratedChunk {
readonly chunkId: number;
readonly documentId: number;
readonly path: string;
readonly title: string | null;
readonly content: string;
readonly startLine: number;
readonly endLine: number;
readonly mtime: number;
/**
* Transcript turn instant (unix seconds) or null (conversation
* chronology, S1). It is the ranker's freshness anchor - age is
* measured from it, falling back to {@link HydratedChunk.mtime} when it
* is null - it separates an exact score tie, and it is surfaced on the
* search result. Null for notes with no turn instant, which keeps their
* ranking and their ordering byte-identical.
*
* BOTH reads project it (D1). It was once absent from the
* representative-chunk read, which left that row unable to distinguish
* "this document declares no instant" from "this read did not ask" -
* and a consumer that routed such a row through the ranker would have
* silently got the storage clock back. Optional in the type only
* because a caller may construct a row without one.
*/
readonly authoredAt?: number | null;
/**
* sha256 of the chunk content, written on every index run since schema
* v1 (`chunks.content_hash`). Projected by {@link hydrateChunks} so the
* assembly stage can fold byte-identical passages into one result row.
* ABSENT - not null - on reads that do not project it (the
* representative-chunk read behind traversal expansion), keeping those
* rows byte-identical to what they were before the projection existed.
* That absence is load-bearing: {@link HydratedChunk.authoredAt} is the
* one field that has since left this rule, and it left it deliberately,
* for the reason stated above.
*/
readonly contentHash?: string;
/**
* sha256 of the WHOLE source file of this chunk's document, frontmatter
* included (`documents.content_hash`, also schema v1). The chunk hash
* alone cannot tell a copied file from two distinct records that happen
* to share a body: frontmatter is stripped before chunking, so two notes
* with the same prose but different `authored_at`, `freshness_trend` or
* declared dates hash their chunks identically while ranking - correctly -
* differently. Pairing the two hashes makes the duplicate merge fire only
* on a genuine copy. Absent under the same rule as `contentHash`.
*/
readonly documentHash?: string;
}
export function getChunksByDocument(db: Database, documentId: number): ChunkRow[] {
const rows = db
.query<
{
id: number;
document_id: number;
chunk_index: number;
content: string;
content_hash: string;
start_line: number;
end_line: number;
token_count: number;
},
[number]
>(
"SELECT id, document_id, chunk_index, content, content_hash, start_line, end_line, token_count " +
"FROM chunks WHERE document_id = ? ORDER BY chunk_index",
)
.all(documentId);
return rows.map((r) => ({
id: r.id,
documentId: r.document_id,
chunkIndex: r.chunk_index,
content: r.content,
contentHash: r.content_hash,
startLine: r.start_line,
endLine: r.end_line,
tokenCount: r.token_count,
}));
}
/** What one document's chunk replacement wrote. */
export interface ChunkReplacement {
/** The chunk ids, in `chunkIndex` order: kept rows keep theirs. */
readonly chunkIds: number[];
/**
* Chunks that kept a stored vector: kept in place with a vector of the
* recorded identity, or new rows that carried the vector of an old chunk
* with the same `content_hash` (vector carry-over). The embedding phase
* will not pay for them again.
*/
readonly embeddingsReused: number;
/**
* Ids of the chunks kept in place: same position, same content, same
* lines and heading. Nothing derived from their content (full-text row,
* entities, vector) needs to be written again.
*/
readonly keptChunkIds: ReadonlySet<number>;
}
interface StoredChunk {
id: number;
chunk_index: number;
content_hash: string;
start_line: number;
end_line: number;
token_count: number;
heading_path: string;
fts_content: string;
}
/**
* Replace a document's chunks, rewriting only the ones that changed.
*
* A chunk at the same `chunkIndex` with the same content hash, lines,
* token count, heading and full-text form is KEPT: its row, id, full-text row, entities
* and vector stay as they are (a kept vector of another embedding
* identity is dropped, as a replaced one would be). Every other old chunk
* is deleted and every other new chunk inserted, carrying the stored
* vector of an old chunk with the same content where the identity allows
* (the rule and its guard live in `vector-carry-over.ts`). A daily log
* that grows all day therefore costs its appended tail per run, not the
* whole file. The old `chunk_vec_map` and `embeddings` rows of deleted
* chunks go with them through the FK cascade; FTS5 stays in sync through
* the chunks_ai/ad/au triggers.
*/
export function replaceDocumentChunks(
db: Database,
vecLoaded: boolean,
documentId: number,
chunks: ReadonlyArray<ChunkInput>,
): ChunkReplacement {
const ids: number[] = [];
let embeddingsReused = 0;
const kept = new Set<number>();
db.exec("BEGIN");
try {
const stored = db
.query<StoredChunk, [number]>(
"SELECT id, chunk_index, content_hash, start_line, end_line, token_count, heading_path, " +
"fts_content FROM chunks WHERE document_id = ?",
)
.all(documentId);
const byIndex = new Map(stored.map((r) => [r.chunk_index, r]));
const keepAt = chunks.map((c) => {
const old = byIndex.get(c.chunkIndex);
const same =
old !== undefined &&
old.content_hash === c.contentHash &&
old.start_line === c.startLine &&
old.end_line === c.endLine &&
old.token_count === c.tokenCount &&
old.heading_path === (c.headingPath ?? "") &&
// The stored full-text form is derived from the content; a release
// that derives it differently must not leave kept rows behind.
old.fts_content === (c.ftsContent ?? c.content);
if (same) kept.add(old.id);
return same ? old.id : null;
});
const candidates = readCarryCandidates(db, vecLoaded, documentId);
// A kept chunk whose vector is not of the recorded identity loses it,
// exactly as a replaced chunk's vector would not be carried.
const validKept = new Set(candidates.filter((c) => kept.has(c.chunkId)).map((c) => c.chunkId));
const keptStale = [...kept].filter((id) => !validKept.has(id));
if (keptStale.length > 0) {
purgeVecRowsByChunkIds(db, vecLoaded, keptStale);
const placeholders = sqlPlaceholders(keptStale);
db.run(`DELETE FROM chunk_vec_map WHERE chunk_id IN (${placeholders})`, keptStale);
db.run(`DELETE FROM embeddings WHERE chunk_id IN (${placeholders})`, keptStale);
}
const newPositions: number[] = [];
keepAt.forEach((id, position) => {
if (id === null) newPositions.push(position);
});
const carried = matchCarriedVectors(
candidates.filter((c) => !kept.has(c.chunkId)),
newPositions.map((p) => chunks[p]!.contentHash),
).map((m) => ({ position: newPositions[m.position]!, candidate: m.candidate }));
const carriedOld = new Set(carried.map((m) => m.candidate.chunkId));
const removed = stored.map((r) => r.id).filter((id) => !kept.has(id));
purgeVecRowsByChunkIds(
db,
vecLoaded,
removed.filter((id) => !carriedOld.has(id)),
);
if (removed.length > 0) {
db.run(`DELETE FROM chunks WHERE id IN (${sqlPlaceholders(removed)})`, removed);
}
// The page-head rule (schema v14's backfill twin): chunk 0 carries the
// frontmatter block when it starts with the `---` fence, and the head is
// then the first content chunk - unless the document has no content
// chunk at all, in which case chunk 0 is all the page there is. A
// document without frontmatter starts its content at chunk 0.
const headIndex =
chunks[0] !== undefined && chunks[0].content.startsWith("---")
? Math.min(1, chunks.length - 1)
: 0;
const insert = db.prepare<
{ id: number },
[
number,
number,
string,
string,
string,
number,
number,
number,
string,
string,
string,
string,
]
>(
"INSERT INTO chunks(document_id, chunk_index, content, fts_content, content_hash, start_line, end_line, token_count, heading_path, created_at, updated_at, chunk_kind) " +
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id",
);
const now = nowIso();
chunks.forEach((c, position) => {
const keptId = keepAt[position];
if (keptId !== null && keptId !== undefined) {
ids.push(keptId);
return;
}
const row = insert.get(
documentId,
c.chunkIndex,
c.content,
c.ftsContent ?? c.content,
c.contentHash,
c.startLine,
c.endLine,
c.tokenCount,
c.headingPath ?? "",
now,
now,
position === headIndex ? "page" : "section",
);
if (!row) throw new SearchError("INDEX_UNREADABLE", "chunk insert returned no id");
ids.push(row.id);
});
restoreCarriedVectors(db, ids, carried);
embeddingsReused = validKept.size + carried.length;
db.exec("COMMIT");
} catch (e) {
db.exec("ROLLBACK");
throw e;
}
return { chunkIds: ids, embeddingsReused, keptChunkIds: kept };
}
/**
* {@link replaceDocumentChunks} for callers that need only the new ids
* (in `chunkIndex` order). The carry-over applies all the same.
*/
export function replaceChunks(
db: Database,
vecLoaded: boolean,
documentId: number,
chunks: ReadonlyArray<ChunkInput>,
): number[] {
return replaceDocumentChunks(db, vecLoaded, documentId, chunks).chunkIds;
}
/**
* Delete a set of chunks by id. Vec rows removed first.
*/
export function deleteChunks(
db: Database,
vecLoaded: boolean,
chunkIds: ReadonlyArray<number>,
): void {
if (chunkIds.length === 0) return;
db.exec("BEGIN");
try {
purgeVecRowsByChunkIds(db, vecLoaded, chunkIds);
const placeholders = sqlPlaceholders(chunkIds);
db.run(`DELETE FROM chunks WHERE id IN (${placeholders})`, chunkIds as number[]);
db.exec("COMMIT");
} catch (e) {
db.exec("ROLLBACK");
throw e;
}
}
/** Ordered chunk ids for one document (surprisal, t_fddfe64a). */
export function chunksForDocument(
db: Database,
documentId: number,
): ReadonlyArray<{ id: number; chunkIndex: number }> {
return db
.query<{ id: number; chunk_index: number }, [number]>(
"SELECT id, chunk_index FROM chunks WHERE document_id = ? ORDER BY chunk_index ASC",
)
.all(documentId)
.map((r) => ({ id: r.id, chunkIndex: r.chunk_index }));
}
/** Total indexed chunk count - used to judge trigram-prefilter selectivity. */
export function countChunks(db: Database): number {
return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM chunks").get()?.n ?? 0;
}
/**
* Optional path scope of the pending-vector census: only chunks whose
* document path starts with one of the prefixes. An absent or empty list
* means vault-wide, and then the census SQL is exactly the unscoped form.
*/
export interface PendingVectorScope {
readonly pathPrefixes?: ReadonlyArray<string>;
}
const PENDING_ANTI_JOIN =
"FROM chunks c LEFT JOIN embeddings e ON e.chunk_id = c.id WHERE e.chunk_id IS NULL";
/**
* The identity a persistent chunk hold is matched under (chunk
* skip-and-hold, F2): the (model, window, signature) triple whose proof
* put the chunk in `chunk_embedding_holds`. The pending anti-join
* excludes a held chunk ONLY while this triple still describes the
* active configuration, so a window or model change releases the hold
* back into pending without any delete pass.
*/
export interface ChunkHoldExclusion {
readonly model: string;
readonly windowTokens: number;
readonly embeddingSignature: string;
}
/** One recorded hold, as read back for reporting. */
export interface ChunkHoldRow {
readonly chunkId: number;
readonly model: string;
readonly windowTokens: number;
readonly embeddingSignature: string;
readonly estimatedTokens: number;
readonly heldAt: string;
}
/**
* The anti-join's exclusion of holds made under `hold`: a chunk with a
* row matching the whole key is owed no vector by this configuration.
* Bound after the scope's bindings, matching clause order.
*/
const HOLD_NOT_EXISTS =
` AND NOT EXISTS (SELECT 1 FROM ${CHUNK_HOLDS_TABLE} h ` +
"WHERE h.chunk_id = c.id AND h.model = ? AND h.window_tokens = ? " +
"AND h.embedding_signature = ?)";
/**
* The anti-join's FROM/WHERE text and bindings for a scope. A scoped
* census joins `documents` and matches each prefix with the sargable
* range form every other prefix reader uses (`path-prefix.ts`), which
* needs no LIKE wildcard escaping and lets `idx_documents_path` answer
* the match. A prefix whose last code point cannot increment has a null
* bound and contributes its lower-bound-only clause.
*/
function pendingCensusClause(
scope: PendingVectorScope | undefined,
hold: ChunkHoldExclusion | undefined,
): { sql: string; bindings: (string | number)[] } {
const prefixes = scope?.pathPrefixes ?? [];
const holdBindings: (string | number)[] = hold
? [hold.model, hold.windowTokens, hold.embeddingSignature]
: [];
const holdClause = hold ? HOLD_NOT_EXISTS : "";
if (prefixes.length === 0) {
return { sql: PENDING_ANTI_JOIN + holdClause, bindings: holdBindings };
}
const ranges = prefixes.map((prefix) => pathPrefixRange(prefix));
const matches = ranges
.map((range) => (range.hi !== null ? "(d.path >= ? AND d.path < ?)" : "(d.path >= ?)"))
.join(" OR ");
return {
sql:
"FROM chunks c JOIN documents d ON d.id = c.document_id " +
"LEFT JOIN embeddings e ON e.chunk_id = c.id " +
`WHERE e.chunk_id IS NULL AND (${matches})${holdClause}`,
bindings: [
...ranges.flatMap((range) => (range.hi !== null ? [range.lo, range.hi] : [range.lo])),
...holdBindings,
],
};
}
/**
* Chunks that have no row in `embeddings`. Used by the indexer to
* populate vectors after a fresh index or after the model-change drop,
* optionally scoped to path prefixes and optionally excluding the chunks
* held back under `hold`'s key (chunk skip-and-hold, F2).
*/
export function findChunksWithoutEmbeddings(
db: Database,
scope?: PendingVectorScope,
hold?: ChunkHoldExclusion,
): Array<{ chunkId: number; content: string }> {
const { sql, bindings } = pendingCensusClause(scope, hold);
const rows = db
.query<{ id: number; content: string }, (string | number)[]>(
`SELECT c.id AS id, c.content AS content ${sql} ORDER BY c.id`,
)
.all(...bindings);
return rows.map((r) => ({ chunkId: r.id, content: r.content }));
}
/**
* How MANY chunks have no row in `embeddings`, without materialising a
* single one of their bodies.
*
* The same anti-join {@link findChunksWithoutEmbeddings} walks, under the
* same scope and hold exclusion, as a `COUNT(*)`. The row-returning form
* is the indexer's work queue and loads every pending chunk's full
* `content` into JS memory to hand it to a provider; a diagnostic that
* only wants the number must not pay that, which is the whole reason this
* second query exists rather than a `.length` on the first.
*/
export function countChunksWithoutEmbeddings(
db: Database,
scope?: PendingVectorScope,
hold?: ChunkHoldExclusion,
): number {
const { sql, bindings } = pendingCensusClause(scope, hold);
return (
db.query<{ n: number }, (string | number)[]>(`SELECT COUNT(*) AS n ${sql}`).get(...bindings)
?.n ?? 0
);
}
/** One hold decision to record: the chunk and the proof's numbers. */
export interface ChunkHoldEntry {
readonly chunkId: number;
readonly estimatedTokens: number;
readonly windowTokens: number;
}
/** The (model, signature) pair every hold of one phase run shares. */
export interface ChunkHoldStamp {
readonly model: string;
readonly embeddingSignature: string;
}
/**
* Record the embedding phase's hold decisions (chunk skip-and-hold, F2).
*
* `INSERT OR REPLACE` on the (chunk, key) primary key: re-deciding the
* same hold refreshes its estimate and timestamp rather than stacking a
* second row, which is what keeps a hand-in-plan re-run of the phase
* free of duplicate state.
*/
export function recordChunkHolds(
db: Database,
entries: ReadonlyArray<ChunkHoldEntry>,
stamp: ChunkHoldStamp,
): void {
if (entries.length === 0) return;
const heldAt = nowIso();
const insert = db.query<never, [number, string, number, string, number, string]>(
`INSERT OR REPLACE INTO ${CHUNK_HOLDS_TABLE}` +
"(chunk_id, model, window_tokens, embedding_signature, estimated_tokens, held_at) " +
"VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
);
// One transaction, the way the sibling bulk writers commit: after a
// model or window change a large held set is hundreds of rows, and one
// autocommit (WAL fsync) per row prices the run in syscalls while a
// failure halfway leaves a partial record beside an in-memory tally
// that already counts every hold.
db.transaction(() => {
for (const entry of entries) {
insert.run(
entry.chunkId,
stamp.model,
entry.windowTokens,
stamp.embeddingSignature,
entry.estimatedTokens,
heldAt,
);
}
})();
}
/** The hold rows recorded for one chunk, whatever key made them. */
export function chunkHoldRowsFor(db: Database, chunkId: number): ChunkHoldRow[] {
return chunkHoldRowsForMany(db, [chunkId]);
}
/** Hold rows for many chunks in one read, keyed by chunk id. */
export function chunkHoldRowsForMany(
db: Database,
chunkIds: ReadonlyArray<number>,
): ChunkHoldRow[] {
if (chunkIds.length === 0) return [];
const placeholders = sqlPlaceholders(chunkIds);
return db
.query<
{
chunk_id: number;
model: string;
window_tokens: number;
embedding_signature: string;
estimated_tokens: number;
held_at: string;
},
number[]
>(
`SELECT chunk_id, model, window_tokens, embedding_signature, estimated_tokens, held_at ` +
`FROM ${CHUNK_HOLDS_TABLE} WHERE chunk_id IN (${placeholders}) ORDER BY chunk_id, model`,
)
.all(...chunkIds)
.map((r) => ({
chunkId: r.chunk_id,
model: r.model,
windowTokens: r.window_tokens,
embeddingSignature: r.embedding_signature,
estimatedTokens: r.estimated_tokens,
heldAt: r.held_at,
}));
}
/** One held chunk with the position that names it outside the index. */
export interface HeldChunkCensusRow {
readonly path: string;
readonly chunkIndex: number;
readonly estimatedTokens: number;
readonly windowTokens: number;
}
/**
* The held chunks recorded under one key, with the UNCAPPED total. The
* rows come back position-ordered and at most `limit` of them, so the
* caller caps without loading a vault's whole hold list.
*/
export function heldChunkCensus(
db: Database,
hold: ChunkHoldExclusion,
limit: number,
): { total: number; rows: HeldChunkCensusRow[] } {
// A hold row whose chunk has a vector is not a hold any more (window
// up, chunk embedded, window back): the report answers to the same fact
// the pending anti-join answers to, so the operator is never warned
// about work that is already done.
const total =
db
.query<{ n: number }, (string | number)[]>(
`SELECT COUNT(*) AS n FROM ${CHUNK_HOLDS_TABLE} h ` +
"WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
"AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id)",
)
.get(hold.model, hold.windowTokens, hold.embeddingSignature)?.n ?? 0;
if (total === 0) return { total: 0, rows: [] };
const rows = db
.query<
{ path: string; chunk_index: number; estimated_tokens: number; window_tokens: number },
(string | number)[]
>(
"SELECT d.path AS path, c.chunk_index AS chunk_index, " +
"h.estimated_tokens AS estimated_tokens, h.window_tokens AS window_tokens " +
`FROM ${CHUNK_HOLDS_TABLE} h ` +
"JOIN chunks c ON c.id = h.chunk_id " +
"JOIN documents d ON d.id = c.document_id " +
"WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
"AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id) " +
"ORDER BY d.path, c.chunk_index LIMIT ?",
)
.all(hold.model, hold.windowTokens, hold.embeddingSignature, limit)
.map((r) => ({
path: r.path,
chunkIndex: r.chunk_index,
estimatedTokens: r.estimated_tokens,
windowTokens: r.window_tokens,
}));
return { total, rows };
}
/** Held chunks aggregated to the source document that carries them. */
export interface HeldChunkDocRow {
readonly path: string;
readonly heldChunks: number;
readonly maxEstimatedTokens: number;
}
/**
* Per-document counts of the holds recorded under one key, document-
* ordered. The doctor's held-sources check speaks per document, and the
* aggregation belongs to the SQL rather than to a caller that would
* have to load every held row to do it.
*/
export function heldChunkDocCensus(db: Database, hold: ChunkHoldExclusion): HeldChunkDocRow[] {
return db
.query<{ path: string; held: number; max_est: number }, (string | number)[]>(
"SELECT d.path AS path, COUNT(*) AS held, MAX(h.estimated_tokens) AS max_est " +
`FROM ${CHUNK_HOLDS_TABLE} h ` +
"JOIN chunks c ON c.id = h.chunk_id " +
"JOIN documents d ON d.id = c.document_id " +
"WHERE h.model = ? AND h.window_tokens = ? AND h.embedding_signature = ? " +
"AND NOT EXISTS (SELECT 1 FROM embeddings e WHERE e.chunk_id = h.chunk_id) " +
"GROUP BY d.path ORDER BY d.path",
)
.all(hold.model, hold.windowTokens, hold.embeddingSignature)
.map((r) => ({ path: r.path, heldChunks: r.held, maxEstimatedTokens: r.max_est }));
}
export function hydrateChunks(
db: Database,
chunkIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
const out = new Map<number, HydratedChunk>();
if (chunkIds.length === 0) return out;
const placeholders = sqlPlaceholders(chunkIds);
const rows = db
.query<
{
chunk_id: number;
document_id: number;
path: string;
title: string | null;
content: string;
start_line: number;
end_line: number;
mtime: number;
authored_at: number | null;
content_hash: string;
document_hash: string;
},
number[]
>(
"SELECT c.id AS chunk_id, c.document_id, d.path AS path, d.title AS title, " +
" c.content AS content, c.start_line AS start_line, c.end_line AS end_line, d.mtime AS mtime, " +
" d.authored_at AS authored_at, c.content_hash AS content_hash, " +
" d.content_hash AS document_hash " +
"FROM chunks c JOIN documents d ON d.id = c.document_id " +
`WHERE c.id IN (${placeholders})`,
)
.all(...(chunkIds as number[]));
for (const r of rows) {
out.set(r.chunk_id, {
chunkId: r.chunk_id,
documentId: r.document_id,
path: r.path,
title: r.title,
content: r.content,
startLine: r.start_line,
endLine: r.end_line,
mtime: r.mtime,
authoredAt: r.authored_at,
contentHash: r.content_hash,
documentHash: r.document_hash,
});
}
return out;
}
/**
* The METADATA-ONLY pool hydration (recall-quality, C5): the same
* projection as {@link hydrateChunks} minus `c.content` - the one column
* the candidate pool never consumes before ranking (scoring reads ids,
* paths, hashes, mtimes and authored instants), and by far the most
* expensive one to pull for every candidate.
*
* The rows type as {@link HydratedChunk} with `content: ""` so the
* downstream signatures do not fork; the assembly's content tier replaces
* the placeholder for the ranked rows before the first content consumer
* (MMR) reads it. A consumer reading `content` off the pool map directly
* would get the placeholder - which is why the pool map flows only to the
* scoring stages.
*/
export function hydrateChunkRefs(
db: Database,
chunkIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
const out = new Map<number, HydratedChunk>();
if (chunkIds.length === 0) return out;
const placeholders = sqlPlaceholders(chunkIds);
const rows = db
.query<
{
chunk_id: number;
document_id: number;
path: string;
title: string | null;
start_line: number;
end_line: number;
mtime: number;
authored_at: number | null;
content_hash: string;
document_hash: string;
},
number[]
>(
"SELECT c.id AS chunk_id, c.document_id, d.path AS path, d.title AS title, " +
" c.start_line AS start_line, c.end_line AS end_line, d.mtime AS mtime, " +
" d.authored_at AS authored_at, c.content_hash AS content_hash, " +
" d.content_hash AS document_hash " +
"FROM chunks c JOIN documents d ON d.id = c.document_id " +
`WHERE c.id IN (${placeholders})`,
)
.all(...(chunkIds as number[]));
for (const r of rows) {
out.set(r.chunk_id, {
chunkId: r.chunk_id,
documentId: r.document_id,
path: r.path,
title: r.title,
content: "",
startLine: r.start_line,
endLine: r.end_line,
mtime: r.mtime,
authoredAt: r.authored_at,
contentHash: r.content_hash,
documentHash: r.document_hash,
});
}
return out;
}
/**
* One representative chunk per document - the lowest `chunk_index`,
* which for markdown is the document head (title / opening section).
* The traversal layer surfaces this when a linked document is not
* already a relevance hit.
*
* Projects `authored_at` (D1) on the join that already fetches `mtime`,
* so both producers of a {@link HydratedChunk} report the same freshness
* anchor. Today's consumers - traversal expansion, the graph pre-pass and
* the relational arm - read `path`, `content` and the ids, so this
* changes no current ranking; what it removes is the trap that a row from
* here silently answers the storage clock for a question about content
* age. The chunk and document hashes stay unprojected, for the reason
* given on {@link HydratedChunk.contentHash}.
*/
export function representativeChunks(
db: Database,
documentIds: ReadonlyArray<number>,
): Map<number, HydratedChunk> {
const out = new Map<number, HydratedChunk>();
if (documentIds.length === 0) return out;
const placeholders = sqlPlaceholders(documentIds);
const rows = db
.query<
{
chunk_id: number;
document_id: number;
path: string;
title: string | null;
content: string;
start_line: number;
end_line: number;
mtime: number;
authored_at: number | null;
},
number[]
>(
"SELECT c.id AS chunk_id, c.document_id AS document_id, d.path AS path, " +
"d.title AS title, c.content AS content, c.start_line AS start_line, " +
"c.end_line AS end_line, d.mtime AS mtime, d.authored_at AS authored_at " +
"FROM chunks c JOIN documents d ON d.id = c.document_id " +
`WHERE c.document_id IN (${placeholders}) ` +
"ORDER BY c.document_id, c.chunk_index ASC",
)
.all(...(documentIds as number[]));
for (const r of rows) {
if (out.has(r.document_id)) continue; // first row per doc = lowest chunk_index
out.set(
r.document_id,
Object.freeze({
chunkId: r.chunk_id,
documentId: r.document_id,
path: r.path,
title: r.title,
content: r.content,
startLine: r.start_line,
endLine: r.end_line,
mtime: r.mtime,
authoredAt: r.authored_at,
}),
);
}
return out;
}