Local-first π§ memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly π΄ dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* F2 chunk skip-and-hold (Lane B, task B1): the embedding phase holds
* chunks whose token count PROVABLY exceeds the model's effective
* declared input window.
*
* "Provably" is the whole unit. The hold acts on the conservative bound
* (`tokenHoldFloor`: a code point charged at no more than a twelfth of a
* token), because the chars-over-four average is an average, not a bound
* - real tokenizers absorb 4.5-9 code points per token on prose, tables
* and code, so the average read whole English chunks that fit the window
* as over it and held them from embedding for nothing. Everything between
* that bound and the average estimate is the undecided band: reported by
* the window census, still embedded, exactly as before this feature. A
* model with no declared window (outside the curated preset table and no
* operator `embedding_input_window_tokens`) proves nothing, so nothing is
* held for it; declaring the window is what arms the hold.
*
* The hold is a FILTER before batching, so whatever the provider does
* receive still answers 1:1 (the batch invariant in `runEmbeddingPhase`
* compares vector counts against its own batch), and what is held never
* reaches `vecUpsert` - no embeddings row, no vector, no spend. A run
* that holds every pending chunk therefore also prices nothing: the cost
* gate reads the embeddable census, not the held one, so held work can
* neither trip a refusal nor appear on the receipt.
*/
import { afterEach, beforeEach, expect, test } from "bun:test";
import { Database } from "bun:sqlite";
import { runEmbeddingPhase, type EmbeddingPhaseTally } from "../../../src/core/search/indexer.ts";
import { Store } from "../../../src/core/search/store.ts";
import {
declaredInputWindowTokens,
RECOMMENDED_EMBEDDING_MODEL,
} from "../../../src/core/search/embeddings/presets.ts";
import { chunkHoldExclusion } from "../../../src/core/search/embedding-spend.ts";
import {
signatureIdentityKnown,
textExtent,
tokenHoldFloor,
} from "../../../src/core/search/embeddings/signature.ts";
import {
charLengthOverTokenBudget,
estimateTokens,
LOCAL_EMBEDDING_MODEL,
} from "../../../src/core/search/embeddings/signature.ts";
import { CHUNK_HOLD_REASON } from "../../../src/core/search/types.ts";
import type {
ResolvedEmbeddingConfig,
ResolvedSearchConfig,
} from "../../../src/core/search/types.ts";
import { createTempVault, makeConfig } from "../../helpers/search-fixtures.ts";
import { startFakeHttp, type FakeHttp } from "../../helpers/fake-http.ts";
import { FAKE_PROVIDER_KEY } from "../../helpers/fake-credentials.ts";
/** The curated window of the shipped recommendation, read from the table. */
const WINDOW = declaredInputWindowTokens(RECOMMENDED_EMBEDDING_MODEL)!;
/** A model string deliberately outside the curated preset table. */
const UNCURATED_MODEL = "acme/custom-embed-9000";
/** Comfortably under any window. */
const FITTING = "small chunk of ordinary prose";
/**
* An ordinary English sentence, assembled into fixtures of an exact
* code-point size. Prose, not a padded run: the hold's proof reads code
* points, and a fixture shaped like the text the phase actually sees is
* what keeps these tests honest about which chunks fit a real tokenizer.
*/
const PROSE_SENTENCE =
"The survey team recorded every measurement in the ledger before the storm reached the harbour wall. ";
function proseOfCodePoints(codePoints: number): string {
return PROSE_SENTENCE.repeat(Math.ceil(codePoints / PROSE_SENTENCE.length)).slice(0, codePoints);
}
/**
* The held fixture: even at the conservative bound - one token per
* twelve code points, far below what this prose costs a real tokenizer -
* it cannot fit the window. A run of dense ASCII would prove the same
* thing about the estimator and nothing about prose; this proves the
* hold on text shaped like the corpus.
*/
const OVER_CONTENT = proseOfCodePoints(WINDOW * 12 + 1);
const HELD_TOKENS = tokenHoldFloor(textExtent(OVER_CONTENT));
/**
* The undecided band, Latin half: ordinary English prose whose average
* estimate (`ceil(codePoints / 4)`) reads over the window while the
* conservative bound clears it comfortably. The estimator this feature
* shipped with HELD this chunk; real tokenizers cost it far fewer tokens
* than the average claims, and it is embedded.
*/
const BAND_CONTENT = proseOfCodePoints(WINDOW * 6);
/**
* The undecided band, non-Latin half: the conservative bound fits (the
* hold never acts on it) while the ceiling charges each Cyrillic code
* point a full token (far over). The census reports this band; the hold
* must never act on it.
*/
const UNDECIDED_CONTENT = "Π΅".repeat(charLengthOverTokenBudget(WINDOW));
let vault: string;
let dbPath: string;
let cleanup: () => void;
let server: FakeHttp | null = null;
beforeEach(() => {
({ vault, dbPath, cleanup } = createTempVault("embed-window-hold"));
});
afterEach(async () => {
await server?.close();
server = null;
cleanup();
});
/**
* Start the fake provider capturing every input text it receives and
* answering exactly one 4-dimensional vector per input.
*/
async function captureProvider(): Promise<string[]> {
server = await startFakeHttp();
const sent: string[] = [];
server.setHandler((req) => {
const body = (req.body ?? {}) as { input?: unknown };
const inputs = Array.isArray(body.input) ? (body.input as string[]) : [];
sent.push(...inputs);
return {
status: 200,
body: {
data: inputs.map((text, index) => ({
object: "embedding",
embedding: [text.length, index, 1, 1],
index,
})),
model: "fake",
},
};
});
return sent;
}
function cfg(semantic: Partial<ResolvedEmbeddingConfig> = {}): ResolvedSearchConfig {
return makeConfig({
vault,
dbPath,
semantic: {
enabled: true,
provider: "openai-compat",
// Never contacted by the local embedder; only the openai-compat
// tests point it at the fake server.
baseUrl: server?.url ?? "https://embeddings.invalid/v1",
model: RECOMMENDED_EMBEDDING_MODEL,
apiKey: FAKE_PROVIDER_KEY,
...semantic,
},
});
}
/** A write-mode store seeded with one document carrying `contents`. */
async function openSeeded(config: ResolvedSearchConfig, contents: string[]): Promise<Store> {
const store = await Store.open(config, { mode: "write" });
const docId = store.upsertDocument({
path: "Brain/note.md",
title: "note",
contentHash: "h1",
mtime: 1_700_000_000,
size: 10,
});
store.replaceChunks(
docId,
contents.map((content, i) => ({
chunkIndex: i,
content,
contentHash: `h1-${i}`,
startLine: 1,
endLine: 2,
tokenCount: content.length,
})),
);
return store;
}
/** The chunk row of exactly `length` code points (SQL counts code points). */
function chunkIdByLength(length: number): number {
const db = new Database(dbPath, { readonly: true });
try {
const row = db
.query<{ id: number }, [number]>("SELECT id FROM chunks WHERE length(content) = ?")
.get(length);
if (!row) throw new Error(`no chunk of length ${length} in the seeded index`);
return row.id;
} finally {
db.close();
}
}
/** Chunk ids that carry a stored vector after the run. */
function embeddedChunkIds(): number[] {
const db = new Database(dbPath, { readonly: true });
try {
return db
.query<{ chunk_id: number }, []>("SELECT chunk_id FROM embeddings ORDER BY chunk_id")
.all()
.map((r) => r.chunk_id);
} finally {
db.close();
}
}
const EMPTY_TALLY = (): EmbeddingPhaseTally => ({ embeddingsComputed: 0, embeddingsRetries: 0 });
// ββ the hold filter ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
test("a provably-over chunk is never sent while fitting and undecided-band chunks are", async () => {
const sent = await captureProvider();
const store = await openSeeded(cfg(), [FITTING, OVER_CONTENT, BAND_CONTENT, UNDECIDED_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, cfg(), tally, {});
} finally {
await store.close();
}
expect(sent).toContain(FITTING);
expect(sent).toContain(BAND_CONTENT);
expect(sent).toContain(UNDECIDED_CONTENT);
expect(sent).not.toContain(OVER_CONTENT);
// One vector per input survived the filter: the phase would have thrown
// on a count mismatch, and the three texts it did send are stored.
expect(tally.embeddingsComputed).toBe(3);
const heldId = chunkIdByLength(OVER_CONTENT.length);
expect(embeddedChunkIds()).not.toContain(heldId);
});
test("held prose reaches no vector store: embeddings rows exactly match what was sent", async () => {
await captureProvider();
const store = await openSeeded(cfg(), [FITTING, OVER_CONTENT, BAND_CONTENT, UNDECIDED_CONTENT]);
try {
await runEmbeddingPhase(store, cfg(), EMPTY_TALLY(), {});
} finally {
await store.close();
}
const ids = embeddedChunkIds().toSorted();
expect(ids).toEqual(
[
chunkIdByLength(FITTING.length),
chunkIdByLength(BAND_CONTENT.length),
chunkIdByLength(UNDECIDED_CONTENT.length),
].toSorted(),
);
});
test("English prose the average estimate would hold fits the window and is embedded", async () => {
// The defect that moved the hold to a conservative bound: this chunk's
// chars-over-four estimate reads over the window - the old trigger -
// while the conservative bound clears it, and real tokenizers cost
// prose far fewer tokens than the average claims. Holding it was
// semantic recall lost on an arithmetic guess.
expect(estimateTokens([BAND_CONTENT])).toBeGreaterThan(WINDOW);
expect(tokenHoldFloor(textExtent(BAND_CONTENT))).toBeLessThanOrEqual(WINDOW);
const sent = await captureProvider();
const store = await openSeeded(cfg(), [BAND_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, cfg(), tally, {});
} finally {
await store.close();
}
expect(sent).toContain(BAND_CONTENT);
expect(tally.embeddingsHeld).toBeUndefined();
expect(tally.embeddingsComputed).toBe(1);
});
// ββ the tally ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
test("the phase records each held chunk with its bound, the window and the reason", async () => {
await captureProvider();
const store = await openSeeded(cfg(), [FITTING, OVER_CONTENT, BAND_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, cfg(), tally, {});
} finally {
await store.close();
}
expect(tally.embeddingsHeld).toBe(1);
expect(tally.heldChunks).toEqual([
{
chunkId: chunkIdByLength(OVER_CONTENT.length),
estimatedTokens: HELD_TOKENS,
windowTokens: WINDOW,
reason: CHUNK_HOLD_REASON.overDeclaredWindow,
},
]);
});
test("a run that holds nothing records no hold tally at all", async () => {
await captureProvider();
const store = await openSeeded(cfg(), [FITTING]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, cfg(), tally, {});
} finally {
await store.close();
}
expect(tally.embeddingsHeld).toBeUndefined();
expect(tally.heldChunks).toBeUndefined();
});
// ββ holding is not spend βββββββββββββββββββββββββββββββββββββββββββββββββββββ
test("everything held: no provider call, no receipt, no refusal under a gate the held census would blow", async () => {
const sent = await captureProvider();
// Priced so OVER_CONTENT's floor alone exceeds the gate: pre-feature
// this census tripped EMBEDDING_COST_GATE, which is the false alarm a
// held chunk must never cause.
const config = cfg({
costGateUsd: 0.000001,
priceOverride: { model: RECOMMENDED_EMBEDDING_MODEL, usdPerMtok: 0.05 },
});
const store = await openSeeded(config, [OVER_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, config, tally, {});
} finally {
await store.close();
}
expect(sent).toEqual([]);
expect(server!.callCount()).toBe(0);
expect(tally.embeddingsComputed).toBe(0);
expect(tally.embeddingsHeld).toBe(1);
expect(tally.spend).toBeUndefined();
});
test("the gate and the receipt read the embeddable census, not the held one", async () => {
await captureProvider();
const config = cfg({
costGateUsd: 0.000001,
priceOverride: { model: RECOMMENDED_EMBEDDING_MODEL, usdPerMtok: 0.05 },
});
// The fitting chunk's own priced estimate stays under the gate; the
// held chunk would have taken the run's estimate over it.
const store = await openSeeded(config, [FITTING, OVER_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, config, tally, {});
} finally {
await store.close();
}
expect(tally.embeddingsHeld).toBe(1);
expect(tally.spend).toBeDefined();
expect(tally.spend!.tokens).toBe(estimateTokens([FITTING]));
expect(tally.spend!.forced).toBe(false);
});
// ββ the hold only acts on proof ββββββββββββββββββββββββββββββββββββββββββββββ
test("an undeclared window proves nothing: an oversized chunk is embedded as before", async () => {
const sent = await captureProvider();
const config = cfg({ model: UNCURATED_MODEL });
const store = await openSeeded(config, [OVER_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, config, tally, {});
} finally {
await store.close();
}
expect(sent).toContain(OVER_CONTENT);
expect(tally.embeddingsHeld).toBeUndefined();
expect(tally.embeddingsComputed).toBe(1);
});
test("an operator-declared window arms the hold even for an uncurated model", async () => {
const sent = await captureProvider();
const config = cfg({ model: UNCURATED_MODEL, inputWindowTokens: WINDOW });
const store = await openSeeded(config, [OVER_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, config, tally, {});
} finally {
await store.close();
}
expect(sent).not.toContain(OVER_CONTENT);
expect(tally.embeddingsHeld).toBe(1);
expect(tally.embeddingsComputed).toBe(0);
});
test("the hold key changes when the sent passage prefix changes", () => {
// The hold's proof is measured over prefix + content, so the key the
// proof is recorded under has to answer for the prefix too: a shortened
// or removed prefix changes what fits the window, and a key that cannot
// see the change would keep excluding a chunk that now fits.
const withPrefix = cfg({ passagePrefix: "passage: ", dimension: 4 });
const shortened = cfg({ passagePrefix: "", dimension: 4 });
const otherPrefix = cfg({ passagePrefix: "represent passage: ", dimension: 4 });
const a = chunkHoldExclusion(withPrefix)!;
const b = chunkHoldExclusion(shortened)!;
const c = chunkHoldExclusion(otherPrefix)!;
expect(a.embeddingSignature).not.toBe(b.embeddingSignature);
expect(a.embeddingSignature).not.toBe(c.embeddingSignature);
// The composed key still reads as a signature from the end: the
// trailing provider:model:dimension triple is intact.
expect(signatureIdentityKnown(a.embeddingSignature)).toBe(true);
expect(signatureIdentityKnown(b.embeddingSignature)).toBe(true);
});
test("the local embedder has no window and holds nothing", async () => {
const config = cfg({ provider: "local", model: LOCAL_EMBEDDING_MODEL });
const store = await openSeeded(config, [OVER_CONTENT]);
const tally = EMPTY_TALLY();
try {
await runEmbeddingPhase(store, config, tally, {});
} finally {
await store.close();
}
expect(tally.embeddingsHeld).toBeUndefined();
expect(tally.embeddingsComputed).toBe(1);
});