Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The local per-model thresholds store (issue #264): file shape,
* append-only history, per-vault overrides, and the enforce gate keyed by
* the model id - never the provider preset - so a swapped model inherits
* nothing and switching back needs no re-tuning.
*/
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
decisionModelModeFor,
resolveDecisionModelConfig,
} from "../../../src/core/decision-model/config.ts";
import {
decisionThresholdsPath,
effectiveThresholds,
EMPTY_THRESHOLD_STORE,
parseThresholdsStore,
readThresholdsStore,
resetThresholdsStoreCache,
storedThresholdEntry,
writeThresholdEntry,
} from "../../../src/core/decision-model/thresholds.ts";
import {
fitThresholdAgainstTarget,
runThresholdFit,
} from "../../../src/core/decision-model/thresholds-fit.ts";
import { emitDecisionModelCall } from "../../../src/core/decision-model/record.ts";
function tempVault(): string {
const v = mkdtempSync(join(tmpdir(), "osb-dm-thr-vault-"));
extraDirs.push(v);
mkdirSync(join(v, "Brain", "preferences"), { recursive: true });
mkdirSync(join(v, "Brain", "retired"), { recursive: true });
return v;
}
/**
* Writes one pair's sides: both active in preferences, or `bRetired`
* moves b to the retired directory, which the report reads as merged.
*/
function preferencePair(vault: string, id: string, aActive: boolean, bRetired: boolean): void {
if (aActive) {
writeFileSync(
join(vault, "Brain", "preferences", `pref-${id}a.md`),
`---\nid: pref-${id}a\n---\n`,
);
}
if (bRetired) {
writeFileSync(join(vault, "Brain", "retired", `ret-${id}b.md`), `---\nid: ret-${id}b\n---\n`);
} else {
writeFileSync(
join(vault, "Brain", "preferences", `pref-${id}b.md`),
`---\nid: pref-${id}b\n---\n`,
);
}
}
let tmp: string;
let configPath: string;
let storePath: string;
const extraDirs: string[] = [];
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), "osb-dm-thr-"));
configPath = join(tmp, "config.yaml");
storePath = decisionThresholdsPath(configPath);
resetThresholdsStoreCache();
});
afterEach(() => {
rmSync(tmp, { recursive: true, force: true });
for (const d of extraDirs.splice(0)) rmSync(d, { recursive: true, force: true });
resetThresholdsStoreCache();
});
const COMPATIBLE_ENFORCE = {
decision_model_enabled: "true",
decision_model_provider: "compatible",
decision_model_base_url: "http://127.0.0.1:9",
decision_model_id: "self-hosted-1",
decision_model_env_key: "O2B_TEST_THR_KEY",
decision_model_uses: "dedup:enforce,tension:enforce,labels:enforce",
};
function resolveWithStore(config: Record<string, string>, vault: string | null = null) {
return resolveDecisionModelConfig({
env: { O2B_TEST_THR_KEY: "unused" } as NodeJS.ProcessEnv,
config,
vault,
thresholdsPath: storePath,
measuredTunedMark: null,
});
}
describe("the thresholds store file", () => {
test("a missing file reads as the empty store and gates everything off", () => {
expect(readThresholdsStore(storePath)).toEqual(EMPTY_THRESHOLD_STORE);
expect(existsSync(storePath)).toBe(false);
});
test("a malformed file reads as empty (fail closed), never throws", () => {
writeFileSync(storePath, "{not json", "utf8");
expect(readThresholdsStore(storePath)).toEqual(EMPTY_THRESHOLD_STORE);
expect(parseThresholdsStore("[]")).toEqual(EMPTY_THRESHOLD_STORE);
});
test("write then read round-trips; an entry keeps its history on rewrite", () => {
writeThresholdEntry(
storePath,
"self-hosted-1",
"dedup",
{ values: { low_priority_min: 0.7 }, source: "set" },
null,
new Date("2026-10-20T10:00:00Z"),
);
writeThresholdEntry(
storePath,
"self-hosted-1",
"dedup",
{ values: { low_priority_min: 0.9 }, source: "fit", target: 0.05, samples: 12 },
null,
new Date("2026-10-21T10:00:00Z"),
);
const store = readThresholdsStore(storePath);
const entry = storedThresholdEntry(store, "self-hosted-1", "dedup", null);
expect(entry?.values).toEqual({ low_priority_min: 0.9 });
expect(entry?.source).toBe("fit");
expect(entry?.target).toBe(0.05);
expect(entry?.samples).toBe(12);
expect(entry?.history).toEqual([
{ value: 0.7, source: "set", updated_at: "2026-10-20T10:00:00.000Z" },
]);
});
test("entries of every model ever tuned are kept; nothing is deleted on a switch", () => {
writeThresholdEntry(storePath, "model-a", "dedup", {
values: { low_priority_min: 0.8 },
source: "set",
});
writeThresholdEntry(storePath, "model-b", "dedup", {
values: { low_priority_min: 0.6 },
source: "set",
});
// The operator moves to model-b; model-a's entry survives untouched.
expect(
storedThresholdEntry(readThresholdsStore(storePath), "model-a", "dedup", null)?.values,
).toEqual({
low_priority_min: 0.8,
});
expect(
storedThresholdEntry(readThresholdsStore(storePath), "model-b", "dedup", null)?.values,
).toEqual({
low_priority_min: 0.6,
});
});
test("history is capped", () => {
for (let i = 0; i < 25; i++) {
writeThresholdEntry(
storePath,
"m",
"dedup",
{ values: { low_priority_min: 0.5 + i / 100 }, source: "set" },
null,
new Date(Date.UTC(2026, 9, 1, 0, 0, i)),
);
}
const entry = storedThresholdEntry(readThresholdsStore(storePath), "m", "dedup", null);
expect(entry?.history?.length).toBe(20);
});
test("a per-vault override wins for that vault and leaves the global entry", () => {
writeThresholdEntry(storePath, "m", "dedup", {
values: { low_priority_min: 0.8 },
source: "set",
});
writeThresholdEntry(
storePath,
"m",
"dedup",
{ values: { low_priority_min: 0.6 }, source: "set" },
"/vaults/work",
);
const store = readThresholdsStore(storePath);
expect(storedThresholdEntry(store, "m", "dedup", "/vaults/work")?.values).toEqual({
low_priority_min: 0.6,
});
expect(storedThresholdEntry(store, "m", "dedup", "/vaults/other")?.values).toEqual({
low_priority_min: 0.8,
});
expect(storedThresholdEntry(store, "m", "dedup", null)?.values).toEqual({
low_priority_min: 0.8,
});
});
test("effectiveThresholds merges stored values over the built-in defaults", () => {
expect(effectiveThresholds(EMPTY_THRESHOLD_STORE, "m", "dedup", null)).toEqual({
low_priority_min: 0.9,
});
writeThresholdEntry(storePath, "m", "dedup", {
values: { low_priority_min: 0.75 },
source: "fit",
});
expect(effectiveThresholds(readThresholdsStore(storePath), "m", "dedup", null)).toEqual({
low_priority_min: 0.75,
});
});
});
describe("the enforce gate keys on the model id", () => {
test("a stored entry arms enforce for its model; another model stays shadow", () => {
writeThresholdEntry(storePath, "self-hosted-1", "dedup", {
values: { low_priority_min: 0.85 },
source: "fit",
});
const tuned = resolveWithStore(COMPATIBLE_ENFORCE);
expect(tuned.status).toBe("active");
expect(tuned.shadowOnlyUses).toEqual(["tension", "labels"]);
expect(tuned.uses.dedup).toBe("enforce");
expect(decisionModelModeFor(tuned, "dedup")).toBe("enforce");
expect(decisionModelModeFor(tuned, "tension")).toBe("shadow");
// The same preset pointing at a different model inherits nothing.
const swapped = resolveWithStore({
...COMPATIBLE_ENFORCE,
decision_model_id: "self-hosted-2",
});
expect(swapped.status).toBe("active");
expect(swapped.shadowOnlyUses).toEqual(["dedup", "tension", "labels"]);
expect(decisionModelModeFor(swapped, "dedup")).toBe("shadow");
// Switching back restores the tune without re-tuning.
const back = resolveWithStore(COMPATIBLE_ENFORCE);
expect(back.uses.dedup).toBe("enforce");
});
test("the baseline family keeps enforcing with no store at all", () => {
const cfg = resolveDecisionModelConfig({
env: { TYPESAFE_API_KEY: "unused" } as NodeJS.ProcessEnv,
config: {
decision_model_enabled: "true",
decision_model_provider: "typesafe",
decision_model_uses: "dedup:enforce",
},
vault: null,
thresholdsPath: storePath,
measuredTunedMark: null,
});
expect(cfg.uses.dedup).toBe("enforce");
expect(cfg.shadowOnlyUses).toEqual([]);
});
test("the store arms a use even when no threshold profile resolves", () => {
// compatible carries no profile: before the store, every enforce use
// silently ran as shadow on it.
writeThresholdEntry(storePath, "self-hosted-1", "dedup", {
values: { low_priority_min: 0.85 },
source: "set",
});
const cfg = resolveWithStore(COMPATIBLE_ENFORCE);
expect(cfg.thresholdProfile).toBeNull();
expect(cfg.uses.dedup).toBe("enforce");
});
test("a per-vault override arms enforce for that vault only", () => {
writeThresholdEntry(
storePath,
"self-hosted-1",
"dedup",
{ values: { low_priority_min: 0.6 }, source: "set" },
"/vaults/work",
);
const work = resolveWithStore(COMPATIBLE_ENFORCE, "/vaults/work");
expect(work.uses.dedup).toBe("enforce");
const other = resolveWithStore(COMPATIBLE_ENFORCE, "/vaults/other");
expect(other.uses.dedup).toBe("shadow");
});
test("thresholdsPath: null disables the store entirely", () => {
writeThresholdEntry(storePath, "self-hosted-1", "dedup", {
values: { low_priority_min: 0.85 },
source: "set",
});
const cfg = resolveDecisionModelConfig({
env: { O2B_TEST_THR_KEY: "unused" } as NodeJS.ProcessEnv,
config: COMPATIBLE_ENFORCE,
vault: null,
thresholdsPath: null,
measuredTunedMark: null,
});
expect(cfg.uses.dedup).toBe("shadow");
});
});
describe("the threshold fit", () => {
test("sweeps to the threshold that holds the false-action target", () => {
const vault = tempVault();
// Eight true negatives at or above 0.9; two false actions below 0.9
// (a demoted pair that was later merged anyway).
for (let i = 0; i < 8; i++) {
preferencePair(vault, `o${i}`, true, false);
emitDecisionModelCall(vault, {
use: "dedup",
mode: "shadow",
provider: "fake",
model: "fit-model-1",
calibrated: true,
questionCount: 1,
candidateCount: 1,
inputPriceUsdPerMtok: null,
latencyMs: 1,
outcome: "ok",
details: {
pair_kind: "preference",
pairs: [
{
id: `o${i}`,
a: `pref-o${i}a`,
b: `pref-o${i}b`,
verdict: "different",
probability: 0.9 + i / 100,
},
],
},
});
}
for (const [i, p] of [0.6, 0.65].entries()) {
preferencePair(vault, `f${i}`, true, true); // one side retired: merged
emitDecisionModelCall(vault, {
use: "dedup",
mode: "shadow",
provider: "fake",
model: "fit-model-1",
calibrated: true,
questionCount: 1,
candidateCount: 1,
inputPriceUsdPerMtok: null,
latencyMs: 1,
outcome: "ok",
details: {
pair_kind: "preference",
pairs: [
{
id: `f${i}`,
a: `pref-f${i}a`,
b: `pref-f${i}b`,
verdict: "different",
probability: p,
},
],
},
});
}
const result = runThresholdFit(storePath, { vault, use: "dedup" });
expect(result.ok).toBe(true);
if (!result.ok) return;
expect(result.model).toBe("fit-model-1");
expect(result.name).toBe("low_priority_min");
// At 0.9 the demotions are the eight open pairs: zero false actions,
// the most action any feasible threshold in this data achieves.
expect(result.entry.values).toEqual({ low_priority_min: 0.9 });
expect(result.entry.source).toBe("fit");
expect(result.entry.target).toBe(0.05);
expect(result.entry.samples).toBe(8);
});
test("nothing is written when no threshold meets the target", () => {
const vault = tempVault();
preferencePair(vault, "t", true, true);
// Eight true negatives at 0.80-0.87 and one merged pair at 0.95: the
// merged pair sits inside no feasible window, so no threshold both
// acts on eight samples and keeps a 1% false share.
emitDecisionModelCall(vault, {
use: "dedup",
mode: "shadow",
provider: "fake",
model: "fit-model-1",
calibrated: true,
questionCount: 1,
candidateCount: 1,
inputPriceUsdPerMtok: null,
latencyMs: 1,
outcome: "ok",
details: {
pair_kind: "preference",
pairs: [{ id: "f", a: "pref-fa", b: "pref-fb", verdict: "different", probability: 0.95 }],
},
});
for (let i = 0; i < 8; i++) {
preferencePair(vault, `p${i}`, true, true);
emitDecisionModelCall(vault, {
use: "dedup",
mode: "shadow",
provider: "fake",
model: "fit-model-1",
calibrated: true,
questionCount: 1,
candidateCount: 1,
inputPriceUsdPerMtok: null,
latencyMs: 1,
outcome: "ok",
details: {
pair_kind: "preference",
pairs: [
{
id: `p${i}`,
a: `pref-p${i}a`,
b: `pref-p${i}b`,
verdict: "different",
probability: 0.8 + i / 100,
},
],
},
});
}
const before = existsSync(storePath) ? readThresholdsStore(storePath) : EMPTY_THRESHOLD_STORE;
const result = runThresholdFit(storePath, { vault, use: "dedup", target: 0.01 });
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.note).toContain("no threshold meets");
expect(readThresholdsStore(storePath)).toEqual(before);
});
test("records naming several models are refused: one model's calibration per entry", () => {
const vault = tempVault();
preferencePair(vault, "t", true, true);
for (const model of ["a", "b"]) {
emitDecisionModelCall(vault, {
use: "dedup",
mode: "shadow",
provider: "fake",
model,
calibrated: true,
questionCount: 1,
candidateCount: 1,
inputPriceUsdPerMtok: null,
latencyMs: 1,
outcome: "ok",
details: {
pair_kind: "preference",
pairs: [
{
id: `x-${model}`,
a: "pref-ta",
b: "pref-tb",
verdict: "different",
probability: 0.9,
},
],
},
});
}
const result = runThresholdFit(storePath, { vault, use: "dedup" });
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.note).toContain("several models");
expect(result.note).toContain("a");
expect(result.note).toContain("b");
});
test("a use without a labelled report is refused by name", () => {
const vault = tempVault();
const result = runThresholdFit(storePath, { vault, use: "rerank" });
expect(result.ok).toBe(false);
if (result.ok) return;
expect(result.note).toContain("no labelled report");
expect(result.note).toContain("dedup, tension, labels, extract_prefilter");
});
});
describe("the fit sweep itself", () => {
test("at_or_above keeps the most action under the target", () => {
const samples = [
...Array.from({ length: 8 }, (_, i) => ({ score: 0.9 + i / 100, falseAction: false })),
{ score: 0.6, falseAction: true },
];
const fit = fitThresholdAgainstTarget(samples, "at_or_above", 0.05);
expect(fit.ok).toBe(true);
if (!fit.ok) return;
expect(fit.candidate.threshold).toBe(0.9);
expect(fit.candidate.acted).toBe(8);
expect(fit.candidate.falseShare).toBe(0);
});
test("below acts under the threshold and drops nothing else", () => {
const samples = [
...Array.from({ length: 9 }, (_, i) => ({ score: 0.1 + i / 100, falseAction: false })),
{ score: 0.6, falseAction: true },
];
const fit = fitThresholdAgainstTarget(samples, "below", 0.05);
expect(fit.ok).toBe(true);
if (!fit.ok) return;
expect(fit.candidate.threshold).toBe(0.6);
expect(fit.candidate.acted).toBe(9);
expect(fit.candidate.falseActions).toBe(0);
});
test("too few samples refuse with the count", () => {
const fit = fitThresholdAgainstTarget(
[{ score: 0.9, falseAction: false }],
"at_or_above",
0.05,
);
expect(fit.ok).toBe(false);
if (fit.ok) return;
expect(fit.note).toContain("1 decidable sample(s)");
});
});