Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Operator diagnostics for the decision-model feature: the data behind
* `o2b decision-model check`, `o2b decision-model report` and the doctor
* readiness line.
*
* These are the ONLY surfaces that say anything about a missing key or an
* incomplete configuration. Every other surface behaves exactly as it does
* without the feature. Nothing here ever prints a key value: the check
* names the variable and says whether it is set.
*/
import { CHOICE_MAX_OPTIONS } from "./answers.ts";
import { DECISION_MODEL_USES, isDecisionModelUse, type DecisionModelUse } from "./contract.ts";
import {
resolveDecisionModelConfig,
resolvedProviderEntries,
type ResolvedDecisionModelConfig,
} from "./config.ts";
import { makeEntryDecisionProvider } from "./provider.ts";
import { DECISION_MODEL_UNPRICED_ADAPTERS } from "./presets.ts";
import type { DecisionProvider } from "./contract.ts";
import { SecretReferenceError } from "../secret-ref.ts";
import {
SecretStoreKeyfileMissingError,
SecretStoreLockedError,
} from "../brain/secrets/envelope.ts";
import {
DECISION_MODEL_CALL_KIND,
emitDecisionModelCall,
listDecisionModelCalls,
listDecisionModelEvalGrades,
todaySpendUsd,
type DecisionEvalGrade,
} from "./record.ts";
import type { ContinuityRecord } from "../brain/continuity/types.ts";
import { DECISION_THRESHOLD_PROFILES } from "./questions.ts";
import { decisionEvalSpecFor, decisionEvalSpecHash, type DecisionEvalSpec } from "./eval-spec.ts";
import { validatedTuningMark } from "./eval/mark.ts";
import { processDirsEnv, type PlatformDirsEnv } from "../platform-dirs.ts";
import { USE_REPORTS } from "./reports/index.ts";
/**
* Warnings the vault's `_brain.yaml` parser raised about its
* `decision_model:` block (for example `enabled: true`, which a vault may
* not set). Empty when there is no vault or no such warning.
*/
export function vaultDecisionModelWarnings(vault: string | null): string[] {
if (vault === null) return [];
// Lazy, as in the config resolver: the policy loader is large.
// eslint-disable-next-line @typescript-eslint/no-require-imports
const policy = require("../brain/policy.ts") as typeof import("../brain/policy.ts");
try {
return policy
.loadBrainConfigDetailed(vault)
.warnings.map((w) => w.message)
.filter((m) => m.startsWith("decision_model"));
} catch {
return [];
}
}
export interface DecisionModelCheckReport {
readonly enabled: boolean;
readonly status: ResolvedDecisionModelConfig["status"];
readonly provider: string | null;
/**
* The adapter the preset names (`systemone`, `vercel-evaluate`,
* `llm-emulation`, `converse`).
*/
readonly adapter: string;
readonly base_url: string | null;
readonly model: string | null;
/**
* The ordered provider chain (`decision_model_providers`), first entry
* first; the top-level provider fields are the FIRST entry's. With
* `--ping`, each entry also carries its own live `ping` result.
*/
readonly provider_order: ReadonlyArray<DecisionProviderCheck>;
/** NAME of the key variable, never its value. */
readonly env_key: string | null;
readonly key_set: boolean;
/** False only for a loopback self-hosted server, which may run without a key. */
readonly key_required: boolean;
/** Whether the provider's probabilities are calibrated. */
readonly calibrated: boolean;
/** The threshold profile answers are held to; null when unknown (no use enforces). */
readonly threshold_profile: string | null;
/** Uses configured `enforce` that run as `shadow` for want of tuned thresholds. */
readonly shadow_only_uses: ReadonlyArray<DecisionModelUse>;
/**
* The measured tuning state of the resolved threshold profile: what the
* source table claims, and which (profile, use) pairs a valid
* machine-side mark covers (`eval/mark.ts`).
*/
readonly tuning: MeasuredTuning;
/** Cached preregistered-eval verdicts, per use, with the skip notes. */
readonly eval_grades: {
readonly total: number;
readonly skipped: ReadonlyArray<string>;
readonly uses: ReadonlyArray<{
readonly use: string;
readonly runs: number;
readonly last_verdict: string | null;
readonly last_at: string | null;
readonly last_spec_hash: string | null;
}>;
};
/** Most options one `choice` question may carry on this route. */
readonly max_choice_options: number;
readonly output_price_usd_per_mtok: number | null;
/** A model licence restriction, when the route's licence has one. */
readonly licence_note: string | null;
/**
* The uses as configured. While the feature is not active every use
* runs `off` whatever this says; `status` tells which applies.
*/
readonly uses: Readonly<Record<DecisionModelUse, string>>;
/** True when some use is in shadow or enforce: requests carry vault text. */
readonly sends_data: boolean;
readonly vault: string | null;
readonly vault_opt_out: boolean;
readonly timeout_ms: number;
readonly max_state_tokens: number;
readonly cost_gate_usd: number;
readonly today_spend_usd: number | null;
readonly input_price_usd_per_mtok: number | null;
readonly processor: string | null;
readonly rerank: {
readonly kind_configured: boolean;
/** `search_rerank_*` endpoint keys that are set but ignored for this kind. */
readonly ignored_keys: ReadonlyArray<string>;
};
readonly errors: ReadonlyArray<string>;
readonly notes: ReadonlyArray<string>;
readonly warnings: ReadonlyArray<string>;
/** How to proceed, when something is missing. */
readonly hint: string | null;
readonly ping?: {
readonly ok: boolean;
readonly model?: string;
readonly latency_ms?: number;
readonly reason?: string;
};
}
export interface DecisionModelCheckOptions {
readonly config: Readonly<Record<string, string>>;
readonly vault: string | null;
readonly env?: NodeJS.ProcessEnv;
readonly ping?: boolean;
readonly now?: Date;
/**
* Where the machine-side tuning marks are read from. Defaults to the
* running process's platform dirs; tests inject a state root.
*/
readonly markSource?: PlatformDirsEnv | null;
/**
* Where the local thresholds store is read from. Undefined reads the
* file beside the default config path (the caller's `env` decides
* where that is); null disables the store, which is what the tests
* that pin the shadow hold use.
*/
readonly thresholdsPath?: string | null;
}
/** One chain entry as `check` reports it, in configured order. */
export interface DecisionProviderCheck {
readonly name: string;
readonly adapter: string;
readonly base_url: string | null;
readonly model: string | null;
/** NAME of the entry's key variable, never its value. */
readonly env_key: string | null;
readonly key_set: boolean;
readonly key_required: boolean;
/** Whether the entry would be tried: it has its key, or needs none. */
readonly configured: boolean;
readonly calibrated: boolean;
/** Present only for a `check --ping` of a configured entry. */
readonly ping?: {
readonly ok: boolean;
readonly model?: string;
readonly latency_ms?: number;
readonly reason?: string;
};
}
const RERANK_ENDPOINT_KEYS = [
"search_rerank_base_url",
"search_rerank_model",
"search_rerank_env_key",
] as const;
// ----- the measured tuning state ----------------------------------------------
/** One (use, spec hash) pair a valid tuning mark covers. */
export interface MarkedUse {
readonly use: DecisionModelUse;
readonly spec_hash: string;
}
export interface MeasuredTuning {
/** The resolved threshold profile; null when none resolves. */
readonly profile: string | null;
/**
* `none`: no profile. `baseline`: the profile ships tuned for every
* use. `hand`: the source table carries hand-registered uses.
* `measured`: at least one pair carries a valid machine-side mark.
* `measured_elsewhere`: a mark exists but is bound to another endpoint,
* so enforce holds as shadow here. `untuned`: nothing tuned, so
* enforce holds as shadow.
*/
readonly state: "none" | "baseline" | "hand" | "measured" | "measured_elsewhere" | "untuned";
readonly marked_uses: ReadonlyArray<MarkedUse>;
/**
* Pairs whose mark passes every acceptance axis but is bound to a
* different endpoint (provider, model or base URL): the gate holds
* them shadow on this one. Never armed, and never counted in
* {@link marked_uses}.
*/
readonly elsewhere_uses: ReadonlyArray<MarkedUse>;
}
/**
* The measured tuning state of the resolved profile: the source table
* decides first (baseline, or a hand-registered list), and only the
* pairs the table leaves untuned can be carried by a valid machine-side
* mark. Marks are checked against THIS config's endpoint - the same
* provider, model and base URL the resolver's gate reader binds - so a
* promote measured on one model never reads as measured for another.
* The same vocabulary and the same short spec hash render into
* `check`, the report and the doctor probe, so the surfaces cannot
* disagree about what "tuned" means here.
*/
export function measuredTuning(
cfg: ResolvedDecisionModelConfig,
source: PlatformDirsEnv = processDirsEnv(),
): MeasuredTuning {
const profile = cfg.thresholdProfile ?? null;
// A profile name the source table does not carry (a self-hosted family
// the code never registered, issue #264) is not "none": its marks are
// keyed by the same string, so the measured states still apply to it.
if (profile === null) {
return { profile, state: "none", marked_uses: [], elsewhere_uses: [] };
}
const tuned = DECISION_THRESHOLD_PROFILES[profile]?.tuned ?? [];
if (tuned === "baseline") {
return { profile, state: "baseline", marked_uses: [], elsewhere_uses: [] };
}
// The same endpoint shape the resolver's gate reader binds
// (config.ts): a mark measured elsewhere never arms enforce here, so
// it must never read as measured here either.
const endpoint = { provider: cfg.provider, model: cfg.model, baseUrl: cfg.baseUrl };
const marked: MarkedUse[] = [];
const elsewhere: MarkedUse[] = [];
for (const use of DECISION_MODEL_USES) {
const spec: DecisionEvalSpec | null = decisionEvalSpecFor(use);
if (spec === null) continue;
if (validatedTuningMark(profile, use, source, spec, Date.now(), endpoint) !== null) {
marked.push({ use, spec_hash: decisionEvalSpecHash(spec) });
} else if (validatedTuningMark(profile, use, source, spec) !== null) {
// Passes every axis but the endpoint: measured, but not here.
elsewhere.push({ use, spec_hash: decisionEvalSpecHash(spec) });
}
}
if (marked.length > 0) {
return { profile, state: "measured", marked_uses: marked, elsewhere_uses: elsewhere };
}
if (elsewhere.length > 0) {
return { profile, state: "measured_elsewhere", marked_uses: [], elsewhere_uses: elsewhere };
}
if (Array.isArray(tuned) && tuned.length > 0) {
return { profile, state: "hand", marked_uses: [], elsewhere_uses: [] };
}
return { profile, state: "untuned", marked_uses: [], elsewhere_uses: [] };
}
/** The short form the prose surfaces print: first twelve hex characters. */
export function shortSpecHash(hash: string): string {
return `${hash.slice(0, 12)}…`;
}
/** The one tuning sentence `check`, `report` and the doctor probe share. */
export function measuredTuningSummary(measured: MeasuredTuning): string {
switch (measured.state) {
case "none":
return "no threshold profile resolves, so the source table alone gates enforce";
case "baseline":
return `profile '${measured.profile}' is the baseline`;
case "hand":
return `profile '${measured.profile}' carries hand-registered tuned uses`;
case "measured": {
const parts = measured.marked_uses
.map((m) => `${shortSpecHash(m.spec_hash)} (${m.use})`)
.join(", ");
const sentence = `profile '${measured.profile}' measured under spec hash ${parts}`;
if (measured.elsewhere_uses.length === 0) return sentence;
const other = measured.elsewhere_uses
.map((m) => `${shortSpecHash(m.spec_hash)} (${m.use})`)
.join(", ");
return (
sentence +
`; a mark for ${other} was measured on another endpoint ` +
"(provider, model or base-url mismatch) and holds shadow here"
);
}
case "measured_elsewhere": {
const parts = measured.elsewhere_uses
.map((m) => `${shortSpecHash(m.spec_hash)} (${m.use})`)
.join(", ");
return (
`profile '${measured.profile}' measured under spec hash ${parts} on another ` +
"endpoint (provider, model or base-url mismatch); enforce holds as shadow"
);
}
case "untuned":
return (
`profile '${measured.profile}' is untuned; enforce holds as shadow ` +
"(o2b decision-model eval run can measure a use, or " +
"o2b decision-model thresholds fit can tune one from shadow data)"
);
}
}
function hintFor(cfg: ResolvedDecisionModelConfig): string | null {
switch (cfg.status) {
case "disabled": {
// Name only what is missing.
const missing = ['decision_model_enabled: "true"'];
if (cfg.provider === null) missing.push("a decision_model_provider");
if (DECISION_MODEL_USES.every((use) => cfg.configuredUses[use] === "off")) {
missing.push('one use in shadow (for example decision_model_uses: "rerank:shadow")');
}
const list =
missing.length === 1
? missing[0]!
: `${missing.slice(0, -1).join(", ")} and ${missing[missing.length - 1]!}`;
return `the feature is off; to try it, set ${list} in the machine config`;
}
case "no_key":
return cfg.envKey === null
? "no key variable is named: set decision_model_env_key to the NAME of an " +
"environment variable that holds the key"
: `set the environment variable ${cfg.envKey} to the key in the environment that runs ` +
"Open Second Brain (the config holds only the variable's name, never the key)";
case "disabled_by_vault":
return "this vault opted out in Brain/_brain.yaml (decision_model: { enabled: false })";
case "invalid":
return "fix the config errors above; until then every use stays off";
case "active":
return null;
}
}
export async function buildDecisionModelCheck(
opts: DecisionModelCheckOptions,
): Promise<DecisionModelCheckReport> {
const env = opts.env ?? process.env;
// The vault's own custody store backs a `$secret:` reference in the key
// value: without this thread the check reported `active` for a reference
// it could not resolve, deferring the failure to the use site - and the
// ping below built a provider whose Bearer key was the raw `$secret:`
// literal.
const cfg = resolveDecisionModelConfig({
env,
config: opts.config,
vault: opts.vault,
secretsVault: opts.vault ?? undefined,
...(opts.thresholdsPath !== undefined ? { thresholdsPath: opts.thresholdsPath } : {}),
});
const warnings: string[] = [];
const notes: string[] = [];
const sendsData = DECISION_MODEL_USES.some((use) => cfg.uses[use] !== "off");
if (cfg.enabled && sendsData) {
warnings.push(
"shadow and enforce both send data: each request carries the masked, clipped, " +
"redacted text of the candidates to the processor named above",
);
}
if (!cfg.enabled && DECISION_MODEL_USES.some((use) => cfg.configuredUses[use] !== "off")) {
warnings.push("decision_model_enabled is not true, so every use listed above runs off");
}
for (const w of vaultDecisionModelWarnings(opts.vault)) warnings.push(`vault: ${w}`);
if (cfg.enabled && !sendsData) {
warnings.push("no use is in shadow or enforce, so nothing is ever sent");
}
let todaySpend: number | null = null;
if (cfg.enabled && opts.vault !== null) {
try {
todaySpend = todaySpendUsd(opts.vault, opts.now ?? new Date());
} catch {
todaySpend = null;
}
}
// The one place an enforce that runs as shadow is reported: hot paths
// stay silent. Printed once per check.
const shadowOnly = cfg.shadowOnlyUses ?? [];
if (cfg.enabled && shadowOnly.length > 0) {
const profileNote =
cfg.thresholdProfile === null || cfg.thresholdProfile === undefined
? ""
: ` (profile '${cfg.thresholdProfile}')`;
warnings.push(
`enforce runs as shadow for ${shadowOnly.join(", ")}: model ` +
`'${cfg.model ?? "unknown"}'${profileNote} has no tuned thresholds for ` +
(shadowOnly.length === 1 ? "it" : "them") +
`; tune with \`o2b decision-model thresholds fit --use ${shadowOnly[0]!}\` against ` +
"this model's shadow data, or set values by hand with " +
"`o2b decision-model thresholds set`",
);
}
if (cfg.enabled && !cfg.calibrated) {
warnings.push(
`provider '${cfg.provider ?? "?"}' is uncalibrated: a generative model reports its own ` +
"probabilities, every record carries calibrated: false, and each request costs far " +
"more than a decision model; use it for shadow evaluation only",
);
}
if (cfg.enabled && cfg.keyRequired === false && !cfg.keyPresent) {
notes.push("no key is set; requests to this loopback server carry no authorization header");
}
// The routes whose presets publish no price (see
// DECISION_MODEL_UNPRICED_ADAPTERS): until the operator sets one, the
// gate records nothing for the route's calls - a stronger under-count
// than the generic unknown-price warning below describes. Both
// chat-shaped routes read an output price too, so each warning names
// both settings; `llm-emulation` alone may still volunteer a cost, so
// its wording keeps the reported-cost escape hatch.
const unpricedRoute = DECISION_MODEL_UNPRICED_ADAPTERS.includes(cfg.adapter);
if (cfg.enabled && cfg.inputPriceUsdPerMtok === null && unpricedRoute) {
warnings.push(
cfg.adapter === "converse"
? "the daily cost gate counts none of converse's calls (the gateway's cost field is not " +
"trusted) until both prices are set: decision_model_input_price_usd_per_mtok and " +
"decision_model_output_price_usd_per_mtok"
: "no prices are set for llm-emulation, so the daily cost gate counts only requests whose " +
"route reports a cost; set decision_model_input_price_usd_per_mtok and " +
"decision_model_output_price_usd_per_mtok",
);
}
if (cfg.enabled && cfg.inputPriceUsdPerMtok === null && !unpricedRoute) {
warnings.push(
"no input price is known for this provider, so the daily cost gate applies only to " +
"routes that report cost; set decision_model_input_price_usd_per_mtok",
);
}
// The measured tuning state: read only when the feature is enabled and
// a profile resolves - the same condition under which the resolver
// consulted the marks - so a disabled install pays no file reads.
const tuning = cfg.enabled
? measuredTuning(
cfg,
opts.markSource === null ? processDirsEnv() : (opts.markSource ?? processDirsEnv()),
)
: {
profile: cfg.thresholdProfile ?? null,
state: "none" as const,
marked_uses: [],
elsewhere_uses: [],
};
// The cached eval verdicts are review evidence for the mark; a check
// that named the measured state without them would be unauditable.
const gradeUses: {
use: string;
runs: number;
last_verdict: string | null;
last_at: string | null;
last_spec_hash: string | null;
}[] = [];
let gradeTotal = 0;
const gradeSkipped: string[] = [];
if (cfg.enabled && opts.vault !== null) {
try {
const { grades, skipped } = listDecisionModelEvalGrades(opts.vault);
gradeTotal = grades.length;
gradeSkipped.push(...skipped);
const byUse = new Map<string, DecisionEvalGrade[]>();
for (const grade of grades) {
const list = byUse.get(grade.use) ?? [];
list.push(grade);
byUse.set(grade.use, list);
}
for (const [use, list] of [...byUse].toSorted((a, b) => a[0].localeCompare(b[0]))) {
const last = list[list.length - 1]!;
gradeUses.push({
use,
runs: list.length,
last_verdict: last.verdict,
last_at: last.createdAt,
last_spec_hash: last.specHash,
});
}
} catch {
// An unreadable log is reported as no cached grades, the same
// fail-soft direction the spend probe above takes.
}
}
const kindRaw = env["OPEN_SECOND_BRAIN_SEARCH_RERANK_KIND"] || opts.config["search_rerank_kind"];
const kindConfigured = kindRaw === "decision-model";
const ignored = kindConfigured
? RERANK_ENDPOINT_KEYS.filter((key) => {
const value = opts.config[key];
return value !== undefined && value !== "";
})
: [];
const report: DecisionModelCheckReport = {
enabled: cfg.enabled,
status: cfg.status,
provider: cfg.provider,
adapter: cfg.adapter,
base_url: cfg.baseUrl,
model: cfg.model,
provider_order: resolvedProviderEntries(cfg).map((entry): DecisionProviderCheck => ({
name: entry.name,
adapter: entry.adapter,
base_url: entry.baseUrl,
model: entry.model,
env_key: entry.envKey,
key_set: entry.keyPresent,
key_required: entry.keyRequired,
configured: entry.keyPresent || !entry.keyRequired,
calibrated: entry.calibrated,
})),
env_key: cfg.envKey,
key_set: cfg.keyPresent,
key_required: cfg.keyRequired !== false,
calibrated: cfg.calibrated,
threshold_profile: cfg.thresholdProfile ?? null,
shadow_only_uses: shadowOnly,
tuning,
eval_grades: { total: gradeTotal, skipped: gradeSkipped, uses: gradeUses },
max_choice_options: cfg.maxChoiceOptions ?? CHOICE_MAX_OPTIONS,
output_price_usd_per_mtok: cfg.outputPriceUsdPerMtok ?? null,
licence_note: cfg.licenceNote ?? null,
uses: cfg.configuredUses,
sends_data: cfg.status === "active" && sendsData,
vault: opts.vault,
vault_opt_out: cfg.status === "disabled_by_vault",
timeout_ms: cfg.timeoutMs,
max_state_tokens: cfg.maxStateTokens,
cost_gate_usd: cfg.dailyCostGateUsd,
today_spend_usd: todaySpend,
input_price_usd_per_mtok: cfg.inputPriceUsdPerMtok,
processor: cfg.processor,
rerank: { kind_configured: kindConfigured, ignored_keys: ignored },
errors: cfg.errors,
notes: [...cfg.notes, ...notes],
warnings,
hint: hintFor(cfg),
};
if (opts.ping !== true) return report;
// The ping resolves each key the way the real request path does (the
// factory re-resolves at call time). A store that locks between the
// config probe and here - or whose keyfile is missing while entries
// survive - raises the named refusal, degraded into that entry's ping
// reason, never sent as a Bearer key and never left to escape the
// check as an exception.
const order = [...report.provider_order];
const entries = resolvedProviderEntries(cfg);
let firstPing: DecisionModelCheckReport["ping"] | undefined;
let anyConfigured = false;
for (const [index, entry] of entries.entries()) {
const slot = order[index];
if (slot === undefined || !slot.configured) continue;
anyConfigured = true;
let adapter: DecisionProvider | null;
try {
adapter = makeEntryDecisionProvider(
entry,
{ allowInsecureHttp: cfg.allowInsecureHttp, timeoutMs: cfg.timeoutMs },
env,
opts.vault ?? undefined,
);
} catch (err) {
if (
err instanceof SecretStoreLockedError ||
err instanceof SecretStoreKeyfileMissingError ||
err instanceof SecretReferenceError
) {
const ping = { ok: false, reason: `not sent: ${err.message}` };
order[index] = { ...slot, ping };
if (firstPing === undefined) firstPing = ping;
continue;
}
throw err;
}
if (adapter === null) continue;
// eslint-disable-next-line no-await-in-loop -- the order is the point: one ping per entry, in order
const pong = await adapter.ping();
// A ping is a real request: record it (origin `ping`), so it counts
// toward the daily gate and shows in the report. Each entry's ping
// names its own provider and answering model, so the records never
// pool two routes.
emitDecisionModelCall(cfg.vault, {
use: "ping",
mode: "shadow",
provider: adapter.name,
model: pong.model ?? adapter.model,
calibrated: adapter.calibrated,
questionCount: 1,
candidateCount: 0,
...(pong.usage !== undefined ? { usage: pong.usage } : {}),
inputPriceUsdPerMtok: entry.inputPriceUsdPerMtok,
...(entry.outputPriceUsdPerMtok !== undefined && entry.outputPriceUsdPerMtok !== null
? { outputPriceUsdPerMtok: entry.outputPriceUsdPerMtok }
: {}),
latencyMs: pong.latencyMs ?? 0,
outcome: pong.ok ? "ok" : (pong.reason ?? "network"),
origin: "ping",
...(opts.now !== undefined ? { createdAt: opts.now.toISOString() } : {}),
});
const ping: DecisionModelCheckReport["ping"] = {
ok: pong.ok,
...(pong.model !== undefined ? { model: pong.model } : {}),
...(pong.latencyMs !== undefined ? { latency_ms: pong.latencyMs } : {}),
...(pong.reason !== undefined ? { reason: pong.reason } : {}),
};
order[index] = { ...slot, ping };
if (firstPing === undefined) firstPing = ping;
}
const withOrder: DecisionModelCheckReport = { ...report, provider_order: order };
if (!anyConfigured) {
return { ...withOrder, ping: { ok: false, reason: `not sent: status ${cfg.status}` } };
}
return { ...withOrder, ping: firstPing };
}
/** Non-zero only for a broken configured provider. */
export function decisionModelCheckExitCode(report: DecisionModelCheckReport): number {
if (report.status === "invalid") return 1;
if (
report.ping !== undefined &&
report.status === "active" &&
!report.ping.ok &&
// With a chain, a dead FIRST entry is the situation the fallback
// exists for: the check fails only when no configured entry answers.
!report.provider_order.some((entry) => entry.ping?.ok === true)
) {
return 1;
}
return 0;
}
export function renderDecisionModelCheck(report: DecisionModelCheckReport): string {
const lines: string[] = [];
lines.push(`decision model: ${report.status}`);
lines.push(` enabled: ${report.enabled}`);
lines.push(
` provider: ${report.provider ?? "(not set)"} (adapter ${report.adapter}` +
`${report.calibrated ? "" : ", uncalibrated"})`,
);
lines.push(` base url: ${report.base_url ?? "(not set)"}`);
lines.push(` model: ${report.model ?? "(not set)"}`);
const order = report.provider_order;
if (order.length > 1) {
lines.push(` providers (in order): ${order.map((p) => p.name).join(", ")}`);
for (const [i, p] of order.entries()) {
const key = p.env_key === null ? "" : `, key ${p.env_key} ${p.key_set ? "set" : "not set"}`;
lines.push(
` ${i + 1}. ${p.name} (${p.adapter}): ${p.base_url ?? "(no base url)"}, ` +
`model ${p.model ?? "?"}${key}` +
`${p.configured ? "" : ", not configured (no key)"}`,
);
if (p.ping !== undefined) {
lines.push(
p.ping.ok
? ` ping: ok, answered by ${p.ping.model ?? "?"} in ${p.ping.latency_ms ?? 0}ms`
: ` ping: failed (${p.ping.reason ?? "unknown"})`,
);
}
}
}
const optional = report.key_required ? "" : ", optional on this loopback server";
lines.push(
report.env_key === null
? ` key variable: (not named${optional})`
: ` key variable: ${report.env_key} (${report.key_set ? "set" : "not set"}${optional})`,
);
const uses = DECISION_MODEL_USES.map((use) => `${use}:${report.uses[use]}`).join(", ");
lines.push(` uses: ${uses}`);
lines.push(` vault opt-out: ${report.vault_opt_out ? "yes" : "no"}`);
lines.push(` timeout: ${report.timeout_ms}ms, max state: ${report.max_state_tokens} tokens`);
lines.push(
` threshold profile: ${report.threshold_profile ?? "(unknown: no use enforces)"}; ` +
`max choice options: ${report.max_choice_options}`,
);
lines.push(` tuning: ${measuredTuningSummary(report.tuning)}`);
const gradeUse = report.eval_grades.uses
.map((u) => {
const at = u.last_at === null ? "" : ` at ${u.last_at}`;
const spec = u.last_spec_hash === null ? "" : ` under ${shortSpecHash(u.last_spec_hash)}`;
return `${u.use}: ${u.runs} run(s), last ${u.last_verdict ?? "?"}${spec}${at}`;
})
.join("; ");
lines.push(
report.eval_grades.total === 0
? " eval grades: none cached"
: ` eval grades: ${report.eval_grades.total} record(s) (${gradeUse})` +
(report.eval_grades.skipped.length > 0
? `, ${report.eval_grades.skipped.length} skipped`
: ""),
);
const spend =
report.today_spend_usd === null ? "unknown" : `$${report.today_spend_usd.toFixed(6)}`;
const gate = report.cost_gate_usd > 0 ? `$${report.cost_gate_usd}` : "off";
lines.push(` daily cost gate: ${gate}; today (UTC): ${spend}`);
if (report.processor !== null) lines.push(` processor: ${report.processor}`);
if (report.licence_note !== null) lines.push(` licence: ${report.licence_note}`);
if (report.rerank.kind_configured) {
lines.push(
" rerank: search_rerank_kind is decision-model and uses this config " +
"(pays off with a semantic lane and search_rerank_top_k around 30)",
);
if (report.rerank.ignored_keys.length > 0) {
lines.push(` ignored for this kind: ${report.rerank.ignored_keys.join(", ")}`);
}
}
for (const error of report.errors) lines.push(` error: ${error}`);
for (const warning of report.warnings) lines.push(` warning: ${warning}`);
for (const note of report.notes) lines.push(` note: ${note}`);
if (report.hint !== null) lines.push(` hint: ${report.hint}`);
if (report.ping !== undefined) {
lines.push(
report.ping.ok
? ` ping: ok, answered by ${report.ping.model ?? "?"} in ${report.ping.latency_ms ?? 0}ms`
: ` ping: failed (${report.ping.reason ?? "unknown"})`,
);
}
return lines.join("\n");
}
// ----- report -----------------------------------------------------------------
export interface DecisionModelUseSummary {
readonly use: string;
readonly calls: number;
readonly outcomes: Readonly<Record<string, number>>;
readonly latency_p50_ms: number | null;
readonly latency_p95_ms: number | null;
readonly input_tokens: number;
readonly cost_usd: number;
readonly unknown_cost_calls: number;
/**
* Rerank only: agreement between decision and heuristic order over the
* shadow records of ordinary searches (eval and ping records excluded).
*/
readonly agreement?: {
readonly compared: number;
readonly top1_agreement: number;
readonly top5_overlap: number;
};
}
export interface DecisionModelReport {
readonly since: string | null;
readonly total: number;
readonly uses: ReadonlyArray<DecisionModelUseSummary>;
/**
* With `--use`, the use's own evaluation report (`reports/index.ts`);
* absent for `rerank`, whose metric is the summary's shadow agreement.
*/
readonly use_report?: { readonly use: DecisionModelUse; readonly report: unknown };
/**
* Present when the caller supplied the resolved config and a use
* configured `enforce` runs as `shadow` for want of tuned thresholds
* for the model: says so explicitly and names the tuning command.
*/
readonly shadow?: {
readonly model: string | null;
readonly uses: ReadonlyArray<DecisionModelUse>;
/** The command that tunes the first shadow use. */
readonly command: string;
};
/**
* The cached verdict sets of the preregistered evaluation
* (`eval/grade.ts`), oldest first, with the named skip notes. Present
* whenever the vault holds at least one, so a vault with none renders
* no eval section at all rather than an empty one.
*/
readonly eval?: {
readonly grades: ReadonlyArray<DecisionEvalGradeLine>;
readonly skipped: ReadonlyArray<string>;
};
}
/** One cached eval grade, as the prose and JSON surfaces print it. */
export interface DecisionEvalGradeLine {
readonly record_id: string;
readonly created_at: string;
readonly use: string;
readonly profile: string | null;
readonly verdict: string;
readonly spec_hash: string;
readonly counted_n: number;
readonly counted_error: number | null;
readonly baseline_error: number | null;
readonly dev_error: number | null;
}
/** The one-line prose form of a cached eval grade, shared by the surfaces. */
export function renderEvalGradeLine(grade: DecisionEvalGradeLine): string {
const num = (v: number | null): string => (v === null ? "-" : v.toFixed(3));
return (
`${grade.created_at} ${grade.use} (${grade.profile ?? "no profile"}) ${grade.verdict}: ` +
`counted error ${num(grade.counted_error)} over n=${grade.counted_n}, ` +
`baseline ${num(grade.baseline_error)}, dev ${num(grade.dev_error)}, ` +
`spec ${shortSpecHash(grade.spec_hash)}`
);
}
function percentile(sorted: ReadonlyArray<number>, p: number): number | null {
if (sorted.length === 0) return null;
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
return sorted[idx]!;
}
function stringArray(value: unknown): string[] | null {
return Array.isArray(value) && value.every((v) => typeof v === "string")
? (value as string[])
: null;
}
function agreementOf(
records: ReadonlyArray<ContinuityRecord>,
): DecisionModelUseSummary["agreement"] {
let compared = 0;
let top1 = 0;
let overlapSum = 0;
for (const record of records) {
if (record.payload["mode"] !== "shadow" || record.payload["origin"] !== undefined) continue;
const heuristic = stringArray(record.payload["heuristic_order"]);
const decision = stringArray(record.payload["decision_order"]);
if (heuristic === null || decision === null || heuristic.length === 0) continue;
compared++;
if (heuristic[0] === decision[0]) top1++;
const k = Math.min(5, heuristic.length);
const a = new Set(heuristic.slice(0, k));
const shared = decision.slice(0, k).filter((p) => a.has(p)).length;
overlapSum += shared / k;
}
if (compared === 0) return undefined;
return { compared, top1_agreement: top1 / compared, top5_overlap: overlapSum / compared };
}
export function buildDecisionModelReport(
vault: string,
opts: {
readonly since?: string;
readonly use?: string;
readonly origin?: string;
/** Attach the use's own report when `use` names one (default true). */
readonly useReport?: boolean;
/** The resolved config, for the shadow section; absent on record-only callers. */
readonly config?: ResolvedDecisionModelConfig;
} = {},
): DecisionModelReport {
const records = listDecisionModelCalls(vault, opts.since).filter(
(r) =>
r.kind === DECISION_MODEL_CALL_KIND &&
(opts.use === undefined || r.payload["use"] === opts.use) &&
(opts.origin === undefined || r.payload["origin"] === opts.origin),
);
const byUse = new Map<string, ContinuityRecord[]>();
for (const record of records) {
const use = typeof record.payload["use"] === "string" ? record.payload["use"] : "unknown";
const list = byUse.get(use) ?? [];
list.push(record);
byUse.set(use, list);
}
const uses: DecisionModelUseSummary[] = [];
for (const [use, list] of [...byUse].toSorted((a, b) => a[0].localeCompare(b[0]))) {
const outcomes: Record<string, number> = {};
const latencies: number[] = [];
let inputTokens = 0;
let cost = 0;
let unknownCost = 0;
for (const record of list) {
const p = record.payload;
const outcome = typeof p["outcome"] === "string" ? p["outcome"] : "unknown";
outcomes[outcome] = (outcomes[outcome] ?? 0) + 1;
if (typeof p["latency_ms"] === "number" && outcome !== "cost_gate" && outcome !== "budget") {
latencies.push(p["latency_ms"]);
}
if (typeof p["input_tokens"] === "number") inputTokens += p["input_tokens"];
if (typeof p["cost_usd"] === "number") cost += p["cost_usd"];
else if (outcome === "ok") unknownCost++;
}
latencies.sort((a, b) => a - b);
const agreement = use === "rerank" ? agreementOf(list) : undefined;
uses.push({
use,
calls: list.length,
outcomes,
latency_p50_ms: percentile(latencies, 0.5),
latency_p95_ms: percentile(latencies, 0.95),
input_tokens: inputTokens,
cost_usd: cost,
unknown_cost_calls: unknownCost,
...(agreement !== undefined ? { agreement } : {}),
});
}
const shadowOnly = opts.config?.shadowOnlyUses ?? [];
const shadow =
opts.config !== undefined && shadowOnly.length > 0
? {
model: opts.config.model,
uses: shadowOnly,
command: `o2b decision-model thresholds fit --use ${shadowOnly[0]!}`,
}
: undefined;
const summary: DecisionModelReport = {
since: opts.since ?? null,
total: records.length,
uses,
...(shadow !== undefined ? { shadow } : {}),
};
// The eval section is built for every report: `o2b decision-model
// report` is the audit surface for the grades a mark was written from.
const { grades, skipped } = listDecisionModelEvalGrades(vault, opts.since);
const evalSection =
grades.length > 0 || skipped.length > 0
? {
grades: grades.map((g): DecisionEvalGradeLine => ({
record_id: g.recordId,
created_at: g.createdAt,
use: g.use,
profile: g.profile,
verdict: g.verdict,
spec_hash: g.specHash,
counted_n: g.countedN,
counted_error: g.countedError,
baseline_error: g.baselineError,
dev_error: g.devError,
})),
skipped,
}
: undefined;
if (opts.use === undefined || opts.useReport === false || !isDecisionModelUse(opts.use)) {
return evalSection === undefined ? summary : { ...summary, eval: evalSection };
}
const entry = USE_REPORTS[opts.use];
if (entry === null) {
return evalSection === undefined ? summary : { ...summary, eval: evalSection };
}
const since = opts.since !== undefined ? { since: opts.since } : {};
return {
...summary,
...(evalSection !== undefined ? { eval: evalSection } : {}),
use_report: { use: opts.use, report: entry.build(vault, since) },
};
}
export function renderDecisionModelReport(report: DecisionModelReport): string {
const lines: string[] = [];
lines.push(
`decision model calls: ${report.total}${report.since !== null ? ` since ${report.since}` : ""}`,
);
for (const u of report.uses) {
const outcomes = Object.entries(u.outcomes)
.map(([k, v]) => `${k} ${v}`)
.join(", ");
lines.push(` ${u.use}: ${u.calls} call(s) (${outcomes})`);
lines.push(
` latency p50 ${u.latency_p50_ms ?? "-"}ms, p95 ${u.latency_p95_ms ?? "-"}ms; ` +
`input tokens ${u.input_tokens}; cost $${u.cost_usd.toFixed(6)}` +
(u.unknown_cost_calls > 0 ? ` (+${u.unknown_cost_calls} call(s) of unknown cost)` : ""),
);
if (u.agreement !== undefined) {
lines.push(
` shadow agreement over ${u.agreement.compared}: top-1 ` +
`${(u.agreement.top1_agreement * 100).toFixed(1)}%, top-5 overlap ` +
`${(u.agreement.top5_overlap * 100).toFixed(1)}%`,
);
}
}
if (report.shadow !== undefined) {
lines.push(
` shadow (no tuned thresholds for model '${report.shadow.model ?? "unknown"}'): ` +
`${report.shadow.uses.join(", ")}`,
);
lines.push(
` tune with \`${report.shadow.command}\`, or set values by hand with ` +
"`o2b decision-model thresholds set`",
);
}
if (report.eval !== undefined) {
lines.push(
`decision model eval grades: ${report.eval.grades.length} record(s)` +
(report.eval.skipped.length > 0 ? `, ${report.eval.skipped.length} skipped` : ""),
);
for (const grade of report.eval.grades) lines.push(` ${renderEvalGradeLine(grade)}`);
for (const note of report.eval.skipped) lines.push(` skipped: ${note}`);
}
if (report.use_report !== undefined) {
const entry = USE_REPORTS[report.use_report.use];
if (entry !== null) lines.push(entry.render(report.use_report.report));
}
return lines.join("\n");
}