Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.7 KB
Raw Blame
/**
 * `o2b decision-model report --use labels` metrics (issue #213, Part 6).
 *
 * Joins every recorded label suggestion to what the note carries now,
 * per note and dimension. `assign` writes the label into the note's
 * frontmatter and nothing else records it, so the note itself is the
 * log of the operator's later `assign` operations:
 *
 *   - `accepted`: the dimension now holds the suggested value, and it did
 *     not hold it when the suggestion was made;
 *   - `overridden`: the dimension changed since, to another value or to
 *     none;
 *   - `unchanged`: the dimension still holds what it held then;
 *   - `already`: the suggestion equalled the value the note already had
 *     (no action to measure);
 *   - `missing`: the note is gone or unreadable.
 *
 * Suggestions are the would-be suggestions of both modes (shadow records
 * them too, although shadow returns null to the host), so shadow data can
 * be evaluated before `enforce`. When a note and dimension were suggested
 * more than once, the latest record is used. Eval and ping records and
 * degraded calls are left out. Read-only.
 */

import { readLabels } from "../../brain/labels.ts";
import { resolveNotePath } from "../../brain/note-path.ts";
import { parseFrontmatter } from "../../vault.ts";
import { listDecisionModelCalls } from "../record.ts";

export type LabelOutcome = "accepted" | "overridden" | "unchanged" | "already" | "missing";

export interface LabelsModeSummary {
  readonly mode: string;
  readonly suggestions: number;
  readonly outcomes: Readonly<Record<LabelOutcome, number>>;
  /** accepted / (accepted + overridden); null when neither happened. */
  readonly acceptance: number | null;
}

export interface LabelsReport {
  readonly use: "labels";
  /** Suggestions with a non-null value, latest per note and dimension. */
  readonly suggestions: number;
  /** Dimensions the model answered `none` or below the threshold. */
  readonly no_suggestion: number;
  readonly by_mode: ReadonlyArray<LabelsModeSummary>;
}

interface Suggested {
  readonly createdAt: string;
  readonly mode: string;
  readonly path: string;
  readonly dimension: string;
  readonly current: string | null;
  readonly suggested: string | null;
  /** The model's confidence in the suggestion, as the record carries it. */
  readonly confidence: number | null;
  /** The answering model of the record the suggestion came from. */
  readonly model: string;
}

function emptyOutcomes(): Record<LabelOutcome, number> {
  return { accepted: 0, overridden: 0, unchanged: 0, already: 0, missing: 0 };
}

function nowValue(
  vault: string,
  path: string,
  dimension: string,
  cache: Map<string, ReadonlyArray<string> | null>,
): { readonly value: string | null } | null {
  let labels = cache.get(path);
  if (labels === undefined) {
    try {
      labels = readLabels(parseFrontmatter(resolveNotePath(vault, path))[0]);
    } catch {
      labels = null;
    }
    cache.set(path, labels);
  }
  if (labels === null) return null;
  const token = labels.find((t) => t.startsWith(`${dimension}/`));
  return { value: token === undefined ? null : token.slice(dimension.length + 1) };
}

/** One suggestion joined to its later outcome, with the model's confidence. */
export interface LabelSample {
  readonly mode: string;
  /** Whether the model made a suggestion at all (`none` or below threshold otherwise). */
  readonly suggested: boolean;
  readonly confidence: number | null;
  readonly outcome: LabelOutcome;
  /** The answering model of the record the suggestion came from. */
  readonly model: string;
}

/**
 * Every latest suggestion per note and dimension joined to its outcome -
 * the data both the acceptance report and the threshold fit derive from.
 * A suggestion without one (`none` or below the threshold) carries
 * `suggested: false` and outcome `missing` (there is nothing the note
 * could accept or override). Read-only.
 */
export function buildLabelSamples(
  vault: string,
  opts: { readonly since?: string } = {},
): { readonly samples: ReadonlyArray<LabelSample> } {
  const latest = new Map<string, Suggested>();
  for (const record of listDecisionModelCalls(vault, opts.since)) {
    const p = record.payload;
    if (p["use"] !== "labels" || p["outcome"] !== "ok" || p["origin"] !== undefined) continue;
    const path = p["note_path"];
    const mode = typeof p["mode"] === "string" ? p["mode"] : "unknown";
    if (typeof path !== "string" || !Array.isArray(p["dimensions"])) continue;
    for (const raw of p["dimensions"]) {
      if (raw === null || typeof raw !== "object") continue;
      const d = raw as Record<string, unknown>;
      if (typeof d["dimension"] !== "string") continue;
      const key = `${path}\u0000${d["dimension"]}`;
      const seen = latest.get(key);
      // By timestamp, not log order (same-millisecond records are unordered).
      if (seen !== undefined && seen.createdAt > record.createdAt) continue;
      latest.set(key, {
        createdAt: record.createdAt,
        mode,
        path,
        dimension: d["dimension"],
        current: typeof d["current"] === "string" ? d["current"] : null,
        suggested: typeof d["suggested"] === "string" ? d["suggested"] : null,
        confidence:
          typeof d["confidence"] === "number" && Number.isFinite(d["confidence"])
            ? d["confidence"]
            : null,
        model: typeof p["model"] === "string" ? p["model"] : "unknown",
      });
    }
  }
  const cache = new Map<string, ReadonlyArray<string> | null>();
  const samples: LabelSample[] = [];
  for (const s of latest.values()) {
    if (s.suggested === null) {
      samples.push({
        mode: s.mode,
        suggested: false,
        confidence: s.confidence,
        outcome: "missing",
        model: s.model,
      });
      continue;
    }
    const now = nowValue(vault, s.path, s.dimension, cache);
    let outcome: LabelOutcome;
    if (now === null) outcome = "missing";
    else if (s.suggested === s.current) outcome = "already";
    else if (now.value === s.current) outcome = "unchanged";
    else if (now.value === s.suggested) outcome = "accepted";
    else outcome = "overridden";
    samples.push({
      mode: s.mode,
      suggested: true,
      confidence: s.confidence,
      outcome,
      model: s.model,
    });
  }
  return { samples };
}

export function buildLabelsReport(
  vault: string,
  opts: { readonly since?: string } = {},
): LabelsReport {
  const { samples } = buildLabelSamples(vault, opts);
  const byMode = new Map<string, Record<LabelOutcome, number>>();
  let suggestions = 0;
  let none = 0;
  for (const s of samples) {
    if (!s.suggested) {
      none++;
      continue;
    }
    suggestions++;
    const counts = byMode.get(s.mode) ?? emptyOutcomes();
    counts[s.outcome]++;
    byMode.set(s.mode, counts);
  }
  return {
    use: "labels",
    suggestions,
    no_suggestion: none,
    by_mode: [...byMode]
      .toSorted((a, b) => a[0].localeCompare(b[0]))
      .map(([mode, outcomes]) => {
        const decided = outcomes.accepted + outcomes.overridden;
        return {
          mode,
          suggestions: Object.values(outcomes).reduce((a, n) => a + n, 0),
          outcomes,
          acceptance: decided === 0 ? null : outcomes.accepted / decided,
        };
      }),
  };
}

export function renderLabelsReport(report: LabelsReport): string {
  const lines = [
    `  labels: ${report.suggestions} suggestion(s), ${report.no_suggestion} without a suggestion`,
  ];
  for (const m of report.by_mode) {
    const outcomes = Object.entries(m.outcomes)
      .filter(([, n]) => n > 0)
      .map(([k, n]) => `${k} ${n}`)
      .join(", ");
    const acceptance = m.acceptance === null ? "-" : `${(m.acceptance * 100).toFixed(1)}%`;
    lines.push(`    ${m.mode}: ${m.suggestions} (${outcomes}); acceptance ${acceptance}`);
  }
  return lines.join("\n");
}