Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
17.5 KB
Raw Blame
/**
 * Maintenance run journal (write-time-integrity-governance,
 * t_166d1226): bounded append-only JSONL in the vault-local state
 * dir. Every attempt lands here - including gate refusals - so the
 * operator can see WHY the quiet-window lane did or did not run
 * without trusting silence. Newest-N retention with an explicit
 * sweep on append, matching the activation-store discipline.
 */

import {
  isEmbeddingPriceSource,
  type EmbeddingPriceSource,
} from "../../search/embeddings/pricing.ts";
import { appendFileSync, existsSync, readFileSync, writeFileSync } from "node:fs";
import { mkdirSync } from "node:fs";
import { dirname, join } from "node:path";
import {
  interleaveShardRows,
  JSONL_LEDGER_EXT,
  jsonlLedgerGrammar,
  readShardLinesByShard,
  resolveAppendShardId,
  shardedFileName,
} from "../ledger-shards.ts";
import { DERIVED_STORE_DIR } from "../path-constants.ts";
import { assertVaultIdentityForWrite } from "../vault-identity.ts";
import type { StorageForecastTier } from "../storage/forecast.ts";
import type { HostPressureUnmeasurableReason } from "./host-pressure.ts";
import { isCustomLaneTask } from "./custom-task-id.ts";
import { renameWithRetry } from "../../fs-atomic.ts";
import { readLedgerRows } from "../../store/read/ledger-rows.ts";
import { isStoreOnly } from "../../store/store-only.ts";
import { routeWrite } from "../../store/store-write.ts";
import { DUAL_WRITE_LEDGERS, ledgerRowRecord } from "../../store/dual-write-records.ts";
import { rowNamesPrivatePage } from "../../store/reference-privacy.ts";

export const MAINTENANCE_JOURNAL_CAP = 500;

/**
 * What one journal row records.
 *
 * A closed vocabulary rather than a bare union because these values are
 * persisted to `.open-second-brain/maintenance-runs[.<deviceId>].jsonl` and read back
 * by whichever build runs next - across an upgrade, that is not the build
 * that wrote them. The guard is the boundary for a row this build does
 * not understand.
 *
 * Three of the members are not decisions about whether to run:
 * `pressure:unmeasurable` and `capacity:unmeasurable` are NOTICES that a
 * gate could not evaluate, and each is emitted on its own line beside the
 * decision, never instead of it. `refused:streak` is a per-task refusal,
 * so it appears among the task rows rather than in place of them.
 */
export const MAINTENANCE_VERDICT = Object.freeze({
  /** The gates were open; the row's task ran (or the lane ran). */
  run: "run",
  skippedWindow: "skipped:window",
  skippedBusy: "skipped:busy",
  /** Host pressure was measured and stood at or above the configured percentage. */
  skippedPressure: "skipped:pressure",
  /** The storage-capacity gate measured the vault at the critical tier. */
  skippedCapacity: "skipped:capacity",
  skippedLease: "skipped:lease",
  /** This task has failed too many times in a row to retry unforced. */
  refusedStreak: "refused:streak",
  /** The host-pressure gate could not evaluate; it was left open. */
  pressureUnmeasurable: "pressure:unmeasurable",
  /** The storage-capacity gate could not evaluate; it was left open. */
  capacityUnmeasurable: "capacity:unmeasurable",
});

export const MAINTENANCE_VERDICTS: ReadonlyArray<string> = Object.freeze(
  Object.values(MAINTENANCE_VERDICT),
);

export type MaintenanceVerdict = (typeof MAINTENANCE_VERDICT)[keyof typeof MAINTENANCE_VERDICT];

export function isMaintenanceVerdict(value: unknown): value is MaintenanceVerdict {
  return typeof value === "string" && MAINTENANCE_VERDICTS.includes(value);
}

export interface MaintenanceJournalEntry {
  readonly ts: string;
  readonly holder: string;
  readonly verdict: MaintenanceVerdict;
  /** Present on per-task rows; absent on lane-level gate-refusal rows. */
  readonly task?: string;
  readonly ok?: boolean;
  readonly duration_ms?: number;
  readonly error?: string;
  /**
   * True on a failed row whose task hit its safeguard deadline. Journaled
   * and reported like any failure. For a built-in task it is not counted
   * toward the streak that refuses the task: a pass that ran out of time
   * is not a broken pass, and refusing it would stop the keyword index
   * from refreshing at all. For a `custom:<name>` task it IS counted: see
   * {@link consecutiveTaskFailures}.
   */
  readonly timed_out?: boolean;
  /** Host pressure the gate read, on a `skipped:pressure` row. */
  readonly pressure_percent?: number;
  /** Why the gate could not read one, on a `pressure:unmeasurable` row. */
  readonly pressure_reason?: HostPressureUnmeasurableReason;
  /** Storage tier the capacity gate read, on a `skipped:capacity` row. */
  readonly capacity_tier?: StorageForecastTier;
  /**
   * The reading behind that tier, on a `skipped:capacity` row, riding the
   * way `pressure_percent` does: the percent of the configured cap, and
   * the measured bytes and cap it is a percent of. Written together,
   * only when the gate decision carried a reading.
   */
  readonly capacity_percent?: number;
  readonly capacity_measured_bytes?: number;
  readonly capacity_cap_bytes?: number;
  /** Why the capacity gate could not read a tier, on a `capacity:unmeasurable` row. */
  readonly capacity_reason?: string;
  /** Consecutive journaled failures behind a `refused:streak` row. */
  readonly streak?: number;
  /**
   * Model spend the task's run accounted for, on a row whose pass
   * completed and returned a receipt (t_9d155d0e). Priced by the cost
   * kernel before the provider was called; `forced` is true only when a
   * `--force-cost` bypass overrode a positive gate that would have
   * blocked the run. A failed attempt records no receipt: the spend of a
   * pass killed mid-flight is unmeasured, and the row says so by
   * carrying the failure without one.
   */
  readonly receipt?: MaintenanceSpendReceipt;
}

/**
 * What one model-spending pass spent, as the cost kernel priced it
 * before the provider was called. Lives beside the journal entry it
 * persists on, because the row is the audit unit the lane renders - and
 * journal.ts is the lower layer both the lane and its surfaces already
 * read, so the shape cannot grow a second definition.
 */
export interface MaintenanceSpendReceipt {
  /** The model the pass named; null when the config leaves it unset. */
  readonly model: string | null;
  readonly tokens: number;
  /**
   * Null when nobody stated the model's price. A row written before price
   * sources existed keeps the number it was written with.
   */
  readonly estimatedUsd: number | null;
  /** True when `--force-cost` overrode a positive gate that would have blocked the run. */
  readonly forced: boolean;
  /**
   * Who stated the price the estimate used. Absent on rows written before
   * price sources existed: such a row's source is unrecorded, not known.
   */
  readonly priceSource?: EmbeddingPriceSource;
}

/** How a receipt written before price sources existed names its source. */
export const PRICE_SOURCE_UNRECORDED = "unrecorded";

/**
 * The price source a journaled receipt recorded, read back off disk:
 * {@link PRICE_SOURCE_UNRECORDED} for a row written before price sources
 * existed (or carrying a value this build does not know), never a guess.
 */
export function recordedPriceSource(
  receipt: MaintenanceSpendReceipt,
): EmbeddingPriceSource | typeof PRICE_SOURCE_UNRECORDED {
  return isEmbeddingPriceSource(receipt.priceSource)
    ? receipt.priceSource
    : PRICE_SOURCE_UNRECORDED;
}

/**
 * The row a surface below local reach may render. A `skipped:capacity`
 * row carries the reading behind the skip - the whole-vault measured
 * bytes, the configured cap and the percent of one against the other -
 * and those are the same local-reach facts the storage report refuses to
 * take below local reach: totals that move when a page the caller may
 * not read appears or goes. The journal itself keeps them (the
 * operator's own record); this drops exactly the three reading fields
 * and keeps the tier word and the unmeasurable reason, which name a
 * verdict, never a size. A surface at local reach renders the row
 * unchanged.
 */
export function journalEntryWithoutCapacityReading(
  entry: MaintenanceJournalEntry,
): MaintenanceJournalEntry {
  const {
    capacity_percent: _percent,
    capacity_measured_bytes: _measured,
    capacity_cap_bytes: _cap,
    ...rest
  } = entry;
  return rest;
}

/**
 * The metrics surface a completed reindex pass's spend receipt is
 * recorded under. Both lane front doors (`o2b brain maintenance` and
 * `brain_maintenance`) append here, so one surface name covers a run's
 * spend whatever door ran it.
 */
export const MAINTENANCE_SPEND_METRIC = "maintenance_spend";

/** The journal's shard stem: `maintenance-runs[.<deviceId>].jsonl`. */
export const MAINTENANCE_JOURNAL_STEM = "maintenance-runs";

/** The journal's file-name layout, handed to the shared shard grammar. */
const JOURNAL_GRAMMAR = jsonlLedgerGrammar(MAINTENANCE_JOURNAL_STEM);

/**
 * The journal file THIS device appends to, and the only shard the cap
 * sweep rewrites: `maintenance-runs[.<deviceId>].jsonl` (t_774dea61). The
 * empty device id yields the legacy un-sharded name.
 */
function journalPath(vault: string): string {
  return join(
    vault,
    DERIVED_STORE_DIR,
    shardedFileName(MAINTENANCE_JOURNAL_STEM, resolveAppendShardId(), JSONL_LEDGER_EXT),
  );
}

export function appendJournal(vault: string, entry: MaintenanceJournalEntry): void {
  // One `maintenance-journal` ledger row on a store-only machine (design
  // D14 item 1), this device's journal line plus the dual write otherwise.
  // Task ids, verdicts and gate readings; a failed task's `error` text can
  // carry a path, so a row naming a private or unjudgeable page goes to
  // `private` only.
  routeWrite({
    legacy: () => {
      // Vault-identity write guard (context-integrity-gates, Unit J).
      assertVaultIdentityForWrite(vault);
      const path = journalPath(vault);
      mkdirSync(dirname(path), { recursive: true });
      // O_APPEND, one line per call: concurrent gate-refusal writers
      // (which run BEFORE the lease is held) interleave instead of
      // overwriting each other through a read-modify-rewrite race. The
      // cap is enforced separately by `sweepJournal`, which runMaintenance
      // calls while it holds the lease - the only safe rewrite point.
      appendFileSync(path, JSON.stringify(entry) + "\n");
    },
    store: () =>
      ledgerRowRecord(
        DUAL_WRITE_LEDGERS.maintenanceJournal,
        entry,
        rowNamesPrivatePage(vault, entry) ? "private" : "shared",
      ),
    context: { vault },
    // The journal is diagnostic: a row that does not land (a private row
    // on a machine without `private`, a store fault) is counted for
    // doctor and never stops the maintenance lane. Privacy stays fail
    // closed: the row is never written to shared instead.
    failOpen: true,
  });
}

/** Trim the journal to the newest `cap` lines. Lease-holder only. */
export function sweepJournal(vault: string, cap: number = MAINTENANCE_JOURNAL_CAP): void {
  // A store-only machine rewrites nothing under the vault (design D14 item
  // 1): its journal is append-only store rows, and the legacy file it may
  // still hold is read-only until the tree leaves the vault.
  if (isStoreOnly({ vault })) return;
  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertVaultIdentityForWrite(vault);
  const path = journalPath(vault);
  const lines = readLines(path);
  if (lines.length <= cap) return;
  const kept = lines.slice(lines.length - cap);
  const tmp = `${path}.tmp`;
  writeFileSync(tmp, kept.join("\n") + "\n");
  renameWithRetry(tmp, path);
}

/**
 * When a row was written, as the merge orders it. A row whose stamp does
 * not parse sorts as the oldest, so it can never displace a readable row
 * from the head of the newest-first list.
 */
function entryTime(entry: MaintenanceJournalEntry): number {
  const time = Date.parse(entry.ts);
  return Number.isFinite(time) ? time : Number.NEGATIVE_INFINITY;
}

/** One journal line as an entry, or `null` for a torn or non-object line. */
function parseEntry(line: string): MaintenanceJournalEntry | null {
  try {
    const parsed: unknown = JSON.parse(line);
    return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
      ? (parsed as MaintenanceJournalEntry)
      : null;
  } catch {
    // Fail-soft: a torn line never breaks the journal read.
    return null;
  }
}

/**
 * Journal entries, newest first - the one order both this list and
 * {@link consecutiveTaskFailures} walk.
 *
 * Every device's shard (t_774dea61) is interleaved by row timestamp, so a
 * failure on one device that is newer than a success on another reads as
 * newer whatever the shard names are; each shard's own append order is
 * never changed, so a single-shard vault reads exactly as before.
 * Unparseable lines are skipped.
 */
export function listJournal(vault: string, limit?: number): MaintenanceJournalEntry[] {
  // A store-only machine reads the `maintenance-journal` ledger rows in
  // fold order (design D14 item 1): every device's rows in its own append
  // order, interleaved by clock, which is what the legacy shard merge
  // gives.
  const out = readLedgerRows<unknown>(DUAL_WRITE_LEDGERS.maintenanceJournal, readLegacyJournal, {
    vault,
  })
    .filter(
      (row): row is MaintenanceJournalEntry =>
        row !== null && typeof row === "object" && !Array.isArray(row),
    )
    .toReversed();
  return limit !== undefined ? out.slice(0, Math.max(0, limit)) : out;
}

/** The legacy shards, interleaved by row time, oldest first. */
function readLegacyJournal(vault: string): MaintenanceJournalEntry[] {
  const shards = readShardLinesByShard(dirname(journalPath(vault)), JOURNAL_GRAMMAR).map(
    (shard) => ({
      shardId: shard.shardId,
      rows: shard.rows.map(parseEntry).filter((entry) => entry !== null),
    }),
  );
  return interleaveShardRows(shards, entryTime);
}

/**
 * How many times in a row `task` has failed since its newest success.
 *
 * The count is the portable stand-in for a crash-loop counter: nothing in
 * this project survives an invocation except what is on disk, and the
 * journal already records every attempt with its outcome. Two properties
 * make it usable as a refusal input:
 *
 *   - a single success ENDS the count, because the walk stops at the
 *     newest row that is not a failure - a transient fault therefore
 *     cannot accumulate into a permanent refusal;
 *   - only rows that record a completed ATTEMPT are counted. A gate
 *     refusal and a lease skip are not attempts, so they cannot deepen a
 *     streak.
 *   - a failure of a BUILT-IN task that is a safeguard TIMEOUT
 *     (`timed_out`) is skipped: it neither deepens the streak nor ends it,
 *     so a long pass that keeps running out of budget is reported every
 *     night but never refused. A custom (`custom:<name>`) task's timeout
 *     COUNTS: its budget is the operator's own `_timeout_seconds`, so a
 *     timeout there is the command hanging, not a slow vault, and a
 *     command that always hangs must reach the refusal like any failure.
 *
 * A row that ran but recorded no outcome stops the walk as well: refusing
 * work on evidence this build cannot read is the wrong direction to err.
 *
 * ## Why a `refused:streak` row ends the walk by ADDING its own count
 *
 * This journal is a ring buffer ({@link MAINTENANCE_JOURNAL_CAP}), and a
 * lane that writes several rows per pass retires roughly a hundred passes'
 * history. Counted naively, a refused task's three `run/ok:false` rows are
 * pushed off the tail by the very refusal rows they cause, and the count
 * silently returns to zero - the refusal un-refusing itself on a schedule
 * set by how chatty the journal is, with no event anywhere.
 *
 * So the refusal row is read as what it is: the journal's own durable
 * record of the count that was reached. The walk stops there and returns
 * the failures NEWER than it plus the number it recorded, which is the
 * same total those rolled-off rows would have produced. It cannot inflate
 * a streak - a refusal reproduces the number it was given, never one more
 * - and it cannot outlive a recovery, because a success is newer than
 * every refusal it follows and stops the walk first.
 *
 * Residual, stated rather than hidden: below the limit no refusal row
 * exists, so a streak whose rows roll off is counted only from what is
 * retained. That undercount errs toward RUNNING the task, which is the
 * direction this function already errs in for an unreadable outcome.
 */
export function consecutiveTaskFailures(vault: string, task: string): number {
  let streak = 0;
  for (const entry of listJournal(vault)) {
    if (entry.task !== task) continue;
    if (entry.verdict === MAINTENANCE_VERDICT.refusedStreak) {
      // A row from a build that did not record the number carries no
      // evidence, so it is skipped rather than read as a zero.
      if (Number.isInteger(entry.streak) && (entry.streak ?? 0) >= 0) {
        return streak + (entry.streak ?? 0);
      }
      continue;
    }
    if (entry.verdict !== MAINTENANCE_VERDICT.run) continue;
    if (entry.ok !== false) break;
    // A built-in timeout neither counts nor ends the streak: see
    // `timed_out`. A custom task's timeout counts like any failure: the
    // operator chose its budget, so running out of it is the command
    // hanging, and a command that always hangs would otherwise never be
    // refused while costing its full timeout of lease time every pass.
    if (entry.timed_out === true && !isCustomLaneTask(entry.task)) continue;
    streak += 1;
  }
  return streak;
}

function readLines(path: string): string[] {
  if (!existsSync(path)) return [];
  return readFileSync(path, "utf8")
    .split("\n")
    .filter((line) => line.trim().length > 0);
}