Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The distilled and undistilled census over imported sessions
* (session-mining-economy, t_0358af42).
*
* The import ledger (`sessions/discover.ts`) knows what was imported; the
* mining ledger (`session-mining-ledger.ts`) knows what was mined and with
* what yield. This module joins the two and classifies every imported
* session, so "how much of my imported backlog was never distilled" is one
* read, not a re-scan of every transcript on the machine:
*
* - `distilled` - mined with accepted yield (written or staged > 0) on
* the session's CURRENT turn material;
* - `known-nothing-new` - mined barren on the current material (the
* tombstone: every item deduped or durability-rejected). This is a
* different state from backlog - the session is known, it just had
* nothing new to say;
* - `undistilled` - never mined, or mined on material that has since
* changed (a re-import changed the turns, which changes the mined
* content hash and re-opens the session with no clearing step);
* - `refused` - sessions the plan would refuse (capture boundary, zero
* imported turns, zero user turns). They can never be distilled, so
* they are excluded from the backlog: counting them would nag forever
* about unparsable material.
*
* ## The join is the portable identity, and the reads are vault-side
*
* The import ledger's keys are absolute machine paths; the mining ledger is
* keyed by the portable `sessionRefIdentity` basename, which is what the
* mining lane and every written signal's `session_ref` already carry. The
* census joins on that basename, so a session whose log moved to another
* absolute path (a rotated store, another machine's layout) still joins.
* No source log is opened or re-hashed here: the mined-material hash runs
* over the continuity turns the mining lane itself reads, and the store is
* read ONCE for the whole census ({@link buildSessionTurnIndex}) - a
* per-session listing would re-read the whole store per session, which is
* quadratic in the imported backlog. That is what makes the census
* answerable inside a SessionStart hook's budget. The
* census reads only - neither ledger is rewritten, and a report that
* quietly rewrote the state it reports on would not be a report.
*/
import { basename } from "node:path";
import { selectMinedTurns } from "../extract-signals.ts";
import { buildCaptureBoundary, type CaptureBoundary } from "../capture-boundary.ts";
import {
minedTurnsHash,
readSessionMiningLedger,
type MinedTurnMaterial,
type SessionMiningEntry,
} from "../session-mining-ledger.ts";
import type { Safeguard } from "../safeguard.ts";
import { buildSessionTurnIndex, type SessionTurnIndex } from "../session-recall.ts";
import { readSessionLedger } from "./discover.ts";
import type { SessionRuntimeId } from "../../runtime/host-facts.ts";
/** Where an imported session stands with respect to distillation. */
export type SessionDistillationState = "distilled" | "known-nothing-new" | "undistilled";
/** One runtime's backlog: how much of what it imported is still undistilled. */
export interface RuntimeMiningCensus {
readonly runtime: SessionRuntimeId;
/** Every session the import ledger records under this runtime. */
readonly imported: number;
readonly distilled: number;
readonly knownNothingNew: number;
readonly undistilled: number;
/**
* Sessions the mining plan would refuse (boundary, zero imported turns,
* zero user turns). Excluded from the backlog by construction;
* `imported = distilled + knownNothingNew + undistilled + refused`.
*/
readonly refused: number;
/** The undistilled sessions' portable ids, sorted. */
readonly undistilledSessionIds: ReadonlyArray<string>;
}
/** The vault-wide distilled and undistilled census. */
export interface SessionMiningCensus {
readonly imported: number;
readonly distilled: number;
readonly knownNothingNew: number;
readonly undistilled: number;
readonly refused: number;
/** One row per runtime the import ledger records, sorted by runtime id. */
readonly byRuntime: ReadonlyArray<RuntimeMiningCensus>;
}
export interface BuildMiningCensusOptions {
/**
* Cooperative deadline, checked once per imported session. The census is
* vault-side, never hashes a source log and reads the store once, but a
* caller with a host deadline (the SessionStart hook) still passes its
* ceiling here so the read stops in time on an outsized vault.
*/
readonly safeguard?: Safeguard;
/**
* The store read to classify against. Default: one
* {@link buildSessionTurnIndex} pass for this census.
*/
readonly turns?: SessionTurnIndex;
}
/**
* Build the census. A corrupt or unknown-schema IMPORT ledger or MINING
* record refuses by name (each read's own contract): a census that could
* not verify one of its two sources must not report a clean backlog as if
* the state were empty.
*/
export function buildMiningCensus(
vault: string,
opts: BuildMiningCensusOptions = {},
): SessionMiningCensus {
const importedEntries = readSessionLedger(vault).entries;
const miningEntries = readSessionMiningLedger(vault).entries;
const boundary = buildCaptureBoundary(vault);
const turns = opts.turns ?? buildSessionTurnIndex(vault);
// One identity can sit under several absolute paths (the same basename
// imported from two machines' layouts); its state is a fact about the
// identity, so it is classified once and applied to every path entry.
const stateByIdentity = new Map<string, SessionDistillationState | "refused">();
const stateOf = (identity: string): SessionDistillationState | "refused" => {
const cached = stateByIdentity.get(identity);
if (cached !== undefined) return cached;
const state = classifySession(vault, boundary, turns, miningEntries, identity);
stateByIdentity.set(identity, state);
return state;
};
interface Row {
imported: number;
refused: number;
states: Record<SessionDistillationState, number>;
ids: Set<string>;
}
const rows = new Map<SessionRuntimeId, Row>();
const rowOf = (runtime: SessionRuntimeId): Row => {
const existing = rows.get(runtime);
if (existing !== undefined) return existing;
const fresh: Row = {
imported: 0,
refused: 0,
states: { distilled: 0, "known-nothing-new": 0, undistilled: 0 },
ids: new Set(),
};
rows.set(runtime, fresh);
return fresh;
};
for (const [path, entry] of Object.entries(importedEntries)) {
opts.safeguard?.checkpoint();
// The join identity is the portable sessionRefIdentity - what the
// mining lane keys its tombstone and every signal's session_ref on:
// the explicit recall id the import recorded when it pinned one
// (--recall-session-id), else the log's basename.
const identity = entry.session_id ?? basename(path);
const state = stateOf(identity);
const row = rowOf(entry.runtime);
row.imported += 1;
if (state === "refused") {
row.refused += 1;
continue;
}
row.states[state] += 1;
if (state === "undistilled") row.ids.add(identity);
}
const byRuntime: RuntimeMiningCensus[] = [...rows.keys()].toSorted().map((runtime) => {
const row = rows.get(runtime)!;
return {
runtime,
imported: row.imported,
distilled: row.states.distilled,
knownNothingNew: row.states["known-nothing-new"],
undistilled: row.states.undistilled,
refused: row.refused,
undistilledSessionIds: [...row.ids].toSorted(),
};
});
const total = (pick: (row: RuntimeMiningCensus) => number): number =>
byRuntime.reduce((sum, row) => sum + pick(row), 0);
return {
imported: total((row) => row.imported),
distilled: total((row) => row.distilled),
knownNothingNew: total((row) => row.knownNothingNew),
undistilled: total((row) => row.undistilled),
refused: total((row) => row.refused),
byRuntime,
};
}
/**
* The mined turn material the plan would embed for one session, or `null`
* when the session is one the plan refuses (boundary, zero imported turns,
* zero user turns). The selection is the plan's own shared
* {@link selectMinedTurns} - user role, unsuppressed text, trimmed and
* non-empty, the turn's stored timestamp verbatim - so this census hashes
* exactly what the plan would mine and cannot drift from it. `boundary` and
* `index` are injectable so a caller walking many sessions builds each once.
*/
export function minedTurnMaterial(
vault: string,
sessionId: string,
boundary: CaptureBoundary = buildCaptureBoundary(vault),
index?: SessionTurnIndex,
): ReadonlyArray<MinedTurnMaterial> | null {
if (boundary.sessionDecision(sessionId) !== "capture") return null;
const turns = (index ?? buildSessionTurnIndex(vault)).turnsFor(sessionId);
if (turns.length === 0) return null;
const mined = selectMinedTurns(boundary, turns);
return mined.length === 0 ? null : mined;
}
/**
* Classify one session identity: the plan's own refusal order (boundary
* decision, zero imported turns, zero user turns), then the mining join.
*/
function classifySession(
vault: string,
boundary: ReturnType<typeof buildCaptureBoundary>,
turns: SessionTurnIndex,
miningEntries: Readonly<Record<string, SessionMiningEntry>>,
identity: string,
): SessionDistillationState | "refused" {
const material = minedTurnMaterial(vault, identity, boundary, turns);
if (material === null) return "refused";
const entry = miningEntries[identity];
if (entry === undefined || entry.minedContentHash !== minedTurnsHash(material)) {
return "undistilled";
}
return entry.outcome.written + entry.outcome.staged > 0 ? "distilled" : "known-nothing-new";
}