Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Knowledge-gap loop (theme A, t_67d38036).
*
* Closes the loop on recurring recall gaps: recurring gaps promote to
* durable vault task notes, open tasks render as a session-start agenda,
* and a task auto-closes once its topic is recalled with sufficient
* confidence - mirroring the dream freshness auto-resolve precedent (a
* recorded status flip in frontmatter, never a silent mutation).
*
* Two independent recurrence sources feed it (signals-that-survive, unit
* 6): the structural recall-telemetry `gap_counts` aggregate, whose topic
* is a coarse telemetry code, and the recall-adequacy verdict stamped onto
* the cross-query demand log, whose topic is a fine term bucket. They are
* counted and keyed separately and NEVER summed - `occurrences` would
* otherwise mean a different thing per row.
*
* Language-agnostic by construction: a structural topic is a telemetry key
* taken verbatim, and a verdict topic is the IDF-derived bucket key
* `normalizeQueryTerms` computes - never words classified from prose. Task
* notes are PLAIN durable files under the Brain area (`Brain/gap-tasks/`);
* they never touch the Hermes kanban board.
*
* Deliberately I/O-scoped to the vault and free of any hook/config
* concern, so recurrence, promotion, agenda, and auto-close are each
* independently unit-testable. The opt-in flags and wiring live in the
* session-start / session-end hooks.
*/
import { createHash } from "node:crypto";
import { join } from "node:path";
import { brainGapTasksDir, BRAIN_GAP_TASKS_REL } from "../paths.ts";
import { aggregateUnmetRecall } from "../query-demand.ts";
import { summarizeRecallTelemetry } from "../recall-telemetry.ts";
import { fenceUntrustedContent, neutralizeSingleLine } from "../untrusted-source.ts";
import { renderActivityTimeline, type ActivityItem } from "../render/activity-line.ts";
import type { RecallResultSet } from "../recall-inject.ts";
import type { RetrievalDegradationCode } from "../../search/retrieval-trail.ts";
import { isFileAlreadyExists } from "../../fs-atomic.ts";
import { parseFrontmatterText } from "../../vault.ts";
import type { FrontmatterMap } from "../../types.ts";
import {
assertLaneWrite,
writeLanePage,
lanePageExists,
listLaneFiles,
readLaneText,
removeLanePage,
} from "../../store/lane-pages.ts";
/** Default recurrence threshold: a gap must recur this often to promote. */
export const GAP_LOOP_RECURRENCE_THRESHOLD = 3;
/**
* Default match-quality floor a recall must clear to auto-close a task.
*
* Compared against {@link RecallResultSet.idfWeightedCoverage} - the share
* of the topic's IDF mass the vault now covers - and not against a result
* score. Against a score this floor could never refuse anything: the
* keyword lane min-max normalises within the candidate set, so the top
* row of ANY recall with a non-empty keyword lane scores at or above the
* configured `keywordWeight` (`DEFAULT_KEYWORD_WEIGHT`, 0.6 by default,
* measured at 0.65 on a freshly written keyword-only vault once the
* freshness prior is added on top), which clears 0.5 whatever the hit was
* worth. A gap task therefore auto-closed on the first keyword match of
* any quality, which is the outcome this floor exists to prevent.
*/
export const GAP_LOOP_AUTO_CLOSE_FLOOR = 0.5;
/**
* Most gap tasks one promotion run may mint. The cap is load-bearing, not
* polish: without it a single bad afternoon of empty recalls fills the
* vault with notes. Rows above the cap are not dropped - detection is
* ordered most-frequent first, so the remainder is minted by the next run.
*/
export const GAP_LOOP_MAX_PROMOTIONS_PER_RUN = 5;
/**
* Most gap-task notes the directory may hold at once.
*
* The per-run mint cap bounds a BURST; this bounds the TOTAL. Both are
* needed now that the adequacy source exists: the structural source drew
* topics from a fixed set of telemetry codes, so the reachable file count
* was small by construction, while a verdict topic is arbitrary
* normalised query text and the reachable count became "however many
* distinct questions the demand log still remembers". A backlog this size
* is already past the point an operator can work through, so refusing to
* mint past it costs nothing that was going to be read.
*/
export const GAP_TASKS_MAX_NOTES = 200;
/**
* How long a CLOSED gap task is kept before the loop removes it.
*
* A closed task is a finished record, not a backlog item: it stops being
* rendered, stops being auto-closed, and exists only so a re-promotion of
* the same topic is deduped against it. One month is long enough for that
* dedupe to be useful and short enough that closed work does not
* accumulate for the life of the vault. An OPEN task is never removed by
* age - deleting an operator's outstanding work would be the silent loss
* this loop exists to prevent.
*/
export const GAP_TASK_CLOSED_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
/** Recurrence source: the structural recall-telemetry `gap_counts` aggregate. */
export const GAP_SOURCE_TELEMETRY = "recall_telemetry";
/** Recurrence source: unmet recall-adequacy verdicts on the demand log. */
export const GAP_SOURCE_ADEQUACY = "recall_adequacy";
export const GAP_SOURCES = [GAP_SOURCE_TELEMETRY, GAP_SOURCE_ADEQUACY] as const;
export type GapSource = (typeof GAP_SOURCES)[number];
/**
* Agenda and body wording per source, so a coarse telemetry code bucket
* never reads as the same kind of row as a fine term bucket.
*/
const GAP_LABEL_BY_SOURCE: Readonly<Record<GapSource, string>> = Object.freeze({
[GAP_SOURCE_TELEMETRY]: "recall gap",
[GAP_SOURCE_ADEQUACY]: "weak recall",
});
export const GAP_TASK_KIND = "brain-gap-task";
export const GAP_TASK_STATUS_OPEN = "open";
export const GAP_TASK_STATUS_CLOSED = "closed";
/** One gap that recurred often enough to be actionable. */
export interface RecurringGap {
readonly topic: string;
readonly occurrences: number;
/** Which recurrence source counted it; rows are never merged across sources. */
readonly source: GapSource;
}
/**
* Recurring gaps from BOTH recurrence sources, filtered to those recurring
* at least `threshold` times and ordered most-frequent first (source then
* topic as stable tie-breaks).
*
* A topic that appears in both sources yields two rows, each carrying its
* own count. Summing them would produce a number that means neither "this
* telemetry code fired N times" nor "this question was asked N times".
*/
export function detectRecurringGaps(
vault: string,
opts: { threshold?: number } = {},
): ReadonlyArray<RecurringGap> {
const threshold = opts.threshold ?? GAP_LOOP_RECURRENCE_THRESHOLD;
const rows = [...structuralGaps(vault), ...adequacyGaps(vault)].filter(
(gap) => gap.occurrences >= threshold,
);
return Object.freeze(rows.toSorted(compareGaps));
}
/** The structural source: telemetry codes taken verbatim as topics. */
function structuralGaps(vault: string): ReadonlyArray<RecurringGap> {
const { gap_counts } = summarizeRecallTelemetry(vault);
return Object.entries(gap_counts).map(([topic, count]) =>
Object.freeze({ topic, occurrences: count, source: GAP_SOURCE_TELEMETRY }),
);
}
/** The verdict source: unmet-recall term buckets from the demand log. */
function adequacyGaps(vault: string): ReadonlyArray<RecurringGap> {
return aggregateUnmetRecall(vault).map((bucket) =>
Object.freeze({
topic: bucket.key,
occurrences: bucket.occurrences,
source: GAP_SOURCE_ADEQUACY,
}),
);
}
function compareGaps(a: RecurringGap, b: RecurringGap): number {
return (
b.occurrences - a.occurrences ||
compareStrings(a.source, b.source) ||
compareStrings(a.topic, b.topic)
);
}
function compareStrings(a: string, b: string): number {
return a < b ? -1 : a > b ? 1 : 0;
}
/**
* Stable, filesystem-safe, collision-free dedupe key for a gap topic: a
* short hash of the trimmed topic, namespaced by its recurrence source.
* Re-promotion of the same topic from the same source resolves to the same
* key (and thus the same note), so a gap can never fork into two competing
* task files - while a topic that happens to be textually identical across
* the two sources still gets one note each.
*
* BOTH sources are namespaced, and the source is LENGTH-PREFIXED so the
* hashed material is an injective encoding of the (source, topic) pair.
* Namespacing only one of them left the other's topic space able to spell
* the namespace: a telemetry gap is caller-supplied text, so a code
* literally spelled `recall_adequacy:<x>` hashed to the same key as the
* adequacy topic `<x>` and the two rows fought over one note.
*
* A key minted before this encoding therefore differs from the one this
* function now returns for the same telemetry topic. The consequence is
* bounded and self-clearing: the old note is still listed, still
* auto-closes, and is removed by the closed-task retention above, while a
* topic that is still recurring mints one note under the new key.
*/
export function gapTaskKey(topic: string, source: GapSource = GAP_SOURCE_TELEMETRY): string {
const material = `${source.length}:${source}:${topic.trim()}`;
return `gap-${createHash("sha256").update(material).digest("hex").slice(0, 16)}`;
}
export interface GapPromotionResult {
/** Keys of gap tasks created by this run. */
readonly created: ReadonlyArray<string>;
/** Keys skipped because a task with that key already existed. */
readonly skipped: ReadonlyArray<string>;
/** Keys of closed tasks this run removed under the retention window. */
readonly pruned: ReadonlyArray<string>;
/**
* Recurring gaps this run did NOT mint because the directory is at
* {@link GAP_TASKS_MAX_NOTES}. Reported rather than silently dropped:
* a full backlog is an operator condition, not a no-op.
*/
readonly capped: number;
}
/**
* Promote every recurring gap to exactly one durable gap-task note, deduped
* on the stable gap key via an exclusive create - a topic already carrying
* a task (open or closed) is skipped, never overwritten or forked.
*
* At most {@link GAP_LOOP_MAX_PROMOTIONS_PER_RUN} NEW notes are minted per
* run. A skip costs no budget, so an established backlog never starves a
* genuinely new gap.
*
* The run also enforces the directory's two bounds, in this order: closed
* tasks past {@link GAP_TASK_CLOSED_RETENTION_MS} are removed first (so
* the room they free is usable immediately), then minting stops at
* {@link GAP_TASKS_MAX_NOTES} and the refusal is COUNTED on the result.
*/
export function promoteGapsToTasks(
vault: string,
opts: { threshold?: number; now: Date },
): GapPromotionResult {
// Vault-identity write guard (context-integrity-gates, Unit J).
assertLaneWrite(vault);
const dir = brainGapTasksDir(vault);
const created: string[] = [];
const skipped: string[] = [];
const pruned = pruneExpiredGapTasks(vault, opts.now);
let noteCount = listGapTasks(vault).length;
let capped = 0;
for (const gap of detectRecurringGaps(vault, { threshold: opts.threshold })) {
if (created.length >= GAP_LOOP_MAX_PROMOTIONS_PER_RUN) break;
const key = gapTaskKey(gap.topic, gap.source);
const path = join(dir, `${key}.md`);
if (noteCount >= GAP_TASKS_MAX_NOTES) {
// A topic that already carries a note needs no new file, so the cap
// did not refuse it - counting it as refused would report a backlog
// pressure that is not there. The exclusive create below stays the
// authority on skipping; this pre-check only keeps the count honest.
if (lanePageExists(vault, path)) skipped.push(key);
else capped++;
continue;
}
const metadata: FrontmatterMap = {
kind: GAP_TASK_KIND,
gap_key: key,
gap_topic: gap.topic,
gap_source: gap.source,
status: GAP_TASK_STATUS_OPEN,
occurrences: String(gap.occurrences),
created_at: opts.now.toISOString(),
};
const body =
`Recurring ${GAP_LABEL_BY_SOURCE[gap.source]} detected ${gap.occurrences} times. ` +
`Add vault coverage for this topic; the task auto-closes once the ` +
`topic is recalled with sufficient confidence.`;
try {
writeLanePage(vault, path, metadata, body, {
existsErrorKind: "gap task",
vaultForRelativePath: vault,
});
created.push(key);
noteCount++;
} catch (exc) {
if (isFileAlreadyExists(exc)) {
skipped.push(key);
continue;
}
throw exc;
}
}
return Object.freeze({
created: Object.freeze(created),
skipped: Object.freeze(skipped),
pruned,
capped,
});
}
/**
* Remove every CLOSED gap task whose `closed_at` is older than the
* retention window, returning the keys removed. A task with no readable
* `closed_at` is left alone: age that cannot be shown is not assumed, and
* an unremoved note still counts against the cap, so the bound holds
* either way.
*/
function pruneExpiredGapTasks(vault: string, now: Date): ReadonlyArray<string> {
const cutoffMs = now.getTime() - GAP_TASK_CLOSED_RETENTION_MS;
const removed: string[] = [];
for (const task of listGapTasks(vault, { status: GAP_TASK_STATUS_CLOSED })) {
const closedAtMs = Date.parse(task.closedAt);
if (!Number.isFinite(closedAtMs) || closedAtMs >= cutoffMs) continue;
removeLanePage(vault, task.path);
removed.push(task.key);
}
return Object.freeze(removed);
}
export interface GapTask {
readonly key: string;
readonly topic: string;
readonly status: string;
readonly occurrences: number;
/** Which recurrence source minted it (see {@link GAP_SOURCES}). */
readonly source: GapSource;
readonly createdAt: string;
/** `closed_at` as the page carries it; empty when absent. */
readonly closedAt: string;
readonly path: string;
}
/**
* All gap-task notes under the Brain area, optionally filtered by status,
* ordered most-recent first (topic tie-break). A missing directory yields
* an empty list.
*/
export function listGapTasks(
vault: string,
opts: { status?: string } = {},
): ReadonlyArray<GapTask> {
const dir = brainGapTasksDir(vault);
// A missing directory lists nothing, in both modes.
const tasks: GapTask[] = [];
for (const name of listLaneFiles(vault, dir)) {
const path = join(dir, name);
const [fm] = parseFrontmatterText(readLaneText(vault, path) ?? "");
if (fm["kind"] !== GAP_TASK_KIND) continue;
const status = stringField(fm["status"]);
if (opts.status !== undefined && status !== opts.status) continue;
tasks.push(
Object.freeze({
key: stringField(fm["gap_key"]) || name.replace(/\.md$/, ""),
topic: stringField(fm["gap_topic"]),
status,
occurrences: Number.parseInt(stringField(fm["occurrences"]) || "0", 10) || 0,
source: coerceGapSource(fm["gap_source"]),
createdAt: stringField(fm["created_at"]),
closedAt: stringField(fm["closed_at"]),
path,
}),
);
}
return Object.freeze(
tasks.toSorted(
(a, b) => b.createdAt.localeCompare(a.createdAt) || compareStrings(a.topic, b.topic),
),
);
}
/**
* A note minted before the verdict source existed carries no `gap_source`;
* it is structural by construction, so that is the only safe default
* (signals-that-survive, unit 6).
*/
function coerceGapSource(value: unknown): GapSource {
return GAP_SOURCES.includes(value as GapSource) ? (value as GapSource) : GAP_SOURCE_TELEMETRY;
}
/** `origin` label stamped on the agenda block's untrusted-content fence. */
const GAP_AGENDA_FENCE_ORIGIN = "recall-gap-agenda";
/**
* Render open gap tasks as a compact session-start agenda through the
* shared activity helper (each task is an `openQuestion` item, so it carries
* the fixed structural `open` marker and a relative-age label). Returns the
* empty string when there is nothing open.
*
* Each row is labelled by its recurrence source, so a reader can tell a
* coarse telemetry code bucket from a fine term bucket and never reads the
* two counts as one number.
*
* Topics are untrusted: they originate in recall telemetry, which records
* query text this module never vetted. Each topic rides the shared
* single-line neutralizer, and the whole block rides the shared
* untrusted-content fence - a topic line that reads like an instruction
* reaches the session as framed data, and cannot break the block out of
* its one-row-per-task shape.
*/
export function renderGapAgenda(vault: string, now: Date): string {
const open = listGapTasks(vault, { status: GAP_TASK_STATUS_OPEN });
if (open.length === 0) return "";
const items: ReadonlyArray<ActivityItem> = open.map((task) => ({
kind: "openQuestion",
text: `${neutralizeSingleLine(task.topic)} (${GAP_LABEL_BY_SOURCE[task.source]} x${task.occurrences})`,
timestamp: task.createdAt,
}));
return fenceUntrustedContent(
`# Open recall-gap tasks\n${renderActivityTimeline(items, now)}`,
GAP_AGENDA_FENCE_ORIGIN,
);
}
/**
* Retrieval for the auto-close gate: the topic, and the membership rule
* the answer will be judged by.
*
* The predicate is an ARGUMENT rather than a post-filter the loop applies
* afterwards, and that is the whole point. A gap-task note carries its own
* topic verbatim, so it matches its own recall better than anything else
* in the vault. Filtering it out of the returned rows afterwards - which
* is what this gate used to do - removes it from the membership test and
* leaves it inside the number: `idfWeightedCoverage` is computed over the
* rows the retrieval returned, so the note the loop is trying not to count
* was still supplying the IDF mass the floor read. Every task then cleared
* the floor on its own text.
*
* So the rule travels WITH the query, and the two answers - "is there
* anything" and "how much of the topic does it cover" - are measured over
* one row set. A retriever that returns a row the predicate rejects has
* measured something else, and {@link autoCloseRecalledGaps} refuses its
* answer by name rather than reading the number anyway.
*
* A plain recall retriever is still assignable (it ignores the
* second argument) and is still correct for a retrieval that cannot
* surface a rejected path at all - which is what the vault-identity
* guard's fixture is.
*/
export type GapRecallRetriever = (
topic: string,
admits: (path: string) => boolean,
) => Promise<RecallResultSet>;
/** Thrown when a {@link GapRecallRetriever} answers outside its membership rule. */
export class GapRecallScopeError extends Error {
constructor(topic: string, paths: ReadonlyArray<string>) {
super(
`gap auto-close: retriever for "${topic}" returned ${paths.length} row(s) the membership rule excludes (${paths.join(", ")}); its match quality measures rows the gate rejected`,
);
this.name = "GapRecallScopeError";
}
}
export interface GapAutoCloseResult {
/** Keys of gap tasks closed by this run. */
readonly closed: ReadonlyArray<string>;
/** Keys left open (recall below the floor, or the recall failed). */
readonly kept: ReadonlyArray<string>;
/**
* Why any of this run's recalls narrowed, as the search trail's codes:
* first-seen order, each code once, present only when non-empty. A hook
* resolves to remote reach, so a gated query embed lands here and the
* hook names it on its LOCAL audit line. Reported only: a degraded
* recall is judged by the same floor as any other.
*/
readonly retrievalDegraded?: ReadonlyArray<RetrievalDegradationCode>;
}
/**
* Auto-close every open gap task whose topic the vault now covers at or
* above the match-quality floor, flipping its frontmatter status to closed
* and stamping `closed_at` / `closed_reason` (mirroring the dream freshness
* auto-resolve precedent). A recall that fails, returns nothing outside the
* gap-task directory, or covers the topic below the floor keeps the task
* open - fail-safe, never a silent close.
*
* Two conditions, deliberately separate. The gap-task notes themselves are
* excluded by PATH, because a task note carries its own topic verbatim and
* would otherwise close itself; that is a membership rule, and expressing
* it as "its score must clear a floor" is what let the floor pretend to be
* a quality test while only ever testing presence. The floor then judges
* the quality of the retrieval, which it reads from the retrieval itself.
*
* Both conditions are measured over ONE row set - see
* {@link GapRecallRetriever} for why the membership rule is handed to the
* retriever instead of applied to its answer. An unmeasurable match
* quality never closes a task: "the vault covers this" is a claim, and a
* retrieval that could not weigh the topic has not made it.
*/
export async function autoCloseRecalledGaps(
vault: string,
retriever: GapRecallRetriever,
opts: { confidenceFloor?: number; now: Date },
): Promise<GapAutoCloseResult> {
// Vault-identity write guard (context-integrity-gates, Unit J), like
// its sibling `promoteGapsToTasks`: closing a task rewrites the note's
// frontmatter, so it is a write path, not a read.
assertLaneWrite(vault);
const floor = opts.confidenceFloor ?? GAP_LOOP_AUTO_CLOSE_FLOOR;
const closed: string[] = [];
const kept: string[] = [];
const degraded: RetrievalDegradationCode[] = [];
for (const task of listGapTasks(vault, { status: GAP_TASK_STATUS_OPEN })) {
let coveredElsewhere = false;
let matchQuality: number | null = null;
let result: RecallResultSet | null = null;
try {
// eslint-disable-next-line no-await-in-loop -- one recall per open task, sequential by design
result = await retriever(task.topic, admitsGapCoverageRow);
} catch {
kept.push(task.key);
continue;
}
// Outside the catch: a retriever that answered outside its membership
// rule is a broken caller, not a failed recall, and folding it into
// "keep the task open" would hide it behind the fail-safe path.
const rejected = result.candidates
.map((candidate) => candidate.path)
.filter((path) => !admitsGapCoverageRow(path));
if (rejected.length > 0) throw new GapRecallScopeError(task.topic, rejected);
for (const code of result.degraded ?? []) if (!degraded.includes(code)) degraded.push(code);
// Only genuine vault coverage elsewhere may close a task - and the
// quality now describes the same rows, because the retriever was told
// which rows count before it measured them.
coveredElsewhere = result.candidates.length > 0;
matchQuality = result.idfWeightedCoverage;
if (coveredElsewhere && matchQuality !== null && matchQuality >= floor) {
closeGapTask(vault, task, opts.now);
closed.push(task.key);
} else {
kept.push(task.key);
}
}
return Object.freeze({
closed: Object.freeze(closed),
kept: Object.freeze(kept),
...(degraded.length > 0 ? { retrievalDegraded: Object.freeze(degraded) } : {}),
});
}
function closeGapTask(vault: string, task: GapTask, now: Date): void {
const text = readLaneText(vault, task.path);
// Removed since it was listed (pruned, or a synced tombstone): nothing to
// close, and writing it would bring back a page with no gap fields.
if (text === null) return;
const [fm, body] = parseFrontmatterText(text);
const updated: FrontmatterMap = {
...fm,
status: GAP_TASK_STATUS_CLOSED,
closed_at: now.toISOString(),
closed_reason: "recalled",
};
writeLanePage(vault, task.path, updated, body, { overwrite: true });
}
function stringField(value: unknown): string {
return typeof value === "string" ? value : "";
}
/** Whether a recall candidate path points at a gap-task note (any origin). */
function isGapTaskPath(path: string): boolean {
return path.includes(BRAIN_GAP_TASKS_REL);
}
/**
* The membership rule the auto-close gate judges by, as the predicate a
* {@link GapRecallRetriever} receives. One definition, used both to shape
* the retrieval and to verify what came back, so the two cannot disagree.
*/
function admitsGapCoverageRow(path: string): boolean {
return !isGapTaskPath(path);
}