Local-first π§ memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly π΄ dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Cross-query demand log (t_97091fff).
*
* A persisted, deterministic record of every recall query against the
* vault, together with three satisfaction signals already computed
* elsewhere: the result count, the IDF-weighted coverage score from
* {@link ../search/coverage.ts}, and β when the caller graded the attempt
* β the unmet recall-adequacy verdict (signals-that-survive, unit 6).
* Aggregated over time, the log surfaces
* the *demand* gap β queries operators keep asking that the vault
* answers poorly (weak or empty) β as a prioritized backlog of what to
* write into `Brain/` next.
*
* This is distinct from the existing structural gaps (dangling wikilink
* targets in deep-synthesis) and from the in-the-moment per-query
* coverage guard (coverage.ts / evidence-pack.ts): those look at one
* query or one link at a time and are never persisted. Here the signal
* is cross-query and durable.
*
* No LLM, no ranking model β pure counting and the reused coverage
* score. Privacy: only the normalized significant terms are stored
* (never the raw query), each run through the secret redactor, so a
* query carrying a token or path cannot leak into the log.
*
* Storage: one shard per device, `Brain/log/query-demand[.<deviceId>].jsonl`
* (the empty device id keeps the legacy un-sharded name), one JSON record
* per line. Each device appends to and compacts only its own shard, within
* the byte budget (see {@link DEMAND_LOG_MAX_BYTES}); reads merge every
* shard.
*/
import { appendFileSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { dirname, join } from "node:path";
import { atomicWriteFileSync } from "../fs-atomic.ts";
import { clamp01 } from "../math.ts";
import { redactRawOutput } from "../redactor.ts";
import { legacyVaultOf } from "./no-vault.ts";
import { BRAIN_LOG_REL } from "./path-constants.ts";
import { jsonlLedgerGrammar, readShardedLines } from "./ledger-shards.ts";
import { QUERY_DEMAND_LEDGER_STEM, queryDemandLogPath } from "./paths.ts";
import { acquireLockSync } from "./sync-lockfile.ts";
import {
COMPLETENESS_COMPLETE_THRESHOLD,
COMPLETENESS_PARTIAL_THRESHOLD,
significantTerms,
} from "../search/coverage.ts";
import type { CompletenessVerdict } from "../search/types.ts";
import { isRecallAdequacyLevel } from "./recall-adequacy.ts";
import type { RecallAdequacyLevel, RecallAdequacyVerdict } from "./recall-adequacy.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
import { readLedgerRows } from "../store/read/ledger-rows.ts";
import { routeWrite } from "../store/store-write.ts";
import { DUAL_WRITE_LEDGERS, ledgerRowRecord } from "../store/dual-write-records.ts";
/** Longest single normalized term kept; longer tokens are dropped as a
* privacy guard (a 40+ char alnum run is far likelier a secret/id than a
* word). */
export const DEMAND_TERM_MAX_LEN = 40;
/** Most terms retained per record β bounds one pathological query's line
* size while keeping the bucket key stable for realistic queries. */
export const DEMAND_MAX_TERMS_PER_RECORD = 24;
/** Compaction trigger: once the log file exceeds this many bytes, the
* oldest lines are dropped so the tail fits in {@link DEMAND_LOG_COMPACT_BYTES}.
* Bounds unbounded growth without reading the whole file on every append
* (the common path only stats the file). */
export const DEMAND_LOG_MAX_BYTES = 1_000_000;
/** Target size after compaction β keep the newest lines whose combined
* size is at most this. Kept below {@link DEMAND_LOG_MAX_BYTES} so
* compaction is amortized, not run on every subsequent append. */
export const DEMAND_LOG_COMPACT_BYTES = 750_000;
/** A query must recur at least this many times to count as demand. */
export const DEMAND_DEFAULT_MIN_OCCURRENCES = 2;
/**
* Adequacy levels that count as unmet recall (signals-that-survive, unit
* 6). A `sufficient` attempt is not demand, so it is never stamped.
*/
export const DEMAND_UNMET_ADEQUACY_LEVELS: ReadonlyArray<RecallAdequacyLevel> = Object.freeze([
"weak",
"insufficient",
]);
/** Whether an adequacy level is unmet recall, i.e. actionable demand. */
export function isUnmetAdequacyLevel(level: RecallAdequacyLevel): boolean {
return DEMAND_UNMET_ADEQUACY_LEVELS.includes(level);
}
/**
* One persisted recall observation. `coverage` is absent when the caller
* did not compute the evidence-pack coverage (the common non-verified
* search path); the aggregator falls back to the result-presence signal
* for those buckets. `adequacy` is absent unless the caller graded the
* attempt through {@link ../brain/recall-adequacy.ts} and the grade came
* back unmet (signals-that-survive, unit 6).
*/
export interface QueryDemandRecord {
readonly ts: string;
readonly terms: ReadonlyArray<string>;
readonly results: number;
readonly coverage?: number;
readonly adequacy?: RecallAdequacyLevel;
}
export interface RecordQueryDemandInput {
/** Raw query text β normalized/redacted to terms before it is stored. */
readonly query?: string;
/** Pre-normalized terms; used verbatim (after redaction) if given. */
readonly terms?: ReadonlyArray<string>;
readonly resultCount: number;
/** IDF-weighted coverage (0..1) from coverage.ts, when available. */
readonly coverage?: number | null;
/** Graded recall adequacy, when the caller ran the verdict layer. */
readonly adequacy?: RecallAdequacyLevel;
/** ISO timestamp; defaults to now. */
readonly at?: string;
}
/** One recurring question the vault keeps answering with unmet grounding. */
export interface UnmetRecallBucket {
/** The shared bucket key: sorted normalized terms joined by a space. */
readonly key: string;
readonly terms: ReadonlyArray<string>;
readonly occurrences: number;
}
export interface QueryDemandFilter {
readonly since?: string;
readonly until?: string;
}
export interface AggregateQueryDemandOptions extends QueryDemandFilter {
/** Minimum occurrences to surface a bucket (default 2). */
readonly minOccurrences?: number;
/**
* Surface only buckets whose mean satisfaction is at or below this
* (default {@link COMPLETENESS_COMPLETE_THRESHOLD}) β i.e. queries not
* already answered well. Ranking by demand pushes the worst to the top.
*/
readonly maxSatisfaction?: number;
/** Cap on returned gaps. */
readonly limit?: number;
}
export interface QueryDemandGap {
/** Sorted normalized terms β the bucket key, split for display. */
readonly terms: ReadonlyArray<string>;
readonly occurrences: number;
/** Mean IDF-weighted coverage over records that carried one, else null. */
readonly meanCoverage: number | null;
readonly meanResultCount: number;
/** Records with zero results. */
readonly emptyCount: number;
readonly emptyRate: number;
/**
* Satisfaction in [0,1]: the coverage mean when any record carried
* coverage, otherwise the fraction of runs that returned any result.
*/
readonly satisfaction: number;
/** Whether satisfaction came from coverage (true) or the result-count
* fallback (false). */
readonly coverageKnown: boolean;
/** occurrences Γ (1 β satisfaction): recurring + unmet ranks highest. */
readonly demandScore: number;
/** Completeness verdict over the satisfaction score. */
readonly verdict: CompletenessVerdict;
readonly firstSeen: string;
readonly lastSeen: string;
}
export interface QueryDemandReport {
readonly totalRecords: number;
readonly distinctQueries: number;
readonly gaps: ReadonlyArray<QueryDemandGap>;
}
/** Length at/above which a mixed letter+digit token is treated as a
* high-entropy secret/id (API key, UUID, hash) rather than a query word
* and dropped. Real words this long are essentially never both alpha and
* numeric. */
export const DEMAND_SECRET_TOKEN_MIN_LEN = 20;
/**
* Normalize a raw query into the terms actually stored: the significant
* terms, deduped and sorted for a stable bucket key, with the privacy
* filters below applied. Capped at {@link DEMAND_MAX_TERMS_PER_RECORD}.
*/
export function normalizeQueryTerms(query: string): string[] {
return sanitizeTerms(significantTerms(query));
}
function sanitizeTerms(terms: ReadonlyArray<string>): string[] {
const out = new Set<string>();
for (const raw of terms) {
const term = raw.toLocaleLowerCase().trim();
// Upper bound only. The lower bound of three characters used to sit
// here too, mirroring the one `significantTerms` carried, and it
// carried the same defect: a query written in a script whose words
// are one or two characters produced no terms, so it could never
// form a demand bucket and its recall gaps were invisible to the
// promotion loop - a whole language silently unrepresented in the
// measurement. Nothing about a short term makes it more identifying,
// which is what the remaining filters here are for.
if (term.length === 0 || term.length > DEMAND_TERM_MAX_LEN) continue;
if (isSecretShapedTerm(term)) continue;
// Belt-and-suspenders: a term the key/value redactor still rewrites
// is secret-shaped β never a legitimate query word, so drop it rather
// than store a `***REDACTED***` bucket key.
if (redactRawOutput(term) !== term) continue;
out.add(term);
}
return [...out].toSorted().slice(0, DEMAND_MAX_TERMS_PER_RECORD);
}
/**
* A bare token is secret-shaped when it is long AND mixes letters with
* digits β the signature of an API key, UUID, or hash. The redactor's
* passes are context-based (`key: value`, bearer headers, env), so a
* single high-entropy word carries no shape they can match; this catches
* it before it becomes a persisted bucket key. Kept deliberately narrow
* (length gate + both classes) so ordinary words and short alnum terms
* (`oauth2`, `sha256`, `utf8`) are never dropped.
*/
function isSecretShapedTerm(term: string): boolean {
if (term.length < DEMAND_SECRET_TOKEN_MIN_LEN) return false;
return /[a-z]/.test(term) && /[0-9]/.test(term);
}
/** The demand ledger's file-name layout, handed to the shared shard grammar. */
const DEMAND_LEDGER_GRAMMAR = jsonlLedgerGrammar(QUERY_DEMAND_LEDGER_STEM);
/**
* Append one recall observation to the demand log. Terms are derived
* from `terms` (redacted) or `query` (normalized+redacted); a query with
* no significant terms is not recorded (returns null). Best-effort by
* contract β callers gate this behind opt-in telemetry and must not let
* a log write fail the search. On a store-only machine a failed store
* write is counted device-locally, not thrown: the returned record is
* what was attempted, not proof that it was stored. An unswitched
* machine's legacy write fault still propagates.
*/
export function recordQueryDemand(
vault: string | null,
input: RecordQueryDemandInput,
): QueryDemandRecord | null {
const terms =
input.terms !== undefined ? sanitizeTerms(input.terms) : normalizeQueryTerms(input.query ?? "");
if (terms.length === 0) return null;
const record: QueryDemandRecord = {
ts: normalizeDemandTimestamp(input.at),
terms,
results: Number.isFinite(input.resultCount) ? Math.max(0, Math.floor(input.resultCount)) : 0,
...(typeof input.coverage === "number" && Number.isFinite(input.coverage)
? { coverage: clamp01(input.coverage) }
: {}),
...(input.adequacy !== undefined ? { adequacy: input.adequacy } : {}),
};
// One `query-demand` ledger row on a store-only machine (design D14
// item 1), the vault's demand shard plus the dual write otherwise. The
// terms are what the user searched for, so the row goes to `private`
// when this machine holds it. Telemetry never fails the search it
// observes, so a store-only failure is counted and the search goes on.
routeWrite({
legacy: () => appendLegacyDemand(legacyVaultOf(vault, "recordQueryDemand"), record),
store: () => ledgerRowRecord(DUAL_WRITE_LEDGERS.queryDemand, record, "private-when-held"),
context: { vault },
failOpen: true,
});
return record;
}
/** The legacy append: this device's demand shard under the vault. */
function appendLegacyDemand(vault: string, record: QueryDemandRecord): void {
// Vault-identity write guard (context-integrity-gates, Unit J).
assertVaultIdentityForWrite(vault);
// Per-device shard (t_774dea61): this device appends to a file no other
// device writes, and the per-file lock plus the in-place compaction are
// scoped to that one shard.
const path = queryDemandLogPath(vault);
mkdirSync(dirname(path), { recursive: true });
const handle = acquireLockSync(path);
try {
appendFileSync(path, `${JSON.stringify(record)}\n`, { encoding: "utf8" });
compactIfNeeded(path);
} finally {
handle.release();
}
}
/**
* Stamp one graded recall attempt onto the demand log
* (signals-that-survive, unit 6). The bucket key is whatever
* {@link normalizeQueryTerms} derives from the query β the identity concept
* this module already owns β so recurrence aggregates without a second one.
*
* Returns null, rather than writing, when the attempt is not demand: a
* `sufficient` verdict is met recall, and a query with no significant terms
* has no bucket to key on. Both are documented non-cases, not swallowed
* failures. A genuine write fault of an unswitched machine still
* propagates from {@link recordQueryDemand}; a store-only machine counts a
* failed store write in the device-local status and the search goes on
* (design D14 item 1), so the returned record may not have been stored.
*/
export function recordRecallAdequacyDemand(
vault: string,
input: {
readonly query: string;
readonly verdict: RecallAdequacyVerdict;
readonly at?: string;
},
): QueryDemandRecord | null {
if (!isUnmetAdequacyLevel(input.verdict.level)) return null;
return recordQueryDemand(vault, {
query: input.query,
resultCount: input.verdict.resultCount,
adequacy: input.verdict.level,
...(input.at !== undefined ? { at: input.at } : {}),
});
}
/**
* Count how often each bucket came back with unmet grounding. This is the
* verdict-side recurrence source the knowledge-gap loop aggregates
* alongside its structural `gap_counts` source (signals-that-survive, unit
* 6); the two are deliberately counted separately, never summed, because a
* telemetry code bucket and a term bucket do not measure the same thing.
*/
export function aggregateUnmetRecall(
vault: string,
filter: QueryDemandFilter = {},
): ReadonlyArray<UnmetRecallBucket> {
const buckets = new Map<string, { terms: ReadonlyArray<string>; occurrences: number }>();
for (const record of readQueryDemand(vault, filter)) {
if (record.adequacy === undefined || !isUnmetAdequacyLevel(record.adequacy)) continue;
const key = demandBucketKey(record.terms);
const bucket = buckets.get(key);
if (bucket === undefined) {
buckets.set(key, { terms: record.terms, occurrences: 1 });
} else {
bucket.occurrences += 1;
}
}
return Object.freeze(
[...buckets].map(([key, bucket]) =>
Object.freeze({ key, terms: bucket.terms, occurrences: bucket.occurrences }),
),
);
}
/** The bucket key both aggregators group by: sorted terms, space-joined. */
function demandBucketKey(terms: ReadonlyArray<string>): string {
return terms.join(" ");
}
/** Read all demand records, filtered by [since, until] and sorted by ts. */
export function readQueryDemand(
vault: string | null,
filter: QueryDemandFilter = {},
): ReadonlyArray<QueryDemandRecord> {
// The `query-demand` ledger rows on a store-only machine; otherwise a
// merged read over every device's shard (t_774dea61), where the absent
// directory lists nothing, the same empty result as before.
const lines = readLedgerRows<unknown>(
DUAL_WRITE_LEDGERS.queryDemand,
(legacyVault) => readShardedLines(join(legacyVault, BRAIN_LOG_REL), DEMAND_LEDGER_GRAMMAR),
{ vault },
);
// Normalize the filter bounds to the same millisecond-precision form the
// stored `ts` already uses (records are normalized at write time). The
// comparison below is lexical, so a second-precision `--since
// 2026-07-01T00:00:00Z` would otherwise drop every ms-precision record in
// `[00:00:00.000Z, 00:00:00.999Z]` β `.` sorts before `Z`. Running the
// bounds through the same normalizer keeps both sides at one precision.
const since = filter.since !== undefined ? normalizeDemandTimestamp(filter.since) : undefined;
const until = filter.until !== undefined ? normalizeDemandTimestamp(filter.until) : undefined;
const out: QueryDemandRecord[] = [];
for (const line of lines) {
const record = coerceRecord(line);
if (record === null) continue;
if (since !== undefined && record.ts < since) continue;
if (until !== undefined && record.ts > until) continue;
out.push(record);
}
out.sort((a, b) => (a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0));
return Object.freeze(out);
}
/**
* Aggregate the demand log into ranked knowledge gaps: recurring queries
* the vault answers poorly. Buckets by the sorted-terms key, scores each
* by occurrences Γ (1 β satisfaction), and returns the worst first.
*/
export function aggregateQueryDemand(
vault: string,
options: AggregateQueryDemandOptions = {},
): QueryDemandReport {
const records = readQueryDemand(vault, {
...(options.since !== undefined ? { since: options.since } : {}),
...(options.until !== undefined ? { until: options.until } : {}),
});
const minOccurrences = Math.max(
1,
Math.floor(options.minOccurrences ?? DEMAND_DEFAULT_MIN_OCCURRENCES),
);
const maxSatisfaction = clamp01(options.maxSatisfaction ?? COMPLETENESS_COMPLETE_THRESHOLD);
interface Bucket {
terms: ReadonlyArray<string>;
occurrences: number;
resultSum: number;
emptyCount: number;
coverageSum: number;
coverageSamples: number;
firstSeen: string;
lastSeen: string;
}
const buckets = new Map<string, Bucket>();
for (const record of records) {
const key = demandBucketKey(record.terms);
let bucket = buckets.get(key);
if (!bucket) {
bucket = {
terms: record.terms,
occurrences: 0,
resultSum: 0,
emptyCount: 0,
coverageSum: 0,
coverageSamples: 0,
firstSeen: record.ts,
lastSeen: record.ts,
};
buckets.set(key, bucket);
}
bucket.occurrences += 1;
bucket.resultSum += record.results;
if (record.results === 0) bucket.emptyCount += 1;
if (typeof record.coverage === "number") {
bucket.coverageSum += record.coverage;
bucket.coverageSamples += 1;
}
if (record.ts < bucket.firstSeen) bucket.firstSeen = record.ts;
if (record.ts > bucket.lastSeen) bucket.lastSeen = record.ts;
}
const gaps: QueryDemandGap[] = [];
for (const bucket of buckets.values()) {
if (bucket.occurrences < minOccurrences) continue;
const coverageKnown = bucket.coverageSamples > 0;
const meanCoverage = coverageKnown ? bucket.coverageSum / bucket.coverageSamples : null;
// Prefer the reused coverage score; only when no record in this
// bucket carried one do we fall back to "did it return anything".
const satisfaction = coverageKnown
? meanCoverage!
: (bucket.occurrences - bucket.emptyCount) / bucket.occurrences;
if (satisfaction > maxSatisfaction) continue;
gaps.push(
Object.freeze({
terms: bucket.terms,
occurrences: bucket.occurrences,
meanCoverage: meanCoverage === null ? null : round4(meanCoverage),
meanResultCount: round4(bucket.resultSum / bucket.occurrences),
emptyCount: bucket.emptyCount,
emptyRate: round4(bucket.emptyCount / bucket.occurrences),
satisfaction: round4(satisfaction),
coverageKnown,
demandScore: round4(bucket.occurrences * (1 - satisfaction)),
verdict: verdictFor(satisfaction),
firstSeen: bucket.firstSeen,
lastSeen: bucket.lastSeen,
}),
);
}
gaps.sort(
(a, b) =>
b.demandScore - a.demandScore ||
b.occurrences - a.occurrences ||
a.terms.join(" ").localeCompare(b.terms.join(" ")),
);
const limited =
options.limit !== undefined ? gaps.slice(0, Math.max(0, Math.floor(options.limit))) : gaps;
return Object.freeze({
totalRecords: records.length,
distinctQueries: buckets.size,
gaps: Object.freeze(limited),
});
}
/**
* Snake_case JSON projection of a report for the MCP tool and CLI
* `--json` surfaces, matching the sibling recall-telemetry convention.
* Kept here so both surfaces serialize identically.
*/
export function serializeQueryDemandReport(report: QueryDemandReport): Record<string, unknown> {
return {
total_records: report.totalRecords,
distinct_queries: report.distinctQueries,
gaps: report.gaps.map((gap) => ({
terms: gap.terms,
occurrences: gap.occurrences,
mean_coverage: gap.meanCoverage,
mean_result_count: gap.meanResultCount,
empty_count: gap.emptyCount,
empty_rate: gap.emptyRate,
satisfaction: gap.satisfaction,
coverage_known: gap.coverageKnown,
demand_score: gap.demandScore,
verdict: gap.verdict,
first_seen: gap.firstSeen,
last_seen: gap.lastSeen,
})),
};
}
function verdictFor(satisfaction: number): CompletenessVerdict {
if (satisfaction >= COMPLETENESS_COMPLETE_THRESHOLD) return "complete";
if (satisfaction >= COMPLETENESS_PARTIAL_THRESHOLD) return "partial";
return "sparse";
}
/** One demand row: a legacy JSONL line, or a store row as its writer wrote it. */
function coerceRecord(line: unknown): QueryDemandRecord | null {
let parsed: unknown = line;
if (typeof line === "string") {
try {
parsed = JSON.parse(line);
} catch {
return null;
}
}
if (parsed === null || typeof parsed !== "object") return null;
const obj = parsed as Record<string, unknown>;
const ts = obj["ts"];
const terms = obj["terms"];
const results = obj["results"];
if (typeof ts !== "string") return null;
if (!Array.isArray(terms) || !terms.every((t) => typeof t === "string")) return null;
if (typeof results !== "number" || !Number.isFinite(results)) return null;
const coverage = obj["coverage"];
const adequacy = obj["adequacy"];
return {
ts,
terms: Object.freeze(terms as string[]),
results: Math.max(0, Math.floor(results)),
...(typeof coverage === "number" && Number.isFinite(coverage)
? { coverage: clamp01(coverage) }
: {}),
// An unrecognised level is dropped rather than trusted: the record
// predates the field, or was hand-edited.
...(isRecallAdequacyLevel(adequacy) ? { adequacy } : {}),
};
}
/**
* Drop the oldest lines once the file grows past
* {@link DEMAND_LOG_MAX_BYTES}, keeping the newest lines that fit in
* {@link DEMAND_LOG_COMPACT_BYTES}. Called under the append lock. The
* common path only stats the file β no read β so per-append cost stays
* O(1) until the rare compaction.
*/
function compactIfNeeded(path: string): void {
// Only a vanished shard is "nothing to compact"; any other stat failure
// (EACCES, EIO) propagates by its code instead of leaving the cap unchecked.
const size = statSync(path, { throwIfNoEntry: false })?.size;
if (size === undefined || size <= DEMAND_LOG_MAX_BYTES) return;
const lines = readFileSync(path, "utf8")
.split("\n")
.filter((line) => line.trim().length > 0);
const kept: string[] = [];
let bytes = 0;
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i]!;
const lineBytes = Buffer.byteLength(line, "utf8") + 1;
if (bytes + lineBytes > DEMAND_LOG_COMPACT_BYTES && kept.length > 0) break;
kept.push(line);
bytes += lineBytes;
}
kept.reverse();
atomicWriteFileSync(path, kept.length > 0 ? `${kept.join("\n")}\n` : "");
}
/**
* Normalize a caller-supplied (or absent) timestamp into a canonical
* millisecond-precision ISO string. `readQueryDemand` and
* `aggregateQueryDemand` compare `ts` as raw strings (sort + range
* filter), which is only stable when every record uses the same precision
* β `2026-07-01T00:00:00.000Z` must sort after `2026-07-01T00:00:00Z`,
* but lexically `.` < `Z` flips them. Parsing-and-reformatting through
* `Date` collapses the difference and also rejects non-ISO input by
* falling back to now, so a malformed `at` can't silently misorder the
* log. Non-finite `Date` (invalid input) resolves to the current time.
*/
function normalizeDemandTimestamp(at: string | undefined): string {
if (at === undefined) return new Date().toISOString();
const parsed = new Date(at);
const ms = parsed.getTime();
return Number.isFinite(ms) ? parsed.toISOString() : new Date().toISOString();
}
function round4(value: number): number {
return Math.round(value * 10_000) / 10_000;
}