Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
9.4 KB
Raw Blame
/**
 * Pending mining surveys: what a plan put in front of a model, remembered
 * until the commit that answers it.
 *
 * The documented mining flow is plan, answer, commit - and the commit is
 * not required to relay the plan's `surveyed_turns_hash`. A commit that
 * relays nothing used to hash the session's turns as they stood at COMMIT
 * time, so an import landing between the plan and the commit made a
 * zero-yield commit tombstone turns no model had seen. This store closes
 * that gap without asking the caller for anything: every plan that hands
 * out a mining envelope records the hash of the material it offered, and
 * the commit that answers it clears the record. A zero-yield commit without a relay
 * writes its tombstone only when exactly one distinct material was
 * surveyed since the last commit and the session still holds it. The
 * commit reads the survey first and clears what it read only once its
 * outcome is final, so a commit that fails transiently (an unreadable
 * session, a ledger write that refused) leaves the survey for its retry.
 *
 * Kept apart from the mining ledger on purpose. The ledger fails closed
 * because a silent reset would unsuppress every barren session; a lost
 * survey can only withhold one tombstone (the session stays minable), so
 * the two have different failure economics and different owners - plans
 * write here, commits write there.
 *
 * A corrupt or unknown-schema file still refuses by name rather than
 * reading as empty: an empty read could make a session with two pending
 * surveys look like it had one, which is exactly the ambiguity the store
 * exists to detect.
 */

import { existsSync, readFileSync } from "node:fs";

import { atomicWriteFileSync } from "../fs-atomic.ts";
import { DERIVED_STORE_DIR } from "./path-constants.ts";
import { acquireLockSyncWithRetry, ingestLockRemedy } from "./sync-lockfile.ts";
import { isoSecond } from "./time.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
import { machineStatePath } from "../state/machine-state.ts";

/** Only survey-store schema this build understands. */
export const SESSION_MINING_SURVEYS_SCHEMA_VERSION = 1 as const;

const SESSION_MINING_SURVEYS_FILE = "session-mining-surveys.json";

/**
 * Distinct hashes kept per session. Past two the answer is already
 * "ambiguous", so the cap only bounds the file; keeping the newest few
 * keeps the record honest about there being several.
 */
export const MAX_SURVEYED_HASHES_PER_SESSION = 8;

/** Sessions kept; the oldest surveys beyond this are dropped at write time. */
export const MAX_SURVEYED_SESSIONS = 512;

/** The remedy every file-level refusal names. */
const CORRUPT_SURVEYS_REMEDY =
  "; to recover, move the file aside or delete it - only pending plan surveys are lost, so " +
  "the next barren commit of an affected session records no tombstone until it is planned again";

/**
 * A store failure in a closed form: the store name plus the errno code,
 * never the raised message, which names the absolute host path. For every
 * surface that answers a caller below local reach (the same form as
 * `closedCollectError`).
 */
export function closedSurveyFailure(err: unknown): string {
  const code = (err as NodeJS.ErrnoException | null)?.code;
  return `session mining surveys store failed (${typeof code === "string" ? code : "error"})`;
}

/** Absolute path of the pending-survey store for a vault. */
export function sessionMiningSurveysPath(vault: string): string {
  return machineStatePath(vault, DERIVED_STORE_DIR, SESSION_MINING_SURVEYS_FILE);
}

interface SurveyRow {
  readonly hashes: ReadonlyArray<string>;
  readonly surveyedAt: string;
}

/**
 * Session ids are caller data, so every map keyed by one is
 * null-prototype: an id such as `__proto__` is an own key like any other
 * instead of an assignment to the prototype.
 */
type SurveyMap = Record<string, SurveyRow>;

function emptySurveyMap(): SurveyMap {
  return Object.create(null) as SurveyMap;
}

function readSurveys(vault: string): SurveyMap {
  const path = sessionMiningSurveysPath(vault);
  if (!existsSync(path)) return emptySurveyMap();
  let parsed: unknown;
  try {
    parsed = JSON.parse(readFileSync(path, "utf8"));
  } catch (e) {
    throw new Error(
      `session mining surveys file is corrupted JSON: ${path}${CORRUPT_SURVEYS_REMEDY}`,
      {
        cause: e,
      },
    );
  }
  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
    throw new Error(
      `session mining surveys file is not an object: ${path}${CORRUPT_SURVEYS_REMEDY}`,
    );
  }
  const obj = parsed as Record<string, unknown>;
  if (obj["schema_version"] !== SESSION_MINING_SURVEYS_SCHEMA_VERSION) {
    throw new Error(
      `session mining surveys schema_version ${String(obj["schema_version"])} not supported ` +
        `(expected ${SESSION_MINING_SURVEYS_SCHEMA_VERSION}): ${path}${CORRUPT_SURVEYS_REMEDY}`,
    );
  }
  const raw = obj["surveys"];
  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) {
    throw new Error(
      `session mining surveys container is not an object: ${path}${CORRUPT_SURVEYS_REMEDY}`,
    );
  }
  const out = emptySurveyMap();
  for (const [key, value] of Object.entries(raw as Record<string, unknown>)) {
    if (value === null || typeof value !== "object" || Array.isArray(value)) continue;
    const row = value as Record<string, unknown>;
    const hashes = row["hashes"];
    if (!Array.isArray(hashes) || !hashes.every((h) => typeof h === "string" && h.length > 0)) {
      // A damaged row is kept as "several surveys" rather than dropped: a
      // dropped row would read as "no survey", and the commit would then
      // name the absence - but a row reduced to one hash could wrongly
      // attest a single survey. Two placeholder hashes force ambiguity.
      out[key] = { hashes: ["<unreadable>", "<unreadable-2>"], surveyedAt: "" };
      continue;
    }
    const surveyedAt = row["surveyedAt"];
    out[key] = {
      hashes: hashes as string[],
      surveyedAt: typeof surveyedAt === "string" ? surveyedAt : "",
    };
  }
  return out;
}

function writeSurveys(vault: string, surveys: SurveyMap): void {
  const path = sessionMiningSurveysPath(vault);
  const sorted = emptySurveyMap();
  for (const key of Object.keys(surveys).toSorted()) sorted[key] = surveys[key]!;
  const next =
    JSON.stringify(
      { schema_version: SESSION_MINING_SURVEYS_SCHEMA_VERSION, surveys: sorted },
      null,
      2,
    ) + "\n";
  if (existsSync(path) && readFileSync(path, "utf8") === next) return;
  atomicWriteFileSync(path, next);
}

/**
 * Record that a plan offered `hash` for `sessionId`. A hash already pending
 * for the session is not added twice, so re-planning unchanged material
 * keeps the survey unambiguous.
 */
export function recordMiningSurvey(
  vault: string,
  sessionId: string,
  hash: string,
  now: Date = new Date(),
): void {
  assertVaultIdentityForWrite(vault);
  const path = sessionMiningSurveysPath(vault);
  const key = sessionId.trim();
  const handle = acquireLockSyncWithRetry(path, undefined, { remedy: ingestLockRemedy });
  try {
    const surveys = readSurveys(vault);
    const prior = surveys[key]?.hashes ?? [];
    const hashes = [...prior.filter((h) => h !== hash), hash].slice(
      -MAX_SURVEYED_HASHES_PER_SESSION,
    );
    surveys[key] = { hashes, surveyedAt: isoSecond(now) };
    const keys = Object.keys(surveys);
    if (keys.length > MAX_SURVEYED_SESSIONS) {
      // A damaged row reads with an empty surveyedAt and so goes first.
      // Evicting it is the safe direction: the session then reads as
      // having no survey, which withholds a tombstone rather than writing one.
      const oldestFirst = keys.toSorted(
        (a, b) =>
          surveys[a]!.surveyedAt.localeCompare(surveys[b]!.surveyedAt) || a.localeCompare(b),
      );
      for (const drop of oldestFirst.slice(0, keys.length - MAX_SURVEYED_SESSIONS)) {
        if (drop !== key) delete surveys[drop];
      }
    }
    writeSurveys(vault, surveys);
  } finally {
    handle.release();
  }
}

/**
 * The pending surveys of one session: the distinct hashes plans offered
 * since its last commit, oldest first, or `[]` when none was recorded.
 * Read-only - the commit clears them with {@link clearMiningSurvey} once
 * its outcome is final. Throws on a corrupt or unknown-schema store.
 */
export function readMiningSurvey(vault: string, sessionId: string): ReadonlyArray<string> {
  return readSurveys(vault)[sessionId.trim()]?.hashes ?? [];
}

/**
 * Remove `hashes` from one session's pending surveys - the ones a commit
 * read and answered. A hash a plan added after that read stays pending, so
 * a plan racing the commit is not lost. The row goes once it is empty.
 */
export function clearMiningSurvey(
  vault: string,
  sessionId: string,
  hashes: ReadonlyArray<string>,
): void {
  if (hashes.length === 0) return;
  const path = sessionMiningSurveysPath(vault);
  if (!existsSync(path)) return;
  assertVaultIdentityForWrite(vault);
  const key = sessionId.trim();
  const handle = acquireLockSyncWithRetry(path, undefined, { remedy: ingestLockRemedy });
  try {
    const surveys = readSurveys(vault);
    const row = surveys[key];
    if (row === undefined) return;
    const answered = new Set(hashes);
    const left = row.hashes.filter((h) => !answered.has(h));
    if (left.length === row.hashes.length) return;
    if (left.length === 0) delete surveys[key];
    else surveys[key] = { hashes: left, surveyedAt: row.surveyedAt };
    writeSurveys(vault, surveys);
  } finally {
    handle.release();
  }
}