Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.6 KB
Raw Blame
/**
 * Per-plan ingest checkpoint (Ingestion & Import Robustness suite,
 * t_ba1fa5f6).
 *
 * The content-hash manifest ({@link ../ingest/content-manifest.ts}) records a
 * source only once its full {@link ../ingest/ingest.ts:ingestSource} write has
 * landed, so it answers "have we ever fully ingested this file". This module
 * answers a finer, plan-scoped question: "within THIS batch plan, which items
 * have completed so far" - so an interrupted large-folder ingest resumes at the
 * item boundary instead of re-planning from scratch.
 *
 * Union-as-you-go: each completed item is folded into the checkpoint as it
 * finishes; the content manifest stays the authoritative final state. The
 * checkpoint lives at `<vault>/.open-second-brain/ingest-checkpoints/<plan_id>.json`,
 * a machine artifact (not curated memory), mirroring the manifest's location and
 * atomic-write / no-op-on-unchanged discipline.
 *
 * Opt-out: setting `OSB_INGEST_NO_CHECKPOINT` to a truthy value makes record and
 * read inert - the deterministic-test escape hatch mirroring upstream graphify's
 * `GRAPHIFY_NO_INCREMENTAL_CACHE`.
 *
 * The mechanism below the record - the id validation, the location, the
 * schema-version refusal, the lock / identity / atomic write - lives in
 * {@link ../checkpoint-store.ts}, shared with the session-import lane. What
 * stays here is this lane's record: a plan id, a source dir, and a set of
 * completed paths that no-ops when the SET is unchanged.
 *
 * Language-agnostic: keys are canonical vault-relative paths and content hashes;
 * no natural-language content is inspected.
 */

import { createHash } from "node:crypto";

import { canonicalNotePath } from "../../path-safety.ts";
import {
  checkpointFilePath,
  checkpointingEnabled,
  NO_CHECKPOINT_ENV,
  readCheckpointObject,
  removeCheckpointFile,
  withCheckpointLock,
  writeCheckpointObject,
} from "../checkpoint-store.ts";
import { isoSecond } from "../time.ts";

/** Only schema version currently understood. Unknown versions are refused. */
const SCHEMA_VERSION = 1 as const;

/** Subdirectory holding one JSON checkpoint per batch plan. */
const CHECKPOINT_DIR = "ingest-checkpoints";

/**
 * How a refusal names this lane's id and its file. Exported because the
 * boundaries that accept a caller-supplied plan id validate it against the
 * shared store's grammar and must name the same thing this lane does.
 */
export const PLAN_ID_LABEL = "plan";
const ID_LABEL = PLAN_ID_LABEL;
const FILE_LABEL = "ingest checkpoint";

export { checkpointingEnabled, NO_CHECKPOINT_ENV };

/** The persisted per-plan checkpoint. */
export interface IngestCheckpoint {
  readonly schema_version: typeof SCHEMA_VERSION;
  /** Stable id derived from the source dir and the full discovered path set. */
  readonly plan_id: string;
  /** Canonical vault-relative source directory the plan covers. */
  readonly source_dir: string;
  /** Canonical vault-relative paths completed so far, sorted. */
  readonly completed: readonly string[];
  readonly updated_at: string;
}

/**
 * Deterministic plan id: a short SHA-256 hex over the canonical source dir, the
 * sorted full discovered path set, and the extraction contract
 * ({@link ./contract.ts}) the plan runs under. Keying on the FULL set (not the
 * remaining work) keeps the id stable across a resume even as items complete;
 * keying on the CONTRACT means a changed contract gets a fresh id, so a stale
 * checkpoint recorded under the old contract cannot resume-skip work extracted
 * under the new one - the old file orphans harmlessly.
 */
export function computePlanId(
  sourceDir: string,
  discoveredPaths: readonly string[],
  contractFingerprint: string,
): string {
  const dir = canonicalNotePath(sourceDir);
  const paths = discoveredPaths.map((p) => canonicalNotePath(p)).toSorted();
  const hash = createHash("sha256");
  hash.update(dir);
  hash.update("\0");
  for (const p of paths) {
    hash.update(p);
    hash.update("\n");
  }
  hash.update("\0");
  hash.update(contractFingerprint);
  return hash.digest("hex").slice(0, 16);
}

/** Absolute path of one plan's checkpoint file. */
export function checkpointPath(vault: string, planId: string): string {
  return checkpointFilePath(vault, CHECKPOINT_DIR, ID_LABEL, planId);
}

function serialize(cp: IngestCheckpoint): string {
  return (
    JSON.stringify(
      {
        schema_version: cp.schema_version,
        plan_id: cp.plan_id,
        source_dir: cp.source_dir,
        completed: [...cp.completed].toSorted(),
        updated_at: cp.updated_at,
      },
      null,
      2,
    ) + "\n"
  );
}

/**
 * Read a plan's checkpoint. A missing file (or checkpointing disabled) returns
 * `null`. A corrupt file or an unknown `schema_version` is a hard error - never
 * a silent reset that would masquerade completed items as pending.
 */
export function readCheckpoint(vault: string, planId: string): IngestCheckpoint | null {
  if (!checkpointingEnabled()) return null;
  const path = checkpointPath(vault, planId);
  const obj = readCheckpointObject(path, FILE_LABEL, SCHEMA_VERSION);
  if (obj === null) return null;
  const rawCompleted = obj["completed"];
  const completed = Array.isArray(rawCompleted)
    ? rawCompleted
        .filter((x): x is string => typeof x === "string")
        .map((p) => canonicalNotePath(p))
        .toSorted()
    : [];
  return Object.freeze({
    schema_version: SCHEMA_VERSION,
    plan_id: planId,
    source_dir: typeof obj["source_dir"] === "string" ? obj["source_dir"] : "",
    completed: Object.freeze(completed),
    updated_at: typeof obj["updated_at"] === "string" ? obj["updated_at"] : "",
  });
}

function sameSet(a: readonly string[], b: readonly string[]): boolean {
  if (a.length !== b.length) return false;
  const set = new Set(a);
  return b.every((x) => set.has(x));
}

/**
 * Fold `paths` into a plan's checkpoint (union-as-you-go), atomically. Returns
 * `true` when the checkpoint was written, `false` on a no-op - checkpointing
 * disabled, or the completed set (and hence the serialized bytes) unchanged. The
 * `updated_at` stamp is only bumped when the set actually grows, so a re-record
 * of an already-recorded set leaves the file byte-identical.
 *
 * The read, the union and the write are ONE critical section under the sync
 * lock. A plan's items are dispatched to parallel subagents, so two of them
 * finishing at once would otherwise each write back the completed set they
 * read - and the later write would erase the earlier item, silently turning a
 * done item back into pending work on the next resume.
 */
export function recordCompleted(
  vault: string,
  planId: string,
  sourceDir: string,
  paths: readonly string[],
  now: Date,
): boolean {
  if (!checkpointingEnabled()) return false;
  const path = checkpointPath(vault, planId);
  return withCheckpointLock(path, () => {
    const prev = readCheckpoint(vault, planId);
    const merged = new Set<string>(prev?.completed ?? []);
    for (const p of paths) merged.add(canonicalNotePath(p));
    const completed = [...merged].toSorted();
    if (prev && sameSet(prev.completed, completed)) return false;
    const next: IngestCheckpoint = {
      schema_version: SCHEMA_VERSION,
      plan_id: planId,
      source_dir: canonicalNotePath(sourceDir),
      completed,
      updated_at: isoSecond(now),
    };
    return writeCheckpointObject(vault, path, serialize(next));
  });
}

/**
 * Remove a plan's checkpoint (the authoritative-final cleanup once a plan is
 * fully drained). Returns `true` when a file was removed, `false` when none
 * existed.
 */
export function clearCheckpoint(vault: string, planId: string): boolean {
  return removeCheckpointFile(vault, checkpointPath(vault, planId));
}