Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
4.9 KB
Raw Blame
/**
 * Reach-aware near-duplicate kernel: one pure lookup shared by every
 * point a fact enters or leaves the vault (write-receipt hint, feedback
 * advisory, retire siblings).
 *
 * The caller projects its probe and pool into token sets with
 * `tokenise` (see `similarity.ts`); the kernel never reads the vault,
 * never sees a vector and never calls a model. The `readable`
 * predicate is required and runs before anything else, so a withheld
 * entry is neither scored nor counted: no match and no census number
 * can disclose it. Operator-reach callers pass {@link READ_ALL_REFS}
 * explicitly, so every unfiltered call site stays visible in the diff.
 */

import { jaccard } from "./similarity.ts";

export type NearDuplicateMethod = "lexical" | "embedding";
export type ReadableRef = (ref: string) => boolean;

/** Explicit operator-reach predicate; every unfiltered call site names it. */
export const READ_ALL_REFS: ReadableRef = () => true;

/**
 * The one named threshold table. `writeHint` is the shipped page-lint
 * bar, `retireSiblingLexical` the doctor lint bar and
 * `retireSiblingEmbedding` the entity semantic-dedup bar.
 */
export const NEAR_DUPLICATE_THRESHOLDS = Object.freeze({
  writeHint: 0.8,
  retireSiblingLexical: 0.7,
  retireSiblingEmbedding: 0.92,
} as const);

/** Short texts share most of their tokens by accident; below this nothing is scored. */
export const NEAR_DUPLICATE_MIN_TOKENS = 4;
/** Upper bound on the entries one lookup scores. */
export const NEAR_DUPLICATE_CANDIDATE_CAP = 200;
/** Upper bound on keyword-index chunks a write-side widening pulls. */
export const NEAR_DUPLICATE_WIDENING_TOP_K = 20;

export interface NearDuplicateProbe {
  /** Vault-relative path or preference id. */
  readonly ref: string;
  /** From `similarity.ts` `tokenise`. */
  readonly tokens: ReadonlySet<string>;
}

export interface NearDuplicatePoolEntry {
  readonly ref: string;
  readonly tokens: ReadonlySet<string>;
  /** Optional bucket; when the options carry `bucket`, entries must match it. */
  readonly bucket?: string;
}

export interface NearDuplicateOptions {
  readonly threshold: number;
  /** Required; applied before scoring and counting. */
  readonly readable: ReadableRef;
  /** Default {@link NEAR_DUPLICATE_MIN_TOKENS}. */
  readonly minTokens?: number;
  /** Default {@link NEAR_DUPLICATE_CANDIDATE_CAP}. */
  readonly cap?: number;
  readonly bucket?: string;
}

export interface NearDuplicateMatch {
  readonly ref: string;
  /** Rounded to 3 decimals. */
  readonly score: number;
  readonly method: NearDuplicateMethod;
}

export interface NearDuplicateScan {
  /** Readable entries actually scored. */
  readonly compared: number;
  /** Readable entries left out by the cap. */
  readonly capped: number;
  /** Readable entries skipped as too short. */
  readonly below_min_tokens: number;
}

export interface NearDuplicateResult {
  /** Sorted by score descending, then ref. */
  readonly matches: ReadonlyArray<NearDuplicateMatch>;
  readonly scan: NearDuplicateScan;
}

/**
 * Score `probe` against `pool` by token Jaccard. Entries are filtered in
 * this order: reach, the probe's own ref, the bucket, the minimum token
 * count, then the cap (in pool order). A probe under the minimum token
 * count scores nothing. Matches at or above `threshold` are returned.
 */
export function findNearDuplicates(
  probe: NearDuplicateProbe,
  pool: ReadonlyArray<NearDuplicatePoolEntry>,
  opts: NearDuplicateOptions,
): NearDuplicateResult {
  const minTokens = opts.minTokens ?? NEAR_DUPLICATE_MIN_TOKENS;
  const cap = opts.cap ?? NEAR_DUPLICATE_CANDIDATE_CAP;
  if (probe.tokens.size < minTokens) {
    return { matches: [], scan: { compared: 0, capped: 0, below_min_tokens: 0 } };
  }
  const matches: NearDuplicateMatch[] = [];
  let compared = 0;
  let capped = 0;
  let belowMinTokens = 0;
  for (const entry of pool) {
    if (!opts.readable(entry.ref)) continue;
    if (entry.ref === probe.ref) continue;
    if (opts.bucket !== undefined && entry.bucket !== opts.bucket) continue;
    if (entry.tokens.size < minTokens) {
      belowMinTokens++;
      continue;
    }
    if (compared >= cap) {
      capped++;
      continue;
    }
    compared++;
    const sim = jaccard(probe.tokens, entry.tokens);
    if (sim < opts.threshold) continue;
    matches.push({ ref: entry.ref, score: roundScore(sim), method: "lexical" });
  }
  matches.sort(compareMatches);
  return { matches, scan: { compared, capped, below_min_tokens: belowMinTokens } };
}

/** Round a similarity score to 3 decimals, the precision every surface reports. */
export function roundScore(score: number): number {
  return Math.round(score * 1000) / 1000;
}

/** Score descending, then ref by code unit, so the order never depends on locale. */
export function compareMatches(
  a: { readonly ref: string; readonly score: number },
  b: { readonly ref: string; readonly score: number },
): number {
  if (a.score !== b.score) return b.score - a.score;
  return a.ref < b.ref ? -1 : a.ref > b.ref ? 1 : 0;
}