Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
10.9 KB
Raw Blame
/**
 * Seeded, question-type-proportional sampling for the LongMemEval suite
 * (wiring-shipped-seams wave, t_4501c662).
 *
 * The LongMemEval lane runs its dataset wholesale, and the full official
 * files carry hundreds of questions per run. This module selects a
 * DETERMINISTIC subset so a bench run can trade coverage for cost
 * without losing the type balance the benchmark's aggregate is computed
 * over: the survivors of abstention exclusion are stratified by their
 * present `question_type` values, a cap is apportioned across strata by
 * largest remainder (ties broken by stratum name), and each stratum
 * contributes its share, chosen by a seeded shuffle.
 *
 * Determinism contract: no Math.random, no wall clock, no fs. The
 * selection is a pure function of (entries, seed, cap). Entries are
 * sorted by question id before the PRNG is touched, so dataset file
 * order cannot change the subset; the PRNG is a mulberry32 stream keyed
 * by the sha256 of the TRIMMED seed string. The seed is normalized in
 * exactly one place - HERE, before the digest - so the same input
 * selects the same subset whatever surface (CLI or API) passed it:
 * callers may refuse a blank seed early to name their own surface, but
 * must not trim or otherwise transform the value. The same options
 * always select the same subset; a different seed or cap selects a
 * different one.
 *
 * The abstention rule lives here rather than in longmemeval.ts because
 * abstention exclusion must happen BEFORE selection, and this module
 * must not import the suite adapter - the adapter imports this module,
 * so the reverse edge would close a cycle. longmemeval.ts re-exports
 * the predicate and its constant, leaving the suite's public API
 * unchanged.
 *
 * An all-abstention (or empty) population, a duplicated question id -
 * which would make the selection depend on dataset file order - and a
 * malformed option raise the named {@link LongMemEvalSampleError},
 * never a silent wholesale run.
 */

import { sha256Hex } from "../integrity/digest.ts";

/**
 * The explicit question type whose entries measure abstention. Shared
 * with the suite adapter, which re-exports it.
 */
export const LONGMEMEVAL_ABSTENTION_TYPE = "abstention";

/** Suffix the official dataset puts on abstention question ids. */
const OFFICIAL_ABSTENTION_SUFFIX = "_abs";

/** The entry shape sampling reads; the suite's full entry satisfies it. */
export interface LongMemEvalSampleEntry {
  readonly q_id: string;
  readonly question_type: string;
}

/**
 * True when an entry measures abstention and is excluded from scoring.
 * The official dataset marks abstention by a question id ending in
 * {@link OFFICIAL_ABSTENTION_SUFFIX}; the legacy sample shape uses the
 * explicit {@link LONGMEMEVAL_ABSTENTION_TYPE} question type. Both
 * paths stay live, and both are excluded before any selection runs.
 */
export function isLongMemEvalAbstention(entry: LongMemEvalSampleEntry): boolean {
  return (
    entry.question_type === LONGMEMEVAL_ABSTENTION_TYPE ||
    entry.q_id.endsWith(OFFICIAL_ABSTENTION_SUFFIX)
  );
}

/** Sampling options: an opaque seed string and a positive integer cap. */
export interface LongMemEvalSampleOptions {
  readonly seed: string;
  readonly cap: number;
}

/** One stratum's population and selected count, for the receipt. */
export interface LongMemEvalSampleStratum {
  readonly question_type: string;
  readonly population: number;
  readonly selected: number;
}

/**
 * What one sampling decision decided, surfaced beside the selected
 * entries so a report can state what a run measured instead of letting
 * a reader assume the whole dataset was scored.
 */
export interface LongMemEvalSampleReceipt {
  /** Entries excluded as abstention before any selection ran. */
  readonly abstention_excluded: number;
  /** Survivors the sampler chose from. */
  readonly population: number;
  /** Entries selected: min(cap, population). */
  readonly selected: number;
  /** One row per present question type, sorted by name ascending. */
  readonly strata: ReadonlyArray<LongMemEvalSampleStratum>;
}

export interface LongMemEvalSampleSelection<E> {
  readonly entries: ReadonlyArray<E>;
  readonly receipt: LongMemEvalSampleReceipt;
}

/**
 * The sampler's named error: a population that is empty once abstention
 * entries are excluded, a duplicated question id, or a malformed
 * option. Callers catch it by name; a sampling failure is never
 * downgraded to a wholesale run.
 */
export class LongMemEvalSampleError extends Error {}

/** Hex characters of the seed digest folded into the PRNG state. */
const SEED_HASH_HEX_CHARS = 8;

/**
 * A mulberry32 PRNG: 32 bits of state, a multiply-shift per draw,
 * identical output for a given state on every platform.
 */
function mulberry32(state: number): () => number {
  let current = state >>> 0;
  return () => {
    current = (current + 0x6d2b79f5) | 0;
    let t = current;
    t = Math.imul(t ^ (t >>> 15), t | 1);
    t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

/** Key the PRNG state from the seed string through the shared digest. */
function prngState(seed: string): number {
  return Number.parseInt(sha256Hex(seed).slice(0, SEED_HASH_HEX_CHARS), 16) >>> 0;
}

/** Fisher-Yates over a copy; the input array is never mutated. */
function shuffled<E>(items: ReadonlyArray<E>, random: () => number): E[] {
  const out = [...items];
  for (let i = out.length - 1; i > 0; i -= 1) {
    const j = Math.floor(random() * (i + 1));
    const swapped = out[i]!;
    out[i] = out[j]!;
    out[j] = swapped;
  }
  return out;
}

/**
 * Select a seeded, type-proportional subset of LongMemEval entries.
 *
 * Abstention entries are excluded first (their count rides the
 * receipt), the survivors are stratified by their present
 * `question_type` values, the cap is apportioned across strata by
 * largest remainder with ties broken by stratum name, and each stratum
 * contributes its share chosen by the seeded shuffle. The selected
 * entries come back in canonical question-id order whatever order the
 * dataset carried.
 *
 * Raises {@link LongMemEvalSampleError} for a malformed option, a
 * duplicated question id, or a population that is empty once
 * abstention entries are excluded.
 */
export function sampleLongMemEvalEntries<E extends LongMemEvalSampleEntry>(
  entries: ReadonlyArray<E>,
  opts: LongMemEvalSampleOptions,
): LongMemEvalSampleSelection<E> {
  if (!Number.isInteger(opts.cap) || opts.cap <= 0) {
    throw new LongMemEvalSampleError(
      `LongMemEval sample cap must be a positive integer, got ${opts.cap}`,
    );
  }
  // The seed is normalized in exactly one place - here, before the
  // digest: it is trimmed, so the CLI (which passes the operator's
  // value through untransformed) and the API path hash the same value
  // for the same input. The raw value is echoed in the error so an
  // operator sees what they actually typed.
  const seed = opts.seed.trim();
  if (seed === "") {
    throw new LongMemEvalSampleError(
      `LongMemEval sample seed must be a non-empty string, got '${opts.seed}'`,
    );
  }
  const eligible = entries.filter((entry) => !isLongMemEvalAbstention(entry));
  const abstentionExcluded = entries.length - eligible.length;
  if (eligible.length === 0) {
    throw new LongMemEvalSampleError(
      `LongMemEval dataset: every one of the ${abstentionExcluded} question(s) is an abstention entry and was excluded before sampling; nothing remains to select`,
    );
  }
  // Duplicated question ids would make the selection below depend on
  // dataset file order; the converter refuses duplicates at conversion
  // time, and the sampler refuses them here, before any order-sensitive
  // step runs.
  const seen = new Set<string>();
  for (const entry of eligible) {
    if (seen.has(entry.q_id)) {
      throw new LongMemEvalSampleError(
        `LongMemEval dataset: question id '${entry.q_id}' appears twice; the sampled selection would depend on dataset file order`,
      );
    }
    seen.add(entry.q_id);
  }

  // File order is erased before the PRNG is touched: one canonical
  // order by question id, strata keyed off it.
  const sorted = [...eligible].toSorted((a, b) => (a.q_id < b.q_id ? -1 : a.q_id > b.q_id ? 1 : 0));
  const byStratum = new Map<string, E[]>();
  for (const entry of sorted) {
    const bucket = byStratum.get(entry.question_type);
    if (bucket === undefined) byStratum.set(entry.question_type, [entry]);
    else bucket.push(entry);
  }
  const stratumNames = [...byStratum.keys()].toSorted();
  const populationByStratum = stratumNames.map((name) => byStratum.get(name)!.length);
  const total = sorted.length;
  const effectiveCap = Math.min(opts.cap, total);

  // Largest-remainder apportionment: integer quotas first, then the
  // surplus seats go to the largest remainders, compared as INTEGER
  // remainders ((cap*pop) mod total). Two mathematically equal
  // fractional remainders can differ as doubles (total 6, cap 2: pop 1
  // computes 0.3333333333333333, pop 4 computes 0.33333333333333326),
  // which would silently defeat the documented tie-break. Exact integer
  // ties break to the lexicographically first stratum. With the cap
  // clamped to the population every quota is at most its stratum's
  // size, so a surplus seat never overflows a stratum.
  const base = populationByStratum.map((pop) => Math.floor((effectiveCap * pop) / total));
  const remainderOrder = stratumNames
    .map((name, i) => ({
      name,
      remainder: (effectiveCap * populationByStratum[i]!) % total,
    }))
    .toSorted((a, b) =>
      a.remainder !== b.remainder ? b.remainder - a.remainder : a.name < b.name ? -1 : 1,
    );
  const selectedByStratum = new Map<string, number>(
    stratumNames.map((name, i) => [name, base[i]!]),
  );
  let surplus = effectiveCap - base.reduce((sum, value) => sum + value, 0);
  for (const { name } of remainderOrder) {
    if (surplus === 0) break;
    selectedByStratum.set(name, selectedByStratum.get(name)! + 1);
    surplus -= 1;
  }

  // Every stratum is shuffled in name order, so the PRNG stream's draw
  // count depends only on the populations - never on the cap. The PRNG
  // is keyed by the trimmed seed (the one normalization point above).
  const random = mulberry32(prngState(seed));
  const selected: E[] = [];
  for (const name of stratumNames) {
    const pool = shuffled(byStratum.get(name)!, random);
    selected.push(...pool.slice(0, selectedByStratum.get(name)));
  }
  const canonical = selected.toSorted((a, b) => (a.q_id < b.q_id ? -1 : a.q_id > b.q_id ? 1 : 0));

  return {
    entries: canonical,
    receipt: Object.freeze({
      abstention_excluded: abstentionExcluded,
      population: total,
      selected: canonical.length,
      strata: Object.freeze(
        stratumNames.map((name, i) =>
          Object.freeze({
            question_type: name,
            population: populationByStratum[i]!,
            selected: selectedByStratum.get(name)!,
          }),
        ),
      ),
    }),
  };
}