Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.9 KB
Raw Blame
/**
 * Seeded LongMemEval sampling (wiring-shipped-seams wave, t_4501c662):
 * abstention exclusion before selection, question-type strata with
 * largest-remainder apportionment, and a determinism contract - the
 * same seed and cap select the same subset on every call, dataset file
 * order cannot change it, and no clock or ambient randomness exists.
 */

import { describe, expect, test } from "bun:test";

import {
  LONGMEMEVAL_ABSTENTION_TYPE,
  LongMemEvalSampleError,
  isLongMemEvalAbstention,
  sampleLongMemEvalEntries,
  type LongMemEvalSampleEntry,
} from "../../../src/core/bench/longmemeval-sample.ts";

/** A minimal entry shaped the way the sampler reads it. */
function sampleEntry(qId: string, questionType: string): LongMemEvalSampleEntry {
  return { q_id: qId, question_type: questionType };
}

/** Three strata of four, so apportionment has remainders to work with. */
function population(): LongMemEvalSampleEntry[] {
  const entries: LongMemEvalSampleEntry[] = [];
  for (const stratum of ["a", "b", "c"]) {
    for (let i = 1; i <= 4; i += 1) {
      entries.push(sampleEntry(`${stratum}-${i}`, `type-${stratum}`));
    }
  }
  return entries;
}

test("the abstention rule keeps both spellings: the explicit type and the official id suffix", () => {
  expect(isLongMemEvalAbstention(sampleEntry("q1", LONGMEMEVAL_ABSTENTION_TYPE))).toBe(true);
  expect(isLongMemEvalAbstention(sampleEntry("official_abs", "single-hop"))).toBe(true);
  expect(isLongMemEvalAbstention(sampleEntry("q1", "single-hop"))).toBe(false);
  expect(isLongMemEvalAbstention(sampleEntry("absurd", "single-hop"))).toBe(false);
});

test("abstention entries are excluded before selection and the count flows through", () => {
  const entries = [
    sampleEntry("q-keep", "single-hop"),
    sampleEntry("abs-explicit", "abstention"),
    sampleEntry("official_abs", "single-hop"),
    sampleEntry("q-keep-2", "multi-hop"),
  ];
  const { entries: selected, receipt } = sampleLongMemEvalEntries(entries, {
    seed: "seed-1",
    cap: 10,
  });
  expect(selected.map((e) => e.q_id).toSorted()).toEqual(["q-keep", "q-keep-2"]);
  expect(receipt.abstention_excluded).toBe(2);
  expect(receipt.population).toBe(2);
  expect(receipt.selected).toBe(2);
});

test("strata are the distinct present question_type values, sorted ascending", () => {
  const entries = [
    sampleEntry("q1", "temporal-reasoning"),
    sampleEntry("q2", "single-hop"),
    sampleEntry("q3", "temporal-reasoning"),
  ];
  const { receipt } = sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 3 });
  expect(receipt.strata).toEqual([
    { question_type: "single-hop", population: 1, selected: 1 },
    { question_type: "temporal-reasoning", population: 2, selected: 2 },
  ]);
});

test("the cap is apportioned by largest remainder and its sum is exact", () => {
  // Populations 1/2/4 over a total of 7 with cap 3: quotas 3/7, 6/7 and
  // 12/7 leave bases 0/0/1 and 2 surplus seats for the two largest
  // fractional remainders (beta .857, gamma .714) - alpha's .429 buys
  // nothing.
  const entries = [
    sampleEntry("a-1", "alpha"),
    sampleEntry("b-1", "beta"),
    sampleEntry("b-2", "beta"),
    sampleEntry("c-1", "gamma"),
    sampleEntry("c-2", "gamma"),
    sampleEntry("c-3", "gamma"),
    sampleEntry("c-4", "gamma"),
  ];
  const { entries: selected, receipt } = sampleLongMemEvalEntries(entries, {
    seed: "seed-1",
    cap: 3,
  });
  expect(selected).toHaveLength(3);
  expect(receipt.selected).toBe(3);
  expect(receipt.strata).toEqual([
    { question_type: "alpha", population: 1, selected: 0 },
    { question_type: "beta", population: 2, selected: 1 },
    { question_type: "gamma", population: 4, selected: 2 },
  ]);
});

test("remainder ties break by stratum name ascending", () => {
  // Two strata of 2 with cap 1: both quotas are 0.5, so the one surplus
  // seat goes to the lexicographically first stratum.
  const entries = [
    sampleEntry("b-1", "beta"),
    sampleEntry("b-2", "beta"),
    sampleEntry("a-1", "alpha"),
    sampleEntry("a-2", "alpha"),
  ];
  const { receipt } = sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 1 });
  expect(receipt.strata).toEqual([
    { question_type: "alpha", population: 2, selected: 1 },
    { question_type: "beta", population: 2, selected: 0 },
  ]);
});

test("mathematically equal remainders tie exactly, so the name tie-break applies", () => {
  // The reviewer's case (pr-251 L1): total 6, cap 2. The fractional
  // remainder of pop 4 is 8/6 - 1 and of pop 1 is 2/6 - 0; both are
  // mathematically 1/3, but as doubles they differ (pop 1 computes
  // 0.3333333333333333, pop 4 computes 0.33333333333333326), so a float
  // comparison hands the surplus seat to beta and the documented name
  // tie-break never applies. The remainders are compared as integers
  // ((cap*pop) mod total): all three strata tie at 2 and the seat goes
  // to the lexicographically first stratum, whose quota (4/3) leaves it
  // one seat short of its population.
  const entries = [
    sampleEntry("a-1", "alpha"),
    sampleEntry("a-2", "alpha"),
    sampleEntry("a-3", "alpha"),
    sampleEntry("a-4", "alpha"),
    sampleEntry("b-1", "beta"),
    sampleEntry("c-1", "gamma"),
  ];
  const { receipt } = sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 2 });
  expect(receipt.strata).toEqual([
    { question_type: "alpha", population: 4, selected: 2 },
    { question_type: "beta", population: 1, selected: 0 },
    { question_type: "gamma", population: 1, selected: 0 },
  ]);
  expect(receipt.selected).toBe(2);
});

test("the same seed and cap select the same subset on every call", () => {
  const entries = population();
  const first = sampleLongMemEvalEntries(entries, { seed: "run-9", cap: 4 });
  const second = sampleLongMemEvalEntries(entries, { seed: "run-9", cap: 4 });
  expect(first.entries.map((e) => e.q_id)).toEqual(second.entries.map((e) => e.q_id));
  // The module takes no clock and draws no ambient randomness, so the
  // result is a pure function of (entries, seed, cap) - repeated here
  // after the first call consumed the PRNG stream.
  const third = sampleLongMemEvalEntries(entries.toReversed(), { seed: "run-9", cap: 4 });
  expect(third.entries.map((e) => e.q_id)).toEqual(first.entries.map((e) => e.q_id));
});

test("a different seed selects a different subset at the same cap", () => {
  const entries = population();
  const bySeed = sampleLongMemEvalEntries(entries, { seed: "run-9", cap: 4 });
  const otherSeed = sampleLongMemEvalEntries(entries, { seed: "run-10", cap: 4 });
  expect(otherSeed.entries.map((e) => e.q_id)).not.toEqual(bySeed.entries.map((e) => e.q_id));
});

test("the seed is normalized once, in the sampler: padded input selects the bare subset", () => {
  // pr-251 L2: the CLI used to trim --sample-seed while the API path
  // hashed the seed as given, so the same input produced different
  // subsets depending on the surface. The sampler trims before hashing
  // - the one normalization point - so padded and bare input select the
  // identical subset.
  const entries = population();
  const bare = sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 4 });
  const padded = sampleLongMemEvalEntries(entries, { seed: "  seed-1\t", cap: 4 });
  expect(padded.entries.map((e) => e.q_id)).toEqual(bare.entries.map((e) => e.q_id));
  expect(padded.receipt).toEqual(bare.receipt);
});

test("a different cap selects a different subset at the same seed", () => {
  const entries = population();
  const byCap = sampleLongMemEvalEntries(entries, { seed: "run-9", cap: 4 });
  const smallerCap = sampleLongMemEvalEntries(entries, { seed: "run-9", cap: 3 });
  expect(smallerCap.entries).toHaveLength(3);
  expect(smallerCap.entries.map((e) => e.q_id)).not.toEqual(byCap.entries.map((e) => e.q_id));
});

test("dataset file order cannot change the selected subset", () => {
  const entries = population();
  const inOrder = sampleLongMemEvalEntries(entries, { seed: "order", cap: 5 });
  const ids = inOrder.entries.map((e) => e.q_id);
  expect(sampleLongMemEvalEntries(entries.toReversed(), { seed: "order", cap: 5 }).entries).toEqual(
    inOrder.entries,
  );
  const rotated = [...entries.slice(5), ...entries.slice(0, 5)];
  expect(
    sampleLongMemEvalEntries(rotated, { seed: "order", cap: 5 }).entries.map((e) => e.q_id),
  ).toEqual(ids);
});

test("a cap at or above the population selects everyone", () => {
  const entries = [...population(), sampleEntry("abs-1", "abstention")];
  for (const cap of [12, 50]) {
    const { entries: selected, receipt } = sampleLongMemEvalEntries(entries, {
      seed: "seed-1",
      cap,
    });
    expect(selected).toHaveLength(12);
    expect(receipt.selected).toBe(12);
    expect(receipt.population).toBe(12);
    // The clamped cap lands in the receipt too: every stratum reports
    // exactly its own population - never the raw apportionment of an
    // unclamped cap, which at cap 50 would report 17 selected in a
    // stratum of 4 - and the per-stratum selections sum to the
    // population.
    expect(receipt.strata).toEqual([
      { question_type: "type-a", population: 4, selected: 4 },
      { question_type: "type-b", population: 4, selected: 4 },
      { question_type: "type-c", population: 4, selected: 4 },
    ]);
    expect(receipt.strata.reduce((sum, stratum) => sum + stratum.selected, 0)).toBe(12);
  }
});

test("selected entries come back in canonical question-id order", () => {
  const { entries: selected } = sampleLongMemEvalEntries(population(), {
    seed: "order-out",
    cap: 7,
  });
  const ids = selected.map((e) => e.q_id);
  expect(ids).toEqual(ids.toSorted());
});

test("a population of only abstention entries is a named error", () => {
  const entries = [sampleEntry("x_abs", "single-hop"), sampleEntry("y", "abstention")];
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 2 })).toThrow(
    LongMemEvalSampleError,
  );
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 2 })).toThrow(/abstention/);
});

test("an empty population is a named error", () => {
  expect(() => sampleLongMemEvalEntries([], { seed: "seed-1", cap: 2 })).toThrow(
    LongMemEvalSampleError,
  );
});

test("malformed options are refused by name", () => {
  const entries = population();
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 0 })).toThrow(
    LongMemEvalSampleError,
  );
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 1.5 })).toThrow(/cap/);
  expect(() => sampleLongMemEvalEntries(entries, { seed: "", cap: 2 })).toThrow(/seed/);
  expect(() => sampleLongMemEvalEntries(entries, { seed: "   ", cap: 2 })).toThrow(/seed/);
});

test("a duplicated question id is a named error, before any order-dependent step", () => {
  const entries = [...population(), sampleEntry("a-1", "type-a")];
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 4 })).toThrow(
    LongMemEvalSampleError,
  );
  expect(() => sampleLongMemEvalEntries(entries, { seed: "seed-1", cap: 4 })).toThrow(
    /'a-1' appears twice/,
  );
});

test("the sampler is pure: same inputs, same outputs, no input mutation", () => {
  const entries = population();
  const snapshot = entries.map((e) => e.q_id);
  const first = sampleLongMemEvalEntries(entries, { seed: "pure", cap: 6 });
  const second = sampleLongMemEvalEntries(entries, { seed: "pure", cap: 6 });
  expect(first.entries.map((e) => e.q_id)).toEqual(second.entries.map((e) => e.q_id));
  expect(entries.map((e) => e.q_id)).toEqual(snapshot);
});

describe("receipt fragment", () => {
  test("per-stratum population and selected counts are carried beside the totals", () => {
    const { receipt } = sampleLongMemEvalEntries(population(), { seed: "receipt", cap: 5 });
    expect(receipt.strata).toEqual([
      { question_type: "type-a", population: 4, selected: 2 },
      { question_type: "type-b", population: 4, selected: 2 },
      { question_type: "type-c", population: 4, selected: 1 },
    ]);
    expect(receipt.population).toBe(12);
    expect(receipt.selected).toBe(5);
    expect(receipt.abstention_excluded).toBe(0);
  });
});