Skip to content

Linger is An Open-Source Personal Agent That Runs Fully on Your PC. We aim to break Big Tech’s grip on AI by bringing powerful local AI to everyday computers through efficient model–harness co-design, so everyone can run their own.

16.3 KB
Raw Blame
// Memory/Wiki benchmark on LoCoMo and LongMemEval. Stages each question's
// public history through the product (product.ts), lets the answering model
// use the Another Me's knowledge tools (answer.ts), grades the answers
// (judge.ts) and writes report.md and summary.json. Every answer is stored as
// it completes, so an interrupted run resumes where it stopped. See README.md.
//
// From apps/node-services:
//   node --env-file=../../.env --import tsx scripts/memory-benchmark/run.ts \
//     --plan locomo-validate7 --out .local/memory-benchmark/runs/<name> \
//     --database-url postgresql://[email protected]:<port>/memory_benchmark \
//     [--arms product,search,wiki,memory-md,none] [--rounds 3]
//     [--state <prepared state>] [--skip-nightly] [--nightly-concurrency 4]
//     [--baseline <another run's out>] [--limit N] [--concurrency 8]
//     [--model openai/gpt-5.6-luna] [--judge-model openai/gpt-5.6-luna]
//     [--retry-errors]
import { execFileSync } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { parseArgs } from "node:util";

import { createGatewayModelCall } from "../gateway-model-call.js";
import {
  type AnswerRecord,
  ARMS,
  type Arm,
  answerQuestion,
  NIGHTLY_ARMS,
} from "./answer.js";
import { type Case, loadCases, readPlan, sha256 } from "./datasets.js";
import {
  calibrateJudge,
  type Judgment,
  JUDGE_PROTOCOL,
  judge,
  judgePrompt,
} from "./judge.js";
import { openProduct, type PreparedCorpus, productStages } from "./product.js";

// Memory's days follow the process time zone; sessions are staged as
// Asia/Shanghai wall time, as the Memory tests pin.
process.env.TZ = "Asia/Shanghai";

const { values: args } = parseArgs({
  options: {
    plan: { type: "string" },
    out: { type: "string" },
    state: { type: "string" },
    "database-url": { type: "string" },
    arms: { type: "string", default: "product" },
    rounds: { type: "string", default: "1" },
    limit: { type: "string" },
    concurrency: { type: "string", default: "8" },
    model: { type: "string", default: "openai/gpt-5.6-luna" },
    "judge-model": { type: "string", default: "openai/gpt-5.6-luna" },
    baseline: { type: "string" },
    data: { type: "string" },
    "retry-errors": { type: "boolean", default: false },
    "skip-nightly": { type: "boolean", default: false },
    "nightly-concurrency": { type: "string", default: "4" },
  },
});
if (!args.plan || !args.out)
  throw new Error("--plan and --out are required; see the header of run.ts");
const databaseUrl =
  args["database-url"] ?? process.env.MEMORY_BENCHMARK_DATABASE_URL;
if (!databaseUrl)
  throw new Error(
    "--database-url (or MEMORY_BENCHMARK_DATABASE_URL) is required",
  );
const arms = args.arms.split(",") as Arm[];
if (!arms.length || arms.some((arm) => !ARMS.includes(arm)))
  throw new Error(`--arms takes ${ARMS.join(", ")}`);
const nightly = !args["skip-nightly"];
if (!nightly && arms.some((arm) => NIGHTLY_ARMS.includes(arm)))
  throw new Error(
    `--skip-nightly leaves no Wiki, Daily or memory.md for ${NIGHTLY_ARMS.join(", ")}`,
  );
const stages = productStages(nightly);
const rounds = Number(args.rounds);
const concurrency = Number(args.concurrency);
const appRoot = path.resolve(import.meta.dirname, "../..");
const out = path.resolve(args.out);
const stateRoot = path.resolve(args.state ?? path.join(out, "state"));
const dataDirectory = path.resolve(
  args.data ?? path.join(appRoot, ".local/memory-benchmark/data"),
);

const plan = await readPlan(args.plan);
const cases = (await loadCases(plan, dataDirectory)).slice(
  0,
  args.limit ? Number(args.limit) : undefined,
);

/** The checkout under test: HEAD plus a hash of uncommitted Memory/Wiki changes. */
function sourceIdentity() {
  const git = (...command: string[]) =>
    execFileSync("git", ["-C", appRoot, ...command], {
      encoding: "utf8",
      maxBuffer: 256 * 1024 * 1024,
    });
  const diff = git("diff", "HEAD", "--", "src", "prompts", "scripts");
  const untracked = git(
    "ls-files",
    "--others",
    "--exclude-standard",
    "--",
    "src",
    "prompts",
    "scripts",
  ).trim();
  return {
    commit: git("rev-parse", "HEAD").trim(),
    dirty_sha256: diff || untracked ? sha256(diff + untracked) : null,
  };
}

const source = sourceIdentity();
const fixed = {
  plan: args.plan,
  benchmark: plan.benchmark,
  dataset_sha256: plan.dataset_sha256,
  case_ids: cases.map((item) => item.id),
  answer_model: args.model,
  judge_model: args["judge-model"],
  judge_protocol: JUDGE_PROTOCOL,
  state: stateRoot,
};
await fs.mkdir(out, { recursive: true });
const manifestPath = path.join(out, "manifest.json");
const previous = await fs
  .readFile(manifestPath, "utf8")
  .then((text) => JSON.parse(text) as { fixed: unknown; source: unknown })
  .catch(() => undefined);
if (previous && JSON.stringify(previous.fixed) !== JSON.stringify(fixed))
  throw new Error(
    "This --out holds a run with another plan, model, judge or state; use a new --out",
  );
if (previous && JSON.stringify(previous.source) !== JSON.stringify(source))
  console.log(
    "Note: the checkout changed since this run started; new answers use the current code",
  );

// The state was prepared by one checkout and one database; answers may come
// from later code, and both identities are recorded.
await fs.mkdir(stateRoot, { recursive: true });
const stateIdentityPath = path.join(stateRoot, "identity.json");
const database = new URL(databaseUrl);
const stateIdentity = await fs
  .readFile(stateIdentityPath, "utf8")
  .then((text) => JSON.parse(text) as Record<string, unknown>)
  .catch(async () => {
    const identity = {
      prepared_by: source,
      stages,
      database: `${database.host}${database.pathname}`,
      model: args.model,
    };
    await fs.writeFile(stateIdentityPath, JSON.stringify(identity, null, 2));
    return identity;
  });
if (stateIdentity.database !== `${database.host}${database.pathname}`)
  throw new Error(
    `The state was indexed in ${String(stateIdentity.database)}; pass that database`,
  );
if (JSON.stringify(stateIdentity.stages) !== JSON.stringify(stages))
  throw new Error(
    `The state was prepared with stages ${String(stateIdentity.stages)}; use the same --skip-nightly or another --state`,
  );
await fs.writeFile(
  manifestPath,
  JSON.stringify(
    {
      fixed,
      source,
      state: stateIdentity,
      arms,
      rounds,
      started_at: new Date().toISOString(),
    },
    null,
    2,
  ),
);

const productGateway = createGatewayModelCall({
  model: args.model,
  concurrency,
});
const judgeGateway = createGatewayModelCall({
  model: args["judge-model"],
  concurrency,
});
await calibrateJudge(judgeGateway.modelCall, plan.benchmark);
console.log(`Judge ${args["judge-model"]} passed its known-answer controls`);

const product = await openProduct({
  dataRoot: path.join(stateRoot, "memory"),
  databaseUrl,
  modelCall: productGateway.modelCall,
  nightly,
  nightlyConcurrency: Number(args["nightly-concurrency"]),
});

interface StoredAnswer {
  case_id: string;
  arm: Arm;
  round: number;
  type: string;
  unanswerable: boolean;
  question: string;
  gold: string;
  record?: AnswerRecord;
  judgment?: Judgment;
  error?: string;
  source: typeof source;
}

const answerPath = (arm: Arm, round: number, item: Case) =>
  path.join(
    out,
    "answers",
    arm,
    `r${round}`,
    `${item.id.replaceAll(":", "_")}.json`,
  );

async function runOne(arm: Arm, round: number, item: Case) {
  const file = answerPath(arm, round, item);
  const stored = await fs
    .readFile(file, "utf8")
    .then((text) => JSON.parse(text) as StoredAnswer)
    .catch(() => undefined);
  if (stored && !(stored.error && args["retry-errors"])) return;
  const result: StoredAnswer = {
    case_id: item.id,
    arm,
    round,
    type: item.type,
    unanswerable: item.unanswerable,
    question: item.question,
    gold: item.answer,
    source,
  };
  try {
    result.record = await answerQuestion({
      item,
      arm,
      ...(arm === "none"
        ? {}
        : { prepared: await product.prepare(item.corpus) }),
      product,
      model: args.model,
    });
    result.judgment = await judge(
      judgeGateway.modelCall,
      `${arm}:${round}:${item.id}`,
      judgePrompt(item, result.record.answer),
    );
  } catch (error) {
    // Bounded: the gateway calls already retried; the question is recorded as
    // failed and the run moves on. --retry-errors runs it again.
    result.error = error instanceof Error ? error.message : String(error);
    console.error(
      `ERROR ${arm} r${round} ${item.id}: ${result.error.slice(0, 300)}`,
    );
  }
  await fs.mkdir(path.dirname(file), { recursive: true });
  await fs.writeFile(file, JSON.stringify(result, null, 2));
}

// Prepare each history once before answering on it; a failed history fails
// its questions and the others go on.
const corpora = [
  ...new Map(cases.map((item) => [item.corpus.id, item.corpus])).values(),
];
let preparedCount = 0;
const prepared = (
  await Promise.all(
    (arms.some((arm) => arm !== "none") ? corpora : []).map(async (corpus) => {
      const started = performance.now();
      try {
        const result = await product.prepare(corpus);
        console.log(
          `Prepared history ${++preparedCount}/${corpora.length} (${corpus.sessions.length} sessions) in ${((performance.now() - started) / 1000).toFixed(0)} s`,
        );
        return result;
      } catch (error) {
        console.error(
          `ERROR preparing ${corpus.id.slice(0, 8)}: ${error instanceof Error ? error.message : String(error)}`,
        );
        return undefined;
      }
    }),
  )
).filter((item): item is PreparedCorpus => Boolean(item));

const work =
  rounds && arms.length
    ? Array.from({ length: rounds }, (_, index) => index + 1).flatMap((round) =>
        arms.flatMap((arm) =>
          cases.map((item) => () => runOne(arm, round, item)),
        ),
      )
    : [];
let done = 0;
const workers = Array.from(
  { length: Math.min(concurrency, work.length) },
  async () => {
    for (let next = work.shift(); next; next = work.shift()) {
      await next();
      done += 1;
      if (done % 10 === 0) console.log(`Answered ${done}`);
    }
  },
);
await Promise.all(workers);
await product.close();

// Report.
async function readAnswers(directory: string): Promise<StoredAnswer[]> {
  const answers: StoredAnswer[] = [];
  const root = path.join(directory, "answers");
  for (const arm of await fs.readdir(root).catch(() => []))
    for (const round of await fs.readdir(path.join(root, arm)))
      for (const file of await fs.readdir(path.join(root, arm, round)))
        answers.push(
          JSON.parse(
            await fs.readFile(path.join(root, arm, round, file), "utf8"),
          ) as StoredAnswer,
        );
  return answers;
}

interface ArmSummary {
  rounds: number[];
  mean: number;
  min: number;
  max: number;
  errors: number;
  by_type: Record<string, number>;
  tokens: { prompt: number; completion: number };
  seconds_per_answer: number;
  tool_calls_per_answer: number;
}

function summarize(
  answers: StoredAnswer[],
  cases: number,
): Record<string, ArmSummary> {
  const result: Record<string, ArmSummary> = {};
  for (const arm of [...new Set(answers.map((answer) => answer.arm))].sort()) {
    const own = answers.filter((answer) => answer.arm === arm);
    const roundNumbers = [...new Set(own.map((answer) => answer.round))].sort();
    const accuracy = roundNumbers.map(
      (round) =>
        own.filter(
          (answer) => answer.round === round && answer.judgment?.correct,
        ).length / cases,
    );
    const types = [...new Set(own.map((answer) => answer.type))].sort();
    const answered = own.filter((answer) => answer.record);
    result[arm] = {
      rounds: accuracy,
      mean: accuracy.reduce((sum, value) => sum + value, 0) / accuracy.length,
      min: Math.min(...accuracy),
      max: Math.max(...accuracy),
      errors: own.filter((answer) => answer.error).length,
      by_type: Object.fromEntries(
        types.map((type) => {
          const typed = own.filter((answer) => answer.type === type);
          return [
            type,
            typed.filter((answer) => answer.judgment?.correct).length /
              typed.length,
          ];
        }),
      ),
      tokens: {
        prompt: answered.reduce(
          (sum, answer) => sum + answer.record!.usage.prompt,
          0,
        ),
        completion: answered.reduce(
          (sum, answer) => sum + answer.record!.usage.completion,
          0,
        ),
      },
      seconds_per_answer:
        answered.reduce((sum, answer) => sum + answer.record!.seconds, 0) /
        Math.max(1, answered.length),
      tool_calls_per_answer:
        answered.reduce(
          (sum, answer) => sum + answer.record!.tool_calls.length,
          0,
        ) / Math.max(1, answered.length),
    };
  }
  return result;
}

const percent = (value: number) => `${(value * 100).toFixed(1)}%`;
const summary = {
  plan: args.plan,
  cases: cases.length,
  source,
  state: stateIdentity,
  prepared: {
    histories: prepared.length,
    raw_sources: prepared.reduce((sum, item) => sum + item.raw_sources, 0),
    indexed_chunks: prepared.reduce(
      (sum, item) => sum + item.indexed_chunks,
      0,
    ),
    nights: prepared.reduce((sum, item) => sum + item.nights, 0),
    failed_wiki_nights: prepared.reduce(
      (sum, item) => sum + item.failed_wiki_nights,
      0,
    ),
    failed_day_reads: prepared.reduce(
      (sum, item) => sum + item.failed_day_reads,
      0,
    ),
    wiki_pages: prepared.reduce((sum, item) => sum + item.wiki_pages, 0),
    dailies: prepared.reduce((sum, item) => sum + item.dailies, 0),
    seconds: prepared.reduce((sum, item) => sum + item.seconds, 0),
  },
  model_usage: {
    preparation: productGateway.usage,
    judge: judgeGateway.usage,
  },
  arms: summarize(await readAnswers(out), cases.length),
};
const baseline = args.baseline
  ? (JSON.parse(
      await fs.readFile(
        path.join(path.resolve(args.baseline), "summary.json"),
        "utf8",
      ),
    ) as typeof summary)
  : undefined;
await fs.writeFile(
  path.join(out, "summary.json"),
  JSON.stringify(summary, null, 2),
);

const lines = [
  `# Memory benchmark: ${args.plan}`,
  "",
  `- Checkout: \`${source.commit.slice(0, 12)}\`${source.dirty_sha256 ? ` + uncommitted \`${source.dirty_sha256.slice(0, 12)}\`` : ""}`,
  `- History prepared by: \`${String((stateIdentity.prepared_by as { commit?: string })?.commit ?? "").slice(0, 12)}\`, stages ${stages.join(", ")}`,
  `- ${cases.length} questions, ${prepared.length} histories, answer model ${args.model}, judge ${args["judge-model"]} (${JUDGE_PROTOCOL})`,
  `- Preparation: ${summary.prepared.nights} nights (${summary.prepared.failed_wiki_nights} Wiki and ${summary.prepared.failed_day_reads} day-read failures skipped), ${summary.prepared.wiki_pages} Wiki pages, ${summary.prepared.dailies} Dailies`,
  "",
  "| Arm | Rounds | Mean | Min–max | Errors | Tool calls / answer | Seconds / answer |",
  "| --- | --- | --- | --- | --- | --- | --- |",
  ...Object.entries(summary.arms).map(
    ([arm, value]) =>
      `| ${arm} | ${value.rounds.map(percent).join(", ")} | ${percent(value.mean)} | ${percent(value.min)}–${percent(value.max)} | ${value.errors} | ${value.tool_calls_per_answer.toFixed(1)} | ${value.seconds_per_answer.toFixed(1)} |`,
  ),
  "",
  "## By question type (all rounds)",
  "",
  "| Arm | Type | Accuracy | Baseline |",
  "| --- | --- | --- | --- |",
  ...Object.entries(summary.arms).flatMap(([arm, value]) =>
    Object.entries(value.by_type).map(
      ([type, accuracy]) =>
        `| ${arm} | ${type} | ${percent(accuracy)} | ${baseline?.arms[arm]?.by_type[type] !== undefined ? percent(baseline.arms[arm].by_type[type]) : "—"} |`,
    ),
  ),
];
if (baseline)
  lines.push(
    "",
    `## Against ${path.resolve(args.baseline!)}`,
    "",
    "| Arm | This run | Baseline | Difference | Baseline min–max |",
    "| --- | --- | --- | --- | --- |",
    ...Object.entries(summary.arms).map(([arm, value]) => {
      const other = baseline.arms[arm];
      return other
        ? `| ${arm} | ${percent(value.mean)} | ${percent(other.mean)} | ${((value.mean - other.mean) * 100).toFixed(1)} pt | ${percent(other.min)}–${percent(other.max)} |`
        : `| ${arm} | ${percent(value.mean)} | — | — | — |`;
    }),
  );
await fs.writeFile(path.join(out, "report.md"), `${lines.join("\n")}\n`);
console.log(lines.join("\n"));
console.log(`\nWrote ${path.join(out, "report.md")}`);