Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(benchmarks): add LongMemEval evaluation harness

Christiana220e0f8a99c
diff --git a/benchmarks/longmemeval.sh b/benchmarks/longmemeval.sh
new file mode 100755
index 0000000..8d836f1
--- /dev/null
+++ b/benchmarks/longmemeval.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT="$(cd "$(dirname "$0")" && pwd)"
+PYTHON="${ROOT}/.venv/bin/python"
+if [[ ! -x "$PYTHON" ]]; then
+ echo "Missing ${PYTHON}. Create it with:"
+ echo " cd benchmarks && python3 -m venv .venv && .venv/bin/pip install -r requirements.txt"
+ exit 1
+fi
+exec "$PYTHON" -m longmemeval "$@"
diff --git a/benchmarks/longmemeval/__init__.py b/benchmarks/longmemeval/__init__.py
new file mode 100644
index 0000000..3e84d14
--- /dev/null
+++ b/benchmarks/longmemeval/__init__.py
@@ -0,0 +1 @@
+"""LongMemEval benchmark harness for BrainAPI (HTTP-only)."""
diff --git a/benchmarks/longmemeval/__main__.py b/benchmarks/longmemeval/__main__.py
new file mode 100644
index 0000000..586397a
--- /dev/null
+++ b/benchmarks/longmemeval/__main__.py
@@ -0,0 +1,4 @@
+from longmemeval.cli import main
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/benchmarks/longmemeval/answer.py b/benchmarks/longmemeval/answer.py
new file mode 100644
index 0000000..ff57808
--- /dev/null
+++ b/benchmarks/longmemeval/answer.py
@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from openai import AzureOpenAI, OpenAI
+
+from longmemeval.config import Settings
+from longmemeval.prompts import build_answer_messages
+from longmemeval.sota import majority_vote
+
+
+@dataclass
+class AnswerResult:
+ answer: str
+ model: str
+ latency_ms: float
+ prompt_tokens: int | None
+ completion_tokens: int | None
+ total_tokens: int | None
+ sc_samples_used: int = 1
+
+
+def _make_openai_client(settings: Settings) -> OpenAI | AzureOpenAI:
+ settings.require_llm()
+ if settings.answer_azure_endpoint:
+ return AzureOpenAI(
+ api_key=settings.openai_api_key,
+ azure_endpoint=settings.answer_azure_endpoint,
+ api_version=settings.answer_azure_api_version,
+ )
+ kwargs: dict[str, Any] = {"api_key": settings.openai_api_key}
+ if settings.llm_base_url:
+ kwargs["base_url"] = settings.llm_base_url
+ return OpenAI(**kwargs)
+
+
+def _one_completion(
+ client: OpenAI | AzureOpenAI,
+ *,
+ model_name: str,
+ messages: list[dict[str, str]],
+ temperature: float,
+) -> tuple[str, Any]:
+ response = client.chat.completions.create(
+ model=model_name,
+ messages=messages,
+ temperature=temperature,
+ )
+ content = (response.choices[0].message.content or "").strip()
+ return content, response
+
+
+def answer_question(
+ settings: Settings,
+ question: str,
+ context: dict[str, Any],
+ *,
+ question_date: str | None = None,
+ model: str | None = None,
+ sc_samples: int | None = None,
+ sc_temperature: float | None = None,
+) -> AnswerResult:
+ client = _make_openai_client(settings)
+ model_name = model or settings.answer_model
+ messages = build_answer_messages(
+ question, context, question_date=question_date
+ )
+ n = max(1, int(sc_samples if sc_samples is not None else settings.sc_samples))
+ temperature = (
+ float(sc_temperature)
+ if sc_temperature is not None
+ else (settings.sc_temperature if n > 1 else 0.0)
+ )
+ started = time.perf_counter()
+ prompt_tokens = 0
+ completion_tokens = 0
+ total_tokens = 0
+ samples: list[str] = []
+ last_response: Any = None
+
+ for _ in range(n):
+ content, response = _one_completion(
+ client,
+ model_name=model_name,
+ messages=messages,
+ temperature=0.0 if n == 1 else temperature,
+ )
+ last_response = response
+ if response.usage:
+ prompt_tokens += getattr(response.usage, "prompt_tokens", 0) or 0
+ completion_tokens += getattr(response.usage, "completion_tokens", 0) or 0
+ total_tokens += getattr(response.usage, "total_tokens", 0) or 0
+ if content:
+ samples.append(content)
+
+ content = majority_vote(samples) if samples else ""
+
+ if not content:
+ retry_messages = messages + [
+ {
+ "role": "user",
+ "content": (
+ "Your previous reply was empty. Answer the question now using the "
+ "retrieved context. Prefer a short factual list or phrase."
+ ),
+ }
+ ]
+ content, response = _one_completion(
+ client,
+ model_name=model_name,
+ messages=retry_messages,
+ temperature=0,
+ )
+ last_response = response
+ if response.usage:
+ prompt_tokens += getattr(response.usage, "prompt_tokens", 0) or 0
+ completion_tokens += getattr(response.usage, "completion_tokens", 0) or 0
+ total_tokens += getattr(response.usage, "total_tokens", 0) or 0
+
+ latency_ms = (time.perf_counter() - started) * 1000
+ return AnswerResult(
+ answer=content,
+ model=getattr(last_response, "model", None) or model_name,
+ latency_ms=latency_ms,
+ prompt_tokens=prompt_tokens or None,
+ completion_tokens=completion_tokens or None,
+ total_tokens=total_tokens or None,
+ sc_samples_used=n,
+ )
diff --git a/benchmarks/longmemeval/cli.py b/benchmarks/longmemeval/cli.py
new file mode 100644
index 0000000..10c6091
--- /dev/null
+++ b/benchmarks/longmemeval/cli.py
@@ -0,0 +1,403 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from rich.console import Console
+from rich.table import Table
+
+from longmemeval.answer import answer_question
+from longmemeval.client import BrainAPIClient
+from longmemeval.config import VARIANT_FILES, Settings
+from longmemeval.dataset import (
+ brain_id_for,
+ dataset_stats,
+ download_dataset,
+ get_question,
+ iter_ingest_units,
+ load_dataset,
+ resolve_questions,
+ resolve_variant_settings,
+)
+from longmemeval.evaluate import evaluate_questions
+from longmemeval.ingest import ensure_run_dir, ingest_questions, write_manifest
+from longmemeval.metrics import selftest_metrics, tokenize
+from longmemeval.prompts import ANSWER_SYSTEM
+from longmemeval.provenance import build_provenance
+from longmemeval.report import print_report_table, write_report
+
+console = Console()
+
+
+def _settings_for_args(args: argparse.Namespace, settings: Settings) -> Settings:
+ variant = getattr(args, "variant", None)
+ return resolve_variant_settings(settings, variant)
+
+
+def cmd_download(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ path = download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ console.print(f"[green]Dataset ready:[/green] {path} (variant={settings.variant})")
+ return 0
+
+
+def cmd_dataset_stats(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ if not settings.dataset_path.exists():
+ download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ stats = dataset_stats(load_dataset(settings.dataset_path, variant=settings.variant))
+ table = Table(title=f"LongMemEval ({settings.variant})")
+ table.add_column("Metric")
+ table.add_column("Value")
+ table.add_row("Questions", str(stats["n_questions"]))
+ table.add_row("Abstention", str(stats["n_abstention"]))
+ sess = stats["sessions_per_question"]
+ table.add_row(
+ "Sessions/question",
+ f"min={sess['min']} max={sess['max']} mean={sess['mean']:.1f}",
+ )
+ console.print(table)
+ type_table = Table(title="By question type")
+ type_table.add_column("Type")
+ type_table.add_column("N", justify="right")
+ for qtype, count in (stats.get("by_type") or {}).items():
+ type_table.add_row(qtype, str(count))
+ console.print(type_table)
+ assert brain_id_for("e47b4ab3aa_abs") == "lmee47b4ab3aaabs"
+ console.print("[dim]brain_id_for('e47b4ab3aa_abs') == 'lmee47b4ab3aaabs' OK[/dim]")
+ return 0
+
+
+def cmd_smoke(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ if not settings.dataset_path.exists():
+ download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ questions = resolve_questions(settings, limit=max(1, args.limit))
+ entry = questions[0]
+ question_id = str(entry["question_id"])
+ brain_id = args.brain or brain_id_for(question_id)
+ units = iter_ingest_units(entry, limit_sessions=1)
+ if not units:
+ raise SystemExit("No haystack sessions to smoke-test")
+ unit = units[0]
+ with BrainAPIClient(settings) as client:
+ submitted = client.ingest_text(
+ unit["text"],
+ brain_id,
+ source_timestamp=unit.get("source_timestamp"),
+ )
+ task_id = (submitted.data or {}).get("task_id")
+ console.print(f"Queued ingest task_id={task_id} brain={brain_id}")
+ waited = client.wait_for_task(task_id, brain_id, timeout_s=args.timeout)
+ console.print(f"Task status: {(waited.data or {}).get('status')}")
+ question = str(entry.get("question") or "What do you remember?")
+ ctx = client.retrieve_context(question, brain_id)
+ text_context = (ctx.data or {}).get("text_context") or ""
+ console.print(f"text_context ({len(text_context)} chars): {text_context[:400]}")
+ if not text_context.strip():
+ console.print("[yellow]Warning: empty text_context[/yellow]")
+ return 1
+ console.print("[green]Smoke test passed[/green]")
+ return 0
+
+
+def cmd_ingest(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ if not settings.dataset_path.exists():
+ download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ question_types = (
+ {t.strip() for t in args.question_type.split(",") if t.strip()}
+ if args.question_type
+ else None
+ )
+ questions = resolve_questions(
+ settings,
+ question_ids=args.question_id or None,
+ limit=args.limit,
+ question_types=question_types,
+ )
+ run_id, run_dir = ensure_run_dir(settings, args.run)
+ write_manifest(
+ run_dir,
+ {
+ "run_id": run_id,
+ "command": "ingest",
+ "variant": settings.variant,
+ "n_questions": len(questions),
+ "question_ids": [q["question_id"] for q in questions],
+ "concurrency": args.concurrency,
+ "limit": args.limit,
+ "limit_sessions": args.limit_sessions,
+ "dry_run": args.dry_run,
+ "brainapi_url": settings.brainapi_url,
+ **build_provenance(settings),
+ },
+ )
+ ingest_questions(
+ settings,
+ questions,
+ run_dir=run_dir,
+ concurrency=args.concurrency,
+ limit_sessions=args.limit_sessions,
+ dry_run=args.dry_run,
+ resume=not args.no_resume,
+ task_timeout_s=args.timeout,
+ )
+ console.print(f"[green]Run directory:[/green] {run_dir}")
+ return 0
+
+
+def cmd_answer_once(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ entry = get_question(
+ load_dataset(settings.dataset_path, variant=settings.variant),
+ args.question_id,
+ )
+ brain_id = brain_id_for(str(entry["question_id"]))
+ question = args.question or str(entry.get("question") or "")
+ with BrainAPIClient(settings) as client:
+ retrieved = client.retrieve_context(question, brain_id)
+ result = answer_question(
+ settings,
+ question,
+ retrieved.data or {},
+ question_date=str(entry.get("question_date") or "") or None,
+ )
+ console.print(result.answer)
+ console.print(
+ f"[dim]model={result.model} latency_ms={result.latency_ms:.0f} "
+ f"tokens={result.total_tokens}[/dim]"
+ )
+ return 0
+
+
+def cmd_selftest_metrics(args: argparse.Namespace, settings: Settings) -> int:
+ errors = selftest_metrics()
+ if errors:
+ for err in errors:
+ console.print(f"[red]FAIL[/red] {err}")
+ return 1
+ console.print("[green]selftest-metrics passed[/green]")
+ return 0
+
+
+def cmd_evaluate(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ if not settings.dataset_path.exists():
+ download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ question_types = (
+ {t.strip() for t in args.question_type.split(",") if t.strip()}
+ if args.question_type
+ else None
+ )
+ questions = resolve_questions(
+ settings,
+ question_ids=args.question_id or None,
+ limit=args.limit,
+ question_types=question_types,
+ )
+ run_id, run_dir = ensure_run_dir(settings, args.run)
+ if settings.judge_shares_answer_family:
+ console.print(
+ f"[yellow]Warning: judge model ({settings.judge_model}) is the same "
+ f"family as the answer model ({settings.answer_model}); judge accuracy "
+ "carries self-preference bias.[/yellow]"
+ )
+ write_manifest(
+ run_dir,
+ {
+ "run_id": run_id,
+ "command": "evaluate",
+ "variant": settings.variant,
+ "n_questions": len(questions),
+ "question_ids": [q["question_id"] for q in questions],
+ "concurrency": args.concurrency,
+ "limit": args.limit,
+ "brainapi_url": settings.brainapi_url,
+ **build_provenance(settings),
+ "historical_limit": args.historical_limit,
+ "max_passages": args.max_passages,
+ "max_facts": args.max_facts,
+ "apply_fact_filter": not args.no_fact_filter,
+ "use_ppr": args.use_ppr,
+ "sufficiency_retry": args.sufficiency_retry,
+ },
+ )
+ evaluate_questions(
+ settings,
+ questions,
+ run_dir=run_dir,
+ concurrency=args.concurrency,
+ resume=not args.no_resume,
+ historical_limit=args.historical_limit,
+ max_passages=args.max_passages,
+ max_facts=args.max_facts,
+ apply_fact_filter=not args.no_fact_filter,
+ use_ppr=args.use_ppr,
+ sufficiency_retry=args.sufficiency_retry,
+ )
+ report = write_report(run_dir)
+ print_report_table(report)
+ console.print(f"[green]Report:[/green] {run_dir / 'report.md'}")
+ return 0
+
+
+def cmd_report(args: argparse.Namespace, settings: Settings) -> int:
+ run_dir = settings.runs_dir / args.run
+ if not run_dir.exists():
+ raise SystemExit(f"Run directory not found: {run_dir}")
+ report = write_report(run_dir)
+ print_report_table(report)
+ console.print(report["run_dir"] + "/report.md")
+ if args.json:
+ console.print_json(json.dumps(report))
+ return 0
+
+
+def cmd_prompt_audit(args: argparse.Namespace, settings: Settings) -> int:
+ settings = _settings_for_args(args, settings)
+ if not settings.dataset_path.exists():
+ download_dataset(
+ settings.dataset_path, settings.dataset_url, variant=settings.variant
+ )
+ dataset = load_dataset(settings.dataset_path, variant=settings.variant)
+ prompt_tokens = tokenize(ANSWER_SYSTEM)
+ n = args.ngram
+ prompt_ngrams = {
+ tuple(prompt_tokens[i : i + n])
+ for i in range(0, max(0, len(prompt_tokens) - n + 1))
+ }
+ hits: list[tuple[str, str]] = []
+ for entry in dataset:
+ qid = str(entry.get("question_id"))
+ gold_tokens = tokenize(str(entry.get("answer") or ""))
+ for i in range(0, max(0, len(gold_tokens) - n + 1)):
+ gram = tuple(gold_tokens[i : i + n])
+ if gram in prompt_ngrams:
+ hits.append((qid, " ".join(gram)))
+ break
+ if hits:
+ console.print(
+ f"[red]FAIL[/red] answer prompt shares a {n}-gram with "
+ f"{len(hits)} gold answers"
+ )
+ for qid, gram in hits[:20]:
+ console.print(f" {qid} {gram!r}")
+ return 1
+ console.print(
+ f"[green]prompt-audit passed[/green] no {n}-gram of the answer prompt "
+ f"appears in any gold answer ({len(dataset)} questions)"
+ )
+ return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ prog="python -m longmemeval",
+ description="LongMemEval benchmark harness for BrainAPI",
+ )
+ parser.add_argument(
+ "--env-file",
+ type=Path,
+ default=None,
+ help="Optional path to .env (defaults to benchmarks/.env)",
+ )
+ sub = parser.add_subparsers(dest="command", required=True)
+
+ def add_variant(p: argparse.ArgumentParser) -> None:
+ p.add_argument(
+ "--variant",
+ choices=sorted(VARIANT_FILES),
+ default=None,
+ help="Dataset variant (default: s / BENCH_LME_VARIANT)",
+ )
+
+ p_download = sub.add_parser("download", help="Download LongMemEval JSON")
+ add_variant(p_download)
+ p_download.set_defaults(func=cmd_download)
+
+ p_stats = sub.add_parser("dataset-stats", help="Print dataset statistics")
+ add_variant(p_stats)
+ p_stats.set_defaults(func=cmd_dataset_stats)
+
+ p_smoke = sub.add_parser("smoke", help="Smoke-test ingest + retrieve")
+ add_variant(p_smoke)
+ p_smoke.add_argument("--brain", default=None)
+ p_smoke.add_argument("--limit", type=int, default=1)
+ p_smoke.add_argument("--timeout", type=float, default=600.0)
+ p_smoke.set_defaults(func=cmd_smoke)
+
+ p_ingest = sub.add_parser("ingest", help="Ingest LongMemEval haystacks")
+ add_variant(p_ingest)
+ p_ingest.add_argument("--run", default=None, help="Existing or new run id")
+ p_ingest.add_argument(
+ "--question-id",
+ action="append",
+ help="Question id (repeatable). Default: all questions.",
+ )
+ p_ingest.add_argument("--question-type", default=None, help="Comma-separated types")
+ p_ingest.add_argument("--limit", type=int, default=None)
+ p_ingest.add_argument("--limit-sessions", type=int, default=None)
+ p_ingest.add_argument("--concurrency", type=int, default=2)
+ p_ingest.add_argument("--dry-run", action="store_true")
+ p_ingest.add_argument("--no-resume", action="store_true")
+ p_ingest.add_argument("--timeout", type=float, default=900.0)
+ p_ingest.set_defaults(func=cmd_ingest)
+
+ p_answer = sub.add_parser("answer-once", help="Retrieve + answer one question")
+ add_variant(p_answer)
+ p_answer.add_argument("--question-id", required=True)
+ p_answer.add_argument("--question", default=None)
+ p_answer.set_defaults(func=cmd_answer_once)
+
+ p_self = sub.add_parser("selftest-metrics", help="Run local metrics self-test")
+ p_self.set_defaults(func=cmd_selftest_metrics)
+
+ p_eval = sub.add_parser("evaluate", help="Answer and score LongMemEval QA")
+ add_variant(p_eval)
+ p_eval.add_argument("--run", default=None)
+ p_eval.add_argument("--question-id", action="append")
+ p_eval.add_argument("--question-type", default=None)
+ p_eval.add_argument("--limit", type=int, default=None)
+ p_eval.add_argument("--concurrency", type=int, default=2)
+ p_eval.add_argument("--no-resume", action="store_true")
+ p_eval.add_argument("--historical-limit", type=int, default=10)
+ p_eval.add_argument("--max-passages", type=int, default=8)
+ p_eval.add_argument("--max-facts", type=int, default=40)
+ p_eval.add_argument("--no-fact-filter", action="store_true")
+ p_eval.add_argument("--use-ppr", action="store_true")
+ p_eval.add_argument("--sufficiency-retry", action="store_true")
+ p_eval.set_defaults(func=cmd_evaluate)
+
+ p_report = sub.add_parser("report", help="Rebuild report from a run directory")
+ p_report.add_argument("--run", required=True)
+ p_report.add_argument("--json", action="store_true")
+ p_report.set_defaults(func=cmd_report)
+
+ p_audit = sub.add_parser(
+ "prompt-audit",
+ help="Fail if the answer prompt shares an n-gram with any gold answer",
+ )
+ add_variant(p_audit)
+ p_audit.add_argument("--ngram", type=int, default=3)
+ p_audit.set_defaults(func=cmd_prompt_audit)
+
+ return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = build_parser()
+ args = parser.parse_args(argv)
+ settings = Settings.load(args.env_file)
+ return args.func(args, settings)
diff --git a/benchmarks/longmemeval/client.py b/benchmarks/longmemeval/client.py
new file mode 100644
index 0000000..f69627f
--- /dev/null
+++ b/benchmarks/longmemeval/client.py
@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+from longmemeval.config import TERMINAL_TASK_STATUSES, Settings
+
+
+@dataclass
+class TimedResult:
+ data: Any
+ latency_ms: float
+
+
+@dataclass
+class BrainAPIClient:
+ settings: Settings
+ timeout: float = 120.0
+ max_retries: int = 5
+ _client: httpx.Client = field(init=False, repr=False)
+
+ def __post_init__(self) -> None:
+ self.settings.require_brainapi()
+ self._client = httpx.Client(
+ base_url=self.settings.brainapi_url,
+ headers={
+ "Authorization": f"Bearer {self.settings.brainpat_token}",
+ "Content-Type": "application/json",
+ },
+ timeout=self.timeout,
+ )
+
+ def close(self) -> None:
+ self._client.close()
+
+ def __enter__(self) -> BrainAPIClient:
+ return self
+
+ def __exit__(self, *args: object) -> None:
+ self.close()
+
+ def _request(
+ self,
+ method: str,
+ path: str,
+ *,
+ json: dict | None = None,
+ params: dict | None = None,
+ ) -> TimedResult:
+ last_error: Exception | None = None
+ for attempt in range(self.max_retries):
+ started = time.perf_counter()
+ try:
+ response = self._client.request(
+ method, path, json=json, params=params
+ )
+ latency_ms = (time.perf_counter() - started) * 1000
+ if response.status_code in (429, 500, 502, 503, 504):
+ wait = min(2**attempt * 0.5, 30)
+ time.sleep(wait)
+ last_error = httpx.HTTPStatusError(
+ f"HTTP {response.status_code}",
+ request=response.request,
+ response=response,
+ )
+ continue
+ response.raise_for_status()
+ if response.status_code == 204 or not response.content:
+ return TimedResult(data=None, latency_ms=latency_ms)
+ return TimedResult(data=response.json(), latency_ms=latency_ms)
+ except (httpx.TransportError, httpx.TimeoutException) as exc:
+ last_error = exc
+ wait = min(2**attempt * 0.5, 30)
+ time.sleep(wait)
+ raise RuntimeError(f"Request failed after retries: {method} {path}: {last_error}")
+
+ def ingest_text(
+ self,
+ text: str,
+ brain_id: str,
+ *,
+ observate_for: list[str] | None = None,
+ preferred_extraction_entities: list[str] | None = None,
+ source_timestamp: str | None = None,
+ ) -> TimedResult:
+ body = {
+ "data": {"data_type": "text", "text_data": text},
+ "brain_id": brain_id,
+ "observate_for": observate_for
+ or [
+ "facts about each speaker",
+ "events and when they happened",
+ "relationships between people",
+ ],
+ "preferred_extraction_entities": preferred_extraction_entities
+ or ["Person", "Event", "Location", "Date"],
+ }
+ if source_timestamp:
+ body["source_timestamp"] = source_timestamp
+ return self._request("POST", "/ingest/", json=body)
+
+ def get_task(self, task_id: str, brain_id: str) -> TimedResult:
+ return self._request(
+ "GET", f"/tasks/{task_id}", params={"brain_id": brain_id}
+ )
+
+ def wait_for_task(
+ self,
+ task_id: str,
+ brain_id: str,
+ *,
+ timeout_s: float = 600.0,
+ poll_interval_s: float = 2.0,
+ ) -> TimedResult:
+ started = time.perf_counter()
+ while True:
+ result = self.get_task(task_id, brain_id)
+ status = (result.data or {}).get("status", "unknown")
+ if status in TERMINAL_TASK_STATUSES:
+ return TimedResult(
+ data=result.data,
+ latency_ms=(time.perf_counter() - started) * 1000,
+ )
+ if time.perf_counter() - started > timeout_s:
+ raise TimeoutError(
+ f"Task {task_id} did not finish within {timeout_s}s "
+ f"(last status={status})"
+ )
+ time.sleep(poll_interval_s)
+
+ def retrieve_context(
+ self,
+ text: str,
+ brain_id: str,
+ *,
+ historical_limit: int = 10,
+ max_passages: int = 8,
+ max_facts: int = 40,
+ apply_fact_filter: bool = True,
+ use_ppr: bool = False,
+ sufficiency_retry: bool = False,
+ cross_event_bridges: int = 3,
+ ) -> TimedResult:
+ body: dict[str, Any] = {
+ "text": text,
+ "brain_id": brain_id,
+ "historical_limit": historical_limit,
+ "max_passages": max_passages,
+ "max_facts": max_facts,
+ "apply_fact_filter": apply_fact_filter,
+ "use_ppr": use_ppr,
+ "sufficiency_retry": sufficiency_retry,
+ "cross_event_bridges": cross_event_bridges,
+ }
+ return self._request("POST", "/retrieve/context", json=body)
+
+ def entity_status(self, target: str, brain_id: str) -> TimedResult:
+ return self._request(
+ "GET",
+ "/retrieve/entity/status",
+ params={"target": target, "brain_id": brain_id},
+ )
+
+ def list_entities(self, brain_id: str, *, limit: int = 5) -> TimedResult:
+ return self._request(
+ "GET",
+ "/retrieve/entities",
+ params={"brain_id": brain_id, "limit": limit},
+ )
diff --git a/benchmarks/longmemeval/config.py b/benchmarks/longmemeval/config.py
new file mode 100644
index 0000000..a4aafb5
--- /dev/null
+++ b/benchmarks/longmemeval/config.py
@@ -0,0 +1,261 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+BENCHMARKS_ROOT = Path(__file__).resolve().parent.parent
+DATA_DIR = BENCHMARKS_ROOT / "data"
+RUNS_DIR = BENCHMARKS_ROOT / "runs"
+
+HF_CLEANED_BASE = (
+ "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main"
+)
+
+VARIANT_FILES = {
+ "s": "longmemeval_s_cleaned.json",
+ "oracle": "longmemeval_oracle.json",
+ "m": "longmemeval_m_cleaned.json",
+}
+
+DEFAULT_VARIANT = "s"
+DEFAULT_LLM_BASE_URL = "https://api.deepseek.com"
+DEFAULT_ANSWER_MODEL = "deepseek-v4-flash"
+DEFAULT_SOTA_ANSWER_MODEL = "deepseek-v4-flash"
+DEFAULT_AZURE_JUDGE_MODEL = "gpt-4o"
+DEFAULT_AZURE_API_VERSION = "2024-12-01-preview"
+DEFAULT_BENCH_PROFILE = "product"
+
+QUESTION_TYPES = (
+ "single-session-user",
+ "single-session-assistant",
+ "single-session-preference",
+ "temporal-reasoning",
+ "knowledge-update",
+ "multi-session",
+)
+
+_MODEL_FAMILY_PREFIXES = {
+ "deepseek": "deepseek",
+ "gpt": "openai",
+ "o1": "openai",
+ "o3": "openai",
+ "o4": "openai",
+ "chatgpt": "openai",
+ "claude": "anthropic",
+ "gemini": "google",
+ "grok": "xai",
+ "qwen": "qwen",
+ "llama": "meta",
+ "mistral": "mistral",
+}
+
+TERMINAL_TASK_STATUSES = frozenset({"completed", "failed", "partial_failed"})
+
+
+def model_family(model: str) -> str:
+ name = (model or "").strip().lower().rsplit("/", 1)[-1]
+ for prefix, family in _MODEL_FAMILY_PREFIXES.items():
+ if name.startswith(prefix):
+ return family
+ return name.split("-", 1)[0] or "unknown"
+
+
+def variant_url(variant: str) -> str:
+ key = (variant or DEFAULT_VARIANT).strip().lower()
+ if key not in VARIANT_FILES:
+ raise ValueError(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")
+ return f"{HF_CLEANED_BASE}/{VARIANT_FILES[key]}"
+
+
+def variant_path(variant: str) -> Path:
+ key = (variant or DEFAULT_VARIANT).strip().lower()
+ if key not in VARIANT_FILES:
+ raise ValueError(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")
+ return DATA_DIR / VARIANT_FILES[key]
+
+
+def profile_defaults(profile: str) -> dict:
+ name = (profile or "product").strip().lower()
+ if name == "sota":
+ return {
+ "bench_profile": "sota",
+ "sc_samples": int(os.getenv("BENCH_SC_SAMPLES", "5")),
+ "sc_temperature": float(os.getenv("BENCH_SC_TEMPERATURE", "0.7")),
+ "gap_fill": os.getenv("BENCH_GAP_FILL", "1") not in {"0", "false", "False"},
+ }
+ return {
+ "bench_profile": "product",
+ "sc_samples": int(os.getenv("BENCH_SC_SAMPLES", "1")),
+ "sc_temperature": 0.0,
+ "gap_fill": os.getenv("BENCH_GAP_FILL", "0") in {"1", "true", "True"},
+ }
+
+
+@dataclass
+class Settings:
+ brainapi_url: str
+ brainpat_token: str
+ openai_api_key: str
+ llm_base_url: str | None
+ answer_model: str
+ judge_model: str
+ judge_api_key: str
+ judge_base_url: str | None
+ judge_azure_endpoint: str | None
+ judge_azure_api_version: str
+ dataset_path: Path
+ dataset_url: str
+ variant: str
+ runs_dir: Path
+ bench_profile: str
+ sc_samples: int
+ sc_temperature: float
+ gap_fill: bool
+ answer_azure_endpoint: str | None = None
+ answer_azure_api_version: str = DEFAULT_AZURE_API_VERSION
+
+ @classmethod
+ def load(cls, env_file: Path | None = None, *, variant: str | None = None) -> "Settings":
+ if env_file is None:
+ env_file = BENCHMARKS_ROOT / ".env"
+ if env_file.exists():
+ load_dotenv(env_file, override=False)
+ else:
+ load_dotenv(override=False)
+
+ profile = (
+ os.getenv("BENCH_PROFILE", DEFAULT_BENCH_PROFILE) or DEFAULT_BENCH_PROFILE
+ ).strip().lower()
+ if profile not in {"product", "sota"}:
+ profile = DEFAULT_BENCH_PROFILE
+ defaults = profile_defaults(profile)
+
+ api_key = (
+ os.getenv("DEEPSEEK_API_KEY")
+ or os.getenv("OPENAI_API_KEY")
+ or ""
+ )
+ llm_base_url = os.getenv("BENCH_LLM_BASE_URL", DEFAULT_LLM_BASE_URL) or None
+ answer_model = os.getenv("BENCH_ANSWER_MODEL", DEFAULT_ANSWER_MODEL)
+
+ azure_endpoint = (
+ os.getenv("BENCH_JUDGE_AZURE_ENDPOINT")
+ or os.getenv("AZURE_LARGE_LLM_ENDPOINT")
+ or ""
+ ).rstrip("/")
+ azure_key = (
+ os.getenv("BENCH_JUDGE_AZURE_KEY")
+ or os.getenv("AZURE_LARGE_LLM_SUBSCRIPTION_KEY")
+ or ""
+ )
+ azure_available = bool(azure_endpoint and azure_key)
+
+ answer_azure_endpoint: str | None = None
+ if profile == "sota":
+ sota_answer = os.getenv("BENCH_SOTA_ANSWER_MODEL", "").strip()
+ if sota_answer:
+ answer_model = sota_answer
+ sota_base = os.getenv("BENCH_SOTA_LLM_BASE_URL", "").strip()
+ if sota_base:
+ llm_base_url = sota_base
+ else:
+ answer_model = os.getenv(
+ "BENCH_ANSWER_MODEL", DEFAULT_SOTA_ANSWER_MODEL
+ )
+
+ judge_model = os.getenv("BENCH_JUDGE_MODEL", "")
+ judge_base_url = os.getenv("BENCH_JUDGE_BASE_URL", "")
+ judge_api_key = os.getenv("BENCH_JUDGE_API_KEY", "")
+ use_azure_judge = azure_available and not judge_model and not judge_base_url
+ if use_azure_judge:
+ judge_model = os.getenv(
+ "AZURE_LARGE_LLM_MODEL", DEFAULT_AZURE_JUDGE_MODEL
+ )
+ if profile == "sota":
+ sota_judge = os.getenv("BENCH_SOTA_JUDGE_MODEL", "").strip()
+ if sota_judge:
+ judge_model = sota_judge
+ use_azure_judge = False
+ elif not os.getenv("BENCH_JUDGE_MODEL"):
+ judge_model = answer_model
+ use_azure_judge = False
+ judge_base_url = llm_base_url or ""
+ judge_api_key = api_key
+
+ resolved_variant = (
+ variant
+ or os.getenv("BENCH_LME_VARIANT", DEFAULT_VARIANT)
+ or DEFAULT_VARIANT
+ ).strip().lower()
+ if resolved_variant not in VARIANT_FILES:
+ resolved_variant = DEFAULT_VARIANT
+ default_path = variant_path(resolved_variant)
+ default_url = variant_url(resolved_variant)
+
+ return cls(
+ brainapi_url=os.getenv("BRAINAPI_URL", "http://localhost:8000").rstrip("/"),
+ brainpat_token=os.getenv("BRAINPAT_TOKEN", ""),
+ openai_api_key=api_key,
+ llm_base_url=llm_base_url,
+ answer_model=answer_model,
+ judge_model=judge_model or answer_model,
+ judge_api_key=(
+ azure_key if use_azure_judge else (judge_api_key or api_key)
+ ),
+ judge_base_url=(
+ None if use_azure_judge else (judge_base_url or llm_base_url)
+ ),
+ judge_azure_endpoint=azure_endpoint if use_azure_judge else None,
+ judge_azure_api_version=os.getenv(
+ "BENCH_JUDGE_AZURE_API_VERSION",
+ os.getenv("AZURE_LARGE_LLM_API_VERSION", DEFAULT_AZURE_API_VERSION),
+ ),
+ dataset_path=Path(
+ os.getenv("BENCH_LME_DATASET_PATH", str(default_path))
+ ),
+ dataset_url=os.getenv("BENCH_LME_DATASET_URL", default_url),
+ variant=resolved_variant,
+ runs_dir=Path(os.getenv("BENCH_RUNS_DIR", str(RUNS_DIR))),
+ bench_profile=str(defaults["bench_profile"]),
+ sc_samples=max(1, int(defaults["sc_samples"])),
+ sc_temperature=float(defaults["sc_temperature"]),
+ gap_fill=bool(defaults["gap_fill"]),
+ answer_azure_endpoint=answer_azure_endpoint,
+ answer_azure_api_version=os.getenv(
+ "BENCH_JUDGE_AZURE_API_VERSION",
+ os.getenv("AZURE_LARGE_LLM_API_VERSION", DEFAULT_AZURE_API_VERSION),
+ ),
+ )
+
+ @property
+ def judge_shares_answer_family(self) -> bool:
+ return model_family(self.judge_model) == model_family(self.answer_model)
+
+ def judge_provider(self) -> str:
+ if self.judge_azure_endpoint:
+ return "azure"
+ return self.judge_base_url or "openai-default"
+
+ def require_brainapi(self) -> None:
+ if not self.brainpat_token:
+ raise SystemExit(
+ "BRAINPAT_TOKEN is required. Copy .env.example to .env and set it."
+ )
+
+ def require_llm(self) -> None:
+ if not self.openai_api_key:
+ raise SystemExit(
+ "DEEPSEEK_API_KEY (or OPENAI_API_KEY) is required for answer/judge "
+ "LLM calls. Copy .env.example to .env and set it."
+ )
+
+ def require_judge_llm(self) -> None:
+ if not self.judge_api_key:
+ raise SystemExit(
+ "No judge API key. Set BENCH_JUDGE_API_KEY (with BENCH_JUDGE_BASE_URL "
+ "and BENCH_JUDGE_MODEL), or DEEPSEEK_API_KEY / OPENAI_API_KEY to reuse "
+ "the answerer's provider."
+ )
diff --git a/benchmarks/longmemeval/dataset.py b/benchmarks/longmemeval/dataset.py
new file mode 100644
index 0000000..93f0adf
--- /dev/null
+++ b/benchmarks/longmemeval/dataset.py
@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Any
+from urllib.request import urlretrieve
+
+from longmemeval.config import (
+ DEFAULT_VARIANT,
+ VARIANT_FILES,
+ Settings,
+ variant_path,
+ variant_url,
+)
+
+
+def brain_id_for(question_id: str) -> str:
+ slug = re.sub(r"[^a-zA-Z0-9]", "", f"lme{question_id}")
+ if not slug.isalnum():
+ raise ValueError(f"Could not derive alphanumeric brain_id from {question_id!r}")
+ return slug.lower()
+
+
+def is_abstention(question_id: str) -> bool:
+ return str(question_id).endswith("_abs")
+
+
+def download_dataset(
+ dest: Path | None = None,
+ url: str | None = None,
+ *,
+ variant: str = DEFAULT_VARIANT,
+) -> Path:
+ dest = dest or variant_path(variant)
+ url = url or variant_url(variant)
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ if dest.exists() and dest.stat().st_size > 0:
+ return dest
+ urlretrieve(url, dest)
+ return dest
+
+
+def load_dataset(path: Path | None = None, *, variant: str = DEFAULT_VARIANT) -> list[dict[str, Any]]:
+ path = path or variant_path(variant)
+ if not path.exists():
+ raise FileNotFoundError(
+ f"Dataset not found at {path}. Run: python -m longmemeval download --variant {variant}"
+ )
+ with path.open(encoding="utf-8") as f:
+ data = json.load(f)
+ if not isinstance(data, list):
+ raise ValueError(f"Expected {path.name} to be a JSON array")
+ return data
+
+
+def get_question(dataset: list[dict[str, Any]], question_id: str) -> dict[str, Any]:
+ for entry in dataset:
+ if entry.get("question_id") == question_id:
+ return entry
+ raise KeyError(f"question_id {question_id!r} not found")
+
+
+def format_turn(turn: dict[str, Any]) -> str:
+ role = str(turn.get("role") or "unknown").strip()
+ content = str(turn.get("content") or "").strip()
+ return f"{role}: {content}"
+
+
+def format_session(
+ session_id: str,
+ session_date: str,
+ turns: list[Any],
+) -> str:
+ lines = [
+ f"Session time: {session_date or 'unknown time'}.",
+ f"Session id: {session_id}.",
+ "",
+ ]
+ for turn in turns or []:
+ if not isinstance(turn, dict):
+ continue
+ clean = {
+ "role": turn.get("role"),
+ "content": turn.get("content"),
+ }
+ lines.append(format_turn(clean))
+ return "\n".join(lines)
+
+
+def iter_ingest_units(
+ entry: dict[str, Any],
+ *,
+ limit_sessions: int | None = None,
+) -> list[dict[str, Any]]:
+ session_ids = list(entry.get("haystack_session_ids") or [])
+ dates = list(entry.get("haystack_dates") or [])
+ sessions = list(entry.get("haystack_sessions") or [])
+ n = min(len(session_ids), len(dates), len(sessions))
+ if limit_sessions is not None:
+ n = min(n, limit_sessions)
+
+ units: list[dict[str, Any]] = []
+ for i in range(n):
+ sid = str(session_ids[i])
+ units.append(
+ {
+ "unit_id": sid,
+ "session_id": sid,
+ "text": format_session(sid, str(dates[i]), sessions[i]),
+ "source_timestamp": str(dates[i]) if dates[i] else None,
+ }
+ )
+ return units
+
+
+def dataset_stats(dataset: list[dict[str, Any]]) -> dict[str, Any]:
+ type_counts = Counter(str(e.get("question_type") or "unknown") for e in dataset)
+ abstention = sum(1 for e in dataset if is_abstention(str(e.get("question_id") or "")))
+ sessions = [
+ len(e.get("haystack_session_ids") or [])
+ for e in dataset
+ ]
+ turns = []
+ for e in dataset:
+ for sess in e.get("haystack_sessions") or []:
+ if isinstance(sess, list):
+ turns.append(len(sess))
+ return {
+ "n_questions": len(dataset),
+ "n_abstention": abstention,
+ "by_type": dict(sorted(type_counts.items())),
+ "sessions_per_question": {
+ "min": min(sessions) if sessions else 0,
+ "max": max(sessions) if sessions else 0,
+ "mean": (sum(sessions) / len(sessions)) if sessions else 0.0,
+ },
+ "turns_per_session": {
+ "min": min(turns) if turns else 0,
+ "max": max(turns) if turns else 0,
+ "mean": (sum(turns) / len(turns)) if turns else 0.0,
+ },
+ }
+
+
+def resolve_questions(
+ settings: Settings,
+ *,
+ question_ids: list[str] | None = None,
+ limit: int | None = None,
+ question_types: set[str] | None = None,
+) -> list[dict[str, Any]]:
+ dataset = load_dataset(settings.dataset_path, variant=settings.variant)
+ if question_ids:
+ wanted = set(question_ids)
+ dataset = [e for e in dataset if e.get("question_id") in wanted]
+ missing = wanted - {e.get("question_id") for e in dataset}
+ if missing:
+ raise KeyError(f"question_id(s) not found: {sorted(missing)}")
+ if question_types:
+ dataset = [
+ e for e in dataset if str(e.get("question_type") or "") in question_types
+ ]
+ if limit is not None:
+ dataset = dataset[: max(0, limit)]
+ return dataset
+
+
+def resolve_variant_settings(
+ settings: Settings, variant: str | None
+) -> Settings:
+ if not variant or variant == settings.variant:
+ return settings
+ key = variant.strip().lower()
+ if key not in VARIANT_FILES:
+ raise SystemExit(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")
+ return Settings(
+ brainapi_url=settings.brainapi_url,
+ brainpat_token=settings.brainpat_token,
+ openai_api_key=settings.openai_api_key,
+ llm_base_url=settings.llm_base_url,
+ answer_model=settings.answer_model,
+ judge_model=settings.judge_model,
+ judge_api_key=settings.judge_api_key,
+ judge_base_url=settings.judge_base_url,
+ judge_azure_endpoint=settings.judge_azure_endpoint,
+ judge_azure_api_version=settings.judge_azure_api_version,
+ dataset_path=variant_path(key),
+ dataset_url=variant_url(key),
+ variant=key,
+ runs_dir=settings.runs_dir,
+ bench_profile=settings.bench_profile,
+ sc_samples=settings.sc_samples,
+ sc_temperature=settings.sc_temperature,
+ gap_fill=settings.gap_fill,
+ answer_azure_endpoint=settings.answer_azure_endpoint,
+ answer_azure_api_version=settings.answer_azure_api_version,
+ )
diff --git a/benchmarks/longmemeval/evaluate.py b/benchmarks/longmemeval/evaluate.py
new file mode 100644
index 0000000..e625a56
--- /dev/null
+++ b/benchmarks/longmemeval/evaluate.py
@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import re
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.progress import Progress
+
+from longmemeval.answer import answer_question
+from longmemeval.client import BrainAPIClient
+from longmemeval.config import Settings
+from longmemeval.dataset import brain_id_for, is_abstention
+from longmemeval.ingest import append_jsonl, load_jsonl
+from longmemeval.judge import judge_answer
+from longmemeval.prompts import enrich_paths_from_triples, flatten_path, flatten_triple
+from longmemeval.sota import merge_contexts, plan_gap_fill
+
+console = Console()
+_write_lock = threading.Lock()
+
+_GRAPH_CONTEXT_CAP = 60000
+_TRIPLES_CAP = 500
+_PASSAGE_PREFIX = "[passage] "
+_SESSION_ID_RE = re.compile(r"Session id:\s*([^\s.]+)", re.IGNORECASE)
+
+
+@dataclass
+class AnswerRecord:
+ question_id: str
+ brain_id: str
+ question: str
+ gold: str
+ prediction: str
+ question_type: str
+ question_date: str
+ answer_session_ids: list[str]
+ judge_correct: bool
+ judge_reason: str
+ judge_raw: str
+ retrieve_latency_ms: float | None
+ answer_latency_ms: float | None
+ judge_latency_ms: float | None
+ answer_model: str | None
+ judge_model: str | None
+ answer_total_tokens: int | None
+ judge_total_tokens: int | None
+ graph_context: str
+ triples: list[str] = field(default_factory=list)
+ paths: list[str] = field(default_factory=list)
+ source_passages: list[str] = field(default_factory=list)
+ historical_context: list[str] = field(default_factory=list)
+ retrieved_session_ids: list[str] = field(default_factory=list)
+ context_truncated: dict[str, int] = field(default_factory=dict)
+ is_abstention: bool = False
+ error: str | None = None
+ timestamp: str = ""
+
+
+def _utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+def _extract_session_ids(*blobs: Any, known_ids: list[str] | None = None) -> list[str]:
+ found: list[str] = []
+ seen: set[str] = set()
+ texts: list[str] = []
+ for blob in blobs:
+ if isinstance(blob, (list, tuple)):
+ texts.append("\n".join(str(item) for item in blob))
+ else:
+ texts.append(str(blob or ""))
+ joined = "\n".join(texts)
+ for match in _SESSION_ID_RE.finditer(joined):
+ sid = match.group(1).rstrip(".")
+ if sid and sid not in seen:
+ seen.add(sid)
+ found.append(sid)
+ if known_ids:
+ lower_joined = joined.lower()
+ for sid in known_ids:
+ if sid and sid not in seen and sid.lower() in lower_joined:
+ seen.add(sid)
+ found.append(sid)
+ return found
+
+
+def _graph_channel(text_context: str, source_passages: list[str]) -> str:
+ passage_block = "\n".join(f"{_PASSAGE_PREFIX}{p}" for p in source_passages)
+ if passage_block and text_context.startswith(passage_block):
+ return text_context[len(passage_block) :].lstrip("\n")
+ return "\n".join(
+ line
+ for line in text_context.splitlines()
+ if not line.startswith(_PASSAGE_PREFIX)
+ )
+
+
+def extract_context_channels(
+ context: dict[str, Any],
+ *,
+ known_session_ids: list[str] | None = None,
+) -> dict[str, Any]:
+ context = enrich_paths_from_triples(context)
+ source_passages = [str(p) for p in (context.get("source_passages") or []) if p]
+ historical_context = [
+ str(h) for h in (context.get("historical_context") or []) if h
+ ]
+ triples = [flatten_triple(t) for t in (context.get("triples") or []) if t]
+ paths = [flatten_path(p) for p in (context.get("paths") or []) if p]
+ graph_context = _graph_channel(
+ str(context.get("text_context") or ""), source_passages
+ )
+ truncated: dict[str, int] = {}
+ if len(graph_context) > _GRAPH_CONTEXT_CAP:
+ truncated["graph_context"] = len(graph_context)
+ graph_context = graph_context[:_GRAPH_CONTEXT_CAP]
+ if len(triples) > _TRIPLES_CAP:
+ truncated["triples"] = len(triples)
+ triples = triples[:_TRIPLES_CAP]
+ session_ids = _extract_session_ids(
+ graph_context,
+ triples,
+ paths,
+ source_passages,
+ historical_context,
+ known_ids=known_session_ids,
+ )
+ return {
+ "graph_context": graph_context,
+ "triples": triples,
+ "paths": paths,
+ "source_passages": source_passages,
+ "historical_context": historical_context,
+ "retrieved_session_ids": session_ids,
+ "context_truncated": truncated,
+ }
+
+
+def completed_question_ids(answers_path: Path) -> set[str]:
+ done = set()
+ for row in load_jsonl(answers_path):
+ if row.get("error"):
+ continue
+ qid = row.get("question_id")
+ if qid:
+ done.add(str(qid))
+ return done
+
+
+def evaluate_questions(
+ settings: Settings,
+ questions: list[dict[str, Any]],
+ *,
+ run_dir: Path,
+ concurrency: int = 2,
+ resume: bool = True,
+ historical_limit: int = 10,
+ max_passages: int = 8,
+ max_facts: int = 40,
+ apply_fact_filter: bool = True,
+ use_ppr: bool = False,
+ sufficiency_retry: bool = False,
+) -> list[AnswerRecord]:
+ run_dir.mkdir(parents=True, exist_ok=True)
+ answers_path = run_dir / "answers.jsonl"
+ already_done = completed_question_ids(answers_path) if resume else set()
+
+ jobs = [
+ {
+ "entry": entry,
+ "question_id": str(entry["question_id"]),
+ "brain_id": brain_id_for(str(entry["question_id"])),
+ }
+ for entry in questions
+ if str(entry["question_id"]) not in already_done
+ ]
+
+ if not jobs:
+ console.print("[green]Nothing to evaluate (all questions already scored).[/green]")
+ return []
+
+ console.print(f"Evaluating {len(jobs)} questions (concurrency={concurrency})")
+
+ records: list[AnswerRecord] = []
+
+ def _one(job: dict[str, Any]) -> AnswerRecord:
+ entry = job["entry"]
+ question_id = job["question_id"]
+ question = str(entry.get("question") or "")
+ gold = str(entry.get("answer") or "")
+ question_type = str(entry.get("question_type") or "")
+ question_date = str(entry.get("question_date") or "")
+ answer_session_ids = [str(s) for s in (entry.get("answer_session_ids") or [])]
+ abstention = is_abstention(question_id)
+ known_ids = [str(s) for s in (entry.get("haystack_session_ids") or [])]
+
+ try:
+ with BrainAPIClient(settings) as client:
+ retrieved = client.retrieve_context(
+ question,
+ job["brain_id"],
+ historical_limit=historical_limit,
+ max_passages=max_passages,
+ max_facts=max_facts,
+ apply_fact_filter=apply_fact_filter,
+ use_ppr=use_ppr,
+ sufficiency_retry=sufficiency_retry,
+ )
+ context = retrieved.data or {}
+ retrieve_ms = retrieved.latency_ms
+ draft = answer_question(
+ settings, question, context, question_date=question_date
+ )
+ if settings.gap_fill:
+ plan = plan_gap_fill(question, draft.answer, context)
+ if plan.needs_retry:
+ with BrainAPIClient(settings) as client:
+ second = client.retrieve_context(
+ plan.reformulated_query,
+ job["brain_id"],
+ historical_limit=historical_limit,
+ max_passages=max_passages,
+ max_facts=max_facts,
+ apply_fact_filter=apply_fact_filter,
+ use_ppr=use_ppr,
+ sufficiency_retry=sufficiency_retry,
+ )
+ retrieve_ms = (retrieve_ms or 0) + (second.latency_ms or 0)
+ context = merge_contexts(context, second.data or {})
+ answered = answer_question(
+ settings, question, context, question_date=question_date
+ )
+ else:
+ answered = draft
+ else:
+ answered = draft
+ channels = extract_context_channels(
+ context, known_session_ids=known_ids
+ )
+ judged = judge_answer(
+ settings,
+ question,
+ gold,
+ answered.answer,
+ question_type=question_type,
+ abstention=abstention,
+ )
+ record = AnswerRecord(
+ question_id=question_id,
+ brain_id=job["brain_id"],
+ question=question,
+ gold=gold,
+ prediction=answered.answer,
+ question_type=question_type,
+ question_date=question_date,
+ answer_session_ids=answer_session_ids,
+ judge_correct=judged.correct,
+ judge_reason=judged.reason,
+ judge_raw=judged.raw,
+ retrieve_latency_ms=retrieve_ms,
+ answer_latency_ms=answered.latency_ms,
+ judge_latency_ms=judged.latency_ms,
+ answer_model=answered.model,
+ judge_model=judged.model,
+ answer_total_tokens=answered.total_tokens,
+ judge_total_tokens=judged.total_tokens,
+ is_abstention=abstention,
+ error=None,
+ timestamp=_utc_now(),
+ **channels,
+ )
+ except Exception as exc:
+ record = AnswerRecord(
+ question_id=question_id,
+ brain_id=job["brain_id"],
+ question=question,
+ gold=gold,
+ prediction="",
+ question_type=question_type,
+ question_date=question_date,
+ answer_session_ids=answer_session_ids,
+ judge_correct=False,
+ judge_reason="",
+ judge_raw="",
+ retrieve_latency_ms=None,
+ answer_latency_ms=None,
+ judge_latency_ms=None,
+ answer_model=None,
+ judge_model=None,
+ answer_total_tokens=None,
+ judge_total_tokens=None,
+ is_abstention=abstention,
+ error=str(exc),
+ timestamp=_utc_now(),
+ **extract_context_channels({}),
+ )
+
+ with _write_lock:
+ append_jsonl(answers_path, asdict(record))
+ return record
+
+ with Progress(console=console) as progress:
+ task = progress.add_task("evaluate", total=len(jobs))
+ with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
+ futures = [pool.submit(_one, job) for job in jobs]
+ for future in as_completed(futures):
+ records.append(future.result())
+ progress.advance(task)
+
+ ok = sum(1 for r in records if not r.error)
+ failed = sum(1 for r in records if r.error)
+ console.print(f"Evaluate done: ok={ok} failed={failed} -> {answers_path}")
+ return records
diff --git a/benchmarks/longmemeval/ingest.py b/benchmarks/longmemeval/ingest.py
new file mode 100644
index 0000000..1be715f
--- /dev/null
+++ b/benchmarks/longmemeval/ingest.py
@@ -0,0 +1,222 @@
+from __future__ import annotations
+
+import json
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.progress import Progress
+
+from longmemeval.client import BrainAPIClient
+from longmemeval.config import Settings
+from longmemeval.dataset import brain_id_for, iter_ingest_units
+
+console = Console()
+_write_lock = threading.Lock()
+
+
+@dataclass
+class IngestRecord:
+ question_id: str
+ brain_id: str
+ unit_id: str
+ session_id: str
+ task_id: str | None
+ status: str
+ error: str | None
+ submit_latency_ms: float | None
+ wait_latency_ms: float | None
+ dry_run: bool
+ timestamp: str
+ cost: dict[str, Any] | None = None
+ total_llm_tokens: int | None = None
+
+
+def _utc_now() -> str:
+ return datetime.now(timezone.utc).isoformat()
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+ if not path.exists():
+ return []
+ rows = []
+ with path.open(encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if line:
+ rows.append(json.loads(line))
+ return rows
+
+
+def append_jsonl(path: Path, row: dict[str, Any]) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ with _write_lock:
+ with path.open("a", encoding="utf-8") as f:
+ f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def completed_unit_ids(ingest_path: Path) -> set[str]:
+ done = set()
+ for row in load_jsonl(ingest_path):
+ status = row.get("status")
+ if status in {"completed", "partial_failed", "dry_run"}:
+ key = f"{row.get('question_id')}::{row.get('unit_id')}"
+ done.add(key)
+ return done
+
+
+def ingest_questions(
+ settings: Settings,
+ questions: list[dict[str, Any]],
+ *,
+ run_dir: Path,
+ concurrency: int = 2,
+ limit_sessions: int | None = None,
+ dry_run: bool = False,
+ resume: bool = True,
+ task_timeout_s: float = 900.0,
+) -> list[IngestRecord]:
+ run_dir.mkdir(parents=True, exist_ok=True)
+ ingest_path = run_dir / "ingest.jsonl"
+ already_done = completed_unit_ids(ingest_path) if resume else set()
+
+ jobs: list[dict[str, Any]] = []
+ for entry in questions:
+ question_id = str(entry["question_id"])
+ brain_id = brain_id_for(question_id)
+ for unit in iter_ingest_units(entry, limit_sessions=limit_sessions):
+ key = f"{question_id}::{unit['unit_id']}"
+ if key in already_done:
+ continue
+ jobs.append(
+ {
+ "question_id": question_id,
+ "brain_id": brain_id,
+ "unit": unit,
+ }
+ )
+
+ if not jobs:
+ console.print("[green]Nothing to ingest (all units already completed).[/green]")
+ return []
+
+ console.print(
+ f"Ingesting {len(jobs)} units "
+ f"(concurrency={concurrency}, dry_run={dry_run})"
+ )
+
+ records: list[IngestRecord] = []
+
+ def _one(job: dict[str, Any]) -> IngestRecord:
+ unit = job["unit"]
+ if dry_run:
+ record = IngestRecord(
+ question_id=job["question_id"],
+ brain_id=job["brain_id"],
+ unit_id=unit["unit_id"],
+ session_id=unit["session_id"],
+ task_id=None,
+ status="dry_run",
+ error=None,
+ submit_latency_ms=None,
+ wait_latency_ms=None,
+ dry_run=True,
+ timestamp=_utc_now(),
+ )
+ append_jsonl(ingest_path, asdict(record))
+ return record
+
+ with BrainAPIClient(settings) as client:
+ try:
+ submitted = client.ingest_text(
+ unit["text"],
+ job["brain_id"],
+ source_timestamp=unit.get("source_timestamp"),
+ )
+ task_id = (submitted.data or {}).get("task_id")
+ if not task_id:
+ raise RuntimeError(f"No task_id in ingest response: {submitted.data}")
+ waited = client.wait_for_task(
+ task_id, job["brain_id"], timeout_s=task_timeout_s
+ )
+ status = (waited.data or {}).get("status", "unknown")
+ error = (waited.data or {}).get("error")
+ cost = (waited.data or {}).get("cost")
+ total_llm_tokens = None
+ if isinstance(cost, dict):
+ total_llm_tokens = cost.get("total_llm_tokens")
+ record = IngestRecord(
+ question_id=job["question_id"],
+ brain_id=job["brain_id"],
+ unit_id=unit["unit_id"],
+ session_id=unit["session_id"],
+ task_id=task_id,
+ status=status,
+ error=error,
+ submit_latency_ms=submitted.latency_ms,
+ wait_latency_ms=waited.latency_ms,
+ dry_run=False,
+ timestamp=_utc_now(),
+ cost=cost if isinstance(cost, dict) else None,
+ total_llm_tokens=(
+ int(total_llm_tokens) if total_llm_tokens is not None else None
+ ),
+ )
+ except Exception as exc:
+ record = IngestRecord(
+ question_id=job["question_id"],
+ brain_id=job["brain_id"],
+ unit_id=unit["unit_id"],
+ session_id=unit["session_id"],
+ task_id=None,
+ status="failed",
+ error=str(exc),
+ submit_latency_ms=None,
+ wait_latency_ms=None,
+ dry_run=False,
+ timestamp=_utc_now(),
+ )
+ append_jsonl(ingest_path, asdict(record))
+ return record
+
+ with Progress(console=console) as progress:
+ task = progress.add_task("ingest", total=len(jobs))
+ with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
+ futures = [pool.submit(_one, job) for job in jobs]
+ for future in as_completed(futures):
+ records.append(future.result())
+ progress.advance(task)
+
+ ok = sum(1 for r in records if r.status in {"completed", "dry_run"})
+ partial = sum(1 for r in records if r.status == "partial_failed")
+ failed = sum(1 for r in records if r.status == "failed")
+ console.print(
+ f"Ingest done: ok={ok} partial_failed={partial} failed={failed} "
+ f"-> {ingest_path}"
+ )
+ return records
+
+
+def ensure_run_dir(settings: Settings, run_id: str | None = None) -> tuple[str, Path]:
+ settings.runs_dir.mkdir(parents=True, exist_ok=True)
+ if not run_id:
+ run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
+ run_dir = settings.runs_dir / run_id
+ run_dir.mkdir(parents=True, exist_ok=True)
+ return run_id, run_dir
+
+
+def write_manifest(run_dir: Path, manifest: dict[str, Any]) -> None:
+ path = run_dir / "manifest.json"
+ existing: dict[str, Any] = {}
+ if path.exists():
+ existing = json.loads(path.read_text(encoding="utf-8"))
+ existing.update(manifest)
+ existing["updated_at"] = _utc_now()
+ if "created_at" not in existing:
+ existing["created_at"] = _utc_now()
+ path.write_text(json.dumps(existing, indent=2), encoding="utf-8")
diff --git a/benchmarks/longmemeval/judge.py b/benchmarks/longmemeval/judge.py
new file mode 100644
index 0000000..d8ff6ee
--- /dev/null
+++ b/benchmarks/longmemeval/judge.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from openai import AzureOpenAI, OpenAI
+
+from longmemeval.config import Settings
+from longmemeval.prompts import get_anscheck_prompt
+
+
+@dataclass
+class JudgeResult:
+ correct: bool
+ reason: str
+ raw: str
+ model: str
+ latency_ms: float
+ prompt_tokens: int | None
+ completion_tokens: int | None
+ total_tokens: int | None
+
+
+def _make_openai_client(settings: Settings) -> OpenAI | AzureOpenAI:
+ settings.require_judge_llm()
+ if settings.judge_azure_endpoint:
+ return AzureOpenAI(
+ api_key=settings.judge_api_key,
+ azure_endpoint=settings.judge_azure_endpoint,
+ api_version=settings.judge_azure_api_version,
+ )
+ kwargs: dict[str, Any] = {"api_key": settings.judge_api_key}
+ if settings.judge_base_url:
+ kwargs["base_url"] = settings.judge_base_url
+ return OpenAI(**kwargs)
+
+
+def parse_yes_no(raw: str) -> bool:
+ return "yes" in (raw or "").strip().lower()
+
+
+def judge_answer(
+ settings: Settings,
+ question: str,
+ gold: str,
+ prediction: str,
+ *,
+ question_type: str,
+ abstention: bool = False,
+ model: str | None = None,
+) -> JudgeResult:
+ client = _make_openai_client(settings)
+ model_name = model or settings.judge_model
+ prompt = get_anscheck_prompt(
+ question_type,
+ question,
+ gold,
+ prediction,
+ abstention=abstention,
+ )
+ started = time.perf_counter()
+ response = client.chat.completions.create(
+ model=model_name,
+ messages=[{"role": "user", "content": prompt}],
+ temperature=0,
+ max_tokens=10,
+ )
+ latency_ms = (time.perf_counter() - started) * 1000
+ raw = (response.choices[0].message.content or "").strip()
+ correct = parse_yes_no(raw)
+ usage = response.usage
+ return JudgeResult(
+ correct=correct,
+ reason=raw,
+ raw=raw,
+ model=getattr(response, "model", None) or model_name,
+ latency_ms=latency_ms,
+ prompt_tokens=getattr(usage, "prompt_tokens", None) if usage else None,
+ completion_tokens=getattr(usage, "completion_tokens", None) if usage else None,
+ total_tokens=getattr(usage, "total_tokens", None) if usage else None,
+ )
diff --git a/benchmarks/longmemeval/metrics.py b/benchmarks/longmemeval/metrics.py
new file mode 100644
index 0000000..111bbd5
--- /dev/null
+++ b/benchmarks/longmemeval/metrics.py
@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+import math
+import string
+from typing import Any, Iterable
+
+
+_ARTICLES = {"a", "an", "the"}
+
+
+def normalize_answer(text: str) -> str:
+ text = (text or "").lower()
+ text = "".join(ch if ch not in string.punctuation else " " for ch in text)
+ tokens = [t for t in text.split() if t and t not in _ARTICLES]
+ return " ".join(tokens)
+
+
+def tokenize(text: str) -> list[str]:
+ return normalize_answer(text).split()
+
+
+def percentile(values: list[float], p: float) -> float | None:
+ if not values:
+ return None
+ ordered = sorted(values)
+ if len(ordered) == 1:
+ return ordered[0]
+ k = (len(ordered) - 1) * (p / 100.0)
+ f = math.floor(k)
+ c = math.ceil(k)
+ if f == c:
+ return ordered[int(k)]
+ return ordered[f] * (c - k) + ordered[c] * (k - f)
+
+
+def mean(values: Iterable[float]) -> float | None:
+ vals = list(values)
+ if not vals:
+ return None
+ return sum(vals) / len(vals)
+
+
+def wilson_ci(successes: int, n: int, z: float = 1.96) -> tuple[float | None, float | None]:
+ if n <= 0:
+ return None, None
+ p = successes / n
+ denom = 1.0 + (z * z) / n
+ center = (p + (z * z) / (2.0 * n)) / denom
+ margin = (
+ z
+ * math.sqrt((p * (1.0 - p) / n) + (z * z) / (4.0 * n * n))
+ / denom
+ )
+ return max(0.0, center - margin), min(1.0, center + margin)
+
+
+def session_recall_coverage(row: dict[str, Any]) -> str | None:
+ if row.get("is_abstention"):
+ return None
+ needed = {str(s) for s in (row.get("answer_session_ids") or []) if s}
+ if not needed:
+ return None
+ have = {str(s) for s in (row.get("retrieved_session_ids") or []) if s}
+ if needed <= have:
+ return "full"
+ if needed & have:
+ return "partial"
+ return "none"
+
+
+def _bucket(items: list[dict[str, Any]]) -> dict[str, Any]:
+ if not items:
+ return {
+ "n": 0,
+ "judge_accuracy": None,
+ "judge_accuracy_ci95": None,
+ "session_recall_full": None,
+ "session_recall_partial": None,
+ "n_with_evidence": 0,
+ }
+ correct = [1.0 if r.get("judge_correct") else 0.0 for r in items]
+ successes = int(sum(correct))
+ lo, hi = wilson_ci(successes, len(items))
+ coverages = [session_recall_coverage(r) for r in items]
+ with_evidence = [c for c in coverages if c is not None]
+ n_ev = len(with_evidence)
+ full = sum(1 for c in with_evidence if c == "full")
+ partial = sum(1 for c in with_evidence if c in {"full", "partial"})
+ return {
+ "n": len(items),
+ "judge_accuracy": mean(correct),
+ "judge_accuracy_ci95": (
+ {"low": lo, "high": hi} if lo is not None and hi is not None else None
+ ),
+ "session_recall_full": (full / n_ev) if n_ev else None,
+ "session_recall_partial": (partial / n_ev) if n_ev else None,
+ "n_with_evidence": n_ev,
+ }
+
+
+def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:
+ by_type: dict[str, list[dict[str, Any]]] = {}
+ for row in rows:
+ qtype = str(row.get("question_type") or "unknown")
+ by_type.setdefault(qtype, []).append(row)
+
+ all_bucket = _bucket(rows)
+ abstention_rows = [r for r in rows if r.get("is_abstention")]
+ non_abs = [r for r in rows if not r.get("is_abstention")]
+
+ retrieval_latencies = [
+ float(r["retrieve_latency_ms"])
+ for r in rows
+ if r.get("retrieve_latency_ms") is not None
+ ]
+ total_tokens = 0
+ for r in rows:
+ for key in ("answer_total_tokens", "judge_total_tokens"):
+ if r.get(key) is not None:
+ total_tokens += int(r[key])
+
+ return {
+ "n_questions": len(rows),
+ "n_abstention": len(abstention_rows),
+ "n_non_abstention": len(non_abs),
+ "headline_judge_accuracy": all_bucket["judge_accuracy"],
+ "headline_judge_accuracy_ci95": all_bucket["judge_accuracy_ci95"],
+ "session_recall_full": _bucket(non_abs)["session_recall_full"],
+ "session_recall_partial": _bucket(non_abs)["session_recall_partial"],
+ "n_with_evidence": _bucket(non_abs)["n_with_evidence"],
+ "abstention": {
+ "n": len(abstention_rows),
+ "accuracy": _bucket(abstention_rows)["judge_accuracy"],
+ },
+ "by_type": {
+ qtype: _bucket(items) for qtype, items in sorted(by_type.items())
+ },
+ "retrieval_latency_ms": {
+ "p50": percentile(retrieval_latencies, 50),
+ "p95": percentile(retrieval_latencies, 95),
+ "mean": mean(retrieval_latencies),
+ },
+ "total_llm_tokens": total_tokens,
+ }
+
+
+def selftest_metrics() -> list[str]:
+ errors: list[str] = []
+ from longmemeval.dataset import brain_id_for, format_session, is_abstention
+ from longmemeval.judge import parse_yes_no
+ from longmemeval.prompts import get_anscheck_prompt
+
+ if brain_id_for("e47b4ab3aa_abs") != "lmee47b4ab3aaabs":
+ errors.append("brain_id_for should strip non-alnum and prefix lme")
+ if not is_abstention("abc_abs"):
+ errors.append("is_abstention should detect _abs suffix")
+ if is_abstention("abc"):
+ errors.append("non-abs question_id should not be abstention")
+
+ text = format_session(
+ "sess-1",
+ "2023/05/07",
+ [{"role": "user", "content": "hi", "has_answer": True}],
+ )
+ if "has_answer" in text:
+ errors.append("format_session must strip has_answer labels")
+ if "Session id: sess-1" not in text:
+ errors.append("format_session must include session id")
+
+ abs_prompt = get_anscheck_prompt(
+ "multi-session", "q", "expl", "resp", abstention=True
+ )
+ if "unanswerable" not in abs_prompt.lower():
+ errors.append("abstention judge prompt missing")
+ temp_prompt = get_anscheck_prompt(
+ "temporal-reasoning", "q", "a", "r", abstention=False
+ )
+ if "off-by-one" not in temp_prompt:
+ errors.append("temporal judge prompt missing off-by-one clause")
+ ku_prompt = get_anscheck_prompt(
+ "knowledge-update", "q", "a", "r", abstention=False
+ )
+ if "updated answer" not in ku_prompt:
+ errors.append("knowledge-update judge prompt missing")
+ pref_prompt = get_anscheck_prompt(
+ "single-session-preference", "q", "rubric", "r", abstention=False
+ )
+ if "Rubric:" not in pref_prompt:
+ errors.append("preference judge prompt missing Rubric")
+
+ if not parse_yes_no("Yes"):
+ errors.append("parse_yes_no Yes failed")
+ if parse_yes_no("No"):
+ errors.append("parse_yes_no No failed")
+
+ full = session_recall_coverage(
+ {
+ "is_abstention": False,
+ "answer_session_ids": ["a", "b"],
+ "retrieved_session_ids": ["a", "b", "c"],
+ }
+ )
+ if full != "full":
+ errors.append(f"expected full recall, got {full}")
+ partial = session_recall_coverage(
+ {
+ "is_abstention": False,
+ "answer_session_ids": ["a", "b"],
+ "retrieved_session_ids": ["a"],
+ }
+ )
+ if partial != "partial":
+ errors.append(f"expected partial recall, got {partial}")
+ none = session_recall_coverage(
+ {
+ "is_abstention": False,
+ "answer_session_ids": ["a"],
+ "retrieved_session_ids": ["z"],
+ }
+ )
+ if none != "none":
+ errors.append(f"expected none recall, got {none}")
+ skipped = session_recall_coverage(
+ {"is_abstention": True, "answer_session_ids": ["a"], "retrieved_session_ids": []}
+ )
+ if skipped is not None:
+ errors.append("abstention rows should skip session recall")
+
+ agg = aggregate_answers(
+ [
+ {
+ "question_type": "multi-session",
+ "judge_correct": True,
+ "is_abstention": False,
+ "answer_session_ids": ["a"],
+ "retrieved_session_ids": ["a"],
+ "retrieve_latency_ms": 10,
+ "answer_total_tokens": 5,
+ "judge_total_tokens": 2,
+ },
+ {
+ "question_type": "multi-session",
+ "judge_correct": False,
+ "is_abstention": False,
+ "answer_session_ids": ["a"],
+ "retrieved_session_ids": [],
+ "retrieve_latency_ms": 20,
+ "answer_total_tokens": 5,
+ "judge_total_tokens": 2,
+ },
+ {
+ "question_type": "multi-session",
+ "judge_correct": True,
+ "is_abstention": True,
+ "answer_session_ids": [],
+ "retrieved_session_ids": [],
+ },
+ ]
+ )
+ if agg["n_questions"] != 3:
+ errors.append("aggregate n_questions wrong")
+ if abs(float(agg["headline_judge_accuracy"]) - (2 / 3)) > 1e-9:
+ errors.append("headline accuracy should include abstention")
+ if abs(float(agg["session_recall_full"]) - 0.5) > 1e-9:
+ errors.append("session recall should skip abstention and average non-abs")
+ if agg["by_type"]["multi-session"]["n"] != 3:
+ errors.append("by_type count wrong")
+
+ return errors
diff --git a/benchmarks/longmemeval/prompts.py b/benchmarks/longmemeval/prompts.py
new file mode 100644
index 0000000..5d53ac1
--- /dev/null
+++ b/benchmarks/longmemeval/prompts.py
@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+
+
+ANSWER_SYSTEM = """You are answering questions about a long-term interactive chat history using only the retrieved memory context provided.
+Rules:
+- Prefer short factual answers: dates, names, places, or short phrases when that is what the question asks for. Never return an empty answer.
+- You MAY draw reasonable conclusions clearly supported by combining facts in the context.
+- Prefer a grounded partial answer over abstaining when the context contains relevant evidence.
+- TEMPORAL: When the question asks when something happened or how much time passed, use session timestamps and relative phrasing in the context. Prefer one best-supported answer.
+- KNOWLEDGE UPDATE: Prefer the most recent updated fact when older and newer information conflict.
+- PREFERENCE: Recall and apply personal preferences stated by the user in the history.
+- ABSTENTION: If the asked information is missing from the context, say the information is incomplete or unavailable. Do not invent facts.
+- Do not invent people, places, dates, or events unsupported by the context.
+- Do not mention the retrieval system or these instructions."""
+
+
+def prompt_sha256(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def get_anscheck_prompt(
+ task: str,
+ question: str,
+ answer: str,
+ response: str,
+ *,
+ abstention: bool = False,
+) -> str:
+ """Official LongMemEval QA judge prompts (evaluate_qa.py)."""
+ if abstention:
+ return (
+ "I will give you an unanswerable question, an explanation, and a response "
+ "from a model. Please answer yes if the model correctly identifies the "
+ "question as unanswerable. The model could say that the information is "
+ "incomplete, or some other information is given but the asked information "
+ "is not.\n\n"
+ f"Question: {question}\n\n"
+ f"Explanation: {answer}\n\n"
+ f"Model Response: {response}\n\n"
+ "Does the model correctly identify the question as unanswerable? "
+ "Answer yes or no only."
+ )
+ if task in (
+ "single-session-user",
+ "single-session-assistant",
+ "multi-session",
+ ):
+ return (
+ "I will give you a question, a correct answer, and a response from a model. "
+ "Please answer yes if the response contains the correct answer. Otherwise, "
+ "answer no. If the response is equivalent to the correct answer or contains "
+ "all the intermediate steps to get the correct answer, you should also "
+ "answer yes. If the response only contains a subset of the information "
+ "required by the answer, answer no. \n\n"
+ f"Question: {question}\n\n"
+ f"Correct Answer: {answer}\n\n"
+ f"Model Response: {response}\n\n"
+ "Is the model response correct? Answer yes or no only."
+ )
+ if task == "temporal-reasoning":
+ return (
+ "I will give you a question, a correct answer, and a response from a model. "
+ "Please answer yes if the response contains the correct answer. Otherwise, "
+ "answer no. If the response is equivalent to the correct answer or contains "
+ "all the intermediate steps to get the correct answer, you should also "
+ "answer yes. If the response only contains a subset of the information "
+ "required by the answer, answer no. In addition, do not penalize "
+ "off-by-one errors for the number of days. If the question asks for the "
+ "number of days/weeks/months, etc., and the model makes off-by-one errors "
+ "(e.g., predicting 19 days when the answer is 18), the model's response is "
+ "still correct. \n\n"
+ f"Question: {question}\n\n"
+ f"Correct Answer: {answer}\n\n"
+ f"Model Response: {response}\n\n"
+ "Is the model response correct? Answer yes or no only."
+ )
+ if task == "knowledge-update":
+ return (
+ "I will give you a question, a correct answer, and a response from a model. "
+ "Please answer yes if the response contains the correct answer. Otherwise, "
+ "answer no. If the response contains some previous information along with "
+ "an updated answer, the response should be considered as correct as long as "
+ "the updated answer is the required answer.\n\n"
+ f"Question: {question}\n\n"
+ f"Correct Answer: {answer}\n\n"
+ f"Model Response: {response}\n\n"
+ "Is the model response correct? Answer yes or no only."
+ )
+ if task == "single-session-preference":
+ return (
+ "I will give you a question, a rubric for desired personalized response, "
+ "and a response from a model. Please answer yes if the response satisfies "
+ "the desired response. Otherwise, answer no. The model does not need to "
+ "reflect all the points in the rubric. The response is correct as long as "
+ "it recalls and utilizes the user's personal information correctly.\n\n"
+ f"Question: {question}\n\n"
+ f"Rubric: {answer}\n\n"
+ f"Model Response: {response}\n\n"
+ "Is the model response correct? Answer yes or no only."
+ )
+ raise NotImplementedError(f"Unknown question_type for judge: {task}")
+
+
+def flatten_triple(triple_obj: Any) -> str:
+ if isinstance(triple_obj, dict):
+ identified = triple_obj.get("identified_entity", "")
+ triple = triple_obj.get("triple")
+ if isinstance(triple, (list, tuple)) and len(triple) >= 5:
+ parts = []
+ for item in triple:
+ if isinstance(item, dict):
+ parts.append(
+ item.get("name")
+ or item.get("type")
+ or item.get("description")
+ or str(item)
+ )
+ else:
+ parts.append(str(item))
+ return f"{identified}: {' | '.join(parts)}" if identified else " | ".join(parts)
+ return str(triple_obj)
+ return str(triple_obj)
+
+
+def flatten_path(path_obj: Any) -> str:
+ if not isinstance(path_obj, dict):
+ return str(path_obj)
+ via = (
+ str(path_obj.get("shared_entity_name") or "").strip()
+ or str(path_obj.get("shared_entity") or "").strip()
+ or "shared entity"
+ )
+ legs = path_obj.get("legs") or []
+ leg_texts = [str(leg).strip() for leg in legs if str(leg).strip()]
+ if leg_texts:
+ return f"via {via}: " + " --> ".join(leg_texts)
+ hubs = [str(h).strip() for h in (path_obj.get("hubs") or []) if str(h).strip()]
+ if hubs:
+ return f"via {via}: hubs {' + '.join(hubs)}"
+ return f"via {via}"
+
+
+def enrich_paths_from_triples(context: dict[str, Any]) -> dict[str, Any]:
+ paths = context.get("paths") or []
+ triples = context.get("triples") or []
+ if not paths:
+ return context
+ if any(isinstance(p, dict) and p.get("legs") for p in paths):
+ return context
+
+ hub_texts: dict[str, str] = {}
+ for triple in triples:
+ if not isinstance(triple, dict):
+ continue
+ raw = triple.get("triple")
+ if not isinstance(raw, (list, tuple)) or len(raw) < 3:
+ continue
+ event = raw[2]
+ hub = ""
+ if isinstance(event, dict):
+ hub = str(event.get("uuid") or "").strip()
+ parts = []
+ for item in raw:
+ if isinstance(item, dict):
+ parts.append(
+ item.get("name")
+ or item.get("type")
+ or item.get("description")
+ or str(item)
+ )
+ else:
+ parts.append(str(item))
+ text = " | ".join(parts).strip()
+ if hub and text and hub not in hub_texts:
+ hub_texts[hub] = text
+ if not hub_texts:
+ return context
+
+ enriched: list[dict[str, Any]] = []
+ for path in paths:
+ if not isinstance(path, dict):
+ continue
+ hubs = [str(h) for h in (path.get("hubs") or []) if h]
+ if len(hubs) < 2 or not all(h in hub_texts for h in hubs):
+ continue
+ enriched.append({**path, "legs": [hub_texts[h] for h in hubs]})
+ if not enriched:
+ return context
+ out = dict(context)
+ out["paths"] = enriched
+ return out
+
+
+def build_context_block(context: dict[str, Any]) -> str:
+ context = enrich_paths_from_triples(context)
+ text_context = (context.get("text_context") or "").strip()
+ triples = context.get("triples") or []
+ paths = context.get("paths") or []
+ historical = context.get("historical_context") or []
+ source_passages = context.get("source_passages") or []
+ topics = context.get("topics") or []
+
+ sections = ["## Retrieved text context", text_context or "(empty)"]
+ if topics:
+ sections.append("## Topics")
+ for topic in topics[:20]:
+ if isinstance(topic, dict):
+ label = topic.get("label") or topic.get("topic_id") or topic
+ sessions = topic.get("sessions") or []
+ sess = ", ".join(str(s) for s in sessions[:12])
+ sections.append(f"- {label}" + (f" ({sess})" if sess else ""))
+ else:
+ sections.append(f"- {topic}")
+ if source_passages:
+ sections.append("## Source passages")
+ for passage in source_passages[:20]:
+ sections.append(f"- {passage}")
+ if paths:
+ sections.append("## Event hub paths")
+ for path in paths[:40]:
+ sections.append(f"- {flatten_path(path)}")
+ if triples:
+ sections.append("## Graph triples")
+ for t in triples[:80]:
+ sections.append(f"- {flatten_triple(t)}")
+ if historical:
+ sections.append("## Historical context")
+ for h in historical[:40]:
+ sections.append(f"- {h}")
+ return "\n".join(sections)
+
+
+def build_answer_messages(
+ question: str,
+ context: dict[str, Any],
+ *,
+ question_date: str | None = None,
+) -> list[dict[str, str]]:
+ date_line = ""
+ if question_date:
+ date_line = f"## Question date\n{question_date}\n\n"
+ user = (
+ f"{build_context_block(context)}\n\n"
+ f"{date_line}"
+ f"## Question\n{question}\n\n"
+ "Answer:"
+ )
+ return [
+ {"role": "system", "content": ANSWER_SYSTEM},
+ {"role": "user", "content": user},
+ ]
+
+
+def judge_prompt_sha256() -> str:
+ samples = [
+ get_anscheck_prompt("multi-session", "q", "a", "r", abstention=False),
+ get_anscheck_prompt("temporal-reasoning", "q", "a", "r", abstention=False),
+ get_anscheck_prompt("knowledge-update", "q", "a", "r", abstention=False),
+ get_anscheck_prompt(
+ "single-session-preference", "q", "a", "r", abstention=False
+ ),
+ get_anscheck_prompt("multi-session", "q", "a", "r", abstention=True),
+ ]
+ return prompt_sha256("\n---\n".join(samples))
diff --git a/benchmarks/longmemeval/provenance.py b/benchmarks/longmemeval/provenance.py
new file mode 100644
index 0000000..676eb41
--- /dev/null
+++ b/benchmarks/longmemeval/provenance.py
@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import hashlib
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from longmemeval.config import BENCHMARKS_ROOT, Settings, model_family
+from longmemeval.prompts import ANSWER_SYSTEM, judge_prompt_sha256, prompt_sha256
+
+REPO_ROOT = BENCHMARKS_ROOT.parent
+
+
+def _git(*args: str) -> str | None:
+ try:
+ result = subprocess.run(
+ ["git", *args],
+ cwd=REPO_ROOT,
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return None
+ if result.returncode != 0:
+ return None
+ return result.stdout.strip()
+
+
+def git_sha() -> str | None:
+ return _git("rev-parse", "HEAD")
+
+
+def git_dirty() -> bool | None:
+ status = _git("status", "--porcelain")
+ if status is None:
+ return None
+ return bool(status.strip())
+
+
+def file_sha256(path: Path) -> str | None:
+ if not path.exists():
+ return None
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def build_provenance(settings: Settings) -> dict[str, Any]:
+ return {
+ "git_sha": git_sha(),
+ "git_dirty": git_dirty(),
+ "answer_prompt_sha256": prompt_sha256(ANSWER_SYSTEM),
+ "judge_prompt_sha256": judge_prompt_sha256(),
+ "dataset_path": str(settings.dataset_path),
+ "dataset_sha256": file_sha256(settings.dataset_path),
+ "variant": settings.variant,
+ "answer_model": settings.answer_model,
+ "answer_model_family": model_family(settings.answer_model),
+ "answer_llm_base_url": settings.llm_base_url,
+ "judge_model": settings.judge_model,
+ "judge_model_family": model_family(settings.judge_model),
+ "judge_provider": settings.judge_provider(),
+ "judge_shares_answer_family": settings.judge_shares_answer_family,
+ "bench_profile": settings.bench_profile,
+ "sc_samples": settings.sc_samples,
+ "sc_temperature": settings.sc_temperature,
+ "gap_fill": settings.gap_fill,
+ }
diff --git a/benchmarks/longmemeval/report.py b/benchmarks/longmemeval/report.py
new file mode 100644
index 0000000..8d5ec38
--- /dev/null
+++ b/benchmarks/longmemeval/report.py
@@ -0,0 +1,542 @@
+from __future__ import annotations
+
+import json
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from rich.console import Console
+from rich.table import Table
+
+from longmemeval.config import BENCHMARKS_ROOT
+from longmemeval.ingest import load_jsonl
+from longmemeval.metrics import aggregate_answers
+
+console = Console()
+
+REPORTS_PATH = BENCHMARKS_ROOT / "REPORTS.json"
+_BENCHMARK_ID = "longmemeval"
+_BENCHMARK_NAME = "LongMemEval"
+_REPORTS_DESCRIPTION = (
+ "BrainAPI benchmark results. Top published scores across suites. "
+ "Updated when a suite evaluate/report completes successfully."
+)
+_LEADERBOARD_ENTRY_KEYS = (
+ "run_id",
+ "track",
+ "scope",
+ "variant",
+ "n_questions",
+ "headline_judge_accuracy",
+ "headline_judge_accuracy_pct",
+ "answer_model",
+ "judge_model",
+ "bench_profile",
+ "git_sha",
+ "report_path",
+ "recorded_at",
+)
+_MIN_LEADERBOARD_N = 50
+
+
+def _latest_ingest_rows(ingest_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ latest: dict[tuple[str, str], dict[str, Any]] = {}
+ for row in ingest_rows:
+ key = (
+ str(row.get("question_id") or ""),
+ str(row.get("unit_id") or row.get("session_id") or ""),
+ )
+ latest[key] = row
+ return list(latest.values())
+
+
+def _latest_answer_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ latest: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ key = str(row.get("question_id") or "")
+ if key:
+ latest[key] = row
+ return list(latest.values())
+
+
+def build_report(run_dir: Path) -> dict[str, Any]:
+ answers_path = run_dir / "answers.jsonl"
+ ingest_path = run_dir / "ingest.jsonl"
+ manifest_path = run_dir / "manifest.json"
+
+ all_answer_rows = load_jsonl(answers_path)
+ scored_rows = [r for r in all_answer_rows if not r.get("error")]
+ answers = _latest_answer_rows(scored_rows)
+ errored = [r for r in all_answer_rows if r.get("error")]
+ duplicates_dropped = len(scored_rows) - len(answers)
+ empty_predictions = sum(
+ 1 for r in answers if not str(r.get("prediction") or "").strip()
+ )
+ truncated_rows = sum(1 for r in answers if r.get("context_truncated"))
+ ingest_rows = load_jsonl(ingest_path)
+ ingest_latest = _latest_ingest_rows(ingest_rows)
+ manifest = {}
+ if manifest_path.exists():
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+
+ metrics = aggregate_answers(answers)
+ ingest_summary = {
+ "n": len(ingest_latest),
+ "completed": sum(1 for r in ingest_latest if r.get("status") == "completed"),
+ "partial_failed": sum(
+ 1 for r in ingest_latest if r.get("status") == "partial_failed"
+ ),
+ "failed": sum(1 for r in ingest_latest if r.get("status") == "failed"),
+ "dry_run": sum(1 for r in ingest_latest if r.get("status") == "dry_run"),
+ "attempts": len(ingest_rows),
+ }
+ wait_vals = [
+ float(r["wait_latency_ms"])
+ for r in ingest_latest
+ if r.get("wait_latency_ms") is not None
+ ]
+ token_vals = [
+ int(r["total_llm_tokens"])
+ for r in ingest_latest
+ if r.get("total_llm_tokens") is not None
+ ]
+ if wait_vals:
+ ordered = sorted(wait_vals)
+ mid = len(ordered) // 2
+ ingest_summary["wait_latency_p50_ms"] = (
+ ordered[mid]
+ if len(ordered) % 2
+ else (ordered[mid - 1] + ordered[mid]) / 2
+ )
+ ingest_summary["wait_latency_mean_ms"] = sum(ordered) / len(ordered)
+ if token_vals:
+ ingest_summary["mean_ingest_llm_tokens"] = sum(token_vals) / len(token_vals)
+ ingest_summary["total_ingest_llm_tokens"] = sum(token_vals)
+ stage_keys = (
+ "scout_total_tokens",
+ "architect_total_tokens",
+ "janitor_total_tokens",
+ "observations_total_tokens",
+ "consolidation_total_tokens",
+ )
+ stage_sums = {k: 0 for k in stage_keys}
+ stage_n = 0
+ for row in ingest_latest:
+ cost = row.get("cost") or {}
+ stages = cost.get("stages") if isinstance(cost, dict) else None
+ if not isinstance(stages, dict):
+ continue
+ stage_n += 1
+ for key in stage_keys:
+ stage_name = key.replace("_total_tokens", "")
+ stage_sums[key] += int(
+ (stages.get(stage_name) or {}).get("total_tokens") or 0
+ )
+ if stage_n:
+ ingest_summary["mean_tokens_by_stage"] = {
+ k.replace("_total_tokens", ""): stage_sums[k] / stage_n
+ for k in stage_keys
+ }
+
+ integrity = {
+ "rows_in_file": len(all_answer_rows),
+ "errored_rows": len(errored),
+ "scored_rows": len(answers),
+ "duplicates_dropped": duplicates_dropped,
+ "empty_predictions": empty_predictions,
+ "rows_with_truncated_context": truncated_rows,
+ }
+
+ report: dict[str, Any] = {
+ "run_dir": str(run_dir),
+ "manifest": manifest,
+ "ingest": ingest_summary,
+ "integrity": integrity,
+ "metrics": metrics,
+ "status": "ok",
+ "warnings": [],
+ }
+ if ingest_summary["partial_failed"] or ingest_summary["failed"]:
+ report["warnings"].append(
+ "Some ingest units failed or partially failed; scores may be skewed."
+ )
+ if duplicates_dropped:
+ report["warnings"].append(
+ f"{duplicates_dropped} duplicate answer rows dropped."
+ )
+ if errored:
+ report["warnings"].append(
+ f"{len(errored)} rows errored and are excluded from every metric."
+ )
+ if truncated_rows:
+ report["warnings"].append(
+ f"{truncated_rows} rows had a retrieved-context channel truncated "
+ "before logging."
+ )
+
+ failure_reasons: list[str] = []
+ if not answers:
+ failure_reasons.append("no answer rows were scored")
+ else:
+ if empty_predictions == len(answers):
+ failure_reasons.append("every scored answer is empty")
+ if metrics.get("headline_judge_accuracy") == 0.0:
+ failure_reasons.append("every scored answer was judged incorrect")
+ if failure_reasons:
+ report["status"] = "failed"
+ report["warnings"].insert(
+ 0,
+ "RUN FAILED WHOLESALE: "
+ + "; ".join(failure_reasons)
+ + ". The percentages below are not a measurement of quality.",
+ )
+ elif errored or duplicates_dropped:
+ report["status"] = "degraded"
+ return report
+
+
+def _pct(value: Any) -> str:
+ if value is None:
+ return "n/a"
+ return f"{100.0 * float(value):.1f}%"
+
+
+def _num(value: Any) -> str:
+ if value is None:
+ return "n/a"
+ return f"{float(value):.2f}"
+
+
+def _ci(ci: Any) -> str:
+ if not isinstance(ci, dict) or ci.get("low") is None or ci.get("high") is None:
+ return ""
+ return f"[{100.0 * float(ci['low']):.1f}%, {100.0 * float(ci['high']):.1f}%]"
+
+
+def render_markdown(report: dict[str, Any]) -> str:
+ metrics = report["metrics"]
+ manifest = report.get("manifest") or {}
+ integrity = report.get("integrity") or {}
+ lines = [
+ "# LongMemEval BrainAPI Benchmark Report",
+ "",
+ ]
+ if report.get("status") == "failed":
+ lines.extend(
+ [
+ "> **RUN FAILED WHOLESALE — the numbers below are not a measurement.**",
+ f"> {report['warnings'][0]}",
+ "",
+ ]
+ )
+ elif report.get("status") == "degraded":
+ lines.extend(
+ [
+ "> **Run degraded** — errored or duplicate rows were dropped; "
+ "see Warnings.",
+ "",
+ ]
+ )
+ lines.extend(
+ [
+ f"- Run dir: `{report['run_dir']}`",
+ f"- Created: {manifest.get('created_at', 'n/a')}",
+ f"- Variant: {manifest.get('variant', 'n/a')}",
+ f"- Answer model: {manifest.get('answer_model', 'n/a')}",
+ f"- Judge model: {manifest.get('judge_model', 'n/a')} "
+ f"(provider: {manifest.get('judge_provider', 'n/a')}, "
+ f"same family as answerer: "
+ f"{manifest.get('judge_shares_answer_family', 'n/a')})",
+ f"- Git SHA: {manifest.get('git_sha', 'n/a')} "
+ f"(dirty: {manifest.get('git_dirty', 'n/a')})",
+ f"- Answer prompt sha256: {manifest.get('answer_prompt_sha256', 'n/a')}",
+ f"- Judge prompt sha256: {manifest.get('judge_prompt_sha256', 'n/a')}",
+ f"- Dataset sha256: {manifest.get('dataset_sha256', 'n/a')}",
+ "",
+ "## Run integrity",
+ "",
+ f"- Rows in answers.jsonl: {integrity.get('rows_in_file', 'n/a')}",
+ f"- Scored unique questions: {integrity.get('scored_rows', 'n/a')}",
+ f"- Errored rows (excluded): {integrity.get('errored_rows', 'n/a')}",
+ f"- Duplicate rows dropped: {integrity.get('duplicates_dropped', 'n/a')}",
+ f"- Empty predictions: {integrity.get('empty_predictions', 'n/a')}",
+ "",
+ "## Headline",
+ "",
+ f"- Judge accuracy (all questions, incl. abstention): "
+ f"**{_pct(metrics.get('headline_judge_accuracy'))}** "
+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))} "
+ f"(n={metrics.get('n_questions', 0)})",
+ f"- Session recall (full, excl. abstention): "
+ f"**{_pct(metrics.get('session_recall_full'))}** "
+ f"(n={metrics.get('n_with_evidence', 0)})",
+ f"- Session recall (partial+full): "
+ f"**{_pct(metrics.get('session_recall_partial'))}**",
+ f"- Abstention accuracy: "
+ f"**{_pct((metrics.get('abstention') or {}).get('accuracy'))}** "
+ f"(n={(metrics.get('abstention') or {}).get('n', 0)})",
+ "",
+ "## Per question type",
+ "",
+ "| Type | N | Judge Acc | 95% CI | Session recall full |",
+ "|---|---:|---:|---|---:|",
+ ]
+ )
+ for qtype, bucket in (metrics.get("by_type") or {}).items():
+ lines.append(
+ f"| {qtype} | {bucket.get('n', 0)} | "
+ f"{_pct(bucket.get('judge_accuracy'))} | "
+ f"{_ci(bucket.get('judge_accuracy_ci95'))} | "
+ f"{_pct(bucket.get('session_recall_full'))} |"
+ )
+
+ lat = metrics.get("retrieval_latency_ms") or {}
+ lines.extend(
+ [
+ "",
+ "## Latency & tokens",
+ "",
+ f"- Retrieval latency p50: {_num(lat.get('p50'))} ms",
+ f"- Retrieval latency p95: {_num(lat.get('p95'))} ms",
+ f"- Retrieval latency mean: {_num(lat.get('mean'))} ms",
+ f"- Total LLM tokens (answer+judge): {metrics.get('total_llm_tokens', 0)}",
+ "",
+ "## Ingest",
+ "",
+ f"- Units: {report['ingest'].get('n', 0)}",
+ f"- Completed: {report['ingest'].get('completed', 0)}",
+ f"- Partial failed: {report['ingest'].get('partial_failed', 0)}",
+ f"- Failed: {report['ingest'].get('failed', 0)}",
+ *(
+ [
+ f"- Ingest wait latency p50: "
+ f"{_num(report['ingest'].get('wait_latency_p50_ms'))} ms"
+ ]
+ if report["ingest"].get("wait_latency_p50_ms") is not None
+ else []
+ ),
+ *(
+ [
+ f"- Mean ingest LLM tokens/unit: "
+ f"{_num(report['ingest'].get('mean_ingest_llm_tokens'))}"
+ ]
+ if report["ingest"].get("mean_ingest_llm_tokens") is not None
+ else []
+ ),
+ *(
+ [
+ f"- Total ingest LLM tokens: "
+ f"{report['ingest'].get('total_ingest_llm_tokens')}"
+ ]
+ if report["ingest"].get("total_ingest_llm_tokens") is not None
+ else []
+ ),
+ "",
+ ]
+ )
+ if report.get("warnings"):
+ lines.append("## Warnings")
+ lines.append("")
+ for warning in report["warnings"]:
+ lines.append(f"- {warning}")
+ lines.append("")
+ return "\n".join(lines)
+
+
+def entry_from_report(report: dict[str, Any]) -> dict[str, Any] | None:
+ if report.get("status") == "failed":
+ return None
+ metrics = report.get("metrics") or {}
+ headline = metrics.get("headline_judge_accuracy")
+ n_questions = int(metrics.get("n_questions") or 0)
+ if headline is None or n_questions < _MIN_LEADERBOARD_N:
+ return None
+
+ manifest = report.get("manifest") or {}
+ run_id = str(manifest.get("run_id") or "").strip()
+ if not run_id:
+ run_dir = str(report.get("run_dir") or "").rstrip("/")
+ run_id = Path(run_dir).name if run_dir else ""
+ if not run_id:
+ return None
+
+ variant = str(manifest.get("variant") or "s")
+ limit = manifest.get("limit")
+ if limit:
+ scope = f"{variant}:limit{limit}"
+ else:
+ scope = variant
+
+ bench_profile = manifest.get("bench_profile") or "product"
+ track = "sota" if bench_profile == "sota" else "product"
+ acc = float(headline)
+ now = datetime.now(timezone.utc).isoformat()
+
+ return {
+ "run_id": run_id,
+ "track": track,
+ "scope": scope,
+ "variant": variant,
+ "n_questions": n_questions,
+ "headline_judge_accuracy": acc,
+ "headline_judge_accuracy_pct": round(acc * 100, 2),
+ "answer_model": manifest.get("answer_model"),
+ "judge_model": manifest.get("judge_model"),
+ "bench_profile": bench_profile,
+ "git_sha": manifest.get("git_sha"),
+ "report_path": f"runs/{run_id}/report.json",
+ "recorded_at": now,
+ }
+
+
+def _empty_reports_ledger() -> dict[str, Any]:
+ return {
+ "schema_version": 2,
+ "updated_at": datetime.now(timezone.utc).isoformat(),
+ "description": _REPORTS_DESCRIPTION,
+ "benchmarks": {
+ _BENCHMARK_ID: {
+ "name": _BENCHMARK_NAME,
+ "leaderboard": [],
+ }
+ },
+ }
+
+
+def _public_leaderboard_entry(row: dict[str, Any]) -> dict[str, Any]:
+ return {key: row[key] for key in _LEADERBOARD_ENTRY_KEYS if key in row}
+
+
+def _sorted_leaderboard(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ leaderboard = [
+ _public_leaderboard_entry(row)
+ for row in rows
+ if isinstance(row, dict) and row.get("run_id")
+ ]
+ leaderboard.sort(
+ key=lambda row: float(row.get("headline_judge_accuracy") or 0.0),
+ reverse=True,
+ )
+ return leaderboard
+
+
+def _normalize_reports_ledger(data: dict[str, Any]) -> dict[str, Any]:
+ benchmarks: dict[str, Any] = {}
+ raw_benchmarks = data.get("benchmarks")
+ if isinstance(raw_benchmarks, dict):
+ for suite_id, suite in raw_benchmarks.items():
+ if not isinstance(suite, dict):
+ continue
+ benchmarks[str(suite_id)] = {
+ "name": suite.get("name") or str(suite_id),
+ "leaderboard": _sorted_leaderboard(list(suite.get("leaderboard") or [])),
+ }
+
+ if _BENCHMARK_ID not in benchmarks:
+ benchmarks[_BENCHMARK_ID] = {
+ "name": _BENCHMARK_NAME,
+ "leaderboard": [],
+ }
+
+ return {
+ "schema_version": 2,
+ "updated_at": data.get("updated_at")
+ or datetime.now(timezone.utc).isoformat(),
+ "description": data.get("description") or _REPORTS_DESCRIPTION,
+ "benchmarks": benchmarks,
+ }
+
+
+def update_reports_json(report: dict[str, Any]) -> None:
+ entry = entry_from_report(report)
+ if entry is None:
+ return
+
+ if REPORTS_PATH.exists():
+ data = json.loads(REPORTS_PATH.read_text(encoding="utf-8"))
+ else:
+ data = _empty_reports_ledger()
+
+ data = _normalize_reports_ledger(data)
+
+ suite = data["benchmarks"].setdefault(
+ _BENCHMARK_ID,
+ {"name": _BENCHMARK_NAME, "leaderboard": []},
+ )
+ run_id = entry["run_id"]
+ leaderboard = list(suite.get("leaderboard") or [])
+ lb_idx = next(
+ (i for i, row in enumerate(leaderboard) if row.get("run_id") == run_id),
+ None,
+ )
+ if lb_idx is not None:
+ leaderboard[lb_idx] = entry
+ else:
+ leaderboard.append(entry)
+ suite["name"] = _BENCHMARK_NAME
+ suite["leaderboard"] = _sorted_leaderboard(leaderboard)
+ data["benchmarks"][_BENCHMARK_ID] = suite
+ data["updated_at"] = datetime.now(timezone.utc).isoformat()
+ data["description"] = _REPORTS_DESCRIPTION
+
+ REPORTS_PATH.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.NamedTemporaryFile(
+ "w",
+ encoding="utf-8",
+ dir=str(REPORTS_PATH.parent),
+ delete=False,
+ suffix=".tmp",
+ ) as tmp:
+ json.dump(data, tmp, indent=2)
+ tmp.write("\n")
+ tmp_path = Path(tmp.name)
+ tmp_path.replace(REPORTS_PATH)
+
+
+def write_report(run_dir: Path) -> dict[str, Any]:
+ report = build_report(run_dir)
+ (run_dir / "report.json").write_text(
+ json.dumps(report, indent=2), encoding="utf-8"
+ )
+ (run_dir / "report.md").write_text(render_markdown(report), encoding="utf-8")
+ if report.get("status") == "ok":
+ update_reports_json(report)
+ return report
+
+
+def print_report_table(report: dict[str, Any]) -> None:
+ metrics = report.get("metrics") or {}
+ table = Table(title="LongMemEval results")
+ table.add_column("Metric")
+ table.add_column("Value")
+ table.add_row(
+ "Headline judge accuracy",
+ f"{_pct(metrics.get('headline_judge_accuracy'))} "
+ f"(n={metrics.get('n_questions', 0)})",
+ )
+ table.add_row(
+ "Session recall (full)",
+ f"{_pct(metrics.get('session_recall_full'))} "
+ f"(n={metrics.get('n_with_evidence', 0)})",
+ )
+ table.add_row(
+ "Abstention accuracy",
+ f"{_pct((metrics.get('abstention') or {}).get('accuracy'))} "
+ f"(n={(metrics.get('abstention') or {}).get('n', 0)})",
+ )
+ table.add_row("Status", str(report.get("status")))
+ console.print(table)
+ by_type = metrics.get("by_type") or {}
+ if by_type:
+ type_table = Table(title="By question type")
+ type_table.add_column("Type")
+ type_table.add_column("N", justify="right")
+ type_table.add_column("Acc")
+ for qtype, bucket in by_type.items():
+ type_table.add_row(
+ qtype,
+ str(bucket.get("n", 0)),
+ _pct(bucket.get("judge_accuracy")),
+ )
+ console.print(type_table)
diff --git a/benchmarks/longmemeval/sota.py b/benchmarks/longmemeval/sota.py
new file mode 100644
index 0000000..a1ef4a7
--- /dev/null
+++ b/benchmarks/longmemeval/sota.py
@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any
+
+
+_NORMALIZE_RE = re.compile(r"[^a-z0-9\s]+")
+_WS_RE = re.compile(r"\s+")
+
+
+def normalize_answer(text: str) -> str:
+ lowered = (text or "").strip().lower()
+ lowered = _NORMALIZE_RE.sub(" ", lowered)
+ return _WS_RE.sub(" ", lowered).strip()
+
+
+def majority_vote(answers: list[str]) -> str:
+ nonempty = [a.strip() for a in answers if (a or "").strip()]
+ if not nonempty:
+ return ""
+ counts = Counter(normalize_answer(a) for a in nonempty)
+ winner_norm, _ = max(
+ counts.items(),
+ key=lambda item: (
+ item[1],
+ -next(i for i, a in enumerate(nonempty) if normalize_answer(a) == item[0]),
+ ),
+ )
+ for answer in nonempty:
+ if normalize_answer(answer) == winner_norm:
+ return answer
+ return nonempty[0]
+
+
+@dataclass(frozen=True)
+class GapFillPlan:
+ needs_retry: bool
+ reformulated_query: str
+ missing_slots: list[str]
+
+
+def plan_gap_fill(
+ question: str,
+ draft_answer: str,
+ context: dict[str, Any],
+) -> GapFillPlan:
+ draft = (draft_answer or "").strip()
+ text_blob = " ".join(
+ [
+ str(context.get("text_context") or ""),
+ " ".join(str(p) for p in (context.get("source_passages") or [])[:20]),
+ " ".join(str(t) for t in (context.get("triples") or [])[:40]),
+ ]
+ ).lower()
+ abstained = bool(
+ re.search(
+ r"not mentioned|no information|incomplete|unknown|cannot (tell|determine)|"
+ r"not available|do not have",
+ draft,
+ re.I,
+ )
+ )
+ q_tokens = [t for t in re.findall(r"[a-z0-9]{3,}", question.lower()) if t]
+ covered = sum(1 for t in q_tokens if t in text_blob) if q_tokens else 0
+ coverage = covered / len(q_tokens) if q_tokens else 1.0
+ missing = [t for t in q_tokens if t not in text_blob][:8]
+ needs = abstained or (coverage < 0.45 and len(draft) < 40)
+ reform = question.strip()
+ if missing:
+ reform = f"{question.strip()} (focus: {', '.join(missing[:5])})"
+ return GapFillPlan(
+ needs_retry=needs,
+ reformulated_query=reform,
+ missing_slots=missing,
+ )
+
+
+def merge_contexts(
+ primary: dict[str, Any], secondary: dict[str, Any]
+) -> dict[str, Any]:
+ out = dict(primary)
+ for key in ("source_passages", "historical_context", "triples", "paths"):
+ a = list(primary.get(key) or [])
+ b = list(secondary.get(key) or [])
+ seen: set[str] = set()
+ merged: list[Any] = []
+ for item in a + b:
+ sig = str(item)
+ if sig in seen:
+ continue
+ seen.add(sig)
+ merged.append(item)
+ out[key] = merged
+ text_a = str(primary.get("text_context") or "")
+ text_b = str(secondary.get("text_context") or "")
+ if text_b and text_b not in text_a:
+ out["text_context"] = (text_a + "\n" + text_b).strip()
+ return out