BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
feat(benchmarks): add LongMemEval evaluation harness
Christiana220e0f8a99c
diff --git a/benchmarks/longmemeval.sh b/benchmarks/longmemeval.shnew file mode 100755index 0000000..8d836f1--- /dev/null+++ b/benchmarks/longmemeval.sh@@ -0,0 +1,10 @@+#!/usr/bin/env bash+set -euo pipefail+ROOT="$(cd "$(dirname "$0")" && pwd)"+PYTHON="${ROOT}/.venv/bin/python"+if [[ ! -x "$PYTHON" ]]; then+ echo "Missing ${PYTHON}. Create it with:"+ echo " cd benchmarks && python3 -m venv .venv && .venv/bin/pip install -r requirements.txt"+ exit 1+fi+exec "$PYTHON" -m longmemeval "$@"diff --git a/benchmarks/longmemeval/__init__.py b/benchmarks/longmemeval/__init__.pynew file mode 100644index 0000000..3e84d14--- /dev/null+++ b/benchmarks/longmemeval/__init__.py@@ -0,0 +1 @@+"""LongMemEval benchmark harness for BrainAPI (HTTP-only)."""diff --git a/benchmarks/longmemeval/__main__.py b/benchmarks/longmemeval/__main__.pynew file mode 100644index 0000000..586397a--- /dev/null+++ b/benchmarks/longmemeval/__main__.py@@ -0,0 +1,4 @@+from longmemeval.cli import main++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/benchmarks/longmemeval/answer.py b/benchmarks/longmemeval/answer.pynew file mode 100644index 0000000..ff57808--- /dev/null+++ b/benchmarks/longmemeval/answer.py@@ -0,0 +1,131 @@+from __future__ import annotations++import time+from dataclasses import dataclass+from typing import Any++from openai import AzureOpenAI, OpenAI++from longmemeval.config import Settings+from longmemeval.prompts import build_answer_messages+from longmemeval.sota import majority_vote+++@dataclass+class AnswerResult:+ answer: str+ model: str+ latency_ms: float+ prompt_tokens: int | None+ completion_tokens: int | None+ total_tokens: int | None+ sc_samples_used: int = 1+++def _make_openai_client(settings: Settings) -> OpenAI | AzureOpenAI:+ settings.require_llm()+ if settings.answer_azure_endpoint:+ return AzureOpenAI(+ api_key=settings.openai_api_key,+ azure_endpoint=settings.answer_azure_endpoint,+ api_version=settings.answer_azure_api_version,+ )+ kwargs: dict[str, Any] = {"api_key": settings.openai_api_key}+ if settings.llm_base_url:+ kwargs["base_url"] = settings.llm_base_url+ return OpenAI(**kwargs)+++def _one_completion(+ client: OpenAI | AzureOpenAI,+ *,+ model_name: str,+ messages: list[dict[str, str]],+ temperature: float,+) -> tuple[str, Any]:+ response = client.chat.completions.create(+ model=model_name,+ messages=messages,+ temperature=temperature,+ )+ content = (response.choices[0].message.content or "").strip()+ return content, response+++def answer_question(+ settings: Settings,+ question: str,+ context: dict[str, Any],+ *,+ question_date: str | None = None,+ model: str | None = None,+ sc_samples: int | None = None,+ sc_temperature: float | None = None,+) -> AnswerResult:+ client = _make_openai_client(settings)+ model_name = model or settings.answer_model+ messages = build_answer_messages(+ question, context, question_date=question_date+ )+ n = max(1, int(sc_samples if sc_samples is not None else settings.sc_samples))+ temperature = (+ float(sc_temperature)+ if sc_temperature is not None+ else (settings.sc_temperature if n > 1 else 0.0)+ )+ started = time.perf_counter()+ prompt_tokens = 0+ completion_tokens = 0+ total_tokens = 0+ samples: list[str] = []+ last_response: Any = None++ for _ in range(n):+ content, response = _one_completion(+ client,+ model_name=model_name,+ messages=messages,+ temperature=0.0 if n == 1 else temperature,+ )+ last_response = response+ if response.usage:+ prompt_tokens += getattr(response.usage, "prompt_tokens", 0) or 0+ completion_tokens += getattr(response.usage, "completion_tokens", 0) or 0+ total_tokens += getattr(response.usage, "total_tokens", 0) or 0+ if content:+ samples.append(content)++ content = majority_vote(samples) if samples else ""++ if not content:+ retry_messages = messages + [+ {+ "role": "user",+ "content": (+ "Your previous reply was empty. Answer the question now using the "+ "retrieved context. Prefer a short factual list or phrase."+ ),+ }+ ]+ content, response = _one_completion(+ client,+ model_name=model_name,+ messages=retry_messages,+ temperature=0,+ )+ last_response = response+ if response.usage:+ prompt_tokens += getattr(response.usage, "prompt_tokens", 0) or 0+ completion_tokens += getattr(response.usage, "completion_tokens", 0) or 0+ total_tokens += getattr(response.usage, "total_tokens", 0) or 0++ latency_ms = (time.perf_counter() - started) * 1000+ return AnswerResult(+ answer=content,+ model=getattr(last_response, "model", None) or model_name,+ latency_ms=latency_ms,+ prompt_tokens=prompt_tokens or None,+ completion_tokens=completion_tokens or None,+ total_tokens=total_tokens or None,+ sc_samples_used=n,+ )diff --git a/benchmarks/longmemeval/cli.py b/benchmarks/longmemeval/cli.pynew file mode 100644index 0000000..10c6091--- /dev/null+++ b/benchmarks/longmemeval/cli.py@@ -0,0 +1,403 @@+from __future__ import annotations++import argparse+import json+from pathlib import Path++from rich.console import Console+from rich.table import Table++from longmemeval.answer import answer_question+from longmemeval.client import BrainAPIClient+from longmemeval.config import VARIANT_FILES, Settings+from longmemeval.dataset import (+ brain_id_for,+ dataset_stats,+ download_dataset,+ get_question,+ iter_ingest_units,+ load_dataset,+ resolve_questions,+ resolve_variant_settings,+)+from longmemeval.evaluate import evaluate_questions+from longmemeval.ingest import ensure_run_dir, ingest_questions, write_manifest+from longmemeval.metrics import selftest_metrics, tokenize+from longmemeval.prompts import ANSWER_SYSTEM+from longmemeval.provenance import build_provenance+from longmemeval.report import print_report_table, write_report++console = Console()+++def _settings_for_args(args: argparse.Namespace, settings: Settings) -> Settings:+ variant = getattr(args, "variant", None)+ return resolve_variant_settings(settings, variant)+++def cmd_download(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ path = download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ console.print(f"[green]Dataset ready:[/green] {path} (variant={settings.variant})")+ return 0+++def cmd_dataset_stats(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ if not settings.dataset_path.exists():+ download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ stats = dataset_stats(load_dataset(settings.dataset_path, variant=settings.variant))+ table = Table(title=f"LongMemEval ({settings.variant})")+ table.add_column("Metric")+ table.add_column("Value")+ table.add_row("Questions", str(stats["n_questions"]))+ table.add_row("Abstention", str(stats["n_abstention"]))+ sess = stats["sessions_per_question"]+ table.add_row(+ "Sessions/question",+ f"min={sess['min']} max={sess['max']} mean={sess['mean']:.1f}",+ )+ console.print(table)+ type_table = Table(title="By question type")+ type_table.add_column("Type")+ type_table.add_column("N", justify="right")+ for qtype, count in (stats.get("by_type") or {}).items():+ type_table.add_row(qtype, str(count))+ console.print(type_table)+ assert brain_id_for("e47b4ab3aa_abs") == "lmee47b4ab3aaabs"+ console.print("[dim]brain_id_for('e47b4ab3aa_abs') == 'lmee47b4ab3aaabs' OK[/dim]")+ return 0+++def cmd_smoke(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ if not settings.dataset_path.exists():+ download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ questions = resolve_questions(settings, limit=max(1, args.limit))+ entry = questions[0]+ question_id = str(entry["question_id"])+ brain_id = args.brain or brain_id_for(question_id)+ units = iter_ingest_units(entry, limit_sessions=1)+ if not units:+ raise SystemExit("No haystack sessions to smoke-test")+ unit = units[0]+ with BrainAPIClient(settings) as client:+ submitted = client.ingest_text(+ unit["text"],+ brain_id,+ source_timestamp=unit.get("source_timestamp"),+ )+ task_id = (submitted.data or {}).get("task_id")+ console.print(f"Queued ingest task_id={task_id} brain={brain_id}")+ waited = client.wait_for_task(task_id, brain_id, timeout_s=args.timeout)+ console.print(f"Task status: {(waited.data or {}).get('status')}")+ question = str(entry.get("question") or "What do you remember?")+ ctx = client.retrieve_context(question, brain_id)+ text_context = (ctx.data or {}).get("text_context") or ""+ console.print(f"text_context ({len(text_context)} chars): {text_context[:400]}")+ if not text_context.strip():+ console.print("[yellow]Warning: empty text_context[/yellow]")+ return 1+ console.print("[green]Smoke test passed[/green]")+ return 0+++def cmd_ingest(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ if not settings.dataset_path.exists():+ download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ question_types = (+ {t.strip() for t in args.question_type.split(",") if t.strip()}+ if args.question_type+ else None+ )+ questions = resolve_questions(+ settings,+ question_ids=args.question_id or None,+ limit=args.limit,+ question_types=question_types,+ )+ run_id, run_dir = ensure_run_dir(settings, args.run)+ write_manifest(+ run_dir,+ {+ "run_id": run_id,+ "command": "ingest",+ "variant": settings.variant,+ "n_questions": len(questions),+ "question_ids": [q["question_id"] for q in questions],+ "concurrency": args.concurrency,+ "limit": args.limit,+ "limit_sessions": args.limit_sessions,+ "dry_run": args.dry_run,+ "brainapi_url": settings.brainapi_url,+ **build_provenance(settings),+ },+ )+ ingest_questions(+ settings,+ questions,+ run_dir=run_dir,+ concurrency=args.concurrency,+ limit_sessions=args.limit_sessions,+ dry_run=args.dry_run,+ resume=not args.no_resume,+ task_timeout_s=args.timeout,+ )+ console.print(f"[green]Run directory:[/green] {run_dir}")+ return 0+++def cmd_answer_once(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ entry = get_question(+ load_dataset(settings.dataset_path, variant=settings.variant),+ args.question_id,+ )+ brain_id = brain_id_for(str(entry["question_id"]))+ question = args.question or str(entry.get("question") or "")+ with BrainAPIClient(settings) as client:+ retrieved = client.retrieve_context(question, brain_id)+ result = answer_question(+ settings,+ question,+ retrieved.data or {},+ question_date=str(entry.get("question_date") or "") or None,+ )+ console.print(result.answer)+ console.print(+ f"[dim]model={result.model} latency_ms={result.latency_ms:.0f} "+ f"tokens={result.total_tokens}[/dim]"+ )+ return 0+++def cmd_selftest_metrics(args: argparse.Namespace, settings: Settings) -> int:+ errors = selftest_metrics()+ if errors:+ for err in errors:+ console.print(f"[red]FAIL[/red] {err}")+ return 1+ console.print("[green]selftest-metrics passed[/green]")+ return 0+++def cmd_evaluate(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ if not settings.dataset_path.exists():+ download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ question_types = (+ {t.strip() for t in args.question_type.split(",") if t.strip()}+ if args.question_type+ else None+ )+ questions = resolve_questions(+ settings,+ question_ids=args.question_id or None,+ limit=args.limit,+ question_types=question_types,+ )+ run_id, run_dir = ensure_run_dir(settings, args.run)+ if settings.judge_shares_answer_family:+ console.print(+ f"[yellow]Warning: judge model ({settings.judge_model}) is the same "+ f"family as the answer model ({settings.answer_model}); judge accuracy "+ "carries self-preference bias.[/yellow]"+ )+ write_manifest(+ run_dir,+ {+ "run_id": run_id,+ "command": "evaluate",+ "variant": settings.variant,+ "n_questions": len(questions),+ "question_ids": [q["question_id"] for q in questions],+ "concurrency": args.concurrency,+ "limit": args.limit,+ "brainapi_url": settings.brainapi_url,+ **build_provenance(settings),+ "historical_limit": args.historical_limit,+ "max_passages": args.max_passages,+ "max_facts": args.max_facts,+ "apply_fact_filter": not args.no_fact_filter,+ "use_ppr": args.use_ppr,+ "sufficiency_retry": args.sufficiency_retry,+ },+ )+ evaluate_questions(+ settings,+ questions,+ run_dir=run_dir,+ concurrency=args.concurrency,+ resume=not args.no_resume,+ historical_limit=args.historical_limit,+ max_passages=args.max_passages,+ max_facts=args.max_facts,+ apply_fact_filter=not args.no_fact_filter,+ use_ppr=args.use_ppr,+ sufficiency_retry=args.sufficiency_retry,+ )+ report = write_report(run_dir)+ print_report_table(report)+ console.print(f"[green]Report:[/green] {run_dir / 'report.md'}")+ return 0+++def cmd_report(args: argparse.Namespace, settings: Settings) -> int:+ run_dir = settings.runs_dir / args.run+ if not run_dir.exists():+ raise SystemExit(f"Run directory not found: {run_dir}")+ report = write_report(run_dir)+ print_report_table(report)+ console.print(report["run_dir"] + "/report.md")+ if args.json:+ console.print_json(json.dumps(report))+ return 0+++def cmd_prompt_audit(args: argparse.Namespace, settings: Settings) -> int:+ settings = _settings_for_args(args, settings)+ if not settings.dataset_path.exists():+ download_dataset(+ settings.dataset_path, settings.dataset_url, variant=settings.variant+ )+ dataset = load_dataset(settings.dataset_path, variant=settings.variant)+ prompt_tokens = tokenize(ANSWER_SYSTEM)+ n = args.ngram+ prompt_ngrams = {+ tuple(prompt_tokens[i : i + n])+ for i in range(0, max(0, len(prompt_tokens) - n + 1))+ }+ hits: list[tuple[str, str]] = []+ for entry in dataset:+ qid = str(entry.get("question_id"))+ gold_tokens = tokenize(str(entry.get("answer") or ""))+ for i in range(0, max(0, len(gold_tokens) - n + 1)):+ gram = tuple(gold_tokens[i : i + n])+ if gram in prompt_ngrams:+ hits.append((qid, " ".join(gram)))+ break+ if hits:+ console.print(+ f"[red]FAIL[/red] answer prompt shares a {n}-gram with "+ f"{len(hits)} gold answers"+ )+ for qid, gram in hits[:20]:+ console.print(f" {qid} {gram!r}")+ return 1+ console.print(+ f"[green]prompt-audit passed[/green] no {n}-gram of the answer prompt "+ f"appears in any gold answer ({len(dataset)} questions)"+ )+ return 0+++def build_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(+ prog="python -m longmemeval",+ description="LongMemEval benchmark harness for BrainAPI",+ )+ parser.add_argument(+ "--env-file",+ type=Path,+ default=None,+ help="Optional path to .env (defaults to benchmarks/.env)",+ )+ sub = parser.add_subparsers(dest="command", required=True)++ def add_variant(p: argparse.ArgumentParser) -> None:+ p.add_argument(+ "--variant",+ choices=sorted(VARIANT_FILES),+ default=None,+ help="Dataset variant (default: s / BENCH_LME_VARIANT)",+ )++ p_download = sub.add_parser("download", help="Download LongMemEval JSON")+ add_variant(p_download)+ p_download.set_defaults(func=cmd_download)++ p_stats = sub.add_parser("dataset-stats", help="Print dataset statistics")+ add_variant(p_stats)+ p_stats.set_defaults(func=cmd_dataset_stats)++ p_smoke = sub.add_parser("smoke", help="Smoke-test ingest + retrieve")+ add_variant(p_smoke)+ p_smoke.add_argument("--brain", default=None)+ p_smoke.add_argument("--limit", type=int, default=1)+ p_smoke.add_argument("--timeout", type=float, default=600.0)+ p_smoke.set_defaults(func=cmd_smoke)++ p_ingest = sub.add_parser("ingest", help="Ingest LongMemEval haystacks")+ add_variant(p_ingest)+ p_ingest.add_argument("--run", default=None, help="Existing or new run id")+ p_ingest.add_argument(+ "--question-id",+ action="append",+ help="Question id (repeatable). Default: all questions.",+ )+ p_ingest.add_argument("--question-type", default=None, help="Comma-separated types")+ p_ingest.add_argument("--limit", type=int, default=None)+ p_ingest.add_argument("--limit-sessions", type=int, default=None)+ p_ingest.add_argument("--concurrency", type=int, default=2)+ p_ingest.add_argument("--dry-run", action="store_true")+ p_ingest.add_argument("--no-resume", action="store_true")+ p_ingest.add_argument("--timeout", type=float, default=900.0)+ p_ingest.set_defaults(func=cmd_ingest)++ p_answer = sub.add_parser("answer-once", help="Retrieve + answer one question")+ add_variant(p_answer)+ p_answer.add_argument("--question-id", required=True)+ p_answer.add_argument("--question", default=None)+ p_answer.set_defaults(func=cmd_answer_once)++ p_self = sub.add_parser("selftest-metrics", help="Run local metrics self-test")+ p_self.set_defaults(func=cmd_selftest_metrics)++ p_eval = sub.add_parser("evaluate", help="Answer and score LongMemEval QA")+ add_variant(p_eval)+ p_eval.add_argument("--run", default=None)+ p_eval.add_argument("--question-id", action="append")+ p_eval.add_argument("--question-type", default=None)+ p_eval.add_argument("--limit", type=int, default=None)+ p_eval.add_argument("--concurrency", type=int, default=2)+ p_eval.add_argument("--no-resume", action="store_true")+ p_eval.add_argument("--historical-limit", type=int, default=10)+ p_eval.add_argument("--max-passages", type=int, default=8)+ p_eval.add_argument("--max-facts", type=int, default=40)+ p_eval.add_argument("--no-fact-filter", action="store_true")+ p_eval.add_argument("--use-ppr", action="store_true")+ p_eval.add_argument("--sufficiency-retry", action="store_true")+ p_eval.set_defaults(func=cmd_evaluate)++ p_report = sub.add_parser("report", help="Rebuild report from a run directory")+ p_report.add_argument("--run", required=True)+ p_report.add_argument("--json", action="store_true")+ p_report.set_defaults(func=cmd_report)++ p_audit = sub.add_parser(+ "prompt-audit",+ help="Fail if the answer prompt shares an n-gram with any gold answer",+ )+ add_variant(p_audit)+ p_audit.add_argument("--ngram", type=int, default=3)+ p_audit.set_defaults(func=cmd_prompt_audit)++ return parser+++def main(argv: list[str] | None = None) -> int:+ parser = build_parser()+ args = parser.parse_args(argv)+ settings = Settings.load(args.env_file)+ return args.func(args, settings)diff --git a/benchmarks/longmemeval/client.py b/benchmarks/longmemeval/client.pynew file mode 100644index 0000000..f69627f--- /dev/null+++ b/benchmarks/longmemeval/client.py@@ -0,0 +1,172 @@+from __future__ import annotations++import time+from dataclasses import dataclass, field+from typing import Any++import httpx++from longmemeval.config import TERMINAL_TASK_STATUSES, Settings+++@dataclass+class TimedResult:+ data: Any+ latency_ms: float+++@dataclass+class BrainAPIClient:+ settings: Settings+ timeout: float = 120.0+ max_retries: int = 5+ _client: httpx.Client = field(init=False, repr=False)++ def __post_init__(self) -> None:+ self.settings.require_brainapi()+ self._client = httpx.Client(+ base_url=self.settings.brainapi_url,+ headers={+ "Authorization": f"Bearer {self.settings.brainpat_token}",+ "Content-Type": "application/json",+ },+ timeout=self.timeout,+ )++ def close(self) -> None:+ self._client.close()++ def __enter__(self) -> BrainAPIClient:+ return self++ def __exit__(self, *args: object) -> None:+ self.close()++ def _request(+ self,+ method: str,+ path: str,+ *,+ json: dict | None = None,+ params: dict | None = None,+ ) -> TimedResult:+ last_error: Exception | None = None+ for attempt in range(self.max_retries):+ started = time.perf_counter()+ try:+ response = self._client.request(+ method, path, json=json, params=params+ )+ latency_ms = (time.perf_counter() - started) * 1000+ if response.status_code in (429, 500, 502, 503, 504):+ wait = min(2**attempt * 0.5, 30)+ time.sleep(wait)+ last_error = httpx.HTTPStatusError(+ f"HTTP {response.status_code}",+ request=response.request,+ response=response,+ )+ continue+ response.raise_for_status()+ if response.status_code == 204 or not response.content:+ return TimedResult(data=None, latency_ms=latency_ms)+ return TimedResult(data=response.json(), latency_ms=latency_ms)+ except (httpx.TransportError, httpx.TimeoutException) as exc:+ last_error = exc+ wait = min(2**attempt * 0.5, 30)+ time.sleep(wait)+ raise RuntimeError(f"Request failed after retries: {method} {path}: {last_error}")++ def ingest_text(+ self,+ text: str,+ brain_id: str,+ *,+ observate_for: list[str] | None = None,+ preferred_extraction_entities: list[str] | None = None,+ source_timestamp: str | None = None,+ ) -> TimedResult:+ body = {+ "data": {"data_type": "text", "text_data": text},+ "brain_id": brain_id,+ "observate_for": observate_for+ or [+ "facts about each speaker",+ "events and when they happened",+ "relationships between people",+ ],+ "preferred_extraction_entities": preferred_extraction_entities+ or ["Person", "Event", "Location", "Date"],+ }+ if source_timestamp:+ body["source_timestamp"] = source_timestamp+ return self._request("POST", "/ingest/", json=body)++ def get_task(self, task_id: str, brain_id: str) -> TimedResult:+ return self._request(+ "GET", f"/tasks/{task_id}", params={"brain_id": brain_id}+ )++ def wait_for_task(+ self,+ task_id: str,+ brain_id: str,+ *,+ timeout_s: float = 600.0,+ poll_interval_s: float = 2.0,+ ) -> TimedResult:+ started = time.perf_counter()+ while True:+ result = self.get_task(task_id, brain_id)+ status = (result.data or {}).get("status", "unknown")+ if status in TERMINAL_TASK_STATUSES:+ return TimedResult(+ data=result.data,+ latency_ms=(time.perf_counter() - started) * 1000,+ )+ if time.perf_counter() - started > timeout_s:+ raise TimeoutError(+ f"Task {task_id} did not finish within {timeout_s}s "+ f"(last status={status})"+ )+ time.sleep(poll_interval_s)++ def retrieve_context(+ self,+ text: str,+ brain_id: str,+ *,+ historical_limit: int = 10,+ max_passages: int = 8,+ max_facts: int = 40,+ apply_fact_filter: bool = True,+ use_ppr: bool = False,+ sufficiency_retry: bool = False,+ cross_event_bridges: int = 3,+ ) -> TimedResult:+ body: dict[str, Any] = {+ "text": text,+ "brain_id": brain_id,+ "historical_limit": historical_limit,+ "max_passages": max_passages,+ "max_facts": max_facts,+ "apply_fact_filter": apply_fact_filter,+ "use_ppr": use_ppr,+ "sufficiency_retry": sufficiency_retry,+ "cross_event_bridges": cross_event_bridges,+ }+ return self._request("POST", "/retrieve/context", json=body)++ def entity_status(self, target: str, brain_id: str) -> TimedResult:+ return self._request(+ "GET",+ "/retrieve/entity/status",+ params={"target": target, "brain_id": brain_id},+ )++ def list_entities(self, brain_id: str, *, limit: int = 5) -> TimedResult:+ return self._request(+ "GET",+ "/retrieve/entities",+ params={"brain_id": brain_id, "limit": limit},+ )diff --git a/benchmarks/longmemeval/config.py b/benchmarks/longmemeval/config.pynew file mode 100644index 0000000..a4aafb5--- /dev/null+++ b/benchmarks/longmemeval/config.py@@ -0,0 +1,261 @@+from __future__ import annotations++import os+from dataclasses import dataclass+from pathlib import Path++from dotenv import load_dotenv++BENCHMARKS_ROOT = Path(__file__).resolve().parent.parent+DATA_DIR = BENCHMARKS_ROOT / "data"+RUNS_DIR = BENCHMARKS_ROOT / "runs"++HF_CLEANED_BASE = (+ "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main"+)++VARIANT_FILES = {+ "s": "longmemeval_s_cleaned.json",+ "oracle": "longmemeval_oracle.json",+ "m": "longmemeval_m_cleaned.json",+}++DEFAULT_VARIANT = "s"+DEFAULT_LLM_BASE_URL = "https://api.deepseek.com"+DEFAULT_ANSWER_MODEL = "deepseek-v4-flash"+DEFAULT_SOTA_ANSWER_MODEL = "deepseek-v4-flash"+DEFAULT_AZURE_JUDGE_MODEL = "gpt-4o"+DEFAULT_AZURE_API_VERSION = "2024-12-01-preview"+DEFAULT_BENCH_PROFILE = "product"++QUESTION_TYPES = (+ "single-session-user",+ "single-session-assistant",+ "single-session-preference",+ "temporal-reasoning",+ "knowledge-update",+ "multi-session",+)++_MODEL_FAMILY_PREFIXES = {+ "deepseek": "deepseek",+ "gpt": "openai",+ "o1": "openai",+ "o3": "openai",+ "o4": "openai",+ "chatgpt": "openai",+ "claude": "anthropic",+ "gemini": "google",+ "grok": "xai",+ "qwen": "qwen",+ "llama": "meta",+ "mistral": "mistral",+}++TERMINAL_TASK_STATUSES = frozenset({"completed", "failed", "partial_failed"})+++def model_family(model: str) -> str:+ name = (model or "").strip().lower().rsplit("/", 1)[-1]+ for prefix, family in _MODEL_FAMILY_PREFIXES.items():+ if name.startswith(prefix):+ return family+ return name.split("-", 1)[0] or "unknown"+++def variant_url(variant: str) -> str:+ key = (variant or DEFAULT_VARIANT).strip().lower()+ if key not in VARIANT_FILES:+ raise ValueError(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")+ return f"{HF_CLEANED_BASE}/{VARIANT_FILES[key]}"+++def variant_path(variant: str) -> Path:+ key = (variant or DEFAULT_VARIANT).strip().lower()+ if key not in VARIANT_FILES:+ raise ValueError(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")+ return DATA_DIR / VARIANT_FILES[key]+++def profile_defaults(profile: str) -> dict:+ name = (profile or "product").strip().lower()+ if name == "sota":+ return {+ "bench_profile": "sota",+ "sc_samples": int(os.getenv("BENCH_SC_SAMPLES", "5")),+ "sc_temperature": float(os.getenv("BENCH_SC_TEMPERATURE", "0.7")),+ "gap_fill": os.getenv("BENCH_GAP_FILL", "1") not in {"0", "false", "False"},+ }+ return {+ "bench_profile": "product",+ "sc_samples": int(os.getenv("BENCH_SC_SAMPLES", "1")),+ "sc_temperature": 0.0,+ "gap_fill": os.getenv("BENCH_GAP_FILL", "0") in {"1", "true", "True"},+ }+++@dataclass+class Settings:+ brainapi_url: str+ brainpat_token: str+ openai_api_key: str+ llm_base_url: str | None+ answer_model: str+ judge_model: str+ judge_api_key: str+ judge_base_url: str | None+ judge_azure_endpoint: str | None+ judge_azure_api_version: str+ dataset_path: Path+ dataset_url: str+ variant: str+ runs_dir: Path+ bench_profile: str+ sc_samples: int+ sc_temperature: float+ gap_fill: bool+ answer_azure_endpoint: str | None = None+ answer_azure_api_version: str = DEFAULT_AZURE_API_VERSION++ @classmethod+ def load(cls, env_file: Path | None = None, *, variant: str | None = None) -> "Settings":+ if env_file is None:+ env_file = BENCHMARKS_ROOT / ".env"+ if env_file.exists():+ load_dotenv(env_file, override=False)+ else:+ load_dotenv(override=False)++ profile = (+ os.getenv("BENCH_PROFILE", DEFAULT_BENCH_PROFILE) or DEFAULT_BENCH_PROFILE+ ).strip().lower()+ if profile not in {"product", "sota"}:+ profile = DEFAULT_BENCH_PROFILE+ defaults = profile_defaults(profile)++ api_key = (+ os.getenv("DEEPSEEK_API_KEY")+ or os.getenv("OPENAI_API_KEY")+ or ""+ )+ llm_base_url = os.getenv("BENCH_LLM_BASE_URL", DEFAULT_LLM_BASE_URL) or None+ answer_model = os.getenv("BENCH_ANSWER_MODEL", DEFAULT_ANSWER_MODEL)++ azure_endpoint = (+ os.getenv("BENCH_JUDGE_AZURE_ENDPOINT")+ or os.getenv("AZURE_LARGE_LLM_ENDPOINT")+ or ""+ ).rstrip("/")+ azure_key = (+ os.getenv("BENCH_JUDGE_AZURE_KEY")+ or os.getenv("AZURE_LARGE_LLM_SUBSCRIPTION_KEY")+ or ""+ )+ azure_available = bool(azure_endpoint and azure_key)++ answer_azure_endpoint: str | None = None+ if profile == "sota":+ sota_answer = os.getenv("BENCH_SOTA_ANSWER_MODEL", "").strip()+ if sota_answer:+ answer_model = sota_answer+ sota_base = os.getenv("BENCH_SOTA_LLM_BASE_URL", "").strip()+ if sota_base:+ llm_base_url = sota_base+ else:+ answer_model = os.getenv(+ "BENCH_ANSWER_MODEL", DEFAULT_SOTA_ANSWER_MODEL+ )++ judge_model = os.getenv("BENCH_JUDGE_MODEL", "")+ judge_base_url = os.getenv("BENCH_JUDGE_BASE_URL", "")+ judge_api_key = os.getenv("BENCH_JUDGE_API_KEY", "")+ use_azure_judge = azure_available and not judge_model and not judge_base_url+ if use_azure_judge:+ judge_model = os.getenv(+ "AZURE_LARGE_LLM_MODEL", DEFAULT_AZURE_JUDGE_MODEL+ )+ if profile == "sota":+ sota_judge = os.getenv("BENCH_SOTA_JUDGE_MODEL", "").strip()+ if sota_judge:+ judge_model = sota_judge+ use_azure_judge = False+ elif not os.getenv("BENCH_JUDGE_MODEL"):+ judge_model = answer_model+ use_azure_judge = False+ judge_base_url = llm_base_url or ""+ judge_api_key = api_key++ resolved_variant = (+ variant+ or os.getenv("BENCH_LME_VARIANT", DEFAULT_VARIANT)+ or DEFAULT_VARIANT+ ).strip().lower()+ if resolved_variant not in VARIANT_FILES:+ resolved_variant = DEFAULT_VARIANT+ default_path = variant_path(resolved_variant)+ default_url = variant_url(resolved_variant)++ return cls(+ brainapi_url=os.getenv("BRAINAPI_URL", "http://localhost:8000").rstrip("/"),+ brainpat_token=os.getenv("BRAINPAT_TOKEN", ""),+ openai_api_key=api_key,+ llm_base_url=llm_base_url,+ answer_model=answer_model,+ judge_model=judge_model or answer_model,+ judge_api_key=(+ azure_key if use_azure_judge else (judge_api_key or api_key)+ ),+ judge_base_url=(+ None if use_azure_judge else (judge_base_url or llm_base_url)+ ),+ judge_azure_endpoint=azure_endpoint if use_azure_judge else None,+ judge_azure_api_version=os.getenv(+ "BENCH_JUDGE_AZURE_API_VERSION",+ os.getenv("AZURE_LARGE_LLM_API_VERSION", DEFAULT_AZURE_API_VERSION),+ ),+ dataset_path=Path(+ os.getenv("BENCH_LME_DATASET_PATH", str(default_path))+ ),+ dataset_url=os.getenv("BENCH_LME_DATASET_URL", default_url),+ variant=resolved_variant,+ runs_dir=Path(os.getenv("BENCH_RUNS_DIR", str(RUNS_DIR))),+ bench_profile=str(defaults["bench_profile"]),+ sc_samples=max(1, int(defaults["sc_samples"])),+ sc_temperature=float(defaults["sc_temperature"]),+ gap_fill=bool(defaults["gap_fill"]),+ answer_azure_endpoint=answer_azure_endpoint,+ answer_azure_api_version=os.getenv(+ "BENCH_JUDGE_AZURE_API_VERSION",+ os.getenv("AZURE_LARGE_LLM_API_VERSION", DEFAULT_AZURE_API_VERSION),+ ),+ )++ @property+ def judge_shares_answer_family(self) -> bool:+ return model_family(self.judge_model) == model_family(self.answer_model)++ def judge_provider(self) -> str:+ if self.judge_azure_endpoint:+ return "azure"+ return self.judge_base_url or "openai-default"++ def require_brainapi(self) -> None:+ if not self.brainpat_token:+ raise SystemExit(+ "BRAINPAT_TOKEN is required. Copy .env.example to .env and set it."+ )++ def require_llm(self) -> None:+ if not self.openai_api_key:+ raise SystemExit(+ "DEEPSEEK_API_KEY (or OPENAI_API_KEY) is required for answer/judge "+ "LLM calls. Copy .env.example to .env and set it."+ )++ def require_judge_llm(self) -> None:+ if not self.judge_api_key:+ raise SystemExit(+ "No judge API key. Set BENCH_JUDGE_API_KEY (with BENCH_JUDGE_BASE_URL "+ "and BENCH_JUDGE_MODEL), or DEEPSEEK_API_KEY / OPENAI_API_KEY to reuse "+ "the answerer's provider."+ )diff --git a/benchmarks/longmemeval/dataset.py b/benchmarks/longmemeval/dataset.pynew file mode 100644index 0000000..93f0adf--- /dev/null+++ b/benchmarks/longmemeval/dataset.py@@ -0,0 +1,199 @@+from __future__ import annotations++import json+import re+from collections import Counter+from pathlib import Path+from typing import Any+from urllib.request import urlretrieve++from longmemeval.config import (+ DEFAULT_VARIANT,+ VARIANT_FILES,+ Settings,+ variant_path,+ variant_url,+)+++def brain_id_for(question_id: str) -> str:+ slug = re.sub(r"[^a-zA-Z0-9]", "", f"lme{question_id}")+ if not slug.isalnum():+ raise ValueError(f"Could not derive alphanumeric brain_id from {question_id!r}")+ return slug.lower()+++def is_abstention(question_id: str) -> bool:+ return str(question_id).endswith("_abs")+++def download_dataset(+ dest: Path | None = None,+ url: str | None = None,+ *,+ variant: str = DEFAULT_VARIANT,+) -> Path:+ dest = dest or variant_path(variant)+ url = url or variant_url(variant)+ dest.parent.mkdir(parents=True, exist_ok=True)+ if dest.exists() and dest.stat().st_size > 0:+ return dest+ urlretrieve(url, dest)+ return dest+++def load_dataset(path: Path | None = None, *, variant: str = DEFAULT_VARIANT) -> list[dict[str, Any]]:+ path = path or variant_path(variant)+ if not path.exists():+ raise FileNotFoundError(+ f"Dataset not found at {path}. Run: python -m longmemeval download --variant {variant}"+ )+ with path.open(encoding="utf-8") as f:+ data = json.load(f)+ if not isinstance(data, list):+ raise ValueError(f"Expected {path.name} to be a JSON array")+ return data+++def get_question(dataset: list[dict[str, Any]], question_id: str) -> dict[str, Any]:+ for entry in dataset:+ if entry.get("question_id") == question_id:+ return entry+ raise KeyError(f"question_id {question_id!r} not found")+++def format_turn(turn: dict[str, Any]) -> str:+ role = str(turn.get("role") or "unknown").strip()+ content = str(turn.get("content") or "").strip()+ return f"{role}: {content}"+++def format_session(+ session_id: str,+ session_date: str,+ turns: list[Any],+) -> str:+ lines = [+ f"Session time: {session_date or 'unknown time'}.",+ f"Session id: {session_id}.",+ "",+ ]+ for turn in turns or []:+ if not isinstance(turn, dict):+ continue+ clean = {+ "role": turn.get("role"),+ "content": turn.get("content"),+ }+ lines.append(format_turn(clean))+ return "\n".join(lines)+++def iter_ingest_units(+ entry: dict[str, Any],+ *,+ limit_sessions: int | None = None,+) -> list[dict[str, Any]]:+ session_ids = list(entry.get("haystack_session_ids") or [])+ dates = list(entry.get("haystack_dates") or [])+ sessions = list(entry.get("haystack_sessions") or [])+ n = min(len(session_ids), len(dates), len(sessions))+ if limit_sessions is not None:+ n = min(n, limit_sessions)++ units: list[dict[str, Any]] = []+ for i in range(n):+ sid = str(session_ids[i])+ units.append(+ {+ "unit_id": sid,+ "session_id": sid,+ "text": format_session(sid, str(dates[i]), sessions[i]),+ "source_timestamp": str(dates[i]) if dates[i] else None,+ }+ )+ return units+++def dataset_stats(dataset: list[dict[str, Any]]) -> dict[str, Any]:+ type_counts = Counter(str(e.get("question_type") or "unknown") for e in dataset)+ abstention = sum(1 for e in dataset if is_abstention(str(e.get("question_id") or "")))+ sessions = [+ len(e.get("haystack_session_ids") or [])+ for e in dataset+ ]+ turns = []+ for e in dataset:+ for sess in e.get("haystack_sessions") or []:+ if isinstance(sess, list):+ turns.append(len(sess))+ return {+ "n_questions": len(dataset),+ "n_abstention": abstention,+ "by_type": dict(sorted(type_counts.items())),+ "sessions_per_question": {+ "min": min(sessions) if sessions else 0,+ "max": max(sessions) if sessions else 0,+ "mean": (sum(sessions) / len(sessions)) if sessions else 0.0,+ },+ "turns_per_session": {+ "min": min(turns) if turns else 0,+ "max": max(turns) if turns else 0,+ "mean": (sum(turns) / len(turns)) if turns else 0.0,+ },+ }+++def resolve_questions(+ settings: Settings,+ *,+ question_ids: list[str] | None = None,+ limit: int | None = None,+ question_types: set[str] | None = None,+) -> list[dict[str, Any]]:+ dataset = load_dataset(settings.dataset_path, variant=settings.variant)+ if question_ids:+ wanted = set(question_ids)+ dataset = [e for e in dataset if e.get("question_id") in wanted]+ missing = wanted - {e.get("question_id") for e in dataset}+ if missing:+ raise KeyError(f"question_id(s) not found: {sorted(missing)}")+ if question_types:+ dataset = [+ e for e in dataset if str(e.get("question_type") or "") in question_types+ ]+ if limit is not None:+ dataset = dataset[: max(0, limit)]+ return dataset+++def resolve_variant_settings(+ settings: Settings, variant: str | None+) -> Settings:+ if not variant or variant == settings.variant:+ return settings+ key = variant.strip().lower()+ if key not in VARIANT_FILES:+ raise SystemExit(f"Unknown variant {variant!r}; choose from {sorted(VARIANT_FILES)}")+ return Settings(+ brainapi_url=settings.brainapi_url,+ brainpat_token=settings.brainpat_token,+ openai_api_key=settings.openai_api_key,+ llm_base_url=settings.llm_base_url,+ answer_model=settings.answer_model,+ judge_model=settings.judge_model,+ judge_api_key=settings.judge_api_key,+ judge_base_url=settings.judge_base_url,+ judge_azure_endpoint=settings.judge_azure_endpoint,+ judge_azure_api_version=settings.judge_azure_api_version,+ dataset_path=variant_path(key),+ dataset_url=variant_url(key),+ variant=key,+ runs_dir=settings.runs_dir,+ bench_profile=settings.bench_profile,+ sc_samples=settings.sc_samples,+ sc_temperature=settings.sc_temperature,+ gap_fill=settings.gap_fill,+ answer_azure_endpoint=settings.answer_azure_endpoint,+ answer_azure_api_version=settings.answer_azure_api_version,+ )diff --git a/benchmarks/longmemeval/evaluate.py b/benchmarks/longmemeval/evaluate.pynew file mode 100644index 0000000..e625a56--- /dev/null+++ b/benchmarks/longmemeval/evaluate.py@@ -0,0 +1,318 @@+from __future__ import annotations++import re+import threading+from concurrent.futures import ThreadPoolExecutor, as_completed+from dataclasses import asdict, dataclass, field+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++from rich.console import Console+from rich.progress import Progress++from longmemeval.answer import answer_question+from longmemeval.client import BrainAPIClient+from longmemeval.config import Settings+from longmemeval.dataset import brain_id_for, is_abstention+from longmemeval.ingest import append_jsonl, load_jsonl+from longmemeval.judge import judge_answer+from longmemeval.prompts import enrich_paths_from_triples, flatten_path, flatten_triple+from longmemeval.sota import merge_contexts, plan_gap_fill++console = Console()+_write_lock = threading.Lock()++_GRAPH_CONTEXT_CAP = 60000+_TRIPLES_CAP = 500+_PASSAGE_PREFIX = "[passage] "+_SESSION_ID_RE = re.compile(r"Session id:\s*([^\s.]+)", re.IGNORECASE)+++@dataclass+class AnswerRecord:+ question_id: str+ brain_id: str+ question: str+ gold: str+ prediction: str+ question_type: str+ question_date: str+ answer_session_ids: list[str]+ judge_correct: bool+ judge_reason: str+ judge_raw: str+ retrieve_latency_ms: float | None+ answer_latency_ms: float | None+ judge_latency_ms: float | None+ answer_model: str | None+ judge_model: str | None+ answer_total_tokens: int | None+ judge_total_tokens: int | None+ graph_context: str+ triples: list[str] = field(default_factory=list)+ paths: list[str] = field(default_factory=list)+ source_passages: list[str] = field(default_factory=list)+ historical_context: list[str] = field(default_factory=list)+ retrieved_session_ids: list[str] = field(default_factory=list)+ context_truncated: dict[str, int] = field(default_factory=dict)+ is_abstention: bool = False+ error: str | None = None+ timestamp: str = ""+++def _utc_now() -> str:+ return datetime.now(timezone.utc).isoformat()+++def _extract_session_ids(*blobs: Any, known_ids: list[str] | None = None) -> list[str]:+ found: list[str] = []+ seen: set[str] = set()+ texts: list[str] = []+ for blob in blobs:+ if isinstance(blob, (list, tuple)):+ texts.append("\n".join(str(item) for item in blob))+ else:+ texts.append(str(blob or ""))+ joined = "\n".join(texts)+ for match in _SESSION_ID_RE.finditer(joined):+ sid = match.group(1).rstrip(".")+ if sid and sid not in seen:+ seen.add(sid)+ found.append(sid)+ if known_ids:+ lower_joined = joined.lower()+ for sid in known_ids:+ if sid and sid not in seen and sid.lower() in lower_joined:+ seen.add(sid)+ found.append(sid)+ return found+++def _graph_channel(text_context: str, source_passages: list[str]) -> str:+ passage_block = "\n".join(f"{_PASSAGE_PREFIX}{p}" for p in source_passages)+ if passage_block and text_context.startswith(passage_block):+ return text_context[len(passage_block) :].lstrip("\n")+ return "\n".join(+ line+ for line in text_context.splitlines()+ if not line.startswith(_PASSAGE_PREFIX)+ )+++def extract_context_channels(+ context: dict[str, Any],+ *,+ known_session_ids: list[str] | None = None,+) -> dict[str, Any]:+ context = enrich_paths_from_triples(context)+ source_passages = [str(p) for p in (context.get("source_passages") or []) if p]+ historical_context = [+ str(h) for h in (context.get("historical_context") or []) if h+ ]+ triples = [flatten_triple(t) for t in (context.get("triples") or []) if t]+ paths = [flatten_path(p) for p in (context.get("paths") or []) if p]+ graph_context = _graph_channel(+ str(context.get("text_context") or ""), source_passages+ )+ truncated: dict[str, int] = {}+ if len(graph_context) > _GRAPH_CONTEXT_CAP:+ truncated["graph_context"] = len(graph_context)+ graph_context = graph_context[:_GRAPH_CONTEXT_CAP]+ if len(triples) > _TRIPLES_CAP:+ truncated["triples"] = len(triples)+ triples = triples[:_TRIPLES_CAP]+ session_ids = _extract_session_ids(+ graph_context,+ triples,+ paths,+ source_passages,+ historical_context,+ known_ids=known_session_ids,+ )+ return {+ "graph_context": graph_context,+ "triples": triples,+ "paths": paths,+ "source_passages": source_passages,+ "historical_context": historical_context,+ "retrieved_session_ids": session_ids,+ "context_truncated": truncated,+ }+++def completed_question_ids(answers_path: Path) -> set[str]:+ done = set()+ for row in load_jsonl(answers_path):+ if row.get("error"):+ continue+ qid = row.get("question_id")+ if qid:+ done.add(str(qid))+ return done+++def evaluate_questions(+ settings: Settings,+ questions: list[dict[str, Any]],+ *,+ run_dir: Path,+ concurrency: int = 2,+ resume: bool = True,+ historical_limit: int = 10,+ max_passages: int = 8,+ max_facts: int = 40,+ apply_fact_filter: bool = True,+ use_ppr: bool = False,+ sufficiency_retry: bool = False,+) -> list[AnswerRecord]:+ run_dir.mkdir(parents=True, exist_ok=True)+ answers_path = run_dir / "answers.jsonl"+ already_done = completed_question_ids(answers_path) if resume else set()++ jobs = [+ {+ "entry": entry,+ "question_id": str(entry["question_id"]),+ "brain_id": brain_id_for(str(entry["question_id"])),+ }+ for entry in questions+ if str(entry["question_id"]) not in already_done+ ]++ if not jobs:+ console.print("[green]Nothing to evaluate (all questions already scored).[/green]")+ return []++ console.print(f"Evaluating {len(jobs)} questions (concurrency={concurrency})")++ records: list[AnswerRecord] = []++ def _one(job: dict[str, Any]) -> AnswerRecord:+ entry = job["entry"]+ question_id = job["question_id"]+ question = str(entry.get("question") or "")+ gold = str(entry.get("answer") or "")+ question_type = str(entry.get("question_type") or "")+ question_date = str(entry.get("question_date") or "")+ answer_session_ids = [str(s) for s in (entry.get("answer_session_ids") or [])]+ abstention = is_abstention(question_id)+ known_ids = [str(s) for s in (entry.get("haystack_session_ids") or [])]++ try:+ with BrainAPIClient(settings) as client:+ retrieved = client.retrieve_context(+ question,+ job["brain_id"],+ historical_limit=historical_limit,+ max_passages=max_passages,+ max_facts=max_facts,+ apply_fact_filter=apply_fact_filter,+ use_ppr=use_ppr,+ sufficiency_retry=sufficiency_retry,+ )+ context = retrieved.data or {}+ retrieve_ms = retrieved.latency_ms+ draft = answer_question(+ settings, question, context, question_date=question_date+ )+ if settings.gap_fill:+ plan = plan_gap_fill(question, draft.answer, context)+ if plan.needs_retry:+ with BrainAPIClient(settings) as client:+ second = client.retrieve_context(+ plan.reformulated_query,+ job["brain_id"],+ historical_limit=historical_limit,+ max_passages=max_passages,+ max_facts=max_facts,+ apply_fact_filter=apply_fact_filter,+ use_ppr=use_ppr,+ sufficiency_retry=sufficiency_retry,+ )+ retrieve_ms = (retrieve_ms or 0) + (second.latency_ms or 0)+ context = merge_contexts(context, second.data or {})+ answered = answer_question(+ settings, question, context, question_date=question_date+ )+ else:+ answered = draft+ else:+ answered = draft+ channels = extract_context_channels(+ context, known_session_ids=known_ids+ )+ judged = judge_answer(+ settings,+ question,+ gold,+ answered.answer,+ question_type=question_type,+ abstention=abstention,+ )+ record = AnswerRecord(+ question_id=question_id,+ brain_id=job["brain_id"],+ question=question,+ gold=gold,+ prediction=answered.answer,+ question_type=question_type,+ question_date=question_date,+ answer_session_ids=answer_session_ids,+ judge_correct=judged.correct,+ judge_reason=judged.reason,+ judge_raw=judged.raw,+ retrieve_latency_ms=retrieve_ms,+ answer_latency_ms=answered.latency_ms,+ judge_latency_ms=judged.latency_ms,+ answer_model=answered.model,+ judge_model=judged.model,+ answer_total_tokens=answered.total_tokens,+ judge_total_tokens=judged.total_tokens,+ is_abstention=abstention,+ error=None,+ timestamp=_utc_now(),+ **channels,+ )+ except Exception as exc:+ record = AnswerRecord(+ question_id=question_id,+ brain_id=job["brain_id"],+ question=question,+ gold=gold,+ prediction="",+ question_type=question_type,+ question_date=question_date,+ answer_session_ids=answer_session_ids,+ judge_correct=False,+ judge_reason="",+ judge_raw="",+ retrieve_latency_ms=None,+ answer_latency_ms=None,+ judge_latency_ms=None,+ answer_model=None,+ judge_model=None,+ answer_total_tokens=None,+ judge_total_tokens=None,+ is_abstention=abstention,+ error=str(exc),+ timestamp=_utc_now(),+ **extract_context_channels({}),+ )++ with _write_lock:+ append_jsonl(answers_path, asdict(record))+ return record++ with Progress(console=console) as progress:+ task = progress.add_task("evaluate", total=len(jobs))+ with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:+ futures = [pool.submit(_one, job) for job in jobs]+ for future in as_completed(futures):+ records.append(future.result())+ progress.advance(task)++ ok = sum(1 for r in records if not r.error)+ failed = sum(1 for r in records if r.error)+ console.print(f"Evaluate done: ok={ok} failed={failed} -> {answers_path}")+ return recordsdiff --git a/benchmarks/longmemeval/ingest.py b/benchmarks/longmemeval/ingest.pynew file mode 100644index 0000000..1be715f--- /dev/null+++ b/benchmarks/longmemeval/ingest.py@@ -0,0 +1,222 @@+from __future__ import annotations++import json+import threading+from concurrent.futures import ThreadPoolExecutor, as_completed+from dataclasses import asdict, dataclass+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++from rich.console import Console+from rich.progress import Progress++from longmemeval.client import BrainAPIClient+from longmemeval.config import Settings+from longmemeval.dataset import brain_id_for, iter_ingest_units++console = Console()+_write_lock = threading.Lock()+++@dataclass+class IngestRecord:+ question_id: str+ brain_id: str+ unit_id: str+ session_id: str+ task_id: str | None+ status: str+ error: str | None+ submit_latency_ms: float | None+ wait_latency_ms: float | None+ dry_run: bool+ timestamp: str+ cost: dict[str, Any] | None = None+ total_llm_tokens: int | None = None+++def _utc_now() -> str:+ return datetime.now(timezone.utc).isoformat()+++def load_jsonl(path: Path) -> list[dict[str, Any]]:+ if not path.exists():+ return []+ rows = []+ with path.open(encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if line:+ rows.append(json.loads(line))+ return rows+++def append_jsonl(path: Path, row: dict[str, Any]) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ with _write_lock:+ with path.open("a", encoding="utf-8") as f:+ f.write(json.dumps(row, ensure_ascii=False) + "\n")+++def completed_unit_ids(ingest_path: Path) -> set[str]:+ done = set()+ for row in load_jsonl(ingest_path):+ status = row.get("status")+ if status in {"completed", "partial_failed", "dry_run"}:+ key = f"{row.get('question_id')}::{row.get('unit_id')}"+ done.add(key)+ return done+++def ingest_questions(+ settings: Settings,+ questions: list[dict[str, Any]],+ *,+ run_dir: Path,+ concurrency: int = 2,+ limit_sessions: int | None = None,+ dry_run: bool = False,+ resume: bool = True,+ task_timeout_s: float = 900.0,+) -> list[IngestRecord]:+ run_dir.mkdir(parents=True, exist_ok=True)+ ingest_path = run_dir / "ingest.jsonl"+ already_done = completed_unit_ids(ingest_path) if resume else set()++ jobs: list[dict[str, Any]] = []+ for entry in questions:+ question_id = str(entry["question_id"])+ brain_id = brain_id_for(question_id)+ for unit in iter_ingest_units(entry, limit_sessions=limit_sessions):+ key = f"{question_id}::{unit['unit_id']}"+ if key in already_done:+ continue+ jobs.append(+ {+ "question_id": question_id,+ "brain_id": brain_id,+ "unit": unit,+ }+ )++ if not jobs:+ console.print("[green]Nothing to ingest (all units already completed).[/green]")+ return []++ console.print(+ f"Ingesting {len(jobs)} units "+ f"(concurrency={concurrency}, dry_run={dry_run})"+ )++ records: list[IngestRecord] = []++ def _one(job: dict[str, Any]) -> IngestRecord:+ unit = job["unit"]+ if dry_run:+ record = IngestRecord(+ question_id=job["question_id"],+ brain_id=job["brain_id"],+ unit_id=unit["unit_id"],+ session_id=unit["session_id"],+ task_id=None,+ status="dry_run",+ error=None,+ submit_latency_ms=None,+ wait_latency_ms=None,+ dry_run=True,+ timestamp=_utc_now(),+ )+ append_jsonl(ingest_path, asdict(record))+ return record++ with BrainAPIClient(settings) as client:+ try:+ submitted = client.ingest_text(+ unit["text"],+ job["brain_id"],+ source_timestamp=unit.get("source_timestamp"),+ )+ task_id = (submitted.data or {}).get("task_id")+ if not task_id:+ raise RuntimeError(f"No task_id in ingest response: {submitted.data}")+ waited = client.wait_for_task(+ task_id, job["brain_id"], timeout_s=task_timeout_s+ )+ status = (waited.data or {}).get("status", "unknown")+ error = (waited.data or {}).get("error")+ cost = (waited.data or {}).get("cost")+ total_llm_tokens = None+ if isinstance(cost, dict):+ total_llm_tokens = cost.get("total_llm_tokens")+ record = IngestRecord(+ question_id=job["question_id"],+ brain_id=job["brain_id"],+ unit_id=unit["unit_id"],+ session_id=unit["session_id"],+ task_id=task_id,+ status=status,+ error=error,+ submit_latency_ms=submitted.latency_ms,+ wait_latency_ms=waited.latency_ms,+ dry_run=False,+ timestamp=_utc_now(),+ cost=cost if isinstance(cost, dict) else None,+ total_llm_tokens=(+ int(total_llm_tokens) if total_llm_tokens is not None else None+ ),+ )+ except Exception as exc:+ record = IngestRecord(+ question_id=job["question_id"],+ brain_id=job["brain_id"],+ unit_id=unit["unit_id"],+ session_id=unit["session_id"],+ task_id=None,+ status="failed",+ error=str(exc),+ submit_latency_ms=None,+ wait_latency_ms=None,+ dry_run=False,+ timestamp=_utc_now(),+ )+ append_jsonl(ingest_path, asdict(record))+ return record++ with Progress(console=console) as progress:+ task = progress.add_task("ingest", total=len(jobs))+ with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:+ futures = [pool.submit(_one, job) for job in jobs]+ for future in as_completed(futures):+ records.append(future.result())+ progress.advance(task)++ ok = sum(1 for r in records if r.status in {"completed", "dry_run"})+ partial = sum(1 for r in records if r.status == "partial_failed")+ failed = sum(1 for r in records if r.status == "failed")+ console.print(+ f"Ingest done: ok={ok} partial_failed={partial} failed={failed} "+ f"-> {ingest_path}"+ )+ return records+++def ensure_run_dir(settings: Settings, run_id: str | None = None) -> tuple[str, Path]:+ settings.runs_dir.mkdir(parents=True, exist_ok=True)+ if not run_id:+ run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")+ run_dir = settings.runs_dir / run_id+ run_dir.mkdir(parents=True, exist_ok=True)+ return run_id, run_dir+++def write_manifest(run_dir: Path, manifest: dict[str, Any]) -> None:+ path = run_dir / "manifest.json"+ existing: dict[str, Any] = {}+ if path.exists():+ existing = json.loads(path.read_text(encoding="utf-8"))+ existing.update(manifest)+ existing["updated_at"] = _utc_now()+ if "created_at" not in existing:+ existing["created_at"] = _utc_now()+ path.write_text(json.dumps(existing, indent=2), encoding="utf-8")diff --git a/benchmarks/longmemeval/judge.py b/benchmarks/longmemeval/judge.pynew file mode 100644index 0000000..d8ff6ee--- /dev/null+++ b/benchmarks/longmemeval/judge.py@@ -0,0 +1,82 @@+from __future__ import annotations++import time+from dataclasses import dataclass+from typing import Any++from openai import AzureOpenAI, OpenAI++from longmemeval.config import Settings+from longmemeval.prompts import get_anscheck_prompt+++@dataclass+class JudgeResult:+ correct: bool+ reason: str+ raw: str+ model: str+ latency_ms: float+ prompt_tokens: int | None+ completion_tokens: int | None+ total_tokens: int | None+++def _make_openai_client(settings: Settings) -> OpenAI | AzureOpenAI:+ settings.require_judge_llm()+ if settings.judge_azure_endpoint:+ return AzureOpenAI(+ api_key=settings.judge_api_key,+ azure_endpoint=settings.judge_azure_endpoint,+ api_version=settings.judge_azure_api_version,+ )+ kwargs: dict[str, Any] = {"api_key": settings.judge_api_key}+ if settings.judge_base_url:+ kwargs["base_url"] = settings.judge_base_url+ return OpenAI(**kwargs)+++def parse_yes_no(raw: str) -> bool:+ return "yes" in (raw or "").strip().lower()+++def judge_answer(+ settings: Settings,+ question: str,+ gold: str,+ prediction: str,+ *,+ question_type: str,+ abstention: bool = False,+ model: str | None = None,+) -> JudgeResult:+ client = _make_openai_client(settings)+ model_name = model or settings.judge_model+ prompt = get_anscheck_prompt(+ question_type,+ question,+ gold,+ prediction,+ abstention=abstention,+ )+ started = time.perf_counter()+ response = client.chat.completions.create(+ model=model_name,+ messages=[{"role": "user", "content": prompt}],+ temperature=0,+ max_tokens=10,+ )+ latency_ms = (time.perf_counter() - started) * 1000+ raw = (response.choices[0].message.content or "").strip()+ correct = parse_yes_no(raw)+ usage = response.usage+ return JudgeResult(+ correct=correct,+ reason=raw,+ raw=raw,+ model=getattr(response, "model", None) or model_name,+ latency_ms=latency_ms,+ prompt_tokens=getattr(usage, "prompt_tokens", None) if usage else None,+ completion_tokens=getattr(usage, "completion_tokens", None) if usage else None,+ total_tokens=getattr(usage, "total_tokens", None) if usage else None,+ )diff --git a/benchmarks/longmemeval/metrics.py b/benchmarks/longmemeval/metrics.pynew file mode 100644index 0000000..111bbd5--- /dev/null+++ b/benchmarks/longmemeval/metrics.py@@ -0,0 +1,269 @@+from __future__ import annotations++import math+import string+from typing import Any, Iterable+++_ARTICLES = {"a", "an", "the"}+++def normalize_answer(text: str) -> str:+ text = (text or "").lower()+ text = "".join(ch if ch not in string.punctuation else " " for ch in text)+ tokens = [t for t in text.split() if t and t not in _ARTICLES]+ return " ".join(tokens)+++def tokenize(text: str) -> list[str]:+ return normalize_answer(text).split()+++def percentile(values: list[float], p: float) -> float | None:+ if not values:+ return None+ ordered = sorted(values)+ if len(ordered) == 1:+ return ordered[0]+ k = (len(ordered) - 1) * (p / 100.0)+ f = math.floor(k)+ c = math.ceil(k)+ if f == c:+ return ordered[int(k)]+ return ordered[f] * (c - k) + ordered[c] * (k - f)+++def mean(values: Iterable[float]) -> float | None:+ vals = list(values)+ if not vals:+ return None+ return sum(vals) / len(vals)+++def wilson_ci(successes: int, n: int, z: float = 1.96) -> tuple[float | None, float | None]:+ if n <= 0:+ return None, None+ p = successes / n+ denom = 1.0 + (z * z) / n+ center = (p + (z * z) / (2.0 * n)) / denom+ margin = (+ z+ * math.sqrt((p * (1.0 - p) / n) + (z * z) / (4.0 * n * n))+ / denom+ )+ return max(0.0, center - margin), min(1.0, center + margin)+++def session_recall_coverage(row: dict[str, Any]) -> str | None:+ if row.get("is_abstention"):+ return None+ needed = {str(s) for s in (row.get("answer_session_ids") or []) if s}+ if not needed:+ return None+ have = {str(s) for s in (row.get("retrieved_session_ids") or []) if s}+ if needed <= have:+ return "full"+ if needed & have:+ return "partial"+ return "none"+++def _bucket(items: list[dict[str, Any]]) -> dict[str, Any]:+ if not items:+ return {+ "n": 0,+ "judge_accuracy": None,+ "judge_accuracy_ci95": None,+ "session_recall_full": None,+ "session_recall_partial": None,+ "n_with_evidence": 0,+ }+ correct = [1.0 if r.get("judge_correct") else 0.0 for r in items]+ successes = int(sum(correct))+ lo, hi = wilson_ci(successes, len(items))+ coverages = [session_recall_coverage(r) for r in items]+ with_evidence = [c for c in coverages if c is not None]+ n_ev = len(with_evidence)+ full = sum(1 for c in with_evidence if c == "full")+ partial = sum(1 for c in with_evidence if c in {"full", "partial"})+ return {+ "n": len(items),+ "judge_accuracy": mean(correct),+ "judge_accuracy_ci95": (+ {"low": lo, "high": hi} if lo is not None and hi is not None else None+ ),+ "session_recall_full": (full / n_ev) if n_ev else None,+ "session_recall_partial": (partial / n_ev) if n_ev else None,+ "n_with_evidence": n_ev,+ }+++def aggregate_answers(rows: list[dict[str, Any]]) -> dict[str, Any]:+ by_type: dict[str, list[dict[str, Any]]] = {}+ for row in rows:+ qtype = str(row.get("question_type") or "unknown")+ by_type.setdefault(qtype, []).append(row)++ all_bucket = _bucket(rows)+ abstention_rows = [r for r in rows if r.get("is_abstention")]+ non_abs = [r for r in rows if not r.get("is_abstention")]++ retrieval_latencies = [+ float(r["retrieve_latency_ms"])+ for r in rows+ if r.get("retrieve_latency_ms") is not None+ ]+ total_tokens = 0+ for r in rows:+ for key in ("answer_total_tokens", "judge_total_tokens"):+ if r.get(key) is not None:+ total_tokens += int(r[key])++ return {+ "n_questions": len(rows),+ "n_abstention": len(abstention_rows),+ "n_non_abstention": len(non_abs),+ "headline_judge_accuracy": all_bucket["judge_accuracy"],+ "headline_judge_accuracy_ci95": all_bucket["judge_accuracy_ci95"],+ "session_recall_full": _bucket(non_abs)["session_recall_full"],+ "session_recall_partial": _bucket(non_abs)["session_recall_partial"],+ "n_with_evidence": _bucket(non_abs)["n_with_evidence"],+ "abstention": {+ "n": len(abstention_rows),+ "accuracy": _bucket(abstention_rows)["judge_accuracy"],+ },+ "by_type": {+ qtype: _bucket(items) for qtype, items in sorted(by_type.items())+ },+ "retrieval_latency_ms": {+ "p50": percentile(retrieval_latencies, 50),+ "p95": percentile(retrieval_latencies, 95),+ "mean": mean(retrieval_latencies),+ },+ "total_llm_tokens": total_tokens,+ }+++def selftest_metrics() -> list[str]:+ errors: list[str] = []+ from longmemeval.dataset import brain_id_for, format_session, is_abstention+ from longmemeval.judge import parse_yes_no+ from longmemeval.prompts import get_anscheck_prompt++ if brain_id_for("e47b4ab3aa_abs") != "lmee47b4ab3aaabs":+ errors.append("brain_id_for should strip non-alnum and prefix lme")+ if not is_abstention("abc_abs"):+ errors.append("is_abstention should detect _abs suffix")+ if is_abstention("abc"):+ errors.append("non-abs question_id should not be abstention")++ text = format_session(+ "sess-1",+ "2023/05/07",+ [{"role": "user", "content": "hi", "has_answer": True}],+ )+ if "has_answer" in text:+ errors.append("format_session must strip has_answer labels")+ if "Session id: sess-1" not in text:+ errors.append("format_session must include session id")++ abs_prompt = get_anscheck_prompt(+ "multi-session", "q", "expl", "resp", abstention=True+ )+ if "unanswerable" not in abs_prompt.lower():+ errors.append("abstention judge prompt missing")+ temp_prompt = get_anscheck_prompt(+ "temporal-reasoning", "q", "a", "r", abstention=False+ )+ if "off-by-one" not in temp_prompt:+ errors.append("temporal judge prompt missing off-by-one clause")+ ku_prompt = get_anscheck_prompt(+ "knowledge-update", "q", "a", "r", abstention=False+ )+ if "updated answer" not in ku_prompt:+ errors.append("knowledge-update judge prompt missing")+ pref_prompt = get_anscheck_prompt(+ "single-session-preference", "q", "rubric", "r", abstention=False+ )+ if "Rubric:" not in pref_prompt:+ errors.append("preference judge prompt missing Rubric")++ if not parse_yes_no("Yes"):+ errors.append("parse_yes_no Yes failed")+ if parse_yes_no("No"):+ errors.append("parse_yes_no No failed")++ full = session_recall_coverage(+ {+ "is_abstention": False,+ "answer_session_ids": ["a", "b"],+ "retrieved_session_ids": ["a", "b", "c"],+ }+ )+ if full != "full":+ errors.append(f"expected full recall, got {full}")+ partial = session_recall_coverage(+ {+ "is_abstention": False,+ "answer_session_ids": ["a", "b"],+ "retrieved_session_ids": ["a"],+ }+ )+ if partial != "partial":+ errors.append(f"expected partial recall, got {partial}")+ none = session_recall_coverage(+ {+ "is_abstention": False,+ "answer_session_ids": ["a"],+ "retrieved_session_ids": ["z"],+ }+ )+ if none != "none":+ errors.append(f"expected none recall, got {none}")+ skipped = session_recall_coverage(+ {"is_abstention": True, "answer_session_ids": ["a"], "retrieved_session_ids": []}+ )+ if skipped is not None:+ errors.append("abstention rows should skip session recall")++ agg = aggregate_answers(+ [+ {+ "question_type": "multi-session",+ "judge_correct": True,+ "is_abstention": False,+ "answer_session_ids": ["a"],+ "retrieved_session_ids": ["a"],+ "retrieve_latency_ms": 10,+ "answer_total_tokens": 5,+ "judge_total_tokens": 2,+ },+ {+ "question_type": "multi-session",+ "judge_correct": False,+ "is_abstention": False,+ "answer_session_ids": ["a"],+ "retrieved_session_ids": [],+ "retrieve_latency_ms": 20,+ "answer_total_tokens": 5,+ "judge_total_tokens": 2,+ },+ {+ "question_type": "multi-session",+ "judge_correct": True,+ "is_abstention": True,+ "answer_session_ids": [],+ "retrieved_session_ids": [],+ },+ ]+ )+ if agg["n_questions"] != 3:+ errors.append("aggregate n_questions wrong")+ if abs(float(agg["headline_judge_accuracy"]) - (2 / 3)) > 1e-9:+ errors.append("headline accuracy should include abstention")+ if abs(float(agg["session_recall_full"]) - 0.5) > 1e-9:+ errors.append("session recall should skip abstention and average non-abs")+ if agg["by_type"]["multi-session"]["n"] != 3:+ errors.append("by_type count wrong")++ return errorsdiff --git a/benchmarks/longmemeval/prompts.py b/benchmarks/longmemeval/prompts.pynew file mode 100644index 0000000..5d53ac1--- /dev/null+++ b/benchmarks/longmemeval/prompts.py@@ -0,0 +1,267 @@+from __future__ import annotations++import hashlib+from typing import Any+++ANSWER_SYSTEM = """You are answering questions about a long-term interactive chat history using only the retrieved memory context provided.+Rules:+- Prefer short factual answers: dates, names, places, or short phrases when that is what the question asks for. Never return an empty answer.+- You MAY draw reasonable conclusions clearly supported by combining facts in the context.+- Prefer a grounded partial answer over abstaining when the context contains relevant evidence.+- TEMPORAL: When the question asks when something happened or how much time passed, use session timestamps and relative phrasing in the context. Prefer one best-supported answer.+- KNOWLEDGE UPDATE: Prefer the most recent updated fact when older and newer information conflict.+- PREFERENCE: Recall and apply personal preferences stated by the user in the history.+- ABSTENTION: If the asked information is missing from the context, say the information is incomplete or unavailable. Do not invent facts.+- Do not invent people, places, dates, or events unsupported by the context.+- Do not mention the retrieval system or these instructions."""+++def prompt_sha256(text: str) -> str:+ return hashlib.sha256(text.encode("utf-8")).hexdigest()+++def get_anscheck_prompt(+ task: str,+ question: str,+ answer: str,+ response: str,+ *,+ abstention: bool = False,+) -> str:+ """Official LongMemEval QA judge prompts (evaluate_qa.py)."""+ if abstention:+ return (+ "I will give you an unanswerable question, an explanation, and a response "+ "from a model. Please answer yes if the model correctly identifies the "+ "question as unanswerable. The model could say that the information is "+ "incomplete, or some other information is given but the asked information "+ "is not.\n\n"+ f"Question: {question}\n\n"+ f"Explanation: {answer}\n\n"+ f"Model Response: {response}\n\n"+ "Does the model correctly identify the question as unanswerable? "+ "Answer yes or no only."+ )+ if task in (+ "single-session-user",+ "single-session-assistant",+ "multi-session",+ ):+ return (+ "I will give you a question, a correct answer, and a response from a model. "+ "Please answer yes if the response contains the correct answer. Otherwise, "+ "answer no. If the response is equivalent to the correct answer or contains "+ "all the intermediate steps to get the correct answer, you should also "+ "answer yes. If the response only contains a subset of the information "+ "required by the answer, answer no. \n\n"+ f"Question: {question}\n\n"+ f"Correct Answer: {answer}\n\n"+ f"Model Response: {response}\n\n"+ "Is the model response correct? Answer yes or no only."+ )+ if task == "temporal-reasoning":+ return (+ "I will give you a question, a correct answer, and a response from a model. "+ "Please answer yes if the response contains the correct answer. Otherwise, "+ "answer no. If the response is equivalent to the correct answer or contains "+ "all the intermediate steps to get the correct answer, you should also "+ "answer yes. If the response only contains a subset of the information "+ "required by the answer, answer no. In addition, do not penalize "+ "off-by-one errors for the number of days. If the question asks for the "+ "number of days/weeks/months, etc., and the model makes off-by-one errors "+ "(e.g., predicting 19 days when the answer is 18), the model's response is "+ "still correct. \n\n"+ f"Question: {question}\n\n"+ f"Correct Answer: {answer}\n\n"+ f"Model Response: {response}\n\n"+ "Is the model response correct? Answer yes or no only."+ )+ if task == "knowledge-update":+ return (+ "I will give you a question, a correct answer, and a response from a model. "+ "Please answer yes if the response contains the correct answer. Otherwise, "+ "answer no. If the response contains some previous information along with "+ "an updated answer, the response should be considered as correct as long as "+ "the updated answer is the required answer.\n\n"+ f"Question: {question}\n\n"+ f"Correct Answer: {answer}\n\n"+ f"Model Response: {response}\n\n"+ "Is the model response correct? Answer yes or no only."+ )+ if task == "single-session-preference":+ return (+ "I will give you a question, a rubric for desired personalized response, "+ "and a response from a model. Please answer yes if the response satisfies "+ "the desired response. Otherwise, answer no. The model does not need to "+ "reflect all the points in the rubric. The response is correct as long as "+ "it recalls and utilizes the user's personal information correctly.\n\n"+ f"Question: {question}\n\n"+ f"Rubric: {answer}\n\n"+ f"Model Response: {response}\n\n"+ "Is the model response correct? Answer yes or no only."+ )+ raise NotImplementedError(f"Unknown question_type for judge: {task}")+++def flatten_triple(triple_obj: Any) -> str:+ if isinstance(triple_obj, dict):+ identified = triple_obj.get("identified_entity", "")+ triple = triple_obj.get("triple")+ if isinstance(triple, (list, tuple)) and len(triple) >= 5:+ parts = []+ for item in triple:+ if isinstance(item, dict):+ parts.append(+ item.get("name")+ or item.get("type")+ or item.get("description")+ or str(item)+ )+ else:+ parts.append(str(item))+ return f"{identified}: {' | '.join(parts)}" if identified else " | ".join(parts)+ return str(triple_obj)+ return str(triple_obj)+++def flatten_path(path_obj: Any) -> str:+ if not isinstance(path_obj, dict):+ return str(path_obj)+ via = (+ str(path_obj.get("shared_entity_name") or "").strip()+ or str(path_obj.get("shared_entity") or "").strip()+ or "shared entity"+ )+ legs = path_obj.get("legs") or []+ leg_texts = [str(leg).strip() for leg in legs if str(leg).strip()]+ if leg_texts:+ return f"via {via}: " + " --> ".join(leg_texts)+ hubs = [str(h).strip() for h in (path_obj.get("hubs") or []) if str(h).strip()]+ if hubs:+ return f"via {via}: hubs {' + '.join(hubs)}"+ return f"via {via}"+++def enrich_paths_from_triples(context: dict[str, Any]) -> dict[str, Any]:+ paths = context.get("paths") or []+ triples = context.get("triples") or []+ if not paths:+ return context+ if any(isinstance(p, dict) and p.get("legs") for p in paths):+ return context++ hub_texts: dict[str, str] = {}+ for triple in triples:+ if not isinstance(triple, dict):+ continue+ raw = triple.get("triple")+ if not isinstance(raw, (list, tuple)) or len(raw) < 3:+ continue+ event = raw[2]+ hub = ""+ if isinstance(event, dict):+ hub = str(event.get("uuid") or "").strip()+ parts = []+ for item in raw:+ if isinstance(item, dict):+ parts.append(+ item.get("name")+ or item.get("type")+ or item.get("description")+ or str(item)+ )+ else:+ parts.append(str(item))+ text = " | ".join(parts).strip()+ if hub and text and hub not in hub_texts:+ hub_texts[hub] = text+ if not hub_texts:+ return context++ enriched: list[dict[str, Any]] = []+ for path in paths:+ if not isinstance(path, dict):+ continue+ hubs = [str(h) for h in (path.get("hubs") or []) if h]+ if len(hubs) < 2 or not all(h in hub_texts for h in hubs):+ continue+ enriched.append({**path, "legs": [hub_texts[h] for h in hubs]})+ if not enriched:+ return context+ out = dict(context)+ out["paths"] = enriched+ return out+++def build_context_block(context: dict[str, Any]) -> str:+ context = enrich_paths_from_triples(context)+ text_context = (context.get("text_context") or "").strip()+ triples = context.get("triples") or []+ paths = context.get("paths") or []+ historical = context.get("historical_context") or []+ source_passages = context.get("source_passages") or []+ topics = context.get("topics") or []++ sections = ["## Retrieved text context", text_context or "(empty)"]+ if topics:+ sections.append("## Topics")+ for topic in topics[:20]:+ if isinstance(topic, dict):+ label = topic.get("label") or topic.get("topic_id") or topic+ sessions = topic.get("sessions") or []+ sess = ", ".join(str(s) for s in sessions[:12])+ sections.append(f"- {label}" + (f" ({sess})" if sess else ""))+ else:+ sections.append(f"- {topic}")+ if source_passages:+ sections.append("## Source passages")+ for passage in source_passages[:20]:+ sections.append(f"- {passage}")+ if paths:+ sections.append("## Event hub paths")+ for path in paths[:40]:+ sections.append(f"- {flatten_path(path)}")+ if triples:+ sections.append("## Graph triples")+ for t in triples[:80]:+ sections.append(f"- {flatten_triple(t)}")+ if historical:+ sections.append("## Historical context")+ for h in historical[:40]:+ sections.append(f"- {h}")+ return "\n".join(sections)+++def build_answer_messages(+ question: str,+ context: dict[str, Any],+ *,+ question_date: str | None = None,+) -> list[dict[str, str]]:+ date_line = ""+ if question_date:+ date_line = f"## Question date\n{question_date}\n\n"+ user = (+ f"{build_context_block(context)}\n\n"+ f"{date_line}"+ f"## Question\n{question}\n\n"+ "Answer:"+ )+ return [+ {"role": "system", "content": ANSWER_SYSTEM},+ {"role": "user", "content": user},+ ]+++def judge_prompt_sha256() -> str:+ samples = [+ get_anscheck_prompt("multi-session", "q", "a", "r", abstention=False),+ get_anscheck_prompt("temporal-reasoning", "q", "a", "r", abstention=False),+ get_anscheck_prompt("knowledge-update", "q", "a", "r", abstention=False),+ get_anscheck_prompt(+ "single-session-preference", "q", "a", "r", abstention=False+ ),+ get_anscheck_prompt("multi-session", "q", "a", "r", abstention=True),+ ]+ return prompt_sha256("\n---\n".join(samples))diff --git a/benchmarks/longmemeval/provenance.py b/benchmarks/longmemeval/provenance.pynew file mode 100644index 0000000..676eb41--- /dev/null+++ b/benchmarks/longmemeval/provenance.py@@ -0,0 +1,71 @@+from __future__ import annotations++import hashlib+import subprocess+from pathlib import Path+from typing import Any++from longmemeval.config import BENCHMARKS_ROOT, Settings, model_family+from longmemeval.prompts import ANSWER_SYSTEM, judge_prompt_sha256, prompt_sha256++REPO_ROOT = BENCHMARKS_ROOT.parent+++def _git(*args: str) -> str | None:+ try:+ result = subprocess.run(+ ["git", *args],+ cwd=REPO_ROOT,+ capture_output=True,+ text=True,+ timeout=10,+ )+ except (OSError, subprocess.SubprocessError):+ return None+ if result.returncode != 0:+ return None+ return result.stdout.strip()+++def git_sha() -> str | None:+ return _git("rev-parse", "HEAD")+++def git_dirty() -> bool | None:+ status = _git("status", "--porcelain")+ if status is None:+ return None+ return bool(status.strip())+++def file_sha256(path: Path) -> str | None:+ if not path.exists():+ return None+ digest = hashlib.sha256()+ with path.open("rb") as handle:+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def build_provenance(settings: Settings) -> dict[str, Any]:+ return {+ "git_sha": git_sha(),+ "git_dirty": git_dirty(),+ "answer_prompt_sha256": prompt_sha256(ANSWER_SYSTEM),+ "judge_prompt_sha256": judge_prompt_sha256(),+ "dataset_path": str(settings.dataset_path),+ "dataset_sha256": file_sha256(settings.dataset_path),+ "variant": settings.variant,+ "answer_model": settings.answer_model,+ "answer_model_family": model_family(settings.answer_model),+ "answer_llm_base_url": settings.llm_base_url,+ "judge_model": settings.judge_model,+ "judge_model_family": model_family(settings.judge_model),+ "judge_provider": settings.judge_provider(),+ "judge_shares_answer_family": settings.judge_shares_answer_family,+ "bench_profile": settings.bench_profile,+ "sc_samples": settings.sc_samples,+ "sc_temperature": settings.sc_temperature,+ "gap_fill": settings.gap_fill,+ }diff --git a/benchmarks/longmemeval/report.py b/benchmarks/longmemeval/report.pynew file mode 100644index 0000000..8d5ec38--- /dev/null+++ b/benchmarks/longmemeval/report.py@@ -0,0 +1,542 @@+from __future__ import annotations++import json+import tempfile+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++from rich.console import Console+from rich.table import Table++from longmemeval.config import BENCHMARKS_ROOT+from longmemeval.ingest import load_jsonl+from longmemeval.metrics import aggregate_answers++console = Console()++REPORTS_PATH = BENCHMARKS_ROOT / "REPORTS.json"+_BENCHMARK_ID = "longmemeval"+_BENCHMARK_NAME = "LongMemEval"+_REPORTS_DESCRIPTION = (+ "BrainAPI benchmark results. Top published scores across suites. "+ "Updated when a suite evaluate/report completes successfully."+)+_LEADERBOARD_ENTRY_KEYS = (+ "run_id",+ "track",+ "scope",+ "variant",+ "n_questions",+ "headline_judge_accuracy",+ "headline_judge_accuracy_pct",+ "answer_model",+ "judge_model",+ "bench_profile",+ "git_sha",+ "report_path",+ "recorded_at",+)+_MIN_LEADERBOARD_N = 50+++def _latest_ingest_rows(ingest_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:+ latest: dict[tuple[str, str], dict[str, Any]] = {}+ for row in ingest_rows:+ key = (+ str(row.get("question_id") or ""),+ str(row.get("unit_id") or row.get("session_id") or ""),+ )+ latest[key] = row+ return list(latest.values())+++def _latest_answer_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:+ latest: dict[str, dict[str, Any]] = {}+ for row in rows:+ key = str(row.get("question_id") or "")+ if key:+ latest[key] = row+ return list(latest.values())+++def build_report(run_dir: Path) -> dict[str, Any]:+ answers_path = run_dir / "answers.jsonl"+ ingest_path = run_dir / "ingest.jsonl"+ manifest_path = run_dir / "manifest.json"++ all_answer_rows = load_jsonl(answers_path)+ scored_rows = [r for r in all_answer_rows if not r.get("error")]+ answers = _latest_answer_rows(scored_rows)+ errored = [r for r in all_answer_rows if r.get("error")]+ duplicates_dropped = len(scored_rows) - len(answers)+ empty_predictions = sum(+ 1 for r in answers if not str(r.get("prediction") or "").strip()+ )+ truncated_rows = sum(1 for r in answers if r.get("context_truncated"))+ ingest_rows = load_jsonl(ingest_path)+ ingest_latest = _latest_ingest_rows(ingest_rows)+ manifest = {}+ if manifest_path.exists():+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))++ metrics = aggregate_answers(answers)+ ingest_summary = {+ "n": len(ingest_latest),+ "completed": sum(1 for r in ingest_latest if r.get("status") == "completed"),+ "partial_failed": sum(+ 1 for r in ingest_latest if r.get("status") == "partial_failed"+ ),+ "failed": sum(1 for r in ingest_latest if r.get("status") == "failed"),+ "dry_run": sum(1 for r in ingest_latest if r.get("status") == "dry_run"),+ "attempts": len(ingest_rows),+ }+ wait_vals = [+ float(r["wait_latency_ms"])+ for r in ingest_latest+ if r.get("wait_latency_ms") is not None+ ]+ token_vals = [+ int(r["total_llm_tokens"])+ for r in ingest_latest+ if r.get("total_llm_tokens") is not None+ ]+ if wait_vals:+ ordered = sorted(wait_vals)+ mid = len(ordered) // 2+ ingest_summary["wait_latency_p50_ms"] = (+ ordered[mid]+ if len(ordered) % 2+ else (ordered[mid - 1] + ordered[mid]) / 2+ )+ ingest_summary["wait_latency_mean_ms"] = sum(ordered) / len(ordered)+ if token_vals:+ ingest_summary["mean_ingest_llm_tokens"] = sum(token_vals) / len(token_vals)+ ingest_summary["total_ingest_llm_tokens"] = sum(token_vals)+ stage_keys = (+ "scout_total_tokens",+ "architect_total_tokens",+ "janitor_total_tokens",+ "observations_total_tokens",+ "consolidation_total_tokens",+ )+ stage_sums = {k: 0 for k in stage_keys}+ stage_n = 0+ for row in ingest_latest:+ cost = row.get("cost") or {}+ stages = cost.get("stages") if isinstance(cost, dict) else None+ if not isinstance(stages, dict):+ continue+ stage_n += 1+ for key in stage_keys:+ stage_name = key.replace("_total_tokens", "")+ stage_sums[key] += int(+ (stages.get(stage_name) or {}).get("total_tokens") or 0+ )+ if stage_n:+ ingest_summary["mean_tokens_by_stage"] = {+ k.replace("_total_tokens", ""): stage_sums[k] / stage_n+ for k in stage_keys+ }++ integrity = {+ "rows_in_file": len(all_answer_rows),+ "errored_rows": len(errored),+ "scored_rows": len(answers),+ "duplicates_dropped": duplicates_dropped,+ "empty_predictions": empty_predictions,+ "rows_with_truncated_context": truncated_rows,+ }++ report: dict[str, Any] = {+ "run_dir": str(run_dir),+ "manifest": manifest,+ "ingest": ingest_summary,+ "integrity": integrity,+ "metrics": metrics,+ "status": "ok",+ "warnings": [],+ }+ if ingest_summary["partial_failed"] or ingest_summary["failed"]:+ report["warnings"].append(+ "Some ingest units failed or partially failed; scores may be skewed."+ )+ if duplicates_dropped:+ report["warnings"].append(+ f"{duplicates_dropped} duplicate answer rows dropped."+ )+ if errored:+ report["warnings"].append(+ f"{len(errored)} rows errored and are excluded from every metric."+ )+ if truncated_rows:+ report["warnings"].append(+ f"{truncated_rows} rows had a retrieved-context channel truncated "+ "before logging."+ )++ failure_reasons: list[str] = []+ if not answers:+ failure_reasons.append("no answer rows were scored")+ else:+ if empty_predictions == len(answers):+ failure_reasons.append("every scored answer is empty")+ if metrics.get("headline_judge_accuracy") == 0.0:+ failure_reasons.append("every scored answer was judged incorrect")+ if failure_reasons:+ report["status"] = "failed"+ report["warnings"].insert(+ 0,+ "RUN FAILED WHOLESALE: "+ + "; ".join(failure_reasons)+ + ". The percentages below are not a measurement of quality.",+ )+ elif errored or duplicates_dropped:+ report["status"] = "degraded"+ return report+++def _pct(value: Any) -> str:+ if value is None:+ return "n/a"+ return f"{100.0 * float(value):.1f}%"+++def _num(value: Any) -> str:+ if value is None:+ return "n/a"+ return f"{float(value):.2f}"+++def _ci(ci: Any) -> str:+ if not isinstance(ci, dict) or ci.get("low") is None or ci.get("high") is None:+ return ""+ return f"[{100.0 * float(ci['low']):.1f}%, {100.0 * float(ci['high']):.1f}%]"+++def render_markdown(report: dict[str, Any]) -> str:+ metrics = report["metrics"]+ manifest = report.get("manifest") or {}+ integrity = report.get("integrity") or {}+ lines = [+ "# LongMemEval BrainAPI Benchmark Report",+ "",+ ]+ if report.get("status") == "failed":+ lines.extend(+ [+ "> **RUN FAILED WHOLESALE — the numbers below are not a measurement.**",+ f"> {report['warnings'][0]}",+ "",+ ]+ )+ elif report.get("status") == "degraded":+ lines.extend(+ [+ "> **Run degraded** — errored or duplicate rows were dropped; "+ "see Warnings.",+ "",+ ]+ )+ lines.extend(+ [+ f"- Run dir: `{report['run_dir']}`",+ f"- Created: {manifest.get('created_at', 'n/a')}",+ f"- Variant: {manifest.get('variant', 'n/a')}",+ f"- Answer model: {manifest.get('answer_model', 'n/a')}",+ f"- Judge model: {manifest.get('judge_model', 'n/a')} "+ f"(provider: {manifest.get('judge_provider', 'n/a')}, "+ f"same family as answerer: "+ f"{manifest.get('judge_shares_answer_family', 'n/a')})",+ f"- Git SHA: {manifest.get('git_sha', 'n/a')} "+ f"(dirty: {manifest.get('git_dirty', 'n/a')})",+ f"- Answer prompt sha256: {manifest.get('answer_prompt_sha256', 'n/a')}",+ f"- Judge prompt sha256: {manifest.get('judge_prompt_sha256', 'n/a')}",+ f"- Dataset sha256: {manifest.get('dataset_sha256', 'n/a')}",+ "",+ "## Run integrity",+ "",+ f"- Rows in answers.jsonl: {integrity.get('rows_in_file', 'n/a')}",+ f"- Scored unique questions: {integrity.get('scored_rows', 'n/a')}",+ f"- Errored rows (excluded): {integrity.get('errored_rows', 'n/a')}",+ f"- Duplicate rows dropped: {integrity.get('duplicates_dropped', 'n/a')}",+ f"- Empty predictions: {integrity.get('empty_predictions', 'n/a')}",+ "",+ "## Headline",+ "",+ f"- Judge accuracy (all questions, incl. abstention): "+ f"**{_pct(metrics.get('headline_judge_accuracy'))}** "+ f"{_ci(metrics.get('headline_judge_accuracy_ci95'))} "+ f"(n={metrics.get('n_questions', 0)})",+ f"- Session recall (full, excl. abstention): "+ f"**{_pct(metrics.get('session_recall_full'))}** "+ f"(n={metrics.get('n_with_evidence', 0)})",+ f"- Session recall (partial+full): "+ f"**{_pct(metrics.get('session_recall_partial'))}**",+ f"- Abstention accuracy: "+ f"**{_pct((metrics.get('abstention') or {}).get('accuracy'))}** "+ f"(n={(metrics.get('abstention') or {}).get('n', 0)})",+ "",+ "## Per question type",+ "",+ "| Type | N | Judge Acc | 95% CI | Session recall full |",+ "|---|---:|---:|---|---:|",+ ]+ )+ for qtype, bucket in (metrics.get("by_type") or {}).items():+ lines.append(+ f"| {qtype} | {bucket.get('n', 0)} | "+ f"{_pct(bucket.get('judge_accuracy'))} | "+ f"{_ci(bucket.get('judge_accuracy_ci95'))} | "+ f"{_pct(bucket.get('session_recall_full'))} |"+ )++ lat = metrics.get("retrieval_latency_ms") or {}+ lines.extend(+ [+ "",+ "## Latency & tokens",+ "",+ f"- Retrieval latency p50: {_num(lat.get('p50'))} ms",+ f"- Retrieval latency p95: {_num(lat.get('p95'))} ms",+ f"- Retrieval latency mean: {_num(lat.get('mean'))} ms",+ f"- Total LLM tokens (answer+judge): {metrics.get('total_llm_tokens', 0)}",+ "",+ "## Ingest",+ "",+ f"- Units: {report['ingest'].get('n', 0)}",+ f"- Completed: {report['ingest'].get('completed', 0)}",+ f"- Partial failed: {report['ingest'].get('partial_failed', 0)}",+ f"- Failed: {report['ingest'].get('failed', 0)}",+ *(+ [+ f"- Ingest wait latency p50: "+ f"{_num(report['ingest'].get('wait_latency_p50_ms'))} ms"+ ]+ if report["ingest"].get("wait_latency_p50_ms") is not None+ else []+ ),+ *(+ [+ f"- Mean ingest LLM tokens/unit: "+ f"{_num(report['ingest'].get('mean_ingest_llm_tokens'))}"+ ]+ if report["ingest"].get("mean_ingest_llm_tokens") is not None+ else []+ ),+ *(+ [+ f"- Total ingest LLM tokens: "+ f"{report['ingest'].get('total_ingest_llm_tokens')}"+ ]+ if report["ingest"].get("total_ingest_llm_tokens") is not None+ else []+ ),+ "",+ ]+ )+ if report.get("warnings"):+ lines.append("## Warnings")+ lines.append("")+ for warning in report["warnings"]:+ lines.append(f"- {warning}")+ lines.append("")+ return "\n".join(lines)+++def entry_from_report(report: dict[str, Any]) -> dict[str, Any] | None:+ if report.get("status") == "failed":+ return None+ metrics = report.get("metrics") or {}+ headline = metrics.get("headline_judge_accuracy")+ n_questions = int(metrics.get("n_questions") or 0)+ if headline is None or n_questions < _MIN_LEADERBOARD_N:+ return None++ manifest = report.get("manifest") or {}+ run_id = str(manifest.get("run_id") or "").strip()+ if not run_id:+ run_dir = str(report.get("run_dir") or "").rstrip("/")+ run_id = Path(run_dir).name if run_dir else ""+ if not run_id:+ return None++ variant = str(manifest.get("variant") or "s")+ limit = manifest.get("limit")+ if limit:+ scope = f"{variant}:limit{limit}"+ else:+ scope = variant++ bench_profile = manifest.get("bench_profile") or "product"+ track = "sota" if bench_profile == "sota" else "product"+ acc = float(headline)+ now = datetime.now(timezone.utc).isoformat()++ return {+ "run_id": run_id,+ "track": track,+ "scope": scope,+ "variant": variant,+ "n_questions": n_questions,+ "headline_judge_accuracy": acc,+ "headline_judge_accuracy_pct": round(acc * 100, 2),+ "answer_model": manifest.get("answer_model"),+ "judge_model": manifest.get("judge_model"),+ "bench_profile": bench_profile,+ "git_sha": manifest.get("git_sha"),+ "report_path": f"runs/{run_id}/report.json",+ "recorded_at": now,+ }+++def _empty_reports_ledger() -> dict[str, Any]:+ return {+ "schema_version": 2,+ "updated_at": datetime.now(timezone.utc).isoformat(),+ "description": _REPORTS_DESCRIPTION,+ "benchmarks": {+ _BENCHMARK_ID: {+ "name": _BENCHMARK_NAME,+ "leaderboard": [],+ }+ },+ }+++def _public_leaderboard_entry(row: dict[str, Any]) -> dict[str, Any]:+ return {key: row[key] for key in _LEADERBOARD_ENTRY_KEYS if key in row}+++def _sorted_leaderboard(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:+ leaderboard = [+ _public_leaderboard_entry(row)+ for row in rows+ if isinstance(row, dict) and row.get("run_id")+ ]+ leaderboard.sort(+ key=lambda row: float(row.get("headline_judge_accuracy") or 0.0),+ reverse=True,+ )+ return leaderboard+++def _normalize_reports_ledger(data: dict[str, Any]) -> dict[str, Any]:+ benchmarks: dict[str, Any] = {}+ raw_benchmarks = data.get("benchmarks")+ if isinstance(raw_benchmarks, dict):+ for suite_id, suite in raw_benchmarks.items():+ if not isinstance(suite, dict):+ continue+ benchmarks[str(suite_id)] = {+ "name": suite.get("name") or str(suite_id),+ "leaderboard": _sorted_leaderboard(list(suite.get("leaderboard") or [])),+ }++ if _BENCHMARK_ID not in benchmarks:+ benchmarks[_BENCHMARK_ID] = {+ "name": _BENCHMARK_NAME,+ "leaderboard": [],+ }++ return {+ "schema_version": 2,+ "updated_at": data.get("updated_at")+ or datetime.now(timezone.utc).isoformat(),+ "description": data.get("description") or _REPORTS_DESCRIPTION,+ "benchmarks": benchmarks,+ }+++def update_reports_json(report: dict[str, Any]) -> None:+ entry = entry_from_report(report)+ if entry is None:+ return++ if REPORTS_PATH.exists():+ data = json.loads(REPORTS_PATH.read_text(encoding="utf-8"))+ else:+ data = _empty_reports_ledger()++ data = _normalize_reports_ledger(data)++ suite = data["benchmarks"].setdefault(+ _BENCHMARK_ID,+ {"name": _BENCHMARK_NAME, "leaderboard": []},+ )+ run_id = entry["run_id"]+ leaderboard = list(suite.get("leaderboard") or [])+ lb_idx = next(+ (i for i, row in enumerate(leaderboard) if row.get("run_id") == run_id),+ None,+ )+ if lb_idx is not None:+ leaderboard[lb_idx] = entry+ else:+ leaderboard.append(entry)+ suite["name"] = _BENCHMARK_NAME+ suite["leaderboard"] = _sorted_leaderboard(leaderboard)+ data["benchmarks"][_BENCHMARK_ID] = suite+ data["updated_at"] = datetime.now(timezone.utc).isoformat()+ data["description"] = _REPORTS_DESCRIPTION++ REPORTS_PATH.parent.mkdir(parents=True, exist_ok=True)+ with tempfile.NamedTemporaryFile(+ "w",+ encoding="utf-8",+ dir=str(REPORTS_PATH.parent),+ delete=False,+ suffix=".tmp",+ ) as tmp:+ json.dump(data, tmp, indent=2)+ tmp.write("\n")+ tmp_path = Path(tmp.name)+ tmp_path.replace(REPORTS_PATH)+++def write_report(run_dir: Path) -> dict[str, Any]:+ report = build_report(run_dir)+ (run_dir / "report.json").write_text(+ json.dumps(report, indent=2), encoding="utf-8"+ )+ (run_dir / "report.md").write_text(render_markdown(report), encoding="utf-8")+ if report.get("status") == "ok":+ update_reports_json(report)+ return report+++def print_report_table(report: dict[str, Any]) -> None:+ metrics = report.get("metrics") or {}+ table = Table(title="LongMemEval results")+ table.add_column("Metric")+ table.add_column("Value")+ table.add_row(+ "Headline judge accuracy",+ f"{_pct(metrics.get('headline_judge_accuracy'))} "+ f"(n={metrics.get('n_questions', 0)})",+ )+ table.add_row(+ "Session recall (full)",+ f"{_pct(metrics.get('session_recall_full'))} "+ f"(n={metrics.get('n_with_evidence', 0)})",+ )+ table.add_row(+ "Abstention accuracy",+ f"{_pct((metrics.get('abstention') or {}).get('accuracy'))} "+ f"(n={(metrics.get('abstention') or {}).get('n', 0)})",+ )+ table.add_row("Status", str(report.get("status")))+ console.print(table)+ by_type = metrics.get("by_type") or {}+ if by_type:+ type_table = Table(title="By question type")+ type_table.add_column("Type")+ type_table.add_column("N", justify="right")+ type_table.add_column("Acc")+ for qtype, bucket in by_type.items():+ type_table.add_row(+ qtype,+ str(bucket.get("n", 0)),+ _pct(bucket.get("judge_accuracy")),+ )+ console.print(type_table)diff --git a/benchmarks/longmemeval/sota.py b/benchmarks/longmemeval/sota.pynew file mode 100644index 0000000..a1ef4a7--- /dev/null+++ b/benchmarks/longmemeval/sota.py@@ -0,0 +1,100 @@+from __future__ import annotations++import re+from collections import Counter+from dataclasses import dataclass+from typing import Any+++_NORMALIZE_RE = re.compile(r"[^a-z0-9\s]+")+_WS_RE = re.compile(r"\s+")+++def normalize_answer(text: str) -> str:+ lowered = (text or "").strip().lower()+ lowered = _NORMALIZE_RE.sub(" ", lowered)+ return _WS_RE.sub(" ", lowered).strip()+++def majority_vote(answers: list[str]) -> str:+ nonempty = [a.strip() for a in answers if (a or "").strip()]+ if not nonempty:+ return ""+ counts = Counter(normalize_answer(a) for a in nonempty)+ winner_norm, _ = max(+ counts.items(),+ key=lambda item: (+ item[1],+ -next(i for i, a in enumerate(nonempty) if normalize_answer(a) == item[0]),+ ),+ )+ for answer in nonempty:+ if normalize_answer(answer) == winner_norm:+ return answer+ return nonempty[0]+++@dataclass(frozen=True)+class GapFillPlan:+ needs_retry: bool+ reformulated_query: str+ missing_slots: list[str]+++def plan_gap_fill(+ question: str,+ draft_answer: str,+ context: dict[str, Any],+) -> GapFillPlan:+ draft = (draft_answer or "").strip()+ text_blob = " ".join(+ [+ str(context.get("text_context") or ""),+ " ".join(str(p) for p in (context.get("source_passages") or [])[:20]),+ " ".join(str(t) for t in (context.get("triples") or [])[:40]),+ ]+ ).lower()+ abstained = bool(+ re.search(+ r"not mentioned|no information|incomplete|unknown|cannot (tell|determine)|"+ r"not available|do not have",+ draft,+ re.I,+ )+ )+ q_tokens = [t for t in re.findall(r"[a-z0-9]{3,}", question.lower()) if t]+ covered = sum(1 for t in q_tokens if t in text_blob) if q_tokens else 0+ coverage = covered / len(q_tokens) if q_tokens else 1.0+ missing = [t for t in q_tokens if t not in text_blob][:8]+ needs = abstained or (coverage < 0.45 and len(draft) < 40)+ reform = question.strip()+ if missing:+ reform = f"{question.strip()} (focus: {', '.join(missing[:5])})"+ return GapFillPlan(+ needs_retry=needs,+ reformulated_query=reform,+ missing_slots=missing,+ )+++def merge_contexts(+ primary: dict[str, Any], secondary: dict[str, Any]+) -> dict[str, Any]:+ out = dict(primary)+ for key in ("source_passages", "historical_context", "triples", "paths"):+ a = list(primary.get(key) or [])+ b = list(secondary.get(key) or [])+ seen: set[str] = set()+ merged: list[Any] = []+ for item in a + b:+ sig = str(item)+ if sig in seen:+ continue+ seen.add(sig)+ merged.append(item)+ out[key] = merged+ text_a = str(primary.get("text_context") or "")+ text_b = str(secondary.get("text_context") or "")+ if text_b and text_b not in text_a:+ out["text_context"] = (text_a + "\n" + text_b).strip()+ return out