BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
from __future__ import annotations
import re
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from rich.console import Console
from rich.progress import Progress
from locomo.answer import answer_question
from locomo.client import BrainAPIClient
from locomo.config import Settings
from locomo.dataset import brain_id_for
from locomo.ingest import append_jsonl, load_jsonl
from locomo.judge import judge_answer
from locomo.metrics import evidence_coverage, overlap_scores
from locomo.prompts import (
attach_image_cues,
enrich_paths_from_triples,
flatten_path,
flatten_triple,
)
from locomo.sota import merge_contexts, plan_gap_fill, plan_selective_second_reflect
console = Console()
_write_lock = threading.Lock()
_GRAPH_CONTEXT_CAP = 60000
_TRIPLES_CAP = 500
_PASSAGE_PREFIX = "[passage] "
_SESSION_RE = re.compile(r"session_(\d+)", re.IGNORECASE)
_DIALOG_RE = re.compile(r"\bD(\d+):", re.IGNORECASE)
@dataclass
class AnswerRecord:
sample_id: str
brain_id: str
qa_index: int
question: str
gold: str
prediction: str
category: int
evidence: list[str]
judge_correct: bool
judge_reason: str
judge_raw: str
f1: float | None
bleu1: float | None
retrieve_latency_ms: float | None
answer_latency_ms: float | None
judge_latency_ms: float | None
answer_model: str | None
judge_model: str | None
answer_total_tokens: int | None
judge_total_tokens: int | None
graph_context: str
triples: list[str] = field(default_factory=list)
paths: list[str] = field(default_factory=list)
source_passages: list[str] = field(default_factory=list)
historical_context: list[str] = field(default_factory=list)
retrieved_session_ids: list[str] = field(default_factory=list)
retrieved_session_ids_graph: list[str] = field(default_factory=list)
retrieved_session_ids_passages: list[str] = field(default_factory=list)
context_truncated: dict[str, int] = field(default_factory=dict)
is_adversarial: bool = False
error: str | None = None
timestamp: str = ""
def _extract_session_ids(*blobs: Any) -> list[str]:
found: list[str] = []
seen: set[str] = set()
for blob in blobs:
if isinstance(blob, (list, tuple)):
text = "\n".join(str(item) for item in blob)
else:
text = str(blob or "")
for match in _SESSION_RE.finditer(text):
sid = f"session_{match.group(1)}"
if sid not in seen:
seen.add(sid)
found.append(sid)
for match in _DIALOG_RE.finditer(text):
sid = f"session_{match.group(1)}"
if sid not in seen:
seen.add(sid)
found.append(sid)
return found
def _graph_channel(text_context: str, source_passages: list[str]) -> str:
passage_block = "\n".join(f"{_PASSAGE_PREFIX}{p}" for p in source_passages)
if passage_block and text_context.startswith(passage_block):
return text_context[len(passage_block) :].lstrip("\n")
return "\n".join(
line
for line in text_context.splitlines()
if not line.startswith(_PASSAGE_PREFIX)
)
def extract_context_channels(context: dict[str, Any]) -> dict[str, Any]:
context = enrich_paths_from_triples(context)
source_passages = [str(p) for p in (context.get("source_passages") or []) if p]
historical_context = [
str(h) for h in (context.get("historical_context") or []) if h
]
triples = [flatten_triple(t) for t in (context.get("triples") or []) if t]
paths = [flatten_path(p) for p in (context.get("paths") or []) if p]
graph_context = _graph_channel(
str(context.get("text_context") or ""), source_passages
)
session_ids_graph = _extract_session_ids(graph_context, triples, paths)
session_ids_passages = _extract_session_ids(source_passages, historical_context)
truncated: dict[str, int] = {}
if len(graph_context) > _GRAPH_CONTEXT_CAP:
truncated["graph_context"] = len(graph_context)
graph_context = graph_context[:_GRAPH_CONTEXT_CAP]
if len(triples) > _TRIPLES_CAP:
truncated["triples"] = len(triples)
triples = triples[:_TRIPLES_CAP]
return {
"graph_context": graph_context,
"triples": triples,
"paths": paths,
"source_passages": source_passages,
"historical_context": historical_context,
"retrieved_session_ids": _extract_session_ids(
session_ids_passages, session_ids_graph
),
"retrieved_session_ids_graph": session_ids_graph,
"retrieved_session_ids_passages": session_ids_passages,
"context_truncated": truncated,
}
def _utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
def qa_gold(qa: dict[str, Any]) -> tuple[str, bool]:
answer = qa.get("answer")
if answer is not None and str(answer).strip():
return str(answer), int(qa.get("category") or 0) == 5
adversarial = qa.get("adversarial_answer")
if adversarial is not None:
return str(adversarial), True
return "", int(qa.get("category") or 0) == 5
def qa_key(sample_id: str, qa_index: int) -> str:
return f"{sample_id}::{qa_index}"
def completed_qa_keys(answers_path: Path) -> set[str]:
done = set()
for row in load_jsonl(answers_path):
if row.get("error"):
continue
if row.get("prediction") is None:
continue
done.add(qa_key(str(row.get("sample_id")), int(row.get("qa_index"))))
return done
def iter_qa_jobs(
samples: list[dict[str, Any]],
*,
categories: set[int] | None = None,
skip_adversarial: bool = True,
limit: int | None = None,
brain_override: str | None = None,
) -> list[dict[str, Any]]:
if brain_override and len(samples) > 1:
raise SystemExit("--brain can only be used with a single --sample")
jobs: list[dict[str, Any]] = []
for sample in samples:
sample_id = str(sample["sample_id"])
brain_id = brain_override or brain_id_for(sample_id)
for idx, qa in enumerate(sample.get("qa") or []):
cat = int(qa.get("category") or 0)
if skip_adversarial and cat == 5:
continue
if categories is not None and cat not in categories:
continue
jobs.append(
{
"sample_id": sample_id,
"brain_id": brain_id,
"qa_index": idx,
"qa": qa,
"sample": sample,
}
)
if limit is not None and len(jobs) >= limit:
return jobs
return jobs
def selftest_records() -> list[str]:
errors: list[str] = []
passages = [
"Conversation between Caroline and Melanie.\nSession id: session_9.\n"
+ ("filler " * 4000),
"Session id: session_11. Caroline talked about the marathon.",
]
context = {
"text_context": "\n".join(
[f"{_PASSAGE_PREFIX}{p}" for p in passages]
+ [
"Melanie | PUT | keys | IN | slipper (session_3)",
"[dossier] Melanie: runs charity races (session_3)",
]
),
"source_passages": passages,
"historical_context": ["Session id: session_11. Earlier chunk."],
"triples": [
{
"identified_entity": "Melanie",
"triple": ["Melanie", "PUT", "keys", "IN", "slipper"],
}
],
"paths": [
{
"hubs": ["event-a", "event-b"],
"shared_entity_name": "Melanie",
"legs": [
"Actor: Melanie | MADE | Event: put keys | TARGETED | Target: slipper",
"Actor: Melanie | MADE | Event: run | TARGETED | Target: race",
],
}
],
}
channels = extract_context_channels(context)
record = AnswerRecord(
sample_id="conv-26",
brain_id="locomoconv26",
qa_index=0,
question="Where did Melanie put the keys?",
gold="In Melanie's slipper",
prediction="In Melanie's slipper",
category=4,
evidence=["D3:2"],
judge_correct=True,
judge_reason="",
judge_raw="",
f1=1.0,
bleu1=1.0,
retrieve_latency_ms=1.0,
answer_latency_ms=1.0,
judge_latency_ms=1.0,
answer_model="answer-model",
judge_model="judge-model",
answer_total_tokens=1,
judge_total_tokens=1,
error=None,
timestamp=_utc_now(),
**channels,
)
row = asdict(record)
if _PASSAGE_PREFIX in row["graph_context"]:
errors.append("graph_context must not contain passage lines")
if "PUT" not in row["graph_context"]:
errors.append("graph fact line missing from graph_context")
if not row["triples"]:
errors.append("triples must be logged as their own field")
if "slipper" not in row["triples"][0]:
errors.append("triples must be flattened to readable text")
if not row["paths"]:
errors.append("paths must be logged as their own field")
if "via Melanie" not in row["paths"][0] or "-->" not in row["paths"][0]:
errors.append("paths must flatten to via-entity leg chains")
if row["retrieved_session_ids_graph"] != ["session_3"]:
errors.append(
f"graph channel session ids wrong: {row['retrieved_session_ids_graph']}"
)
if sorted(row["retrieved_session_ids_passages"]) != ["session_11", "session_9"]:
errors.append(
"passage channel session ids wrong: "
f"{row['retrieved_session_ids_passages']}"
)
if sorted(row["retrieved_session_ids"]) != [
"session_11",
"session_3",
"session_9",
]:
errors.append("combined session ids should be the union of both channels")
if evidence_coverage(row, "graph") != "full":
errors.append("graph-only evidence should score full recall on the graph channel")
if evidence_coverage(row, "passages") != "none":
errors.append("graph-only evidence should score no recall on the passage channel")
long_context = dict(context)
long_context["text_context"] = "x" * (_GRAPH_CONTEXT_CAP + 10)
long_context["source_passages"] = []
capped = extract_context_channels(long_context)
if len(capped["graph_context"]) != _GRAPH_CONTEXT_CAP:
errors.append("graph_context should be capped")
if capped["context_truncated"].get("graph_context") != _GRAPH_CONTEXT_CAP + 10:
errors.append("truncation must be recorded with the original size")
gold, adversarial = qa_gold(
{"category": 5, "adversarial_answer": "No information available"}
)
if gold != "No information available" or not adversarial:
errors.append("adversarial gold must be read from adversarial_answer")
gold, adversarial = qa_gold({"category": 4, "answer": "7 May 2023"})
if gold != "7 May 2023" or adversarial:
errors.append("non-adversarial gold must be read from answer")
return errors
def evaluate_samples(
settings: Settings,
samples: list[dict[str, Any]],
*,
run_dir: Path,
concurrency: int = 2,
categories: set[int] | None = None,
skip_adversarial: bool = True,
limit: int | None = None,
resume: bool = True,
historical_limit: int = 10,
max_passages: int = 8,
max_facts: int = 40,
apply_fact_filter: bool = True,
use_ppr: bool = False,
sufficiency_retry: bool = False,
brain_override: str | None = None,
) -> list[AnswerRecord]:
run_dir.mkdir(parents=True, exist_ok=True)
answers_path = run_dir / "answers.jsonl"
already_done = completed_qa_keys(answers_path) if resume else set()
jobs = iter_qa_jobs(
samples,
categories=categories,
skip_adversarial=skip_adversarial,
limit=limit,
brain_override=brain_override,
)
jobs = [
j
for j in jobs
if qa_key(j["sample_id"], j["qa_index"]) not in already_done
]
if not jobs:
console.print("[green]Nothing to evaluate (all QA already scored).[/green]")
return []
console.print(
f"Evaluating {len(jobs)} QA items "
f"(concurrency={concurrency}, skip_adversarial={skip_adversarial})"
)
records: list[AnswerRecord] = []
def _one(job: dict[str, Any]) -> AnswerRecord:
qa = job["qa"]
question = str(qa.get("question") or "")
gold, is_adversarial = qa_gold(qa)
category = int(qa.get("category") or 0)
evidence = list(qa.get("evidence") or [])
try:
with BrainAPIClient(settings) as client:
retrieved = client.retrieve_context(
question,
job["brain_id"],
historical_limit=historical_limit,
max_passages=max_passages,
max_facts=max_facts,
apply_fact_filter=apply_fact_filter,
use_ppr=use_ppr,
sufficiency_retry=sufficiency_retry,
)
context = retrieved.data or {}
context = attach_image_cues(
context, job.get("sample"), question=question
)
retrieve_ms = retrieved.latency_ms
draft = answer_question(settings, question, context)
if settings.gap_fill:
plan = plan_gap_fill(question, draft.answer, context)
if plan.needs_retry:
with BrainAPIClient(settings) as client:
second = client.retrieve_context(
plan.reformulated_query,
job["brain_id"],
historical_limit=historical_limit,
max_passages=max_passages,
max_facts=max_facts,
apply_fact_filter=apply_fact_filter,
use_ppr=use_ppr,
sufficiency_retry=sufficiency_retry,
)
retrieve_ms = (retrieve_ms or 0) + (second.latency_ms or 0)
context = merge_contexts(context, second.data or {})
context = attach_image_cues(
context, job.get("sample"), question=question
)
answered = answer_question(settings, question, context)
else:
answered = draft
# Cap: at most one selective second reflect after primary gap-fill.
reflect = plan_selective_second_reflect(
question, answered.answer, context
)
if reflect.needs_retry:
answered = answer_question(settings, question, context)
else:
answered = draft
channels = extract_context_channels(context)
judged = judge_answer(
settings,
question,
gold,
answered.answer,
adversarial=is_adversarial,
)
f1, bleu1 = overlap_scores(
answered.answer, gold, adversarial=is_adversarial
)
record = AnswerRecord(
sample_id=job["sample_id"],
brain_id=job["brain_id"],
qa_index=job["qa_index"],
question=question,
gold=gold,
prediction=answered.answer,
category=category,
evidence=evidence,
judge_correct=judged.correct,
judge_reason=judged.reason,
judge_raw=judged.raw,
f1=f1,
bleu1=bleu1,
retrieve_latency_ms=retrieve_ms,
answer_latency_ms=answered.latency_ms,
judge_latency_ms=judged.latency_ms,
answer_model=answered.model,
judge_model=judged.model,
answer_total_tokens=answered.total_tokens,
judge_total_tokens=judged.total_tokens,
is_adversarial=is_adversarial,
error=None,
timestamp=_utc_now(),
**channels,
)
except Exception as exc:
record = AnswerRecord(
sample_id=job["sample_id"],
brain_id=job["brain_id"],
qa_index=job["qa_index"],
question=question,
gold=gold,
prediction="",
category=category,
evidence=evidence,
judge_correct=False,
judge_reason="",
judge_raw="",
f1=None,
bleu1=None,
retrieve_latency_ms=None,
answer_latency_ms=None,
judge_latency_ms=None,
answer_model=None,
judge_model=None,
answer_total_tokens=None,
judge_total_tokens=None,
is_adversarial=is_adversarial,
error=str(exc),
timestamp=_utc_now(),
**extract_context_channels({}),
)
with _write_lock:
append_jsonl(answers_path, asdict(record))
return record
with Progress(console=console) as progress:
task = progress.add_task("evaluate", total=len(jobs))
with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool:
futures = [pool.submit(_one, job) for job in jobs]
for future in as_completed(futures):
records.append(future.result())
progress.advance(task)
ok = sum(1 for r in records if not r.error)
failed = sum(1 for r in records if r.error)
console.print(f"Evaluate done: ok={ok} failed={failed} -> {answers_path}")
return records