BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
from __future__ import annotations
from typing import Any
from search.dataset import split_corpus
from search.metrics import aggregate_query_metrics, mrr, ndcg_at_k, recall_at_k
from search.rank_pool import doc_texts
DEFAULT_BASE = "sentence-transformers/all-MiniLM-L6-v2"
def score_ranked_first_stage(
ranked: list[str],
query_row: dict[str, Any],
*,
ks: tuple[int, ...],
) -> dict[str, float]:
gold = {str(item) for item in (query_row.get("gold_doc_ids") or [])}
grades = {
str(doc_id): float(gain)
for doc_id, gain in (query_row.get("gold_grades") or {}).items()
if float(gain or 0) > 0
}
if grades:
gold |= set(grades)
metrics = {f"recall@{cut}": recall_at_k(ranked, gold, cut) for cut in ks}
ndcg_ks = tuple(dict.fromkeys((*ks, 10, 20)))
for cut in ndcg_ks:
metrics[f"ndcg@{cut}"] = ndcg_at_k(ranked, gold, cut, grades=grades or None)
metrics["ndcg"] = ndcg_at_k(ranked, gold, max(len(ranked), 1), grades=grades or None)
metrics["mrr"] = mrr(ranked, gold)
return metrics
def hard_negative_texts_from_eval(
eval_result: dict[str, Any],
texts: dict[str, str],
queries: list[dict[str, Any]],
) -> list[str]:
by_qid = {str(query.get("qid") or ""): query for query in queries}
bank: list[str] = []
seen: set[str] = set()
for row in eval_result.get("queries") or []:
qid = str(row.get("qid") or "")
source = by_qid.get(qid) or row
gold = {str(item) for item in (source.get("gold_doc_ids") or [])}
grades = source.get("gold_grades") or {}
if isinstance(grades, dict):
gold |= {
str(doc_id)
for doc_id, gain in grades.items()
if float(gain or 0) > 0
}
for doc_id in row.get("hit_ids") or []:
key = str(doc_id)
if key in gold or key in seen:
continue
text = texts.get(key) or ""
if not text:
continue
seen.add(key)
bank.append(text)
return bank
def retrieve_dense(
model: Any,
queries: list[dict[str, Any]],
docs: list[dict[str, Any]],
*,
k: int,
batch_size: int = 32,
) -> tuple[dict[str, list[str]], float]:
import time
import numpy as np
texts = doc_texts(docs)
doc_ids = [str(doc.get("doc_id") or "") for doc in docs if doc.get("doc_id")]
doc_passages = [texts.get(doc_id) or doc_id for doc_id in doc_ids]
start = time.perf_counter()
doc_emb = model.encode(
doc_passages,
batch_size=batch_size,
convert_to_numpy=True,
normalize_embeddings=True,
show_progress_bar=False,
)
q_texts = [str(query.get("query") or "") for query in queries]
q_emb = model.encode(
q_texts,
batch_size=batch_size,
convert_to_numpy=True,
normalize_embeddings=True,
show_progress_bar=False,
)
elapsed_ms = (time.perf_counter() - start) * 1000.0
scores = np.matmul(q_emb, doc_emb.T)
ranked_by_qid: dict[str, list[str]] = {}
cut = max(1, min(int(k), len(doc_ids)))
for index, query in enumerate(queries):
order = np.argsort(-scores[index])[:cut]
qid = str(query.get("qid") or "")
ranked_by_qid[qid] = [doc_ids[int(pos)] for pos in order]
return ranked_by_qid, elapsed_ms
def evaluate_dense(
ranked_by_qid: dict[str, list[str]],
queries: list[dict[str, Any]],
*,
ks: tuple[int, ...],
encode_ms: float,
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
per_query: list[dict[str, Any]] = []
per_q_ms = encode_ms / max(1, len(queries))
for query in queries:
qid = str(query.get("qid") or "")
ranked = ranked_by_qid.get(qid) or []
metrics = score_ranked_first_stage(ranked, query, ks=ks)
per_query.append(
{
"qid": qid,
"query": query.get("query"),
"slice": query.get("slice") or "unspecified",
"gold_doc_ids": list(query.get("gold_doc_ids") or []),
"gold_grades": query.get("gold_grades") or {},
"hit_ids": ranked,
"metrics": metrics,
"n_hits": len(ranked),
"retrieve_ms": per_q_ms,
"embed_ms": per_q_ms,
"client_wall_ms": per_q_ms,
}
)
metrics = aggregate_query_metrics(per_query, ks=ks) if per_query else {}
return metrics, per_query
def run_local_dense(
rows: list[dict[str, Any]],
*,
model_name: str,
dataset_name: str,
k: int = 50,
ks: tuple[int, ...] = (5, 10, 20, 50),
brain_id: str | None = None,
protocol: str | None = None,
) -> dict[str, Any]:
from sentence_transformers import SentenceTransformer
docs, queries = split_corpus(rows)
model = SentenceTransformer(model_name)
ranked_by_qid, encode_ms = retrieve_dense(model, queries, docs, k=k)
metrics, per_query = evaluate_dense(
ranked_by_qid, queries, ks=ks, encode_ms=encode_ms
)
return {
"status": "ok",
"brain_id": brain_id or "harness-local",
"dataset": dataset_name,
"fusion": "none",
"rerank": "none",
"channels": ["harness-dense"],
"rank_pool": False,
"skip_enrichment": True,
"k": k,
"ks": list(ks),
"n_docs": len(docs),
"n_queries": len(queries),
"n_docs_mapped": len(docs),
"ce_model": model_name,
"protocol": protocol
or (
"local dual-encoder over search_esci_74.jsonl; not BrainAPI embeddings; "
"not Reddy 0.857"
),
"metrics": metrics,
"queries": per_query,
"ingest": {"status": "completed", "n_docs": 0, "tasks": [], "reused": True},
}