BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
from __future__ import annotations
import argparse
import json
import random
from pathlib import Path
from typing import Any
from search.catalog import (
ESCI_CACHE_DIR,
ESCI_EXAMPLES_FILE,
ESCI_GAINS,
ESCI_PRODUCTS_FILE,
PRODUCT_TEXT_MAX_CHARS,
_esci_products,
_iter_parquet,
_stringify_catalog_value,
)
from search.dataset import load_records, split_corpus
DEFAULT_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
REDDY_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
CLASS_ORDER = ("E", "S", "C", "I")
CLASS_TO_INDEX = {label: index for index, label in enumerate(CLASS_ORDER)}
def held_out_query_ids(jsonl_path: Path) -> set[str]:
_, queries = split_corpus(load_records(jsonl_path))
out: set[str] = set()
for row in queries:
qid = str(row.get("qid") or "")
if qid.lower().startswith("esci-"):
out.add(qid[5:])
out.add(qid)
return out
def product_passage(catalog: dict[str, Any], *, fields: str) -> str:
title = _stringify_catalog_value(catalog.get("product_title")) or ""
if fields == "title":
return title
parts: list[str] = []
if title:
parts.append(title)
brand = _stringify_catalog_value(catalog.get("product_brand"))
if brand:
parts.append(f"Brand: {brand}")
color = _stringify_catalog_value(catalog.get("product_color"))
if color:
parts.append(f"Color: {color}")
bullets = _stringify_catalog_value(catalog.get("product_bullet_point"))
if bullets:
parts.append(f"Bullets: {bullets}")
description = _stringify_catalog_value(catalog.get("product_description"))
if description:
parts.append(f"Description: {description}")
body = "\n".join(parts).strip()
if len(body) > PRODUCT_TEXT_MAX_CHARS:
return body[: PRODUCT_TEXT_MAX_CHARS - 1].rstrip() + "…"
return body
def load_train_rows(
*,
holdout: set[str],
locale: str = "us",
max_pairs: int = 80000,
seed: int = 7,
) -> list[dict[str, Any]]:
examples_path = ESCI_CACHE_DIR / ESCI_EXAMPLES_FILE
wanted_locale = locale.strip().lower()
rows: list[dict[str, Any]] = []
for item in _iter_parquet(examples_path):
if int(item.get("small_version") or 0) != 1:
continue
if str(item.get("product_locale") or "").strip().lower() != wanted_locale:
continue
if str(item.get("split") or "").strip().lower() != "train":
continue
qid = str(item.get("query_id") or "").strip()
if not qid or qid in holdout:
continue
pid = str(item.get("product_id") or "").strip()
query = str(item.get("query") or "").strip()
if not pid or not query:
continue
label = str(item.get("esci_label") or "").strip().upper()
if label not in ESCI_GAINS:
continue
rows.append(
{
"query": query,
"product_id": pid,
"label": label,
"gain": float(ESCI_GAINS[label]),
"binary": 1.0 if label == "E" else 0.0,
"class_index": CLASS_TO_INDEX[label],
}
)
rng = random.Random(seed)
rng.shuffle(rows)
return rows[: max(1, max_pairs)]
def load_train_pairs(
*,
holdout: set[str],
locale: str = "us",
max_pairs: int = 30000,
seed: int = 7,
fields: str = "title",
label_mode: str = "binary",
) -> list[tuple[str, str, float]]:
rows = load_train_rows(
holdout=holdout, locale=locale, max_pairs=max_pairs, seed=seed
)
products_path = ESCI_CACHE_DIR / ESCI_PRODUCTS_FILE
needed = {str(row["product_id"]) for row in rows}
products = _esci_products(products_path, needed, locale=locale)
pairs: list[tuple[str, str, float]] = []
for row in rows:
catalog = products.get(str(row["product_id"])) or {}
passage = product_passage(catalog, fields=fields) or str(row["product_id"])
if label_mode == "binary":
target = float(row["binary"])
elif label_mode == "graded":
target = float(row["gain"])
else:
target = float(row["class_index"])
pairs.append((row["query"], passage, target))
return pairs
def finetune(
*,
jsonl_path: Path,
out_dir: Path,
max_pairs: int = 80000,
epochs: int = 1,
batch_size: int = 32,
label_mode: str = "graded",
fields: str = "catalog",
base: str = DEFAULT_MODEL,
lr: float = 7e-6,
max_length: int = 256,
) -> dict[str, Any]:
from sentence_transformers import CrossEncoder, InputExample
from torch.utils.data import DataLoader
if label_mode not in {"binary", "graded", "multiclass"}:
raise ValueError(f"Unknown label_mode {label_mode!r}")
if fields not in {"title", "catalog"}:
raise ValueError(f"Unknown fields {fields!r}")
holdout = held_out_query_ids(jsonl_path)
pairs = load_train_pairs(
holdout=holdout,
max_pairs=max_pairs,
fields=fields,
label_mode=label_mode,
)
if label_mode == "multiclass":
examples = [
InputExample(texts=[query, passage], label=int(target))
for query, passage, target in pairs
]
model = CrossEncoder(base, num_labels=len(CLASS_ORDER), max_length=max_length)
else:
examples = [
InputExample(texts=[query, passage], label=float(target))
for query, passage, target in pairs
]
model = CrossEncoder(
base,
num_labels=1,
max_length=max_length,
default_activation_function=None,
)
n_pos = sum(
1
for _, _, target in pairs
if (
int(target) == CLASS_TO_INDEX["E"]
if label_mode == "multiclass"
else float(target) >= 1.0
)
)
loader = DataLoader(examples, shuffle=True, batch_size=batch_size)
warmup = max(10, len(loader) // 10)
model.fit(
train_dataloader=loader,
epochs=epochs,
warmup_steps=warmup,
optimizer_params={"lr": lr},
show_progress_bar=True,
)
out_dir.mkdir(parents=True, exist_ok=True)
model.save(str(out_dir))
counts: dict[str, int] = {}
if label_mode == "multiclass":
for _, _, target in pairs:
label = CLASS_ORDER[int(target)]
counts[label] = counts.get(label, 0) + 1
meta = {
"n_pairs": len(pairs),
"n_exact": n_pos,
"n_holdout_qids": len(holdout),
"epochs": epochs,
"model": str(out_dir),
"base": base,
"label_mode": label_mode,
"fields": fields,
"lr": lr,
"max_length": max_length,
"class_counts": counts,
"class_order": list(CLASS_ORDER) if label_mode == "multiclass" else None,
}
(out_dir / "finetune_meta.json").write_text(
json.dumps(meta, indent=2) + "\n", encoding="utf-8"
)
return meta
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="finetune_esci_ce")
parser.add_argument(
"--dataset",
type=Path,
default=Path("data/search_esci_74.jsonl"),
)
parser.add_argument(
"--out",
type=Path,
default=Path("data/models/esci-minilm-graded"),
)
parser.add_argument("--max-pairs", type=int, default=80000)
parser.add_argument("--epochs", type=int, default=1)
parser.add_argument("--batch-size", type=int, default=32)
parser.add_argument(
"--label-mode",
choices=["binary", "graded", "multiclass"],
default="graded",
)
parser.add_argument("--fields", choices=["title", "catalog"], default="catalog")
parser.add_argument("--base", default=DEFAULT_MODEL)
parser.add_argument("--lr", type=float, default=7e-6)
parser.add_argument("--max-length", type=int, default=256)
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
meta = finetune(
jsonl_path=args.dataset,
out_dir=args.out,
max_pairs=args.max_pairs,
epochs=args.epochs,
batch_size=args.batch_size,
label_mode=args.label_mode,
fields=args.fields,
base=args.base,
lr=args.lr,
max_length=args.max_length,
)
print(json.dumps(meta, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())