Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

22.5 KB
Raw Blame
from __future__ import annotations

import json
import math
from pathlib import Path
from typing import Any

import numpy as np

from search.dataset import split_corpus
from search.local_dense import evaluate_dense
from search.pool_first_stage import field_value, tokenize

PROTOCOL = "ltr-head-cv"
PROTOCOL_APPLY = "ltr-head-apply"
PAIR_UNLABELED_ZERO = "unlabeled_zero"
PAIR_OTHER_QUERY_NEG = "other_query_neg"
PAIR_POLICIES = (PAIR_UNLABELED_ZERO, PAIR_OTHER_QUERY_NEG)
BASE_FEATURE_NAMES = (
    "rrf_inv",
    "bm25_inv",
    "dense_inv",
    "title_overlap",
    "brand_hit",
    "query_in_title",
)
CE_FEATURE = "ce_gain"
FEATURE_NAMES = BASE_FEATURE_NAMES
HEAD_RANKNET = "ranknet"
HEAD_LIGHTGBM = "lightgbm"
HEADS = (HEAD_RANKNET, HEAD_LIGHTGBM)
N_FOLDS = 5
SEED = 0
EPOCHS = 40
LR = 0.05
L2 = 1e-3
MAX_PAIRS = 400
LGBM_N_ESTIMATORS = 100
LGBM_MAX_DEPTH = 3
LGBM_LEARNING_RATE = 0.05


def feature_names(*, with_ce: bool = False) -> tuple[str, ...]:
    if with_ce:
        return (*BASE_FEATURE_NAMES, CE_FEATURE)
    return BASE_FEATURE_NAMES


def _gain(row: dict[str, Any], doc_id: str) -> float:
    grades = row.get("gold_grades") or {}
    if isinstance(grades, dict) and doc_id in grades:
        return float(grades.get(doc_id) or 0.0)
    gold = {str(item) for item in (row.get("gold_doc_ids") or []) if item}
    return 1.0 if doc_id in gold else 0.0


def _chunk_to_doc(row: dict[str, Any]) -> dict[str, str]:
    mapping: dict[str, str] = {}
    for hit in row.get("hits") or []:
        chunk_id = str(hit.get("id") or "")
        doc_id = str(hit.get("doc_id") or "")
        if chunk_id and doc_id:
            mapping[chunk_id] = doc_id
    return mapping


def _as_docs(ids: list[str], mapping: dict[str, str]) -> list[str]:
    out: list[str] = []
    seen: set[str] = set()
    for raw in ids:
        key = mapping.get(str(raw), str(raw))
        if not key or key in seen:
            continue
        seen.add(key)
        out.append(key)
    return out


def _rank_inv(ids: list[str], doc_id: str) -> float:
    try:
        return 1.0 / (ids.index(doc_id) + 1)
    except ValueError:
        return 0.0


def _title(doc: dict[str, Any]) -> str:
    titled = str(doc.get("title") or "").strip()
    if titled:
        return titled
    return field_value(str(doc.get("text") or ""), "Title")


def _brand(doc: dict[str, Any]) -> str:
    branded = str(doc.get("brand") or "").strip()
    if branded:
        return branded
    return field_value(str(doc.get("text") or ""), "Brand")


def overlap(query: str, text: str) -> float:
    qtoks = [tok for tok in tokenize(query) if len(tok) >= 3]
    if not qtoks:
        return 0.0
    ttoks = set(tokenize(text))
    return len(set(qtoks) & ttoks) / float(len(set(qtoks)))


def ce_cache_name(model_dir: Path) -> str:
    stem = model_dir.name.strip() or "ce"
    safe = "".join(ch if ch.isalnum() or ch in "-_" else "-" for ch in stem)
    return f"ce_gain_{safe}.json"


def load_ce_cache(path: Path) -> dict[str, dict[str, float]] | None:
    if not path.exists():
        return None
    raw = json.loads(path.read_text(encoding="utf-8"))
    if not isinstance(raw, dict):
        return None
    out: dict[str, dict[str, float]] = {}
    for qid, scores in raw.items():
        if not isinstance(scores, dict):
            continue
        out[str(qid)] = {str(doc_id): float(value) for doc_id, value in scores.items()}
    return out


def write_ce_cache(path: Path, scores: dict[str, dict[str, float]]) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(scores, indent=2), encoding="utf-8")


def score_ce_gains(
    eval_result: dict[str, Any],
    docs_by_id: dict[str, dict[str, Any]],
    *,
    model_dir: Path,
    k: int = 50,
    cache_path: Path | None = None,
) -> dict[str, dict[str, float]]:
    if cache_path is not None:
        cached = load_ce_cache(cache_path)
        if cached:
            return cached
    from search.finetune_esci_4class import CLASS_GAINS, weighted_scores
    from search.rank_pool import doc_texts
    from search.rank_pool_4class import load_4class_predict

    predict, _model = load_4class_predict(model_dir)
    texts = doc_texts(list(docs_by_id.values()))
    scored: dict[str, dict[str, float]] = {}
    rows = list(eval_result.get("queries") or [])
    for index, row in enumerate(rows):
        qid = str(row.get("qid") or "")
        query = str(row.get("query") or "")
        mapping = _chunk_to_doc(row)
        hit_ids = [str(item) for item in (row.get("hit_ids") or []) if item][:k]
        doc_ids = _as_docs(hit_ids, mapping)
        if not doc_ids:
            doc_ids = hit_ids
        pairs: list[tuple[str, str]] = []
        for doc_id in doc_ids:
            text = texts.get(doc_id) or _title(docs_by_id.get(doc_id) or {})
            pairs.append((query, text))
        probs = predict(pairs) if pairs else []
        gains = weighted_scores(probs, CLASS_GAINS) if pairs else []
        if len(gains) < len(doc_ids):
            gains = list(gains) + [0.0] * (len(doc_ids) - len(gains))
        scored[qid] = {
            doc_id: float(gains[pos]) for pos, doc_id in enumerate(doc_ids)
        }
        print(f"ltr-ce {index + 1}/{len(rows)} {qid}", flush=True)
    if cache_path is not None:
        write_ce_cache(cache_path, scored)
    return scored


def features_for_doc(
    query: str,
    doc_id: str,
    *,
    rrf_ids: list[str],
    bm25_ids: list[str],
    dense_ids: list[str],
    doc: dict[str, Any] | None,
    ce_gain: float | None = None,
) -> np.ndarray:
    title = _title(doc or {})
    brand = _brand(doc or {})
    q = (query or "").strip().lower()
    in_title = 1.0 if q and len(q) >= 3 and q in title.lower() else 0.0
    qtoks = [tok for tok in tokenize(query) if len(tok) >= 3]
    brand_toks = set(tokenize(brand))
    brand_hit = 1.0 if qtoks and set(qtoks) & brand_toks else 0.0
    values = [
        _rank_inv(rrf_ids, doc_id),
        _rank_inv(bm25_ids, doc_id),
        _rank_inv(dense_ids, doc_id),
        overlap(query, title),
        brand_hit,
        in_title,
    ]
    if ce_gain is not None:
        values.append(float(ce_gain))
    return np.array(values, dtype=np.float64)


def example_from_eval_row(
    row: dict[str, Any],
    docs_by_id: dict[str, dict[str, Any]],
    *,
    k: int = 50,
    ce_scores: dict[str, float] | None = None,
) -> dict[str, Any] | None:
    mapping = _chunk_to_doc(row)
    rrf_ids = [str(item) for item in (row.get("hit_ids") or []) if item][:k]
    if not rrf_ids:
        return None
    bm25_ids = _as_docs(list(row.get("bm25_ids") or []), mapping)
    dense_ids = _as_docs(list(row.get("dense_ids") or []), mapping)
    query = str(row.get("query") or "")
    feats: list[np.ndarray] = []
    gains: list[float] = []
    for doc_id in rrf_ids:
        ce_gain = None
        if ce_scores is not None:
            ce_gain = float(ce_scores.get(doc_id) or 0.0)
        feats.append(
            features_for_doc(
                query,
                doc_id,
                rrf_ids=rrf_ids,
                bm25_ids=bm25_ids,
                dense_ids=dense_ids,
                doc=docs_by_id.get(doc_id),
                ce_gain=ce_gain,
            )
        )
        gains.append(_gain(row, doc_id))
    return {
        "qid": str(row.get("qid") or ""),
        "query": query,
        "ids": rrf_ids,
        "features": np.stack(feats, axis=0),
        "gains": np.array(gains, dtype=np.float64),
        "row": row,
    }


def other_query_gold_ids(examples: list[dict[str, Any]], skip_qid: str) -> set[str]:
    gold: set[str] = set()
    for example in examples:
        if example["qid"] == skip_qid:
            continue
        for doc_id, gain in zip(example["ids"], example["gains"]):
            if float(gain) > 0:
                gold.add(str(doc_id))
    return gold


def training_gains(
    example: dict[str, Any],
    *,
    pair_policy: str,
    other_gold: set[str],
) -> list[float | None]:
    policy = (pair_policy or PAIR_UNLABELED_ZERO).strip()
    out: list[float | None] = []
    for doc_id, gain in zip(example["ids"], example["gains"]):
        value = float(gain)
        if policy == PAIR_OTHER_QUERY_NEG:
            if value > 0:
                out.append(value)
            elif str(doc_id) in other_gold:
                out.append(0.0)
            else:
                out.append(None)
        else:
            out.append(value)
    return out


def collect_pairs(
    example: dict[str, Any],
    *,
    pair_policy: str = PAIR_UNLABELED_ZERO,
    other_gold: set[str] | None = None,
) -> list[tuple[int, int]]:
    labeled = training_gains(
        example,
        pair_policy=pair_policy,
        other_gold=other_gold or set(),
    )
    eligible = [
        (index, gain)
        for index, gain in enumerate(labeled)
        if gain is not None
    ]
    pairs: list[tuple[int, int]] = []
    for left in range(len(eligible)):
        for right in range(left):
            left_i, left_g = eligible[left]
            right_i, right_g = eligible[right]
            if left_g == right_g:
                continue
            hi, lo = (
                (left_i, right_i) if left_g > right_g else (right_i, left_i)
            )
            pairs.append((hi, lo))
    return pairs


def _sigmoid(value: float) -> float:
    if value >= 0:
        z = math.exp(-value)
        return 1.0 / (1.0 + z)
    z = math.exp(value)
    return z / (1.0 + z)


def fit_pairwise(
    examples: list[dict[str, Any]],
    *,
    epochs: int = EPOCHS,
    lr: float = LR,
    l2: float = L2,
    seed: int = SEED,
    pair_policy: str = PAIR_UNLABELED_ZERO,
) -> np.ndarray:
    n_features = int(examples[0]["features"].shape[1]) if examples else len(BASE_FEATURE_NAMES)
    rng = np.random.default_rng(seed)
    weights = np.zeros(n_features, dtype=np.float64)
    if not examples:
        return weights
    for _ in range(max(1, int(epochs))):
        order = rng.permutation(len(examples))
        for index in order:
            example = examples[int(index)]
            feats = example["features"]
            other_gold = other_query_gold_ids(examples, example["qid"])
            pairs = collect_pairs(
                example,
                pair_policy=pair_policy,
                other_gold=other_gold,
            )
            if not pairs:
                continue
            if len(pairs) > MAX_PAIRS:
                chosen = rng.choice(len(pairs), size=MAX_PAIRS, replace=False)
                pairs = [pairs[int(pos)] for pos in chosen]
            for hi, lo in pairs:
                diff = feats[hi] - feats[lo]
                pred = _sigmoid(float(weights.dot(diff)))
                grad = (pred - 1.0) * diff + l2 * weights
                weights -= lr * grad
    return weights


def score_docs(features: np.ndarray, weights: np.ndarray) -> np.ndarray:
    return features.dot(weights)


def rerank_ids(example: dict[str, Any], weights: np.ndarray) -> list[str]:
    scores = score_docs(example["features"], weights)
    order = np.argsort(-scores, kind="stable")
    return [example["ids"][int(pos)] for pos in order]


def gain_to_rank_label(gain: float) -> int:
    if gain >= 0.99:
        return 3
    if gain >= 0.09:
        return 2
    if gain >= 0.005:
        return 1
    return 0


def rank_train_group(
    example: dict[str, Any],
    *,
    pair_policy: str,
    other_gold: set[str],
) -> tuple[np.ndarray, np.ndarray] | None:
    labeled = training_gains(
        example,
        pair_policy=pair_policy,
        other_gold=other_gold,
    )
    idxs = [index for index, gain in enumerate(labeled) if gain is not None]
    if len(idxs) < 2:
        return None
    y = np.array([gain_to_rank_label(float(labeled[index] or 0.0)) for index in idxs])
    if int(np.unique(y).size) < 2:
        return None
    return example["features"][idxs], y


class _ZeroRanker:
    def __init__(self, n_features: int) -> None:
        self.feature_importances_ = np.zeros(max(1, int(n_features)), dtype=np.float64)

    def predict(self, features: np.ndarray) -> np.ndarray:
        return np.zeros(len(features), dtype=np.float64)


class _LgbmRanker:
    def __init__(self, booster: Any, n_features: int) -> None:
        self.booster = booster
        raw = booster.feature_importance(importance_type="gain")
        importances = np.asarray(raw, dtype=np.float64)
        if importances.size < n_features:
            importances = np.pad(importances, (0, n_features - int(importances.size)))
        self.feature_importances_ = importances[:n_features]

    def predict(self, features: np.ndarray) -> np.ndarray:
        return np.asarray(self.booster.predict(features), dtype=np.float64)


def fit_lightgbm(
    examples: list[dict[str, Any]],
    *,
    pair_policy: str = PAIR_UNLABELED_ZERO,
    seed: int = SEED,
    min_data_in_leaf: int | None = None,
) -> Any:
    n_features = (
        int(examples[0]["features"].shape[1]) if examples else len(BASE_FEATURE_NAMES)
    )
    xs: list[np.ndarray] = []
    ys: list[np.ndarray] = []
    groups: list[int] = []
    for example in examples:
        group = rank_train_group(
            example,
            pair_policy=pair_policy,
            other_gold=other_query_gold_ids(examples, example["qid"]),
        )
        if group is None:
            continue
        feats, labels = group
        xs.append(feats)
        ys.append(labels)
        groups.append(int(len(labels)))
    if not groups:
        return _ZeroRanker(n_features)
    import lightgbm as lgb

    dataset = lgb.Dataset(
        np.vstack(xs),
        label=np.concatenate(ys),
        group=groups,
        free_raw_data=False,
    )
    params: dict[str, Any] = {
        "objective": "lambdarank",
        "max_depth": LGBM_MAX_DEPTH,
        "learning_rate": LGBM_LEARNING_RATE,
        "num_leaves": 8,
        "verbosity": -1,
        "seed": int(seed),
    }
    if min_data_in_leaf is not None:
        params["min_data_in_leaf"] = int(min_data_in_leaf)
    booster = lgb.train(
        params,
        dataset,
        num_boost_round=LGBM_N_ESTIMATORS,
    )
    return _LgbmRanker(booster, n_features)


def rerank_ids_model(example: dict[str, Any], model: Any) -> list[str]:
    scores = np.asarray(model.predict(example["features"]), dtype=np.float64)
    order = np.argsort(-scores, kind="stable")
    return [example["ids"][int(pos)] for pos in order]


def overlap_only_ids(example: dict[str, Any]) -> list[str]:
    title_scores = example["features"][:, 3]
    order = np.argsort(-title_scores, kind="stable")
    return [example["ids"][int(pos)] for pos in order]


def grouped_folds(qids: list[str], n_folds: int = N_FOLDS) -> list[set[str]]:
    folds: list[set[str]] = [set() for _ in range(max(2, int(n_folds)))]
    for index, qid in enumerate(sorted(qids)):
        folds[index % len(folds)].add(qid)
    return folds


def build_ltr_examples(
    eval_result: dict[str, Any],
    rows: list[dict[str, Any]],
    *,
    k: int = 50,
    ce_model: Path | None = None,
    ce_cache_path: Path | None = None,
    ce_scores: dict[str, dict[str, float]] | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, dict[str, Any]]]:
    docs, queries = split_corpus(rows)
    docs_by_id = {str(doc.get("doc_id") or ""): doc for doc in docs if doc.get("doc_id")}
    resolved_ce = ce_scores
    ce_model_path = Path(ce_model) if ce_model else None
    if resolved_ce is None and ce_model_path is not None:
        resolved_ce = score_ce_gains(
            eval_result,
            docs_by_id,
            model_dir=ce_model_path,
            k=k,
            cache_path=ce_cache_path,
        )
    with_ce = resolved_ce is not None
    examples: list[dict[str, Any]] = []
    for row in eval_result.get("queries") or []:
        qid = str(row.get("qid") or "")
        example = example_from_eval_row(
            row,
            docs_by_id,
            k=k,
            ce_scores=(resolved_ce or {}).get(qid) if with_ce else None,
        )
        if example is None:
            continue
        examples.append(example)
    return examples, queries, docs_by_id


def run_ltr_head(
    eval_result: dict[str, Any],
    rows: list[dict[str, Any]],
    *,
    dataset_name: str,
    k: int = 50,
    ks: tuple[int, ...] = (5, 10, 20, 50),
    n_folds: int = N_FOLDS,
    brain_id: str | None = None,
    source_run: str | None = None,
    pair_policy: str = PAIR_UNLABELED_ZERO,
    ce_model: Path | None = None,
    ce_cache_path: Path | None = None,
    ce_scores: dict[str, dict[str, float]] | None = None,
    ltr_head: str = HEAD_RANKNET,
    train_eval_result: dict[str, Any] | None = None,
    train_rows: list[dict[str, Any]] | None = None,
    train_source_run: str | None = None,
    train_ce_cache_path: Path | None = None,
    train_ce_scores: dict[str, dict[str, float]] | None = None,
) -> dict[str, Any]:
    policy = (pair_policy or PAIR_UNLABELED_ZERO).strip()
    if policy not in PAIR_POLICIES:
        raise ValueError(f"Unknown pair policy {pair_policy!r}")
    head = (ltr_head or HEAD_RANKNET).strip()
    if head not in HEADS:
        raise ValueError(f"Unknown LTR head {ltr_head!r}")
    if train_eval_result is not None and train_rows is None:
        raise ValueError("train_rows is required when train_eval_result is set")
    apply_mode = train_eval_result is not None
    ce_model_path = Path(ce_model) if ce_model else None
    eval_examples, queries, docs_by_id = build_ltr_examples(
        eval_result,
        rows,
        k=k,
        ce_model=ce_model_path,
        ce_cache_path=ce_cache_path,
        ce_scores=ce_scores,
    )
    names = feature_names(with_ce=ce_model_path is not None or ce_scores is not None)
    if apply_mode:
        train_examples, _, _ = build_ltr_examples(
            train_eval_result or {},
            train_rows or [],
            k=k,
            ce_model=ce_model_path,
            ce_cache_path=train_ce_cache_path,
            ce_scores=train_ce_scores,
        )
        names = feature_names(
            with_ce=bool(train_examples)
            and int(train_examples[0]["features"].shape[1]) > len(BASE_FEATURE_NAMES)
        )
        ranked_cv: dict[str, list[str]] = {}
        ranked_overlap: dict[str, list[str]] = {}
        fold_weights: list[list[float]] = []
        if head == HEAD_LIGHTGBM:
            model = fit_lightgbm(train_examples, pair_policy=policy)
            fold_weights.append([float(item) for item in model.feature_importances_])
            for example in eval_examples:
                ranked_cv[example["qid"]] = rerank_ids_model(example, model)[:k]
                ranked_overlap[example["qid"]] = overlap_only_ids(example)[:k]
        else:
            weights = fit_pairwise(train_examples, pair_policy=policy)
            fold_weights.append([float(item) for item in weights])
            for example in eval_examples:
                ranked_cv[example["qid"]] = rerank_ids(example, weights)[:k]
                ranked_overlap[example["qid"]] = overlap_only_ids(example)[:k]
        n_train = len(train_examples)
    else:
        examples = eval_examples
        by_qid = {example["qid"]: example for example in examples}
        qids = [example["qid"] for example in examples]
        ranked_cv = {}
        ranked_overlap = {}
        fold_weights = []
        for test_qids in grouped_folds(qids, n_folds=n_folds):
            train_examples = [
                example for example in examples if example["qid"] not in test_qids
            ]
            if head == HEAD_LIGHTGBM:
                model = fit_lightgbm(train_examples, pair_policy=policy)
                fold_weights.append([float(item) for item in model.feature_importances_])
                for qid in test_qids:
                    example = by_qid.get(qid)
                    if example is None:
                        continue
                    ranked_cv[qid] = rerank_ids_model(example, model)[:k]
                    ranked_overlap[qid] = overlap_only_ids(example)[:k]
                continue
            weights = fit_pairwise(train_examples, pair_policy=policy)
            fold_weights.append([float(item) for item in weights])
            for qid in test_qids:
                example = by_qid.get(qid)
                if example is None:
                    continue
                ranked_cv[qid] = rerank_ids(example, weights)[:k]
                ranked_overlap[qid] = overlap_only_ids(example)[:k]
        n_train = None
        train_examples = []
    metrics, per_query = evaluate_dense(
        ranked_cv,
        queries,
        ks=ks,
        encode_ms=0.0,
    )
    overlap_metrics, _ = evaluate_dense(
        ranked_overlap,
        queries,
        ks=ks,
        encode_ms=0.0,
    )
    mean_weights = (
        np.mean(np.array(fold_weights, dtype=np.float64), axis=0)
        if fold_weights
        else np.zeros(len(names))
    )
    return {
        "status": "ok" if per_query else "failed",
        "brain_id": brain_id or "harness-local-ltr",
        "dataset": dataset_name,
        "fusion": "none",
        "rerank": "ltr-lightgbm" if head == HEAD_LIGHTGBM else "ltr-pairwise",
        "channels": ["harness-ltr-head"],
        "expand": "none",
        "k": k,
        "ks": list(ks),
        "n_docs": len(docs_by_id),
        "n_queries": len(per_query),
        "n_docs_mapped": 0,
        "skip_enrichment": True,
        "ingest_graph": False,
        "skip_ingest": True,
        "rank_pool": False,
        "ltr_from_run": source_run or eval_result.get("run_id"),
        "ltr_train_run": train_source_run if apply_mode else None,
        "ltr_n_train_queries": n_train if apply_mode else None,
        "protocol": PROTOCOL_APPLY if apply_mode else PROTOCOL,
        "ltr_features": list(names),
        "ltr_pair_policy": policy,
        "ltr_head": head,
        "ltr_ce_model": str(ce_model_path) if ce_model_path else None,
        "ltr_lgbm": (
            {
                "n_estimators": LGBM_N_ESTIMATORS,
                "max_depth": LGBM_MAX_DEPTH,
                "learning_rate": LGBM_LEARNING_RATE,
            }
            if head == HEAD_LIGHTGBM
            else None
        ),
        "ltr_n_folds": 1 if apply_mode else n_folds,
        "ltr_seed": SEED,
        "ltr_epochs": EPOCHS,
        "ltr_mean_weights": {
            name: float(weight)
            for name, weight in zip(names, mean_weights)
        },
        "overlap_only_metrics": overlap_metrics,
        "ingest": {
            "status": "completed",
            "n_docs": len(docs_by_id),
            "tasks": [],
            "reused": True,
        },
        "graph_ingest": {"status": "skipped", "n_triples": 0, "tasks": []},
        "interaction_ingest": {"status": "skipped", "n_triples": 0, "tasks": []},
        "event_probe": None,
        "search_error": None,
        "metrics": metrics,
        "queries": per_query,
    }