Skip to content
Python
21.3 KB
Raw Blame
"""Question-blind pool-wide Episode/Tag bundle planning before local BGE.

The planner treats every post-MMR candidate as a seed.  Write-time hierarchy
and static tag memberships may nominate additional canonical L2 memories, but
the structural labels are navigation-only: only canonical L2 bodies are
rendered in the final bundle.  All decisions are deterministic and bounded.
"""

from __future__ import annotations

from collections import defaultdict
import math
import re
from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple

from .hierarchy_bundle import cosine, estimate_tokens_no_io
from .lemmatize import lemmatize_for_bm25


_GENERIC_TAGS = {
    "activity", "activities", "event", "events", "family", "friend",
    "friends", "health", "life", "memory", "personal", "people",
    "relationship", "relationships", "social", "thing", "things", "work",
}

_COMPOSITE_FIELDS = (
    "_bundle_items", "_graph_merged_items", "related_memories",
    "_capsule_memories", "_timeline_member_ids", "_timeline",
    "graph_merged", "graph_merged_ids",
)


def _node_id(item: Mapping[str, Any]) -> str:
    node = item.get("node")
    return str(item.get("node_id") or getattr(node, "node_id", "") or "")


def _content(item: Mapping[str, Any]) -> str:
    node = item.get("node")
    return str(item.get("content") or getattr(node, "content", "") or "")


def _active_latest_l2(node: Any) -> bool:
    layer = getattr(getattr(node, "layer", None), "value", getattr(node, "layer", None))
    status = getattr(
        getattr(node, "status", None), "value", getattr(node, "status", None)
    )
    return (
        str(layer or "") == "l2_fact"
        and str(status or "active") == "active"
        and bool(getattr(node, "is_latest", True))
    )


def _canonical_singleton(item: Mapping[str, Any]) -> bool:
    node = item.get("node")
    if node is None or not _active_latest_l2(node):
        return False
    return not any(item.get(field) for field in _COMPOSITE_FIELDS)


def canonical_seed_view(items: Sequence[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Expand graph composites into unique canonical L2 candidates.

    This is a parallel planning view only.  Callers retain the original items as
    the byte-for-byte fallback when no multi-seed admission succeeds.
    """
    output: List[Dict[str, Any]] = []
    index_by_id: Dict[str, int] = {}
    for original in items or []:
        pieces = original.get("_graph_merged_items") or [original]
        for piece in pieces:
            item = dict(piece)
            for field in _COMPOSITE_FIELDS:
                item.pop(field, None)
            memory_id = _node_id(item)
            if not memory_id or not _canonical_singleton(item):
                continue
            previous_index = index_by_id.get(memory_id)
            if previous_index is None:
                index_by_id[memory_id] = len(output)
                output.append(item)
                continue
            if float(item.get("score") or 0.0) > float(
                output[previous_index].get("score") or 0.0
            ):
                output[previous_index] = item
    return output


def _unit(value: Any) -> float:
    try:
        number = float(value)
    except (TypeError, ValueError):
        return 0.0
    if not math.isfinite(number):
        return 0.0
    return max(0.0, min(1.0, number))


def bounded_noisy_or(values: Iterable[float]) -> float:
    remaining = 1.0
    for value in values:
        remaining *= 1.0 - _unit(value)
    return 1.0 - remaining


def _terms(text: str) -> set[str]:
    normalized = lemmatize_for_bm25(str(text or ""))
    return {
        token.casefold() for token in re.findall(r"[\w'-]+", normalized)
        if len(token) > 1
    }


def lexical_overlap(query: str, document: str) -> float:
    query_terms = _terms(query)
    if not query_terms:
        return 0.0
    return len(query_terms & _terms(document)) / float(len(query_terms))


def lightweight_hybrid_score(
    query: str,
    document: str,
    *,
    semantic_score: Optional[float],
    lexical_weight: float = 0.35,
) -> float:
    """Fixed additive light scorer on stored vectors plus lexical overlap."""
    semantic = _unit(semantic_score)
    lexical = lexical_overlap(query, document)
    return _unit(semantic + max(0.0, float(lexical_weight)) * lexical)


def _tag_eligible(
    tag: str,
    ner: str,
    *,
    excluded: set[str],
    df: int,
    max_df: int,
) -> bool:
    folded = str(tag or "").strip().casefold()
    if not folded or folded in excluded or folded in _GENERIC_TAGS:
        return False
    if str(ner or "").strip().upper() == "PERSON":
        return False
    return 0 < int(df) <= max(1, int(max_df))


def _bundle_item(
    members: Sequence[Dict[str, Any]],
    *,
    group_id: str,
    group_type: str,
    group_score: float,
) -> Dict[str, Any]:
    anchor = dict(members[0])
    return {
        **anchor,
        "source": "multi_seed_bundle",
        "_bundle_items": list(members),
        "_multi_seed_group_id": group_id,
        "_multi_seed_group_type": group_type,
        "_multi_seed_group_score": float(group_score),
    }


async def build_multi_seed_bundle_pool(
    *,
    graph_store,
    vector_store,
    isolation_key: str,
    query: str,
    query_embedding: Sequence[float],
    seed_pool: Sequence[Dict[str, Any]],
    fallback_pool: Optional[Sequence[Dict[str, Any]]] = None,
    mode: str = "episode_single",
    tag_hits: Sequence[Dict[str, Any]] = (),
    excluded_tags: Sequence[str] = (),
    alpha: float = 0.6,
    beta: float = 0.4,
    gamma: float = 0.4,
    member_floor: float = 0.20,
    group_floor: float = 0.35,
    tag_s1_floor: float = 0.0,
    tau: float = 0.45,
    lexical_weight: float = 0.35,
    max_tag_df: int = 8,
    scan_limit: int = 50,
    max_members: int = 8,
    max_bundles: int = 0,
    min_episode_seeds: int = 1,
    single_tag_seed_ceiling: float = 1.01,
    token_budget: int = 400,
    expected_seed_count: Optional[int] = None,
    min_units: int = 1,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
    """Return rerank units or the exact input pool when admission fails closed."""
    seeds = list(seed_pool or [])
    baseline = list(fallback_pool) if fallback_pool is not None else list(seeds)
    info: Dict[str, Any] = {
        "enabled": True,
        "applied": False,
        "mode": mode,
        "seed_count": len(seeds),
        "fallback_count": len(baseline),
        "tag_s1_floor": float(tag_s1_floor),
        "min_episode_seeds": max(1, int(min_episode_seeds)),
        "single_tag_seed_ceiling": float(single_tag_seed_ceiling),
    }
    if not seeds or not query_embedding:
        return baseline, {**info, "reason": "empty_pool_or_embedding"}
    if expected_seed_count is not None and len(seeds) != int(expected_seed_count):
        return baseline, {
            **info, "reason": "unexpected_seed_count",
            "expected_seed_count": int(expected_seed_count),
        }
    if not all(_canonical_singleton(item) for item in seeds):
        return baseline, {**info, "reason": "noncanonical_seed"}
    valid_modes = {"episode_single", "episode_multi", "tag_multi", "episode_tag"}
    if mode not in valid_modes:
        return baseline, {**info, "reason": "invalid_mode"}
    seed_by_id = {_node_id(item): item for item in seeds if _node_id(item)}
    if len(seed_by_id) != len(seeds):
        return baseline, {**info, "reason": "invalid_or_duplicate_seed"}

    use_episode = mode in {"episode_single", "episode_multi", "episode_tag"}
    use_tag = mode in {"tag_multi", "episode_tag"}
    group_meta: Dict[str, Dict[str, Any]] = {}
    members_by_group: Dict[str, set[str]] = defaultdict(set)
    groups_by_member: Dict[str, set[str]] = defaultdict(set)

    if use_episode:
        topics = await graph_store.get_hierarchy_topics(isolation_key)
        memberships = await graph_store.get_hierarchy_memberships(isolation_key)
        episode_topics = {
            str(topic.get("topic_id") or ""): topic for topic in topics or []
            if str(topic.get("topic_id") or "")
            and str(topic.get("node_type") or "").casefold() == "episode"
        }
        for relation in memberships or []:
            memory_id = str(relation.get("node_id") or "")
            topic_id = str(relation.get("topic_id") or "")
            if memory_id and topic_id in episode_topics:
                group_id = f"episode:{topic_id}"
                members_by_group[group_id].add(memory_id)
                groups_by_member[memory_id].add(group_id)
                topic = episode_topics[topic_id]
                group_meta[group_id] = {
                    "type": "episode",
                    "title": str(topic.get("name") or ""),
                    "semantic": cosine(query_embedding, topic.get("embedding") or []),
                }

    all_nodes: Dict[str, Any] = {
        memory_id: item.get("node") for memory_id, item in seed_by_id.items()
    }
    tag_scores = {
        str(hit.get("tag") or "").strip().casefold(): _unit(hit.get("score"))
        for hit in tag_hits or [] if str(hit.get("tag") or "").strip()
    }
    if use_tag:
        if not tag_hits and not use_episode:
            return baseline, {**info, "reason": "tag_index_unavailable"}
        if tag_hits:
            # Enumerate the user's active/latest corpus exactly; do not silently turn
            # the safety limit into a semantic top-k cutoff.  Only query-matched tag
            # vectors may form groups: treating every tag on every seed as a match
            # assigns s1=0 to unscored groups and lets seed corroboration alone flood
            # the candidate set with unrelated tags.
            listed = await vector_store.list_by_user(
                user_id=getattr(next(iter(all_nodes.values()), None), "user_id", None),
                agent_id=getattr(next(iter(all_nodes.values()), None), "agent_id", None),
                limit=2000,
                with_vectors=False,
            )
            for node in listed or []:
                if not _active_latest_l2(node):
                    continue
                all_nodes[str(node.node_id)] = node
            raw_df: Dict[str, int] = defaultdict(int)
            for node in all_nodes.values():
                for tag in set(str(value or "").strip().casefold() for value in getattr(node, "tags", []) or []):
                    if tag:
                        raw_df[tag] += 1
            excluded = {str(value or "").strip().casefold() for value in excluded_tags if value}
            for memory_id, node in all_nodes.items():
                tags = list(getattr(node, "tags", None) or [])
                ners = list(getattr(node, "tag_ners", None) or [])
                for index, tag in enumerate(tags):
                    folded = str(tag or "").strip().casefold()
                    ner = ners[index] if index < len(ners) else ""
                    if folded not in tag_scores or not _tag_eligible(
                        tag, ner, excluded=excluded, df=raw_df.get(folded, 0),
                        max_df=max_tag_df,
                    ):
                        continue
                    group_id = f"tag:{folded}"
                    members_by_group[group_id].add(memory_id)
                    groups_by_member[memory_id].add(group_id)
                    group_meta[group_id] = {
                        "type": "tag", "title": str(tag),
                        "semantic": tag_scores[folded], "df": raw_df[folded],
                    }

    seed_groups: Dict[str, List[str]] = defaultdict(list)
    for memory_id in seed_by_id:
        for group_id in groups_by_member.get(memory_id, ()):
            seed_groups[group_id].append(memory_id)
    if not seed_groups:
        return baseline, {**info, "reason": "no_seed_groups"}

    group_scores: Dict[str, float] = {}
    group_s1_scores: Dict[str, float] = {}
    group_rows = []
    for group_id in sorted(seed_groups):
        group_seed_ids = sorted(seed_groups[group_id])
        seed_values = [
            _unit(seed_by_id[memory_id].get("score"))
            for memory_id in group_seed_ids
        ]
        s2 = max(seed_values, default=0.0) if mode == "episode_single" else bounded_noisy_or(seed_values)
        meta = group_meta[group_id]
        s1 = lightweight_hybrid_score(
            query, meta.get("title", ""), semantic_score=meta.get("semantic"),
            lexical_weight=lexical_weight,
        )
        c1 = max(0.0, float(alpha)) * s1 + max(0.0, float(beta)) * s2
        group_s1_scores[group_id] = s1
        group_scores[group_id] = c1
        group_rows.append({
            "group_id": group_id, "group_type": meta["type"],
            "seed_count": len(group_seed_ids),
            "seeds": [
                {"node_id": mid, "h": value}
                for mid, value in zip(group_seed_ids, seed_values)
            ],
            "s1": s1, "s2": s2, "c1": c1,
        })

    min_episode_seed_count = max(1, int(min_episode_seeds))

    def meets_seed_consensus(group_id: str) -> bool:
        return (
            group_meta[group_id]["type"] != "episode"
            or len(set(seed_groups[group_id])) >= min_episode_seed_count
        )

    def preserves_high_confidence_single_tag_seed(group_id: str) -> bool:
        if group_meta[group_id]["type"] != "tag":
            return True
        group_seed_ids = seed_groups[group_id]
        if len(set(group_seed_ids)) != 1:
            return True
        seed_score = max(
            _unit(seed_by_id[memory_id].get("score"))
            for memory_id in group_seed_ids
        )
        return seed_score < float(single_tag_seed_ceiling)

    viable_seed_groups = {
        group_id for group_id in seed_groups
        if meets_seed_consensus(group_id)
        and preserves_high_confidence_single_tag_seed(group_id)
        and group_scores[group_id] >= float(group_floor)
        and (
            group_meta[group_id]["type"] != "tag"
            or group_s1_scores[group_id] >= float(tag_s1_floor)
        )
    }
    if not viable_seed_groups:
        return baseline, {
            **info, "reason": "no_member_admitted", "groups": group_rows,
            "novel_candidate_count": 0, "members": [],
        }
    novel_ids = sorted({
        memory_id for group_id in viable_seed_groups
        for memory_id in members_by_group.get(group_id, ())
        if memory_id not in seed_by_id
    })
    if len(novel_ids) > max(0, int(scan_limit)):
        return baseline, {
            **info, "reason": "candidate_overflow", "novel_candidate_count": len(novel_ids),
            "scan_limit": max(0, int(scan_limit)), "groups": group_rows,
        }
    missing_nodes = [memory_id for memory_id in novel_ids if memory_id not in all_nodes]
    if missing_nodes:
        fetched = await vector_store.get_by_ids(missing_nodes)
        all_nodes.update({str(node.node_id): node for node in fetched or []})
    if any(memory_id not in all_nodes for memory_id in novel_ids):
        return baseline, {**info, "reason": "missing_member", "groups": group_rows}
    ineligible_novel_ids = sorted(
        memory_id for memory_id in novel_ids
        if not _active_latest_l2(all_nodes[memory_id])
    )
    novel_ids = [
        memory_id for memory_id in novel_ids if memory_id not in ineligible_novel_ids
    ]
    if not novel_ids:
        return baseline, {
            **info, "reason": "no_active_latest_novel_member",
            "groups": group_rows,
            "ineligible_novel_ids": ineligible_novel_ids,
        }
    embeddings = await vector_store.get_embeddings(novel_ids)
    if any(memory_id not in embeddings for memory_id in novel_ids):
        return baseline, {**info, "reason": "missing_member_embedding", "groups": group_rows}

    admitted: Dict[str, Dict[str, Any]] = {}
    member_rows = []
    for memory_id in novel_ids:
        node = all_nodes[memory_id]
        semantic = cosine(query_embedding, embeddings[memory_id])
        s3 = lightweight_hybrid_score(
            query, getattr(node, "content", ""), semantic_score=semantic,
            lexical_weight=lexical_weight,
        )
        eligible_groups = [
            group for group in groups_by_member[memory_id]
            if group in seed_groups
        ]
        support_groups = [
            group for group in eligible_groups if group in viable_seed_groups
        ]
        if not support_groups:
            continue
        # Admission support and final bundle ownership are intentionally
        # separate.  The contract defines b(m) as the strongest supporting
        # group, regardless of type, while overlap ownership is Episode-first.
        # Coupling the two makes a weaker Episode suppress a stronger Tag and
        # can incorrectly reject an otherwise eligible canonical L2.
        boost_group = min(
            support_groups,
            key=lambda group: (-group_scores[group], group),
        )
        ownership_groups = [
            group for group in eligible_groups
            if meets_seed_consensus(group)
            and preserves_high_confidence_single_tag_seed(group)
        ]
        owner_group = min(
            ownership_groups,
            key=lambda group: (
                0 if group_meta[group]["type"] == "episode" else 1,
                -group_scores[group], group,
            ),
        )
        boost = group_scores[boost_group]
        final_score = s3 + max(0.0, float(gamma)) * boost
        passed = (
            s3 >= float(member_floor)
            and boost >= float(group_floor)
            and final_score >= float(tau)
        )
        member_rows.append({
            "node_id": memory_id,
            "group_id": owner_group,
            "boost_group_id": boost_group,
            "s3": s3,
            "group_boost": boost, "final_score": final_score, "admitted": passed,
        })
        if passed:
            admitted[memory_id] = {
                "group_id": owner_group,
                "boost_group_id": boost_group,
                "score": final_score,
                "item": {
                    "node_id": memory_id, "node": node,
                    "content": getattr(node, "content", ""),
                    "score": s3, "source": "multi_seed_member",
                    "_multi_seed_member_score": final_score,
                },
            }
    if not admitted:
        return baseline, {
            **info, "reason": "no_member_admitted", "groups": group_rows,
            "novel_candidate_count": len(novel_ids), "members": member_rows,
        }

    active_groups = {value["group_id"] for value in admitted.values()}
    if max(0, int(max_bundles)) and len(active_groups) > int(max_bundles):
        return baseline, {
            **info,
            "reason": "bundle_count_exceeded",
            "bundle_count": len(active_groups),
            "max_bundles": int(max_bundles),
            "groups": group_rows,
            "members": member_rows,
        }
    seed_assignment: Dict[str, str] = {}
    for memory_id in seed_by_id:
        eligible = [group for group in groups_by_member.get(memory_id, ()) if group in active_groups]
        if eligible:
            seed_assignment[memory_id] = min(
                eligible,
                key=lambda group: (
                    0 if group_meta[group]["type"] == "episode" else 1,
                    -group_scores[group], group,
                ),
            )
    grouped: Dict[str, List[Tuple[float, Dict[str, Any]]]] = defaultdict(list)
    for memory_id, group_id in seed_assignment.items():
        grouped[group_id].append((float(seed_by_id[memory_id].get("score") or 0.0), seed_by_id[memory_id]))
    for value in admitted.values():
        grouped[value["group_id"]].append((float(value["score"]), value["item"]))

    invalid_groups = set()
    for group_id, rows in grouped.items():
        items = [item for _, item in rows]
        tokens = sum(estimate_tokens_no_io(_content(item)) for item in items)
        if len(items) > max(2, int(max_members)) or tokens > max(1, int(token_budget)):
            invalid_groups.add(group_id)
    if invalid_groups:
        return baseline, {
            **info, "reason": "bundle_budget_exceeded",
            "invalid_groups": sorted(invalid_groups), "groups": group_rows,
            "members": member_rows,
        }

    units: List[Dict[str, Any]] = []
    emitted_groups = set()
    for seed in seeds:
        memory_id = _node_id(seed)
        group_id = seed_assignment.get(memory_id)
        if not group_id:
            units.append(seed)
            continue
        if group_id in emitted_groups:
            continue
        emitted_groups.add(group_id)
        rows = sorted(grouped[group_id], key=lambda row: (-row[0], _node_id(row[1])))
        units.append(_bundle_item(
            [item for _, item in rows], group_id=group_id,
            group_type=group_meta[group_id]["type"],
            group_score=group_scores[group_id],
        ))
    if len(units) > len(seeds):
        return baseline, {**info, "reason": "unit_count_invariant", "groups": group_rows}
    if len(units) < max(1, int(min_units)):
        return baseline, {
            **info, "reason": "insufficient_rerank_units", "groups": group_rows,
            "members": member_rows, "unit_count": len(units),
        }
    return units, {
        **info, "applied": True, "reason": "bundled",
        "group_count": len(seed_groups), "groups": group_rows,
        "novel_candidate_count": len(novel_ids), "members": member_rows,
        "ineligible_novel_ids": ineligible_novel_ids,
        "admitted_count": len(admitted), "bundle_count": len(emitted_groups),
        "bundle_widths": [len(grouped[group]) for group in sorted(emitted_groups)],
        "unit_count": len(units),
    }