Skip to content
Python
9.5 KB
Raw Blame
# -*- coding: utf-8 -*-
"""Light-search diversity helpers (no LLM, no extra embed API).

Historical helper used before top-k truncation:
  - social-boilerplate demotion (meet-up / catch-up templates — structural, not domain lexicon)
  - pool DF demotion (tokens that dominate the candidate pool)
  - near-dup demotion via content token Jaccard
  - MMR selection to reduce homogeneous clusters flooding top-k
  - RRF blend so weak BM25 still moves rank
"""
from __future__ import annotations

import re
from collections import Counter
from typing import Any, Callable, Dict, List, Optional, Sequence, Set

_TOKEN_RE = re.compile(r"[a-z0-9]+")

# Discourse-level social scheduling / check-in templates (not topic lexicons).
_SOCIAL_BOILER_RE = re.compile(
    r"(?i)\b("
    r"plan(?:s|ned|ning)?\s+to\s+catch\s+up|catch\s+up\s+soon|"
    r"get\s+together|"
    r"offered\s+to\s+(?:get\s+together|show|help|meet)|"
    r"accepted\s+\w+\s+offer\s+to\s+get\s+together|"
    r"enjoying\s+(?:his|her|their)\s+new\s+hobbies|"
    r"how\s+\w+\s+is\s+enjoying\s+(?:his|her|their)\s+new\s+hobbies"
    r")\b"
)


def content_tokens(text: str) -> Set[str]:
    return set(_TOKEN_RE.findall((text or "").lower()))


def jaccard(a: Set[str], b: Set[str]) -> float:
    if not a or not b:
        return 0.0
    inter = len(a & b)
    if inter == 0:
        return 0.0
    return inter / float(len(a | b))


def _hit_content(hit: Dict[str, Any]) -> str:
    c = hit.get("content")
    if c:
        return str(c)
    node = hit.get("node")
    if node is not None:
        return str(getattr(node, "content", "") or "")
    return ""


def is_social_boilerplate(text: str) -> bool:
    return bool(_SOCIAL_BOILER_RE.search(text or ""))


def social_boilerplate_demote(
    hits: Sequence[Dict[str, Any]],
    *,
    penalty: float = 0.22,
    score_key: str = "score",
) -> List[Dict[str, Any]]:
    """Multiply scores of meet-up / catch-up template memories.

    Structural discourse prior — not a per-topic expand lexicon.
    """
    if not hits or penalty <= 0:
        return list(hits)
    out: List[Dict[str, Any]] = []
    for h in hits:
        item = dict(h)
        if is_social_boilerplate(_hit_content(item)):
            sc = float(item.get(score_key) or 0.0)
            item[score_key] = sc * (1.0 - float(penalty))
            item["_social_boiler_demoted"] = True
        out.append(item)
    out.sort(key=lambda h: float(h.get(score_key) or 0.0), reverse=True)
    return out


def pool_df_demote(
    hits: Sequence[Dict[str, Any]],
    *,
    df_thr: float = 0.4,
    penalty: float = 0.18,
    score_key: str = "score",
    min_token_len: int = 4,
) -> List[Dict[str, Any]]:
    """Demote hits whose content is dominated by high-DF tokens in this pool.

    If many candidates share the same hub phrases (names + plan verbs), those
    hits lose score — lifts rarer, more specific facts without topic rules.
    """
    if not hits or df_thr <= 0 or penalty <= 0 or len(hits) < 4:
        return list(hits)
    n = float(len(hits))
    tok_lists: List[Set[str]] = []
    df: Counter = Counter()
    for h in hits:
        toks = {
            t for t in content_tokens(_hit_content(h))
            if len(t) >= min_token_len
        }
        tok_lists.append(toks)
        for t in toks:
            df[t] += 1
    hub = {t for t, c in df.items() if (c / n) >= df_thr}
    if not hub:
        return list(hits)
    out: List[Dict[str, Any]] = []
    for h, toks in zip(hits, tok_lists):
        item = dict(h)
        if toks and len(toks & hub) / float(len(toks)) >= 0.35:
            sc = float(item.get(score_key) or 0.0)
            item[score_key] = sc * (1.0 - float(penalty))
            item["_pool_df_demoted"] = True
        out.append(item)
    out.sort(key=lambda h: float(h.get(score_key) or 0.0), reverse=True)
    return out


def near_dup_demote(
    hits: Sequence[Dict[str, Any]],
    *,
    jaccard_thr: float = 0.55,
    penalty: float = 0.28,
    score_key: str = "score",
) -> List[Dict[str, Any]]:
    """Demote near-duplicate contents that trail a higher-scoring sibling.

    Keeps all hits (no hard drop) so later MMR / temporal can still promote;
    multiplies trailing near-dups by ``(1 - penalty)``.
    """
    if not hits or jaccard_thr <= 0 or penalty <= 0:
        return list(hits)
    ordered = sorted(
        hits, key=lambda h: float(h.get(score_key) or 0.0), reverse=True
    )
    kept_toks: List[Set[str]] = []
    out: List[Dict[str, Any]] = []
    for h in ordered:
        toks = content_tokens(_hit_content(h))
        dup = False
        for prev in kept_toks:
            if jaccard(toks, prev) >= jaccard_thr:
                dup = True
                break
        item = dict(h)
        if dup:
            sc = float(item.get(score_key) or 0.0)
            item[score_key] = sc * (1.0 - float(penalty))
            item["_near_dup_demoted"] = True
        else:
            if toks:
                kept_toks.append(toks)
        out.append(item)
    out.sort(key=lambda h: float(h.get(score_key) or 0.0), reverse=True)
    return out


# MMR 只在调用方请求的 top-k(search 的 limit)小于这个值时自动启用。
# mmr_select 的贪心是 O(k^2 * pool) 次 Jaccard,k 大时代价陡增,而大 k 本身已
# 足够容纳多样性,收益很小。权重仍由 MEMORY_HYBRID_V2_MMR_LAMBDA 控制。
MMR_MAX_TOP_K = 20


def mmr_applies(top_k: int, lambda_: float) -> bool:
    """本次检索是否走 MMR:λ 在有效区间内,且 top_k < MMR_MAX_TOP_K。"""
    return 0.0 < float(lambda_) < 0.999 and int(top_k) < MMR_MAX_TOP_K


def near_dup_applies(top_k: int) -> bool:
    """本次检索是否走近重复降权:与 MMR 同一个 top-k 门槛。

    near_dup_demote 让每个候选和此前所有"非重复"候选两两算 Jaccard,候选
    互不相似时是 O(pool^2);池子随 limit 放大(约 max(3*limit, 30) + 关键词
    命中),limit 达到 MMR_MAX_TOP_K 时代价同样陡增。大 k 的调用方(如记忆
    面板翻页)要的是全部相关项,相似项也应照常出现,降权没有收益。
    """
    return int(top_k) < MMR_MAX_TOP_K


def mmr_select(
    hits: Sequence[Dict[str, Any]],
    limit: int,
    *,
    lambda_: float = 0.7,
    score_key: str = "score",
    sim_fn: Optional[Callable[[Dict[str, Any], Dict[str, Any]], float]] = None,
) -> List[Dict[str, Any]]:
    """Greedy MMR: ``λ·rel − (1−λ)·max_sim(selected)``.

    Default similarity is content-token Jaccard (local, ms-level).
    """
    if limit <= 0 or not hits:
        return []
    if len(hits) <= limit or lambda_ >= 0.999:
        return sorted(
            hits, key=lambda h: float(h.get(score_key) or 0.0), reverse=True
        )[:limit]

    if sim_fn is None:
        tok_cache: Dict[int, Set[str]] = {}

        def _tok(h: Dict[str, Any]) -> Set[str]:
            i = id(h)
            if i not in tok_cache:
                tok_cache[i] = content_tokens(_hit_content(h))
            return tok_cache[i]

        def sim_fn(a: Dict[str, Any], b: Dict[str, Any]) -> float:
            return jaccard(_tok(a), _tok(b))

    remaining = list(hits)
    selected: List[Dict[str, Any]] = []
    # seed with best relevance
    remaining.sort(key=lambda h: float(h.get(score_key) or 0.0), reverse=True)
    selected.append(remaining.pop(0))

    while remaining and len(selected) < limit:
        best_i = 0
        best_val = float("-inf")
        for i, cand in enumerate(remaining):
            rel = float(cand.get(score_key) or 0.0)
            max_sim = 0.0
            for s in selected:
                max_sim = max(max_sim, float(sim_fn(cand, s)))
            val = float(lambda_) * rel - (1.0 - float(lambda_)) * max_sim
            if val > best_val:
                best_val = val
                best_i = i
        selected.append(remaining.pop(best_i))
    return selected


def blend_rrf_into_scores(
    hits: Sequence[Dict[str, Any]],
    *,
    blend: float = 0.4,
    bm25_weight: float = 1.25,
    score_key: str = "score",
    sem_key: str = "_semantic",
    bm25_key: str = "_bm25",
) -> List[Dict[str, Any]]:
    """Blend additive scores with RRF(sem, bm25, additive) so weak BM25 still moves rank.

    ``blend=0`` → unchanged; ``blend=1`` → fully RRF-scaled to pool max additive.
    """
    from .rrf import rrf_fuse

    if not hits or blend <= 0:
        return list(hits)
    items = [dict(h) for h in hits]
    sem_ch = sorted(
        items, key=lambda h: float(h.get(sem_key) or 0.0), reverse=True
    )
    bm25_ch = sorted(
        [h for h in items if float(h.get(bm25_key) or 0.0) > 0],
        key=lambda h: float(h.get(bm25_key) or 0.0),
        reverse=True,
    )
    add_ch = sorted(
        items, key=lambda h: float(h.get(score_key) or 0.0), reverse=True
    )
    channels = {"sem": sem_ch, "add": add_ch}
    weights = {"sem": 1.0, "add": 0.85}
    if bm25_ch:
        channels["bm25"] = bm25_ch
        weights["bm25"] = float(bm25_weight)
    fused = rrf_fuse(channels, weights=weights)
    if not fused:
        return items
    rrf_by = {f["node_id"]: float(f.get("rrf_score") or 0.0) for f in fused}
    max_rrf = max(rrf_by.values()) or 1.0
    max_add = max(float(h.get(score_key) or 0.0) for h in items) or 1.0
    b = max(0.0, min(1.0, float(blend)))
    for h in items:
        nid = h.get("node_id")
        r_norm = (rrf_by.get(nid, 0.0) / max_rrf) if nid else 0.0
        add = float(h.get(score_key) or 0.0)
        h[score_key] = (1.0 - b) * add + b * max_add * r_norm
        h["_rrf_norm"] = r_norm
    items.sort(key=lambda h: float(h.get(score_key) or 0.0), reverse=True)
    return items