Skip to content
Python
19.7 KB
Raw Blame
# -*- coding: utf-8 -*-
"""
记忆去重核心逻辑(纯函数,无 I/O,可单测)。

线上 L2_FACT / L4_IDENTITY 经常出现重复(extractor 抽重 / reconcile 漏合并)。
本模块用 embedding cosine 相似度(> 阈值,默认 0.95)识别重复组,并按确定性
规则决定保留/删除,供 extractor / reconcile / search 三条链路共用。

判重范围(调用方负责传入):
  只在「普通 memory + 链头(is_latest=True)」之间判重;链身(SUPERSEDED,
  is_latest=False)不参与——它们是历史,由其链头代表。

隔离维度按 **owner** 决定(见 group_items_by_owner):
  owner='user'(含空值)→ 按 (user_id,) 分批,**跨 agent 合并**:同一用户在不同
    agent 下说的同一件事是同一条事实。
  owner='agent'         → 按 (user_id, agent_id) 分批,**按 agent 隔离**:不同
    assistant 说过的话是独立记忆。

删除决策(每个重复组,确定性,非随机):
  - 全是非链节点          → 保留 gmt_created 最早的一条,删其余
  - 1 个链头 + 其余非链   → 保留链头,删非链
  - n 个链头(n≥2) + 非链  → 保留 gmt_created 最早的链头,删其余链头 + 非链;
                            被删链头**连带删除其整条 SUPERSEDED 历史链**
                            (chain_node_ids 由调用方用 _trace_full_chain 预先补全)

"删除"是**逻辑删除**:置 status=SHADOW + is_latest=False(不被检索召回),
而非硬删,可恢复。见 _shadow_nodes。

相似度计算:把参与节点的向量堆成矩阵一次算 M @ M.T(行先 L2 归一化),
30 条 4096 维约 0.3ms,不调 embedding API、不需预筛。
"""

from __future__ import annotations

import json
import logging
import os
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional

try:
    import numpy as np
except Exception:  # pragma: no cover
    np = None

logger = logging.getLogger(__name__)

DEDUP_LOG_STEP = "DEDUP"

DEFAULT_DEDUP_THRESHOLD = 0.95


def get_dedup_threshold() -> float:
    """去重 cosine 阈值,env MEMORY_DEDUP_THRESHOLD 覆盖,默认 0.95。"""
    try:
        return float(os.environ.get("MEMORY_DEDUP_THRESHOLD", DEFAULT_DEDUP_THRESHOLD))
    except (TypeError, ValueError):
        return DEFAULT_DEDUP_THRESHOLD


@dataclass
class DedupItem:
    """去重输入项(调用方组装)。

    is_chain_head / chain_node_ids 由调用方预先判定/补全:
      - 普通节点:is_chain_head=False, chain_node_ids=[node_id]
      - 链头:    is_chain_head=True,  chain_node_ids=[head, ...SUPERSEDED bodies]
    """
    node_id: str
    embedding: List[float]
    content: str = ""
    is_latest: bool = True
    is_chain_head: bool = False
    gmt_created: Optional[float] = None  # Unix ts;用于确定性保留(越小越早)
    chain_node_ids: List[str] = field(default_factory=list)
    # 分批用(见 group_items_by_owner):owner 决定隔离维度,user_id/agent_id
    # 决定批次键。三者缺失时由调用方的 default_* 兜底,行为退化为"全部一批",
    # 与加入分批之前等价。
    owner: Optional[str] = None
    user_id: str = ""
    agent_id: str = ""
    source: str = "default"


def _gmt_key(item: DedupItem) -> float:
    """gmt_created 排序键:缺失视为 +inf(最晚),保证有时间的优先被保留。"""
    return item.gmt_created if item.gmt_created is not None else float("inf")


# owner 归一化:空 owner 一律按 'user' 处理。
#
# owner 字段 2026-06-16(commit a240189)才引入,之前写入的 L2 节点没有该字段;
# 此外 LLM 也可能漏填。extractor/reconcile prompt 的约定都是 "When unsure,
# default to user",`reconciler.py` 组装 existing_mem_list 时同样用
# `owner or "user"` 兜底 —— 此处沿用同一约定,保持全链路一致。
OWNER_USER = "user"
OWNER_AGENT = "agent"


def _owner_bucket(owner: Optional[str]) -> str:
    """归一化 owner 到 'user' / 'agent' 两个桶;空值/未知值归 'user'。"""
    if not owner:
        return OWNER_USER
    v = str(owner).strip().lower()
    return OWNER_AGENT if v == OWNER_AGENT else OWNER_USER


def group_items_by_owner(
    items: List[DedupItem],
    *,
    default_user_id: str = "",
    default_agent_id: str = "",
) -> List[Dict[str, Any]]:
    """按 owner 语义把候选分成若干互不判重的批次。

    去重的隔离维度应当由 **owner** 决定,而不是无差别地按 agent_id 切分:

      - owner='user'(含空值):记忆属于**用户**这一侧。同一个用户在 agent_a
        和 agent_b 下说的同一件事是同一条事实,必须放进同一批判重,否则
        "The user is a vegetarian." 会在每个 agent 下各存一份且永远无法合并。
        → 批次键 = (user_id,),**跨 agent 合并**
      - owner='agent':记忆是某个 assistant 提供/被要求做的事。agent_a 说过的话
        和 agent_b 说过的话是两条独立记忆,即使语义相近也不应互删。
        → 批次键 = (user_id, agent_id),**按 agent 隔离**

    L4_IDENTITY 的 owner 恒为空(仅 L2_FACT / L7_INTENTION 写 owner),按上面的
    兜底规则落进 user 桶 —— identity 本就是用户画像,跨 agent 合并是正确语义。

    同时解决审计归属错误:此前调用方只能传一个 user_id/agent_id(取
    `effective_user_ids[0]`),跨 user/跨 agent 检索时 DEDUP log 记的归属是错的。
    分批后每批的 user_id/agent_id 是确定的,日志自然准确。

    Args:
        items: 候选项,需带 user_id / agent_id / owner(缺失则用 default_*)。
        default_user_id / default_agent_id: item 上没有对应字段时的兜底值。

    Returns:
        [{"user_id": ..., "agent_id": ..., "owner": "user"|"agent",
          "items": [DedupItem, ...]}, ...]
        只返回 len(items) >= 2 的批次(单条无从判重)。
    """
    buckets: Dict[tuple, Dict[str, Any]] = {}
    for it in items:
        uid = it.user_id or default_user_id
        aid = it.agent_id or default_agent_id
        from ...utils.source_filter import normalize_source
        source = normalize_source(it.source)
        bucket = _owner_bucket(it.owner)
        # user 桶不带 agent_id → 同一 user 的所有 agent 归一批
        key = (
            (uid, "", OWNER_USER, source)
            if bucket == OWNER_USER else (uid, aid, OWNER_AGENT, source)
        )
        entry = buckets.get(key)
        if entry is None:
            entry = {
                "user_id": uid,
                # user 桶跨 agent,没有单一归属的 agent_id:用调用方给的默认值
                # 落审计日志(代表"本次触发去重的 agent"),不参与分批。
                "agent_id": default_agent_id if bucket == OWNER_USER else aid,
                "owner": bucket,
                "source": source,
                "items": [],
            }
            buckets[key] = entry
        entry["items"].append(it)

    return [b for b in buckets.values() if len(b["items"]) >= 2]


def _pairwise_cosine(vectors: List[List[float]]):
    """行 L2 归一化后 M @ M.T,返回相似度矩阵(numpy)。"""
    M = np.asarray(vectors, dtype=np.float32)
    norms = np.linalg.norm(M, axis=1, keepdims=True)
    norms[norms == 0] = 1.0
    M = M / norms
    return M @ M.T


def _union_find_groups(n: int, sim, threshold: float) -> List[List[int]]:
    """对 sim > threshold 的成对索引做并查集连通,返回每组的 index 列表(含单点)。"""
    parent = list(range(n))

    def find(x):
        while parent[x] != x:
            parent[x] = parent[parent[x]]
            x = parent[x]
        return x

    def union(a, b):
        ra, rb = find(a), find(b)
        if ra != rb:
            parent[rb] = ra

    for i in range(n):
        for j in range(i + 1, n):
            if float(sim[i][j]) > threshold:
                union(i, j)

    groups: Dict[int, List[int]] = {}
    for i in range(n):
        groups.setdefault(find(i), []).append(i)
    return list(groups.values())


def _decide_group(members: List[DedupItem]) -> Dict[str, Any]:
    """对一个重复组决定保留/删除,返回组明细(含审计字段)。"""
    heads = [m for m in members if m.is_chain_head]

    if heads:
        # 有链头:保留 gmt 最早的链头;其余(含其他链头 + 非链)全删
        kept = min(heads, key=_gmt_key)
        reason = "earliest_chain_head" if len(heads) > 1 else "chain_head"
    else:
        # 全非链:保留 gmt 最早的一条
        kept = min(members, key=_gmt_key)
        reason = "earliest_gmt"

    deleted = []
    delete_ids: List[str] = []
    for m in members:
        if m.node_id == kept.node_id:
            continue
        # 被删链头连带删除整条历史链(chain_node_ids 含 head 自身 + bodies)
        cascade = [cid for cid in (m.chain_node_ids or [m.node_id]) if cid != kept.node_id]
        if not cascade:
            cascade = [m.node_id]
        also = [cid for cid in cascade if cid != m.node_id]
        deleted.append({
            "node_id": m.node_id,
            "content": m.content,
            "is_chain_head": m.is_chain_head,
            "also_deleted_chain": also,
        })
        delete_ids.extend(cascade)

    return {
        "members": [
            {
                "node_id": m.node_id,
                "content": m.content,
                "is_latest": m.is_latest,
                "is_chain_head": m.is_chain_head,
            }
            for m in members
        ],
        "kept": {"node_id": kept.node_id, "content": kept.content, "reason": reason},
        "deleted": deleted,
        "delete_ids": delete_ids,
    }


def plan_dedup(
    items: List[DedupItem],
    *,
    threshold: Optional[float] = None,
) -> Dict[str, Any]:
    """
    对一组候选项做去重规划(纯函数,不执行删除)。

    只比较 is_latest=True 的项(普通节点 + 链头);is_latest=False 的链身被忽略
    (但若某链头被删,其 chain_node_ids 里的链身会进 delete_ids 连带删)。

    Returns:
        {
          "threshold": float,
          "groups": [ {members, kept, deleted, delete_ids}, ... ]  # 只含有删除的组
          "delete_ids": [所有待删 node_id(含连带链),去重后],
          "kept_ids":   [所有保留的代表 node_id],
        }
    """
    thr = threshold if threshold is not None else get_dedup_threshold()
    empty = {"threshold": thr, "groups": [], "delete_ids": [], "kept_ids": [],
             "compared_count": 0, "max_cosine": None, "avg_cosine": None}

    if np is None or not items:
        return empty

    # 只参与 is_latest=True 且有 embedding 的项
    parts = [it for it in items if it.is_latest and it.embedding]
    if len(parts) < 2:
        return {**empty, "compared_count": len(parts)}

    sim = _pairwise_cosine([it.embedding for it in parts])
    idx_groups = _union_find_groups(len(parts), sim, thr)

    # 上三角余弦统计(排除对角 1.0),供审计:最高 / 平均
    n = len(parts)
    iu = np.triu_indices(n, k=1)
    pair_cos = sim[iu]
    max_cos = float(pair_cos.max()) if pair_cos.size else None
    avg_cos = float(pair_cos.mean()) if pair_cos.size else None

    groups_out: List[Dict[str, Any]] = []
    all_delete: List[str] = []
    all_kept: List[str] = []
    seen_del = set()

    for idxs in idx_groups:
        if len(idxs) < 2:
            continue  # 非重复,跳过
        members = [parts[i] for i in idxs]
        decided = _decide_group(members)
        all_kept.append(decided["kept"]["node_id"])
        for did in decided["delete_ids"]:
            if did not in seen_del:
                seen_del.add(did)
                all_delete.append(did)
        groups_out.append({
            "members": decided["members"],
            "kept": decided["kept"],
            "deleted": decided["deleted"],
        })

    return {
        "threshold": thr,
        "groups": groups_out,
        "delete_ids": all_delete,
        "kept_ids": all_kept,
        "compared_count": len(parts),
        "max_cosine": max_cos,
        "avg_cosine": avg_cos,
    }


def _build_dedup_content(trigger: str, plan: Dict[str, Any]) -> str:
    """构造 DEDUP step 的人类可读 content。

    无删除:trigger + 比较条数 + max/avg cosine + 阈值。
    有删除:上面这些 + 每个重复组「重复的是哪些 / 删了哪些 / 保留哪条」。
    """
    thr = plan.get("threshold")
    compared = plan.get("compared_count", 0)
    max_c = plan.get("max_cosine")
    avg_c = plan.get("avg_cosine")
    delete_ids = plan.get("delete_ids", [])

    def _fmt(v):
        return f"{v:.4f}" if isinstance(v, (int, float)) else "n/a"

    lines = [
        f"[DEDUP] trigger={trigger} threshold={thr}",
        f"compared_count={compared} max_cosine={_fmt(max_c)} avg_cosine={_fmt(avg_c)}",
        f"deleted_count={len(delete_ids)}",
    ]

    groups = plan.get("groups", [])
    if groups:
        for gi, g in enumerate(groups):
            members = g.get("members", [])
            kept = g.get("kept", {})
            deleted = g.get("deleted", [])
            lines.append(f"-- group#{gi} ({len(members)} duplicates) --")
            for m in members:
                lines.append(
                    f"   dup: id={m.get('node_id')} head={m.get('is_chain_head')} "
                    f"content={(m.get('content') or '')[:120]}"
                )
            lines.append(
                f"   KEEP: id={kept.get('node_id')} reason={kept.get('reason')} "
                f"content={(kept.get('content') or '')[:120]}"
            )
            for d in deleted:
                also = d.get("also_deleted_chain") or []
                also_s = f" also_chain={also}" if also else ""
                lines.append(
                    f"   DELETE: id={d.get('node_id')} head={d.get('is_chain_head')}{also_s} "
                    f"content={(d.get('content') or '')[:120]}"
                )
        lines.append(f"all_deleted_ids={delete_ids}")

    return "\n".join(lines)


async def _emit_dedup_log(
    cache: Any,
    *,
    request_id: str,
    user_id: str,
    agent_id: str,
    trigger: str,
    plan: Dict[str, Any],
    extra_parsed: Optional[Dict[str, Any]] = None,
) -> None:
    """写 DEDUP pipeline log(删除审计明细)。best-effort。"""
    if cache is None or not getattr(cache, "store_pipeline_log", None):
        return
    try:
        parsed = {
            "trigger": trigger,
            "threshold": plan.get("threshold"),
            "groups": plan.get("groups", []),
            "deleted_count": len(plan.get("delete_ids", [])),
            "kept_count": len(plan.get("kept_ids", [])),
            "compared_count": plan.get("compared_count", 0),
            "max_cosine": plan.get("max_cosine"),
            "avg_cosine": plan.get("avg_cosine"),
        }
        if extra_parsed:
            parsed.update(extra_parsed)
        await cache.store_pipeline_log(
            request_id=request_id or "",
            user_id=user_id or "",
            agent_id=agent_id or "default_agent",
            step=DEDUP_LOG_STEP,
            prompt="",
            response=_build_dedup_content(trigger, plan),
            parsed=json.dumps(parsed, ensure_ascii=False, default=str),
            memory_ids=plan.get("delete_ids", []) or None,
        )
    except Exception as e:
        logger.debug(f"[dedup] emit DEDUP log failed: {e}")


async def _shadow_nodes(
    vector_store: Any,
    node_ids: List[str],
    *,
    kept_id: str,
    trigger: str,
) -> int:
    """把重复节点置 SHADOW(逻辑删除),而不是硬删。

    此前走 `vector_store.delete(nid)` 真删、不可恢复 —— 而被删的可能是一整条
    演化链(链头被判重时连带删 chain_node_ids),一次阈值误判就永久抹掉一条
    记忆的全部演化轨迹。search 链路的 dedup 还是 fire-and-forget 后台任务,
    用户完全无感知,出错也无从恢复。

    改为 SHADOW 后语义等价(SHADOW 不被任何检索召回,等于逻辑删除)但可恢复,
    与 reconciler UPDATE / L1_RAW 降级用的是同一套状态位。

    用 update_payload 而非 upsert:只改状态位,不重写向量(DedupItem 手里只有
    embedding,没有完整节点;整节点 upsert 会破坏 payload)。
    """
    from ...models.memory import MemoryStatus

    ok = 0
    for nid in node_ids:
        try:
            await vector_store.update_payload(
                nid,
                {
                    "is_latest": False,
                    "status": MemoryStatus.SHADOW.value,
                    # 审计:这条为什么不见了、被哪条取代
                    "shadow_reason": f"dedup:{trigger}",
                    "dedup_kept_id": kept_id,
                },
            )
            ok += 1
        except Exception as e:
            logger.warning(f"[dedup] shadow {nid} failed (trigger={trigger}): {e}")
    return ok


async def execute_dedup(
    items: List[DedupItem],
    *,
    vector_store: Any,
    cache: Any = None,
    trigger: str,
    request_id: str = "",
    user_id: str = "",
    agent_id: str = "",
    delete_from_store: bool = True,
    threshold: Optional[float] = None,
    extra_parsed: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
    """
    去重编排:plan → (可选)删库 → 写 DEDUP log。整体 best-effort,绝不抛错影响主流程。

    Args:
        items: 同一 isolation 范围内的候选(调用方保证不跨 user/agent)。
        delete_from_store: True 时把 plan.delete_ids 置 SHADOW(逻辑删除,可恢复;
            不是硬删——见 _shadow_nodes);extractor 阶段(结果还没入库)传 False,
            只记录被丢弃项。
        trigger: "extractor" / "reconcile" / "search",落进 DEDUP log。
        extra_parsed: 额外写入 DEDUP log.parsed 的字段(如 embed 耗时 / provider meta)。

    Returns:
        plan dict(含 delete_ids / kept_ids / groups)。无重复时 delete_ids 为空。
    """
    try:
        plan = plan_dedup(items, threshold=threshold)
    except Exception as e:
        logger.warning(f"[dedup] plan_dedup failed (trigger={trigger}): {e}")
        return {"threshold": threshold, "groups": [], "delete_ids": [], "kept_ids": []}

    delete_ids = plan.get("delete_ids", [])
    if not delete_ids:
        # 无重复:仍落一条 DEDUP step(记录 compared_count / max_cosine / avg_cosine),
        # 方便确认 dedup 确实跑过、最接近的一对离阈值有多远。
        await _emit_dedup_log(
            cache,
            request_id=request_id,
            user_id=user_id,
            agent_id=agent_id,
            trigger=trigger,
            plan=plan,
            extra_parsed=extra_parsed,
        )
        return plan

    if delete_from_store and vector_store is not None:
        # 逻辑删除(SHADOW),不硬删。每个组的 kept_id 单独记,便于事后追溯
        # "这条被哪条取代"。
        for g in plan.get("groups", []):
            _kept = (g.get("kept") or {}).get("node_id", "")
            _gids: List[str] = []
            for d in (g.get("deleted") or []):
                if d.get("node_id"):
                    _gids.append(d["node_id"])
                _gids.extend(d.get("also_deleted_chain") or [])
            if _gids:
                await _shadow_nodes(
                    vector_store, _gids, kept_id=_kept, trigger=trigger,
                )

    await _emit_dedup_log(
        cache,
        request_id=request_id,
        user_id=user_id,
        agent_id=agent_id,
        trigger=trigger,
        plan=plan,
        extra_parsed=extra_parsed,
    )

    logger.info(
        f"[dedup] trigger={trigger} groups={len(plan.get('groups', []))} "
        f"shadowed={len(delete_ids)} (apply={delete_from_store})"
    )
    return plan