Skip to content
Python
16.5 KB
Raw Blame
# -*- coding: utf-8 -*-
"""
HY Memory - Intention (L7) 召回 + 惰性过期转换 + 复本判定

意图(L7_INTENTION)由 extractor(System 1)直接写入 VDB,带可选 `valid_until`
截止日。reader 在 proactive 路调用本模块召回意图:

  - 召回 status=ACTIVE 的 L7 节点(**超采样**,见下);
  - 对每条命中检查 `valid_until`:若已过期(now > valid_until),则**惰性**把该
    节点的 layer 改写为 l2_fact(持久化 update_payload),并从意图结果里剔除——
    过期意图变成普通历史事实,下次走 normal 召回;
  - **判重(重点)**:对幸存意图按下述规则分组,同组只留一条本次返回:
        同 user_id + 同 agent_id + 同一天 memory_at + valid_until 相同
        (都为 None 或都相等)+ cosine 相似度 ≥ MEMORY_DEDUP_THRESHOLD (默认 0.95)
    → 视为同一个意图的复本(extractor 复读机 / 多 session 重复表达)。
    保留组里 `gmt_created` 最早的(老的先来先得,稳定 id);
  - **异步删库**:其余复本 fire-and-forget 从 VDB 删除,不影响本次调用 RT。

Q: 为什么要看 memory_at 和 valid_until?
A: 意图是 point-in-time 信号。同一句"我周五要开会",如果一条 memory_at 在
   本周一(valid_until=本周五)、另一条 memory_at 在下周一(valid_until=下
   周五),指向的是**两次不同的会议**,绝不能合并。所以只看 cosine 会误伤。

best-effort:过期转换 / dedup / 异步删除任何异常都不影响主 search 流程。
"""

from __future__ import annotations

import asyncio
import logging
import os
import time
from datetime import date, datetime
from typing import Any, Dict, List, Optional, Tuple

from ...models.memory import MemoryLayer, MemoryStatus

logger = logging.getLogger(__name__)


def _get_dedup_threshold() -> float:
    """去重 cosine 阈值:沿用 dedup 内核的 MEMORY_DEDUP_THRESHOLD,默认 0.95。"""
    try:
        return float(os.environ.get("MEMORY_DEDUP_THRESHOLD", 0.95))
    except (TypeError, ValueError):
        return 0.95


def _memory_at_day(node: Any) -> Optional[date]:
    """取 node.memory_at 的日期部分;None 返回 None。"""
    if node is None:
        return None
    ma = getattr(node, "memory_at", None)
    if ma is None:
        return None
    # datetime 或 date 都兼容
    try:
        return ma.date() if hasattr(ma, "date") else ma
    except Exception:
        return None


def _valid_until_key(node: Any) -> Any:
    """valid_until 归一化到"比较键":datetime→date,None→None。

    只要求日粒度一致(valid_until 本身在 writer 里被 pin 到当天日末,同天视为等)。
    """
    if node is None:
        return None
    vu = getattr(node, "valid_until", None)
    if vu is None:
        return None
    try:
        return vu.date() if hasattr(vu, "date") else vu
    except Exception:
        return None


def _cosine(a: List[float], b: List[float]) -> float:
    """朴素 cosine(同事说"从 VDB 拿现成 embed,不重算",这里只是对拿到的向量比一次)。

    重复调用量级:每次 search 内 fetch_limit ≤ 30,两两比较 O(n²) 也就 <500 次
    浮点内积,无需 numpy。返回 [-1, 1],向量为空返回 -inf 避免误判。
    """
    if not a or not b:
        return float("-inf")
    la = sum(x * x for x in a) ** 0.5
    lb = sum(x * x for x in b) ** 0.5
    if la == 0.0 or lb == 0.0:
        return float("-inf")
    dot = sum(x * y for x, y in zip(a, b))
    return dot / (la * lb)


def _partition_duplicates(
    hits: List[Dict[str, Any]],
    threshold: float,
) -> Tuple[List[Dict[str, Any]], List[str]]:
    """把 hits 分成"保留列表"和"待删 node_id 列表"。

    分组键:(memory_at.day, valid_until.day);每组内两两比 cosine,
    ≥ threshold 视为复本。复本组保留 gmt_created 最早的(gmt 缺失视为 +inf
    最晚),其余进入 delete_ids。

    Args:
        hits: recall + 未过期过滤后的 L7 命中
        threshold: cosine 复本阈值

    Returns:
        (kept_hits, delete_ids) — kept_hits 保序(保留项按原顺序)
    """
    if len(hits) < 2:
        return list(hits), []

    # 1. 按 (user_id, agent_id, memory_at day, valid_until day) 分桶。
    #    user_id / agent_id 必须进键:多用户检索时不同用户的同一天意图可能
    #    文本、日期都相同,但它们是两个独立的租户数据;只按日期分桶会把其中
    #    一个判成复本并硬删除。
    buckets: Dict[Tuple[Any, Any, Any, Any], List[int]] = {}
    for idx, h in enumerate(hits):
        node = h.get("node")
        key = (
            getattr(node, "user_id", None) if node else None,
            getattr(node, "agent_id", None) if node else None,
            _memory_at_day(node),
            _valid_until_key(node),
        )
        buckets.setdefault(key, []).append(idx)

    def _gmt(h: Dict[str, Any]) -> float:
        node = h.get("node")
        gc = getattr(node, "gmt_created", None) if node else None
        if gc is None:
            return float("inf")
        try:
            return gc.timestamp() if hasattr(gc, "timestamp") else float(gc)
        except Exception:
            return float("inf")

    dropped_idx: set = set()

    # 2. 每桶内两两比 cosine → 并查集
    for key, idx_list in buckets.items():
        if len(idx_list) < 2:
            continue

        parent = {i: i for i in idx_list}

        def find(x):
            while parent[x] != x:
                parent[x] = parent[parent[x]]
                x = parent[x]
            return x

        def union(a, b):
            ra, rb = find(a), find(b)
            if ra != rb:
                parent[rb] = ra

        # 提取 embedding
        embs: Dict[int, List[float]] = {}
        for i in idx_list:
            node = hits[i].get("node")
            emb = getattr(node, "embedding", None) if node else None
            if emb:
                embs[i] = emb

        # 两两比较
        candidates = [i for i in idx_list if i in embs]
        for a_pos in range(len(candidates)):
            for b_pos in range(a_pos + 1, len(candidates)):
                ia, ib = candidates[a_pos], candidates[b_pos]
                if _cosine(embs[ia], embs[ib]) >= threshold:
                    union(ia, ib)

        # 3. 每个连通分量保留 gmt 最早的
        groups: Dict[int, List[int]] = {}
        for i in idx_list:
            groups.setdefault(find(i), []).append(i)

        for members in groups.values():
            if len(members) < 2:
                continue  # 单点,非复本
            # 保留 gmt 最早
            keeper = min(members, key=lambda i: _gmt(hits[i]))
            for i in members:
                if i != keeper:
                    dropped_idx.add(i)

    kept = [h for i, h in enumerate(hits) if i not in dropped_idx]
    delete_ids = [
        hits[i].get("node_id") for i in sorted(dropped_idx)
        if hits[i].get("node_id")
    ]
    return kept, delete_ids


def _spawn_async_delete(
    vector_store: Any,
    delete_ids: List[str],
    *,
    user_id_for_log: str = "",
) -> None:
    """fire-and-forget 从 VDB 删除复本,不影响主 search RT。

    - 用 asyncio.create_task 挂到当前 loop(recall 已经在 async 上下文);
    - 失败逐条 swallow,最后一条日志汇总;
    - 若拿不到 loop(极少见),静默跳过。
    """
    if not delete_ids or vector_store is None:
        return

    async def _run():
        ok, fail = 0, 0
        for nid in delete_ids:
            try:
                await vector_store.delete(nid)
                ok += 1
            except Exception as e:
                fail += 1
                logger.debug(f"[intention] async delete {nid} failed: {e}")
        logger.info(
            f"[intention] async dedup delete done: ok={ok} fail={fail} "
            f"total={len(delete_ids)} user_id={user_id_for_log or '-'}"
        )

    try:
        asyncio.create_task(_run())
    except RuntimeError:
        # 无 running loop:静默跳过(recall 本身在 async 环境,一般不会走到)
        logger.debug("[intention] no running loop for async delete; skipped")


async def recall_intentions(
    vector_store: Any,
    query_embedding: List[float],
    *,
    user_ids: Optional[List[str]] = None,
    agent_ids: Optional[List[str]] = None,
    isolation_key: str = "",
    isolation_keys: Optional[List[str]] = None,
    limit: int = 10,
    now: Optional[datetime] = None,
    stats_out: Optional[Dict[str, Any]] = None,
    include_sources: Optional[List[str]] = None,
    exclude_sources: Optional[List[str]] = None,
) -> List[Dict[str, Any]]:
    """
    召回未过期的 L7_INTENTION 节点(VDB),惰性过期转 L2_FACT,判重并异步删复本。

    Args:
        vector_store: VectorStoreBase 实例
        query_embedding: 查询向量
        user_ids: 用户隔离(单/多用户)
        agent_ids: 可选 agent 过滤
        isolation_key / isolation_keys: 精确的 (user, agent, session) 隔离键。
            与其它召回通道一致:调用方在按 session 过滤时只生成这两项,
            user_ids / agent_ids 为 None;不把它们透传下去就会退化成全库检索。
        limit: 召回上限(返回给上游的条数)
        now: 当前时间(默认 datetime.now(),测试可注入)
        stats_out: 可选 dict,本函数把分段耗时与数量**原地回填**进去,供调用方落
            READ_INTENTION pipeline log。传 None 则完全不统计(零开销)。

    Returns:
        [{"node_id", "score", "node", "layer": "l7_intention", "source": "vdb_intention"}, ...]

    实现细节:
      - **超采样**:VDB 侧多拿一些,扛住 L7 复读机;判重后截取 limit
      - **判重规则**:同一天 memory_at + valid_until 同日 + cosine≥0.95。
      - **异步删库**:待删复本 fire-and-forget 从 VDB 删,不影响返回 RT。

    埋点字段(回填进 stats_out):
      耗时(ms): vdb_search_ms / expire_ms / backfill_embed_ms / dedup_ms / total_ms
      数量: fetched / expired / survivors / dedup_dropped / returned
      配置: fetch_limit / out_limit / dedup_threshold
      说明: expire_ms、backfill_embed_ms 只有真的发生了对应操作才非 0,
            借此可判断慢在 VDB 检索、逐条过期转换、补拉向量还是判重计算。
    """
    if limit <= 0:
        return []
    _now = now or datetime.now()
    _t_total = time.perf_counter()

    def _stat(**kv: Any) -> None:
        """回填埋点字段;stats_out 为 None 时是 no-op。"""
        if stats_out is not None:
            stats_out.update(kv)

    # 超采样:VDB 侧多拿一些,扛住 L7 复读机;判重后截取 limit
    fetch_limit = max(limit * 6, limit + 15)
    _stat(fetch_limit=fetch_limit, out_limit=limit)

    _t = time.perf_counter()
    try:
        hits = await vector_store.search(
            query_embedding=query_embedding,
            user_ids=user_ids or None,
            agent_ids=agent_ids,
            isolation_key=isolation_key,
            isolation_keys=isolation_keys,
            layers=[MemoryLayer.L7_INTENTION],
            limit=fetch_limit,
            score_threshold=0.0,  # intention 不设语义门槛
            status_filter=[MemoryStatus.ACTIVE],
            only_latest=True,
            include_sources=include_sources,
            exclude_sources=exclude_sources,
        ) or []
    except Exception as e:
        logger.debug(f"[intention] recall search failed: {e}")
        _stat(
            vdb_search_ms=round((time.perf_counter() - _t) * 1000, 2),
            total_ms=round((time.perf_counter() - _t_total) * 1000, 2),
            fetched=0, returned=0, search_failed=True,
        )
        return []
    _stat(vdb_search_ms=round((time.perf_counter() - _t) * 1000, 2), fetched=len(hits))

    # 1. 惰性过期转换:valid_until 已过的 → 转 L2_FACT + 剔除
    _t = time.perf_counter()
    survivors: List[Dict[str, Any]] = []
    expired_count = 0
    for h in hits:
        node = h.get("node")
        node_id = h.get("node_id", "")
        valid_until = getattr(node, "valid_until", None) if node else None

        if valid_until and _now > valid_until:
            try:
                await vector_store.update_payload(node_id, {"layer": MemoryLayer.L2_FACT.value})
                logger.debug(
                    f"[intention] expired L7 → L2_FACT: node_id={node_id} "
                    f"valid_until={valid_until}"
                )
                expired_count += 1
            except Exception as e:
                logger.debug(f"[intention] expire conversion failed for {node_id}: {e}")
            continue

        survivors.append({
            **h,
            "layer": MemoryLayer.L7_INTENTION.value,
            "retrieval_source": "vdb_intention",
        })
    # 无过期节点时这段是纯内存循环、耗时≈0;非 0 即为逐条 update_payload 的开销
    _stat(
        expire_ms=round((time.perf_counter() - _t) * 1000, 2),
        expired=expired_count,
        survivors=len(survivors),
    )

    # 1.5. 关键补丁:不同 VDB 后端的 search() 是否携带 embedding 不一致
    #    - Qdrant/Faiss 部分场景 hit 带 embedding
    #    - Chroma search() 默认 include 不含 embeddings → node.embedding=None
    # 若判重时拿不到 embedding,_partition_duplicates 会静默降级为"不去重"。
    # 这里对缺失 embedding 的节点,用 get_embeddings 批量补拉(VDB 已有向量,
    # 不重算 embed 服务)。失败 → 保持 None → dedup 自然降级,不影响主流程。
    _t = time.perf_counter()
    _backfilled = 0
    _missing_cnt = 0
    if survivors:
        missing_ids = [
            h.get("node_id") for h in survivors
            if h.get("node_id") and not getattr(h.get("node"), "embedding", None)
        ]
        _missing_cnt = len(missing_ids)
        if missing_ids and hasattr(vector_store, "get_embeddings"):
            try:
                emb_map = await vector_store.get_embeddings(missing_ids)
                if emb_map:
                    for h in survivors:
                        node = h.get("node")
                        nid = h.get("node_id")
                        if node is not None and nid in emb_map and not getattr(node, "embedding", None):
                            try:
                                node.embedding = list(emb_map[nid])
                            except Exception:
                                pass
                    _backfilled = len(emb_map)
                    logger.debug(
                        f"[intention] backfilled embeddings for {len(emb_map)}/"
                        f"{len(missing_ids)} nodes before dedup"
                    )
            except Exception as e:
                logger.debug(f"[intention] get_embeddings failed (skip backfill): {e}")
    # 只有真的补拉过才非 0;非 0 且很大 → get_embeddings 是瓶颈
    _stat(
        backfill_embed_ms=round((time.perf_counter() - _t) * 1000, 2),
        embed_missing=_missing_cnt,
        embed_backfilled=_backfilled,
    )

    # 2. 判重:同 user + 同天 memory_at + valid_until 同日 + cosine≥threshold
    _t = time.perf_counter()
    threshold = _get_dedup_threshold()
    try:
        kept, delete_ids = _partition_duplicates(survivors, threshold)
    except Exception as e:
        # 任何 dedup 异常 → 降级为不去重(返回全部 survivors 截断 limit)
        logger.warning(f"[intention] dedup partition failed, degrade to raw: {e}")
        kept, delete_ids = survivors, []
    # 纯 CPU(cosine 两两比较);survivors 多时可能显著
    _stat(
        dedup_ms=round((time.perf_counter() - _t) * 1000, 2),
        dedup_dropped=len(delete_ids),
        dedup_threshold=threshold,
    )

    # 3. 截断到调用方要求的 limit
    result = kept[:limit]
    _stat(
        returned=len(result),
        total_ms=round((time.perf_counter() - _t_total) * 1000, 2),
    )

    # 4. 异步删库(fire-and-forget)
    if delete_ids:
        user_id_for_log = (user_ids[0] if user_ids else "") or ""
        _spawn_async_delete(vector_store, delete_ids, user_id_for_log=user_id_for_log)

    if expired_count or delete_ids or len(hits) != len(result):
        logger.info(
            f"[intention] recall: fetched={len(hits)} expired={expired_count} "
            f"survivors={len(survivors)} dedup_dropped={len(delete_ids)} "
            f"kept_after_dedup={len(kept)} returned={len(result)} "
            f"(fetch_limit={fetch_limit}, out_limit={limit}, thr={threshold})"
        )
    return result