Skip to content
Python
29.9 KB
Raw Blame
"""
Agent Memory V2 - VectorStore (FAISS)

基于 FAISS 的向量存储层。

特点:
- 使用 faiss-cpu,纯 CPU 计算
- 内存 dict 存 payload,search 后手动过滤 metadata
- faiss.write_index() + JSON 持久化 payload
- 同步 API → _run_in_vdb_pool() 包装为异步
"""

from typing import Optional, List, Dict, Any
from pathlib import Path
import asyncio
import concurrent.futures
import json
import logging
import threading
import numpy as np

from ..models.memory import (
    MemoryNode, MemoryLayer, MemoryStatus,
)
from ..config import MemoryConfig
from ..utils.async_executor import run_in_executor_guarded
from .vector_store_base import VectorStoreBase, passes_created_after

logger = logging.getLogger(__name__)

# VDB 独立线程池
from .vector_store_chroma import _vdb_executor


def _run_in_vdb_pool(func, *args, **kwargs):
    """在 VDB 独立线程池中执行同步函数"""
    import functools
    call = functools.partial(func, *args, **kwargs) if args or kwargs else func
    return run_in_executor_guarded(_vdb_executor, call)


class FaissVectorStore(VectorStoreBase):
    """
    FAISS 向量存储实现

    使用 faiss-cpu 做相似度检索,内存 dict 存储 payload,
    持久化到磁盘(faiss index 文件 + JSON payload 文件)。
    """

    def __init__(self, config: MemoryConfig):
        super().__init__(config)
        self._index = None
        # point_id (str) → { "payload": dict, "embedding": list, "idx": int }
        self._data: Dict[str, Dict[str, Any]] = {}
        # faiss 内部索引 int → point_id str 的映射
        self._idx_to_id: Dict[int, str] = {}
        self._next_idx: int = 0
        self._lock = threading.Lock()
        self._dims = config.vector_store.embedding_dims

    async def initialize(self) -> None:
        """初始化 FAISS 索引"""
        try:
            import faiss
        except ImportError:
            raise ImportError(
                "faiss-cpu is required for FAISS backend. "
                "Install with: pip install hy-memory[faiss]"
            )

        def _init():
            persist_dir = Path(self.config.vector_store.persist_directory)
            persist_dir.mkdir(parents=True, exist_ok=True)

            index_path = persist_dir / f"{self._collection_name}.faiss"
            payload_path = persist_dir / f"{self._collection_name}.json"

            if index_path.exists() and payload_path.exists():
                # 从磁盘加载
                self._index = faiss.read_index(str(index_path))
                with open(payload_path, "r", encoding="utf-8") as f:
                    saved = json.load(f)
                self._data = saved.get("data", {})
                self._idx_to_id = {int(k): v for k, v in saved.get("idx_to_id", {}).items()}
                self._next_idx = saved.get("next_idx", 0)
                logger.info(
                    f"FAISS index loaded from {index_path}, "
                    f"points={self._index.ntotal}"
                )
            else:
                # 创建新索引 (使用 IndexFlatIP,余弦相似度需先归一化)
                self._index = faiss.IndexFlatIP(self._dims)
                logger.info(f"Created new FAISS index, dims={self._dims}")

        await _run_in_vdb_pool(_init)
        logger.debug(f"VectorStore initialized (FAISS), collection={self._collection_name}")

    # ================================================================
    # 持久化
    # ================================================================

    def _persist(self) -> None:
        """持久化索引和 payload 到磁盘"""
        import faiss

        persist_dir = Path(self.config.vector_store.persist_directory)
        persist_dir.mkdir(parents=True, exist_ok=True)

        index_path = persist_dir / f"{self._collection_name}.faiss"
        payload_path = persist_dir / f"{self._collection_name}.json"

        faiss.write_index(self._index, str(index_path))
        saved = {
            "data": self._data,
            "idx_to_id": {str(k): v for k, v in self._idx_to_id.items()},
            "next_idx": self._next_idx,
        }
        with open(payload_path, "w", encoding="utf-8") as f:
            json.dump(saved, f, ensure_ascii=False)

    @staticmethod
    def _normalize(vec: List[float]) -> np.ndarray:
        """L2 归一化,用于余弦相似度"""
        arr = np.array(vec, dtype=np.float32).reshape(1, -1)
        norm = np.linalg.norm(arr)
        if norm > 0:
            arr /= norm
        return arr

    # ================================================================
    # 写入
    # ================================================================

    async def upsert(self, node: MemoryNode) -> str:
        """写入或更新一个 MemoryNode"""
        if node.embedding is None:
            raise ValueError(f"Node {node.node_id} has no embedding, cannot upsert to vector store")

        payload = self._node_to_payload(node)
        point_id = self._node_id_to_point_id(node.node_id)

        def _upsert():
            with self._lock:
                vec = self._normalize(node.embedding)

                if point_id in self._data:
                    # 更新: FAISS 不支持原地更新,记录新索引位置
                    old_idx = self._data[point_id]["idx"]
                    # 标记旧索引无效
                    if old_idx in self._idx_to_id:
                        del self._idx_to_id[old_idx]

                idx = self._next_idx
                self._next_idx += 1
                self._index.add(vec)
                self._idx_to_id[idx] = point_id
                self._data[point_id] = {
                    "payload": payload,
                    "idx": idx,
                }
                self._persist()

        await _run_in_vdb_pool(_upsert)
        logger.debug(
            f"[vector-store] upsert: id={node.node_id} "
            f"layer={node.layer.value if hasattr(node.layer, 'value') else node.layer} "
            f"isolation_key={payload.get('isolation_key', '')} "
            f"content={node.content[:200]}"
        )
        return node.node_id

    async def upsert_batch(self, nodes: List[MemoryNode]) -> List[str]:
        """批量写入"""
        valid_nodes = [n for n in nodes if n.embedding is not None]
        if not valid_nodes:
            return []

        for n in nodes:
            if n.embedding is None:
                logger.warning(f"Skipping node {n.node_id}: no embedding")

        def _upsert_batch():
            with self._lock:
                ids = []
                vecs = []
                for node in valid_nodes:
                    point_id = self._node_id_to_point_id(node.node_id)
                    payload = self._node_to_payload(node)
                    vec = self._normalize(node.embedding)

                    if point_id in self._data:
                        old_idx = self._data[point_id]["idx"]
                        if old_idx in self._idx_to_id:
                            del self._idx_to_id[old_idx]

                    idx = self._next_idx
                    self._next_idx += 1
                    self._idx_to_id[idx] = point_id
                    self._data[point_id] = {
                        "payload": payload,
                        "idx": idx,
                    }
                    ids.append(node.node_id)
                    vecs.append(vec)

                if vecs:
                    batch = np.vstack(vecs)
                    self._index.add(batch)
                    self._persist()

                return ids

        return await _run_in_vdb_pool(_upsert_batch)

    async def update_embedding(self, node_id: str, embedding: List[float]) -> bool:
        """仅更新向量(payload 不变)"""
        point_id = self._node_id_to_point_id(node_id)
        try:
            def _update():
                with self._lock:
                    if point_id not in self._data:
                        return False

                    old_idx = self._data[point_id]["idx"]
                    if old_idx in self._idx_to_id:
                        del self._idx_to_id[old_idx]

                    vec = self._normalize(embedding)
                    idx = self._next_idx
                    self._next_idx += 1
                    self._index.add(vec)
                    self._idx_to_id[idx] = point_id
                    self._data[point_id]["idx"] = idx
                    self._persist()
                    return True

            return await _run_in_vdb_pool(_update)
        except Exception as e:
            logger.warning(f"Failed to update embedding for {node_id}: {e}")
            return False

    async def update_payload(self, node_id: str, updates: Dict[str, Any]) -> bool:
        """更新 payload 字段;若 updates 含 'embedding',同时更新向量(重建索引项)。"""
        if not updates:
            return True
        point_id = self._node_id_to_point_id(node_id)
        updates = dict(updates)
        new_embedding = updates.pop("embedding", None)  # 取出向量单独处理
        try:
            def _update():
                with self._lock:
                    if point_id not in self._data:
                        return False
                    # 1) payload 字段
                    for k, v in updates.items():
                        self._data[point_id]["payload"][k] = v
                    # 2) 向量:重新 add(FaissFlat 不支持原地改,沿用 update_embedding 思路)
                    if new_embedding is not None:
                        old_idx = self._data[point_id]["idx"]
                        if old_idx in self._idx_to_id:
                            del self._idx_to_id[old_idx]
                        vec = self._normalize(new_embedding)
                        idx = self._next_idx
                        self._next_idx += 1
                        self._index.add(vec)
                        self._idx_to_id[idx] = point_id
                        self._data[point_id]["idx"] = idx
                        self._data[point_id]["embedding"] = list(new_embedding)
                    self._persist()
                    return True
            result = await _run_in_vdb_pool(_update)
            if result:
                logger.debug(
                    f"[vector-store] update_payload: id={node_id} updates={list(updates.keys())} "
                    f"embedding={'yes' if new_embedding is not None else 'no'}"
                )
            return result
        except Exception as e:
            logger.warning(f"Failed to update payload for {node_id}: {e}")
            return False

    # ================================================================
    # 检索
    # ================================================================

    async def search(
        self,
        query_embedding: List[float],
        isolation_key: str = "",
        isolation_keys: Optional[List[str]] = None,
        user_id: Optional[str] = None,
        user_ids: Optional[List[str]] = None,
        agent_ids: Optional[List[str]] = None,
        include_sources: Optional[List[str]] = None,
        exclude_sources: Optional[List[str]] = None,
        layers: Optional[List[MemoryLayer]] = None,
        limit: int = 10,
        score_threshold: float = 0.0,
        status_filter: Optional[List[MemoryStatus]] = None,
        only_latest: bool = True,
        tags_match_any: Optional[List[str]] = None,  # reader_hybrid_tag 路 B;faiss 暂未实现,忽略
        created_after: Optional[float] = None,
    ) -> List[Dict[str, Any]]:
        """语义检索(FAISS 搜索 + 手动 metadata 过滤)"""

        # 预处理过滤集合
        keys_set = set(isolation_keys) if isolation_keys else None
        effective_uids = user_ids if user_ids else ([user_id] if user_id else [])
        uids_set = set(effective_uids) if effective_uids else None
        agent_ids_set = set(agent_ids) if agent_ids else None
        include_sources_set = set(include_sources) if include_sources else None
        exclude_sources_set = set(exclude_sources) if exclude_sources else set()

        layer_values = set(l.value for l in layers) if layers else None
        if status_filter:
            status_values = set(s.value for s in status_filter)
        else:
            status_values = {MemoryStatus.ACTIVE.value}

        def _search():
            with self._lock:
                if self._index.ntotal == 0:
                    return []

                vec = self._normalize(query_embedding)
                # IndexFlatIP 支持 IDSelector:先由 metadata 构造允许的内部 ID,
                # 再让 FAISS 在该子集内做精确 top-k。不能先取 limit*N 再过滤,
                # 否则黑白名单会吃掉名额并改变召回语义。
                import faiss
                allowed_indices = []
                for point_id, entry in self._data.items():
                    idx = entry.get("idx")
                    if idx is None or self._idx_to_id.get(int(idx)) != point_id:
                        continue
                    payload = entry.get("payload") or {}
                    if keys_set:
                        if payload.get("isolation_key") not in keys_set:
                            continue
                    elif isolation_key:
                        if payload.get("isolation_key") != isolation_key:
                            continue
                    elif uids_set:
                        if payload.get("user_id") not in uids_set:
                            continue
                        if agent_ids_set and payload.get("agent_id") not in agent_ids_set:
                            continue
                    source = payload.get("source") or "default"
                    if include_sources_set and source not in include_sources_set:
                        continue
                    if source in exclude_sources_set:
                        continue
                    if layer_values and payload.get("layer") not in layer_values:
                        continue
                    if payload.get("status") not in status_values:
                        continue
                    if only_latest and not payload.get("is_latest", True):
                        continue
                    if not passes_created_after(payload, created_after):
                        continue
                    allowed_indices.append(int(idx))
                if not allowed_indices:
                    return []
                selector = faiss.IDSelectorBatch(
                    np.asarray(allowed_indices, dtype=np.int64)
                )
                params = faiss.SearchParameters(sel=selector)
                k = min(limit, len(allowed_indices))
                scores, indices = self._index.search(vec, k, params=params)

                results = []
                for score, idx in zip(scores[0], indices[0]):
                    if idx < 0:
                        continue
                    point_id = self._idx_to_id.get(int(idx))
                    if point_id is None:
                        # 已被删除或更新的旧索引
                        continue

                    entry = self._data.get(point_id)
                    if entry is None:
                        continue

                    payload = entry["payload"]

                    # metadata 过滤: 精确 keys > 单 key > user_ids/user_id + agent 级
                    if keys_set:
                        if payload.get("isolation_key") not in keys_set:
                            continue
                    elif isolation_key:
                        if payload.get("isolation_key") != isolation_key:
                            continue
                    elif uids_set:
                        if payload.get("user_id") not in uids_set:
                            continue
                        if agent_ids_set and payload.get("agent_id") not in agent_ids_set:
                            continue
                    if layer_values and payload.get("layer") not in layer_values:
                        continue
                    if payload.get("status") not in status_values:
                        continue
                    # 只搜索演化链末端节点
                    if only_latest and not payload.get("is_latest", True):
                        continue
                    # 时间过滤:事件时间 memory_at 优先,为空回退 gmt_created
                    if not passes_created_after(payload, created_after):
                        continue

                    # score 是内积(归一化后 = cosine similarity)
                    sim = float(score)
                    if sim < score_threshold:
                        continue

                    results.append({
                        "point_id": point_id,
                        "payload": payload,
                        "score": sim,
                    })

                    if len(results) >= limit:
                        break

                return results

        raw_results = await _run_in_vdb_pool(_search)

        output = []
        for r in raw_results:
            node = self._payload_to_node(r["payload"])
            output.append({
                "node_id": node.node_id,
                "score": r["score"],
                "node": node,
            })

        logger.debug(
            f"[vector-store] search: isolation_key={isolation_key} "
            f"isolation_keys={isolation_keys} user_id={user_id} agent_ids={agent_ids} "
            f"limit={limit} found={len(output)}"
        )
        for i, item in enumerate(output):
            logger.debug(
                f"[vector-store] hit[{i}] score={item['score']:.4f} "
                f"id={item['node_id']} "
                f"layer={item['node'].layer.value if hasattr(item['node'].layer, 'value') else ''} "
                f"content={item['node'].content[:200]}"
            )

        return output

    async def get_by_id(self, node_id: str) -> Optional[MemoryNode]:
        """按 ID 获取"""
        point_id = self._node_id_to_point_id(node_id)
        try:
            entry = self._data.get(point_id)
            if entry is None:
                return None
            node = self._payload_to_node(entry["payload"])
            # FAISS 不方便按 ID 取向量,跳过 embedding
            return node
        except Exception as e:
            logger.warning(f"Failed to get vector point {node_id}: {e}")
            return None

    async def get_embeddings(self, node_ids: List[str]) -> Dict[str, List[float]]:
        """批量取向量,供去重 / episode 摘要计算 cosine。

        ``_data`` 只保存 payload 与 FAISS 内部下标;普通 upsert 不额外存一份
        embedding,向量本体在 FAISS index 里。这里按当前有效下标 reconstruct,
        并用 ``_idx_to_id`` 校验,避免读到已被替换的旧下标。
        """

        def _get() -> Dict[str, List[float]]:
            out: Dict[str, List[float]] = {}
            with self._lock:
                for nid in node_ids:
                    point_id = self._node_id_to_point_id(nid)
                    entry = self._data.get(point_id)
                    if entry is None:
                        continue
                    embedding = entry.get("embedding")
                    if embedding:
                        out[nid] = list(embedding)
                        continue
                    idx = entry.get("idx")
                    if (
                        idx is None
                        or self._index is None
                        or self._idx_to_id.get(int(idx)) != point_id
                    ):
                        continue
                    try:
                        out[nid] = self._index.reconstruct(int(idx)).tolist()
                    except Exception:
                        continue
            return out

        return await _run_in_vdb_pool(_get)

    # ================================================================
    # 删除
    # ================================================================

    async def delete(self, node_id: str) -> bool:
        """删除单个向量点"""
        point_id = self._node_id_to_point_id(node_id)
        try:
            def _delete():
                with self._lock:
                    if point_id not in self._data:
                        return False
                    idx = self._data[point_id]["idx"]
                    if idx in self._idx_to_id:
                        del self._idx_to_id[idx]
                    del self._data[point_id]
                    self._persist()
                    return True
            return await _run_in_vdb_pool(_delete)
        except Exception as e:
            logger.warning(f"Failed to delete vector point {node_id}: {e}")
            return False

    async def delete_by_isolation_key(self, isolation_key: str) -> int:
        """删除某隔离键下的所有向量"""
        try:
            def _delete():
                with self._lock:
                    to_delete = []
                    for pid, entry in self._data.items():
                        if entry["payload"].get("isolation_key") == isolation_key:
                            to_delete.append(pid)

                    for pid in to_delete:
                        idx = self._data[pid]["idx"]
                        if idx in self._idx_to_id:
                            del self._idx_to_id[idx]
                        del self._data[pid]

                    if to_delete:
                        self._persist()
                    return len(to_delete)
            return await _run_in_vdb_pool(_delete)
        except Exception as e:
            logger.warning(f"Failed to delete vectors for {isolation_key}: {e}")
            return 0

    async def delete_by_metadata(
        self,
        user_id: str,
        agent_id: Optional[str] = None,
        session_id: Optional[str] = None,
    ) -> int:
        """按 metadata 字段组合删除向量"""
        try:
            def _delete():
                with self._lock:
                    to_delete = []
                    for pid, entry in self._data.items():
                        payload = entry["payload"]
                        if payload.get("user_id") != user_id:
                            continue
                        if agent_id is not None and payload.get("agent_id") != agent_id:
                            continue
                        if session_id is not None and payload.get("session_id") != session_id:
                            continue
                        to_delete.append(pid)

                    for pid in to_delete:
                        idx = self._data[pid]["idx"]
                        if idx in self._idx_to_id:
                            del self._idx_to_id[idx]
                        del self._data[pid]

                    if to_delete:
                        self._persist()
                    return len(to_delete)
            return await _run_in_vdb_pool(_delete)
        except Exception as e:
            logger.warning(f"Failed to delete vectors by metadata (user_id={user_id}): {e}")
            return 0

    # ================================================================
    # 枚举
    # ================================================================

    async def list_by_user(
        self,
        user_id: Optional[str] = None,
        agent_id: Optional[str] = None,
        limit: int = 10000,
        status_filter: Optional[List] = None,
        layers: Optional[List] = None,
        user_ids: Optional[List[str]] = None,
        agent_ids: Optional[List[str]] = None,
        include_sources: Optional[List[str]] = None,
        exclude_sources: Optional[List[str]] = None,
        only_latest: bool = False,
        with_vectors: bool = True,
        allow_raw_layer: bool = False,
    ) -> List[MemoryNode]:
        """枚举用户的记忆节点,支持多 user / 多 agent

        allow_raw_layer: 默认 False,即不返回 L1_RAW(对外读路径)。
        """
        import asyncio
        import numpy as np

        effective_uids = list(user_ids) if user_ids else ([user_id] if user_id else [])
        if not effective_uids:
            logger.warning("[vector-store] list_by_user called without user_id/user_ids, refused")
            return []
        uid_set = set(effective_uids)
        aid_set = set(agent_ids) if agent_ids else ({agent_id} if agent_id else None)
        include_set = set(include_sources) if include_sources else None
        exclude_set = set(exclude_sources) if exclude_sources else set()

        status_values = {s.value if hasattr(s, 'value') else str(s) for s in status_filter} if status_filter else None
        layer_values = {l.value if hasattr(l, 'value') else str(l) for l in layers} if layers else None
        _raw_value = MemoryLayer.L1_RAW.value
        if layer_values is not None and not allow_raw_layer:
            layer_values.discard(_raw_value)
            if not layer_values:
                # 只点名了 raw:置为不可能命中的空白名单,结果必然为空集
                layer_values = {"__blocked__"}

        def _list():
            nodes = []
            with self._lock:
                for pid, entry in self._data.items():
                    payload = entry["payload"]
                    if payload.get("user_id") not in uid_set:
                        continue
                    if aid_set and payload.get("agent_id") not in aid_set:
                        continue
                    source = payload.get("source") or "default"
                    if include_set and source not in include_set:
                        continue
                    if source in exclude_set:
                        continue
                    if only_latest and not payload.get("is_latest", True):
                        continue
                    if status_values and payload.get("status", "") not in status_values:
                        continue
                    if layer_values and payload.get("layer", "") not in layer_values:
                        continue
                    if not allow_raw_layer and payload.get("layer", "") == _raw_value:
                        continue
                    node = self._payload_to_node(payload)
                    # 从 FAISS index 取向量
                    idx = entry.get("idx")
                    if with_vectors and idx is not None and self._index is not None:
                        try:
                            vec = self._index.reconstruct(int(idx))
                            node.embedding = vec.tolist()
                        except Exception:
                            pass
                    nodes.append(node)
                    if len(nodes) >= limit:
                        break
            return nodes

        result = await _run_in_vdb_pool(_list)
        logger.info(
            f"[vector-store] list_by_user: user_ids={effective_uids} "
            f"agent_ids={sorted(aid_set) if aid_set else []} found={len(result)}"
        )
        return result

    # ================================================================
    # 统计
    # ================================================================

    async def get_stats(self) -> Dict[str, Any]:
        """获取集合统计"""
        try:
            return {
                "collection": self._collection_name,
                "points_count": len(self._data),
                "vectors_count": self._index.ntotal if self._index else 0,
                "status": "ok",
            }
        except Exception as e:
            return {"error": str(e)}

    async def list_by_episode(
        self,
        episode_ids: List[str],
        *,
        user_id: Optional[str] = None,
        agent_id: Optional[str] = None,
        layers: Optional[List[MemoryLayer]] = None,
        limit: int = 200,
        include_sources: Optional[List[str]] = None,
        exclude_sources: Optional[List[str]] = None,
    ) -> List[MemoryNode]:
        wanted = {value for value in episode_ids if value}
        include_set = set(include_sources or [])
        exclude_set = set(exclude_sources or [])
        layer_set = {layer.value for layer in layers} if layers else None
        if not wanted:
            return []

        def _list():
            nodes = []
            with self._lock:
                for point_id, entry in self._data.items():
                    payload = entry.get("payload") or {}
                    if payload.get("episode_id") not in wanted:
                        continue
                    if user_id and payload.get("user_id") != user_id:
                        continue
                    if agent_id and payload.get("agent_id") != agent_id:
                        continue
                    if layer_set and payload.get("layer") not in layer_set:
                        continue
                    source = payload.get("source") or "default"
                    if include_set and source not in include_set:
                        continue
                    if source in exclude_set:
                        continue
                    idx = entry.get("idx")
                    if idx is None or self._idx_to_id.get(int(idx)) != point_id:
                        continue
                    nodes.append(self._payload_to_node(payload))
                    if len(nodes) >= limit:
                        break
            return nodes

        return await _run_in_vdb_pool(_list)

    async def count(self, isolation_key: str) -> int:
        """统计某隔离键下的向量数量"""
        try:
            c = 0
            for entry in self._data.values():
                if entry["payload"].get("isolation_key") == isolation_key:
                    c += 1
            return c
        except Exception as e:
            logger.warning(f"Failed to count vectors for {isolation_key}: {e}")
            return 0

    async def close(self) -> None:
        """关闭并持久化"""
        if self._index is not None:
            try:
                await _run_in_vdb_pool(self._persist)
            except Exception as e:
                logger.warning(f"Failed to persist FAISS index on close: {e}")
            self._index = None
            self._data.clear()
            self._idx_to_id.clear()