Skip to content
Python

fix(sdk): repair FAISS embedding lookup, dedup pairing, and episode summaries (#84)

u7-u7d6890793ec33
diff --git a/hy_memory/client.py b/hy_memory/client.py
index a81c35d..a4e0bda 100644
--- a/hy_memory/client.py
+++ b/hy_memory/client.py
@@ -2862,6 +2862,81 @@ class HyMemoryClient:
return resp
+ @staticmethod
+ def _public_search_views(
+ result_memories: List[Dict[str, Any]],
+ *,
+ wanted_layers: Optional[set],
+ lite_mode: bool,
+ ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+ """Build the public search items and the private dedup view together.
+
+ The dedup marker must come from the same source element as its public
+ item. Building the dedup view with a second ``zip(result_memories,
+ all_mems)`` is wrong as soon as one result is dropped by the layer
+ filter: every later kind shifts by one, and a canonical_state can enter
+ fuzzy dedup and be shadowed.
+ """
+ from .models.memory import MemoryLayer
+
+ all_mems: List[Dict[str, Any]] = []
+ dedup_mems: List[Dict[str, Any]] = []
+ for mem in result_memories:
+ if wanted_layers is not None and mem.get("layer", "") not in wanted_layers:
+ continue
+ if lite_mode and mem.get("layer", "") != MemoryLayer.L1_RAW.value:
+ continue
+ score = mem.get("score", 0.0)
+ item = {
+ "memory_id": mem.get("memory_id", ""),
+ "content": mem.get("content", ""),
+ "score": round(score, 4),
+ "layer": mem.get("layer", ""),
+ # 多 user 检索时调用方需据此追溯归属(R1)
+ "user_id": mem.get("user_id", "") or "",
+ "agent_id": mem.get("agent_id", "") or "",
+ "source": mem.get("source", "default") or "default",
+ "retrieval_source": mem.get("retrieval_source", "") or "",
+ "owner": mem.get("owner"),
+ "speculate": mem.get("speculate"),
+ # 不透出 source_raw_memory_id:它指向 L1_RAW 节点,而 raw 对外
+ # 读路径一律不可见,该 id 到手也查不到东西(按 ID 取返回"不存在"),
+ # 留着只会让调用方以为能据此追溯原文。
+ # 节点上仍保留该字段,内部溯源与 clone 重映射不受影响。
+ "tags": mem.get("tags") or [],
+ "memory_at": mem.get("memory_at"),
+ "gmt_created": mem.get("gmt_created"),
+ # 调用方自定义 meta(透传负载)。由 reader 从 custom["meta"] 取出,
+ # 未实现该字段的 reader 下为空列表。
+ "meta": mem.get("meta") or [],
+ "_schema_channel": bool(mem.get("_schema_channel")),
+ }
+ if mem.get("schema_type"):
+ item["schema_type"] = mem["schema_type"]
+ if mem.get("workspace_id"):
+ item["workspace_id"] = mem["workspace_id"]
+ if mem.get("branch"):
+ item["branch"] = mem["branch"]
+ if mem.get("evolution_chain"):
+ item["evolution_chain"] = mem["evolution_chain"]
+ if mem.get("capsule_memories") is not None:
+ item["capsule_memories"] = mem["capsule_memories"]
+ if mem.get("episode_id"):
+ item["episode_id"] = mem["episode_id"]
+ if mem.get("member_ids") is not None:
+ item["member_ids"] = mem["member_ids"]
+ if mem.get("timeline") is not None:
+ item["timeline"] = mem["timeline"]
+ if mem.get("rerank_score") is not None:
+ item["rerank_score"] = mem["rerank_score"]
+ all_mems.append(item)
+
+ dedup_item = dict(item)
+ dedup_item["_memory_kind"] = mem.get("_memory_kind", "")
+ dedup_mems.append(dedup_item)
+
+ return all_mems, dedup_mems
+
def _spawn_search_dedup(
self,
all_mems: List[Dict[str, Any]],
@@ -3497,62 +3572,11 @@ class HyMemoryClient:
if effective_search_layers else None
)
- _all_mems = []
- for mem in result.memories:
- if _wanted_layers is not None and mem.get("layer", "") not in _wanted_layers:
- continue
- if lite_mode and mem.get("layer", "") != MemoryLayer.L1_RAW.value:
- continue
- score = mem.get("score", 0.0)
- item = {
- "memory_id": mem.get("memory_id", ""),
- "content": mem.get("content", ""),
- "score": round(score, 4),
- "layer": mem.get("layer", ""),
- # 多 user 检索时调用方需据此追溯归属(R1)
- "user_id": mem.get("user_id", "") or "",
- "agent_id": mem.get("agent_id", "") or "",
- "source": mem.get("source", "default") or "default",
- "retrieval_source": mem.get("retrieval_source", "") or "",
- "owner": mem.get("owner"),
- "speculate": mem.get("speculate"),
- # 不透出 source_raw_memory_id:它指向 L1_RAW 节点,而 raw 对外
- # 读路径一律不可见,该 id 到手也查不到东西(按 ID 取返回"不存在"),
- # 留着只会让调用方以为能据此追溯原文。
- # 节点上仍保留该字段,内部溯源与 clone 重映射不受影响。
- "tags": mem.get("tags") or [],
- "memory_at": mem.get("memory_at"),
- "gmt_created": mem.get("gmt_created"),
- # 调用方自定义 meta(透传负载)。由 reader 从 custom["meta"] 取出,
- # 未实现该字段的 reader 下为空列表。
- "meta": mem.get("meta") or [],
- "_schema_channel": bool(mem.get("_schema_channel")),
- }
- if mem.get("schema_type"):
- item["schema_type"] = mem["schema_type"]
- if mem.get("workspace_id"):
- item["workspace_id"] = mem["workspace_id"]
- if mem.get("branch"):
- item["branch"] = mem["branch"]
- if mem.get("evolution_chain"):
- item["evolution_chain"] = mem["evolution_chain"]
- if mem.get("capsule_memories") is not None:
- item["capsule_memories"] = mem["capsule_memories"]
- if mem.get("episode_id"):
- item["episode_id"] = mem["episode_id"]
- if mem.get("member_ids") is not None:
- item["member_ids"] = mem["member_ids"]
- if mem.get("timeline") is not None:
- item["timeline"] = mem["timeline"]
- if mem.get("rerank_score") is not None:
- item["rerank_score"] = mem["rerank_score"]
- _all_mems.append(item)
-
- _dedup_mems = []
- for source, public in zip(result.memories, _all_mems):
- copy = dict(public)
- copy["_memory_kind"] = source.get("_memory_kind", "")
- _dedup_mems.append(copy)
+ _all_mems, _dedup_mems = self._public_search_views(
+ result.memories,
+ wanted_layers=_wanted_layers,
+ lite_mode=lite_mode,
+ )
# search 链路去重:仅对 L2_FACT + L4_IDENTITY(重复重灾区)判重删库。
# L0 及其他层不参与。fire-and-forget,不阻塞本次返回(清理面向后续查询)。
diff --git a/hy_memory/data/vector_store_faiss.py b/hy_memory/data/vector_store_faiss.py
index ebf4e14..58009a9 100644
--- a/hy_memory/data/vector_store_faiss.py
+++ b/hy_memory/data/vector_store_faiss.py
@@ -462,14 +462,39 @@ class FaissVectorStore(VectorStoreBase):
return None
async def get_embeddings(self, node_ids: List[str]) -> Dict[str, List[float]]:
- """批量取向量(in-memory _data 直接读 embedding),供去重算 cosine。"""
- out: Dict[str, List[float]] = {}
- for nid in node_ids:
- point_id = self._node_id_to_point_id(nid)
- entry = self._data.get(point_id)
- if entry and entry.get("embedding"):
- out[nid] = list(entry["embedding"])
- return out
+ """批量取向量,供去重 / episode 摘要计算 cosine。
+
+ ``_data`` 只保存 payload 与 FAISS 内部下标;普通 upsert 不额外存一份
+ embedding,向量本体在 FAISS index 里。这里按当前有效下标 reconstruct,
+ 并用 ``_idx_to_id`` 校验,避免读到已被替换的旧下标。
+ """
+
+ def _get() -> Dict[str, List[float]]:
+ out: Dict[str, List[float]] = {}
+ with self._lock:
+ for nid in node_ids:
+ point_id = self._node_id_to_point_id(nid)
+ entry = self._data.get(point_id)
+ if entry is None:
+ continue
+ embedding = entry.get("embedding")
+ if embedding:
+ out[nid] = list(embedding)
+ continue
+ idx = entry.get("idx")
+ if (
+ idx is None
+ or self._index is None
+ or self._idx_to_id.get(int(idx)) != point_id
+ ):
+ continue
+ try:
+ out[nid] = self._index.reconstruct(int(idx)).tolist()
+ except Exception:
+ continue
+ return out
+
+ return await _run_in_vdb_pool(_get)
# ================================================================
# 删除
diff --git a/hy_memory/pipelines/writer.py b/hy_memory/pipelines/writer.py
index 6e1b023..23d5a52 100644
--- a/hy_memory/pipelines/writer.py
+++ b/hy_memory/pipelines/writer.py
@@ -968,7 +968,7 @@ class MemoryWriter(WritePipeline):
async def _run():
try:
nid = await summarize_closed_episode(
- vector_store=self.vector_store,
+ vector_store=await self._get_vector_store(),
embed_service=self.embed_service,
llm_provider=self.llm_provider,
episode_id=episode_id,
diff --git a/tests/test_episode.py b/tests/test_episode.py
index 9d10641..943e213 100644
--- a/tests/test_episode.py
+++ b/tests/test_episode.py
@@ -173,6 +173,41 @@ class TestEpisodeCloseTriggersSummary:
await w._resolve_episode_id(_req(t0 + timedelta(hours=3)), 3)
assert scheduled == [ep1], "只在关闭时对刚关闭的那个 episode 触发一次"
+ @pytest.mark.asyncio
+ async def test_summary_uses_the_writer_vector_store(self, monkeypatch):
+ """后台摘要必须拿到 writer 已初始化的 vector store。
+
+ 旧实现访问了不存在的 ``self.vector_store``,AttributeError 被
+ except 吞掉,L3 episode 摘要从不生成。
+ """
+ from types import SimpleNamespace
+
+ import hy_memory.agent.episode_summary as episode_summary
+
+ captured = {}
+
+ async def fake_summarize(**kwargs):
+ captured.update(kwargs)
+ return "episode-node"
+
+ monkeypatch.setattr(
+ episode_summary, "summarize_closed_episode", fake_summarize,
+ )
+
+ w = _writer(_FakeCache())
+ w.config.recall.episode_summary_enabled = True
+ w._mem_agent = SimpleNamespace(llm_provider=object())
+ w._embed_service = object()
+ vector_store = object()
+ w._external_vector_store = vector_store
+ w._vector_store = vector_store
+ w._vector_store_initialized = True
+
+ w._schedule_episode_summary(_req(datetime(2026, 9, 13)), "ep-1")
+ await asyncio.sleep(0.05)
+
+ assert captured["vector_store"] is vector_store
+
# ======================================================================
# Episode 摘要(写侧,省 token)
diff --git a/tests/test_faiss_get_embeddings.py b/tests/test_faiss_get_embeddings.py
new file mode 100644
index 0000000..c17304f
--- /dev/null
+++ b/tests/test_faiss_get_embeddings.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+"""FAISS 后端的 get_embeddings 必须能取回普通 upsert 写入的向量。
+
+``_data`` 只保存 payload 与内部下标,向量本体在 FAISS index 里。去重和
+episode 摘要依赖 get_embeddings,取不到向量会让这些能力静默失效。
+"""
+
+import asyncio
+
+import pytest
+
+pytest.importorskip("faiss")
+
+from hy_memory.config import MemoryConfig # noqa: E402
+from hy_memory.data.vector_store_faiss import FaissVectorStore # noqa: E402
+from hy_memory.models.memory import MemoryLayer, MemoryNode # noqa: E402
+
+
+def _config(tmp_path):
+ return MemoryConfig.from_dict({
+ "vector_store": {
+ "provider": "faiss",
+ "collection_name": "get_embeddings_test",
+ "persist_directory": str(tmp_path / "faiss"),
+ "embedding_dims": 4,
+ },
+ })
+
+
+def test_get_embeddings_reconstructs_upserted_and_updated_vectors(tmp_path):
+ async def _run():
+ store = FaissVectorStore(_config(tmp_path))
+ await store.initialize()
+ try:
+ node = MemoryNode(
+ user_id="u1",
+ agent_id="a1",
+ session_id="s1",
+ source="default",
+ layer=MemoryLayer.L1_RAW,
+ content="hello",
+ embedding=[1.0, 0.0, 0.0, 0.0],
+ )
+ await store.upsert(node)
+
+ got = await store.get_embeddings([node.node_id])
+ assert got[node.node_id] == pytest.approx([1.0, 0.0, 0.0, 0.0])
+
+ assert await store.update_embedding(node.node_id, [0.0, 1.0, 0.0, 0.0])
+ got = await store.get_embeddings([node.node_id])
+ assert got[node.node_id] == pytest.approx([0.0, 1.0, 0.0, 0.0])
+
+ assert await store.get_embeddings(["missing"]) == {}
+ finally:
+ await store.close()
+
+ asyncio.run(_run())
diff --git a/tests/test_search_layer_filter.py b/tests/test_search_layer_filter.py
index d65553c..9106837 100644
--- a/tests/test_search_layer_filter.py
+++ b/tests/test_search_layer_filter.py
@@ -77,6 +77,28 @@ class TestReadRequestWiring:
HyMemoryClient._normalize_layers(["l3_identity"])
+class TestDedupKindPairing:
+ """dedup 的 _memory_kind 必须与它的 public item 来自同一条 reader 结果。"""
+
+ def test_memory_kind_follows_source_after_layer_filter(self):
+ memories = [
+ {"memory_id": "raw", "layer": "l1_raw", "content": "raw"},
+ {
+ "memory_id": "canon",
+ "layer": "l2_fact",
+ "content": "canon",
+ "_memory_kind": "canonical_state",
+ },
+ ]
+ all_mems, dedup_mems = HyMemoryClient._public_search_views(
+ memories,
+ wanted_layers={"l2_fact"},
+ lite_mode=False,
+ )
+ assert [m["memory_id"] for m in all_mems] == ["canon"]
+ assert dedup_mems[0]["_memory_kind"] == "canonical_state"
+
+
class TestHybridDirectedRecall:
"""共享 reader 主流程必须把 request.layers 下推到召回阶段。"""