Python
feat(sdk): re-embed vector stores and the graph into a new embedding space
alvinfei302fd30af175
diff --git a/.env.example b/.env.exampleindex 13ad1ed..a44ef8e 100644--- a/.env.example+++ b/.env.example@@ -40,6 +40,7 @@ MEMORY_EMBEDDING_DIMS=1024# MEMORY_EMBEDDER_TIMEOUT=15# MEMORY_EMBEDDER_MAX_RETRIES=2# MEMORY_EMBEDDER_RETRY_DELAY=0.3+# MEMORY_EMBEDDER_BATCH_SIZE=64# MEMORY_EMBEDDER_EXTRA_HEADERS={}# MEMORY_EMBEDDER_EXTRA_BODY={}diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 00c9fe9..31ed08c 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -4,6 +4,46 @@ Hy Memory and MemHub are versioned and released independently.## Unreleased+- Hy Memory: new `hy_memory.reembed` module for switching the embedding+ model. Writers must be paused for the whole run. Nothing is derived from+ the provider or endpoint: source and target are named explicitly, and the+ identity of a space is the model name plus the width (`space_identity`).+ Only Chroma and Kuzu are supported.+ - `vector_space_names(config, base_name=None)` returns every physical+ collection a client opens for the six vector roles: memory, entities,+ coding keys and working facts, each in the working and coding variants.+ - `reembed_vector_space(source, target, embedding=...)` pages every raw+ source collection, covering all users and all layers including L1_RAW. It+ embeds the text the write path embedded (`reembed_text`: the canonical+ state's `state_retrieval_text`, a schema's `retrieval_text`, otherwise the+ document). It then writes each record into the target with the same id,+ document and metadata. Each batch's widths are checked before anything is+ written. A cancel stops at a batch boundary (`ReembedCancelled`). A rerun+ skips the ids the target already holds, so it resumes, and a merge never+ overwrites the target. The source is never written.+ - `plan_reembed`, `verify_reembed` and `drop_vector_space` complete the+ vector side. `verify_reembed` checks ids, metadata, per-user counts and+ the width, and runs a self-recall sample. `drop_vector_space` refuses+ collections that a live client in this process holds open.+ - `scan_vector_spaces(persist_directory)` is read-only. It lists every+ space on disk with its roles, counts, stored width, per-user counts and+ the models recorded in metadata. `diff_spaces(source, target)` returns the+ ids that the source holds and the target lacks, whatever their status.+ - `reembed_graph(source_path, target_path, embedding=..., target_dims=...)`+ builds a new Kuzu graph at the new width. It copies every node and edge+ and recomputes `Memory.embedding`, `Memory.beh_embedding`, and the Topic+ and HierarchyTopic embeddings, then rebuilds the HNSW index. The source+ graph is opened read-only and kept for rollback. A run that fails or is+ cancelled leaves no target behind, and a rerun starts over.+ `verify_graph_reembed` and `drop_graph` complete the graph side.+- Hy Memory: `EmbedService.embed_batch` now splits large batches into+ requests of `embedder.batch_size` texts each (`MEMORY_EMBEDDER_BATCH_SIZE`,+ default 64); before, it sent them all in one request. On a 429 it waits for+ the `Retry-After` time plus jitter, capped at 60 s. It takes a per-call+ `max_retries`. A returned vector whose width differs from the configured+ `embedding_dims` now raises `EmbeddingDimensionMismatchError` and is not+ retried.+## MemHub 1.0.5 — 2026-10-28Build 36, on `hy-memory==2.0.8`. Linux x64/arm64 Runtime and macOS arm64diff --git a/docs/env_reference.md b/docs/env_reference.mdindex 7bd3841..699d552 100644--- a/docs/env_reference.md+++ b/docs/env_reference.md@@ -48,6 +48,7 @@| `MEMORY_EMBEDDER_TIMEOUT` | `15` | 单次超时,秒 || `MEMORY_EMBEDDER_MAX_RETRIES` | `2` | 最大重试次数 || `MEMORY_EMBEDDER_RETRY_DELAY` | `0.3` | 重试间隔,秒 |+| `MEMORY_EMBEDDER_BATCH_SIZE` | `64` | `embed_batch` 单次请求最多发送的文本条数,超出按此切块 || `MEMORY_EMBEDDER_EXTRA_HEADERS` | — | JSON object,附加 HTTP headers || `MEMORY_EMBEDDER_EXTRA_BODY` | — | JSON object,附加请求 body |diff --git a/hy_memory/__init__.py b/hy_memory/__init__.pyindex e0a1f3a..fa03975 100644--- a/hy_memory/__init__.py+++ b/hy_memory/__init__.py@@ -55,6 +55,33 @@ except Exception:from .client import HyMemoryClientfrom .inspector import MemoryInspectorfrom .runtime import SharedRuntime+from .reembed import (+ EmbeddingSpaceIdentity,+ GraphReembedPlan,+ GraphReembedProgress,+ GraphReembedReport,+ GraphVerification,+ ReembedCancelled,+ ReembedPlan,+ ReembedProgress,+ ReembedReport,+ ReembedVerification,+ ScannedSpace,+ SpaceDiff,+ VectorSpace,+ diff_spaces,+ drop_graph,+ drop_vector_space,+ plan_reembed,+ reembed_graph,+ reembed_text,+ reembed_vector_space,+ scan_vector_spaces,+ space_identity,+ vector_space_names,+ verify_graph_reembed,+ verify_reembed,+)from .snapshot import (DataPlaneRestore,DataPlaneSnapshot,@@ -129,6 +156,32 @@ __all__ = ["DataPlaneSnapshotRejected","restore_local_data_plane","snapshot_local_data_plane",+ # 换 embedding 模型:把向量库和图重新 embed 进新的 embedding 空间+ "EmbeddingSpaceIdentity",+ "GraphReembedPlan",+ "GraphReembedProgress",+ "GraphReembedReport",+ "GraphVerification",+ "ReembedCancelled",+ "ReembedPlan",+ "ReembedProgress",+ "ReembedReport",+ "ReembedVerification",+ "ScannedSpace",+ "SpaceDiff",+ "VectorSpace",+ "diff_spaces",+ "drop_graph",+ "drop_vector_space",+ "plan_reembed",+ "reembed_graph",+ "reembed_text",+ "reembed_vector_space",+ "scan_vector_spaces",+ "space_identity",+ "vector_space_names",+ "verify_graph_reembed",+ "verify_reembed",# 配置"MemoryConfig",# 公共扩展协议 / 精确读取类型diff --git a/hy_memory/config.py b/hy_memory/config.pyindex 4785245..828f0e7 100644--- a/hy_memory/config.py+++ b/hy_memory/config.py@@ -260,6 +260,7 @@ class EmbedderConfig:max_retries: int = None # 最大重试次数retry_delay: float = None # 重试间隔(秒)timeout: int = None # 请求超时(秒)+ batch_size: int = None # 单次 HTTP 请求最多带多少条 input(embed_batch 按此切块)# 内部平台扩展(extra_headers / extra_body 会透传给 OpenAI client)extra_headers: Optional[Dict[str, str]] = None@@ -294,6 +295,10 @@ class EmbedderConfig:# 请求切断并制造重试风暴。SDK 内层重试已关闭,15s 是每个显式 attempt# 的真实上限;部署可通过 MEMORY_EMBEDDER_TIMEOUT 覆盖。self.timeout = _get_env_int("MEMORY_EMBEDDER_TIMEOUT", 15)+ if self.batch_size is None:+ # 大批量(如重新 embed 整个库)不能一次 HTTP 发几千条:多数服务对+ # input 数组有上限,超了直接 400。按 batch_size 切块逐块发送。+ self.batch_size = _get_env_int("MEMORY_EMBEDDER_BATCH_SIZE", 64)if self.extra_headers is None:val = os.getenv("MEMORY_EMBEDDER_EXTRA_HEADERS")if val:diff --git a/hy_memory/core/embed_service.py b/hy_memory/core/embed_service.pyindex 96fec8c..ab80cb2 100644--- a/hy_memory/core/embed_service.py+++ b/hy_memory/core/embed_service.py@@ -38,10 +38,12 @@ import importlib.utilimport jsonfrom json import JSONDecodeErrorimport logging+import randomimport threadingimport timefrom ..config import MemoryConfig+from ..embedding import EmbeddingDimensionMismatchErrorfrom .workload import embedding_slotfrom ..utils.log_setup import get_request_id@@ -187,6 +189,41 @@ def _redact_headers(headers: Optional[Dict[str, Any]]) -> Optional[Dict[str, Anyreturn out+# Retry-After 的上限:服务端给出超长等待(或恶意值)时不让单次调用挂几十分钟。+_RETRY_AFTER_MAX_S = 60.0+++def _retry_after_seconds(exc: Exception) -> Optional[float]:+ """从 429 异常的响应头里取 Retry-After(秒数或 HTTP 日期),取不到返回 None。"""+ response = getattr(exc, "response", None)+ headers = getattr(response, "headers", None)+ if not headers:+ return None+ try:+ raw = headers.get("retry-after") or headers.get("Retry-After")+ except Exception:+ return None+ if raw is None:+ return None+ raw = str(raw).strip()+ try:+ seconds = float(raw)+ except ValueError:+ try:+ from email.utils import parsedate_to_datetime+ from datetime import datetime, timezone++ when = parsedate_to_datetime(raw)+ if when.tzinfo is None:+ when = when.replace(tzinfo=timezone.utc)+ seconds = (when - datetime.now(timezone.utc)).total_seconds()+ except Exception:+ return None+ if seconds != seconds: # NaN+ return None+ return min(max(0.0, seconds), _RETRY_AFTER_MAX_S)++def _embeddings_api_url(base_url: str) -> str:"""OpenAI SDK posts to ``{base_url}/embeddings`` (base_url usually ends with /v1)."""base = (base_url or "").rstrip("/")@@ -205,7 +242,8 @@ class EmbedServiceConfig:api_key: str = ""base_url: str = ""embedding_dims: int = 1536- batch_size: int = 100+ # embed_batch 单次 HTTP 最多带多少条 input;超出按块顺序发送+ batch_size: int = 64enable_cache: bool = Truecache_size: int = 10000max_retries: int = 5@@ -287,6 +325,7 @@ class EmbedService:max_retries=embedder.max_retries if embedder.max_retries is not None else 5,retry_delay=embedder.retry_delay if embedder.retry_delay is not None else 3.0,timeout=embedder.timeout if embedder.timeout is not None else 60,+ batch_size=max(1, int(getattr(embedder, "batch_size", None) or 64)),queue_batch_size=int(os.getenv("MEMORY_EMBED_BATCH_SIZE", "8")),queue_batch_window_ms=float(os.getenv("MEMORY_EMBED_BATCH_WINDOW_MS", "100")),extra_headers=embedder.extra_headers,@@ -640,6 +679,8 @@ class EmbedService:self,texts: List[str],use_cache: bool = True,+ *,+ max_retries: Optional[int] = None,) -> List[List[float]]:"""批量向量化@@ -647,17 +688,23 @@ class EmbedService:Args:texts: 文本列表use_cache: 是否使用缓存+ max_retries: 本次调用的重试次数;None 用配置值。长批量任务(重新+ embed 整个库)可以给更宽的重试预算,不影响在线路径。Returns:向量列表"""- vectors, _metas = await self.embed_batch_with_meta(texts, use_cache=use_cache)+ vectors, _metas = await self.embed_batch_with_meta(+ texts, use_cache=use_cache, max_retries=max_retries,+ )return vectorsasync def embed_batch_with_meta(self,texts: List[str],use_cache: bool = True,+ *,+ max_retries: Optional[int] = None,) -> Tuple[List[List[float]], List[Dict[str, Any]]]:"""批量向量化,同时返回每条的 provider metadata(非 vector)。"""if not texts:@@ -682,7 +729,12 @@ class EmbedService:indices_to_embed.append(i)if texts_to_embed:- vectors, batch_metas = await self._embed_texts_with_meta(texts_to_embed)+ # max_retries 只在显式给出时透传:保持 `_embed_texts_with_meta(texts)`+ # 这个既有调用形状不变(测试与下游替身按它打桩)。+ retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}+ vectors, batch_metas = await self._embed_texts_with_meta(+ texts_to_embed, **retry_kwargs,+ )for j, vector in enumerate(vectors):idx = indices_to_embed[j]@@ -700,9 +752,9 @@ class EmbedService:return results, metasasync def _embed_texts_with_meta(- self, texts: List[str]+ self, texts: List[str], *, max_retries: Optional[int] = None,) -> Tuple[List[List[float]], List[Dict[str, Any]]]:- """实际的向量化调用:**一次 HTTP 带走整批**;返回 (vectors, metas)。+ """实际的向量化调用:**一次 HTTP 带走一块**;返回 (vectors, metas)。早期实现是 `for text in texts: await _embed_openai([text])` 逐条串行,原意是"避免 batch size 限制",但代价极大:线上一批 8 条 × ~5.6s 串行@@ -719,21 +771,34 @@ class EmbedService:截断在这里做(发送前逐条过 `_truncate_for_embedding`):所有 embedding路径(embed / embed_batch / embed_queued / *_with_meta)最终都落到这个方法 —— 放在各入口会漏。理由与阈值见模块头部 `_EMBED_MAX_TOKENS`。++ 切块:一次带走整批只适用于小批。几千条的 input 数组会被多数服务直接+ 400(input 上限),所以按 `batch_size`(默认 64)切块,块之间顺序发送。+ 不超过一块时行为与切块前逐字相同(`batch_mode="single_request"`)。"""if not texts:return [], []safe_texts = [_truncate_for_embedding(t) for t in texts]- vectors, meta = await self._embed_openai(safe_texts)- base_meta = dict(meta or {})- base_meta["batch_mode"] = "single_request"- base_meta["batch_input_count"] = len(safe_texts)-+ chunk = max(1, int(self._embed_config.batch_size or 64))+ vectors: List[List[float]] = []all_metas: List[Dict[str, Any]] = []- for idx in range(len(safe_texts)):- m = dict(base_meta)- m["batch_index"] = idx- all_metas.append(m)+ chunk_count = (len(safe_texts) + chunk - 1) // chunk+ for start in range(0, len(safe_texts), chunk):+ part = safe_texts[start:start + chunk]+ retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}+ part_vectors, meta = await self._embed_openai(part, **retry_kwargs)+ base_meta = dict(meta or {})+ base_meta["batch_mode"] = "single_request" if chunk_count == 1 else "chunked"+ base_meta["batch_input_count"] = len(part)+ if chunk_count > 1:+ base_meta["batch_chunk_index"] = start // chunk+ base_meta["batch_chunk_count"] = chunk_count+ for idx in range(len(part)):+ m = dict(base_meta)+ m["batch_index"] = start + idx+ all_metas.append(m)+ vectors.extend(part_vectors)self._last_embed_meta = all_metas[-1] if all_metas else {}return vectors, all_metas@@ -783,7 +848,7 @@ class EmbedService:)async def _embed_openai(- self, texts: List[str]+ self, texts: List[str], *, max_retries: Optional[int] = None,) -> Tuple[List[List[float]], Dict[str, Any]]:"""使用 OpenAI 兼容接口向量化@@ -873,7 +938,9 @@ class EmbedService:"endpoint_overridden": bool(_ov),}- _max_retries = max(0, int(self._embed_config.max_retries or 0))+ _max_retries = max(0, int(+ (self._embed_config.max_retries or 0) if max_retries is None else max_retries+ ))_retry_delay = max(0.0, float(self._embed_config.retry_delay or 0.0))last_exc: Optional[Exception] = None@@ -919,6 +986,17 @@ class EmbedService:f"got={len(vectors)}")dims = [len(v) for v in vectors]+ expected_dims = self._embed_config.embedding_dims+ if expected_dims:+ bad = [d for d in dims if d != int(expected_dims)]+ if bad:+ # 宽度不对的向量写进集合会把集合的维度钉死在错误值上(Chroma+ # 首次 add 定维),或在已有集合上整批失败。确定性错误,不重试。+ raise EmbeddingDimensionMismatchError(+ "embedding dimension mismatch: "+ f"expected={int(expected_dims)} got={bad[0]} "+ f"model={self._embed_config.model}"+ )meta = {"http_status": raw.status_code,"response_headers": resp_headers,@@ -954,6 +1032,8 @@ class EmbedService:},)return vectors, meta+ except EmbeddingDimensionMismatchError:+ raiseexcept Exception as e:last_exc = e_err_name = type(e).__name__@@ -987,12 +1067,18 @@ class EmbedService:if _retryable and attempt < _max_retries:# 逐出可能卡死的 client,下个 attempt 重建干净连接(恢复原防御语义)self._evict_client(_eff_base_url, _eff_api_key, _eff_timeout)+ _delay = _retry_delay+ _retry_after = _retry_after_seconds(e) if _status == 429 else None+ if _retry_after is not None:+ # 限流时服务端明确告诉了要等多久:按它等,再加一点抖动,+ # 避免多个 worker 在同一时刻一起醒来再次撞限流。+ _delay = max(_delay, _retry_after) + random.uniform(0, 0.25 * _retry_after)logger.warning(f"[embed] attempt {attempt + 1}/{_max_retries + 1} failed "- f"({_err_name}): {_err_msg[:200]}; retry in {_retry_delay}s"+ f"({_err_name}): {_err_msg[:200]}; retry in {_delay:.2f}s")- if _retry_delay > 0:- await asyncio.sleep(_retry_delay)+ if _delay > 0:+ await asyncio.sleep(_delay)continueself._log_embed_call(status="error",diff --git a/hy_memory/data/graph_store_kuzu.py b/hy_memory/data/graph_store_kuzu.pyindex e11eeb4..7c4cddd 100644--- a/hy_memory/data/graph_store_kuzu.py+++ b/hy_memory/data/graph_store_kuzu.py@@ -22,6 +22,7 @@ import loggingimport jsonimport threadingimport time+import weakreffrom ..models.memory import MemoryNode, MemoryLayer, MemoryStatusfrom ..config import MemoryConfig@@ -77,6 +78,21 @@ async def _run_in_kuzu_pool(func, *args, **kwargs):return await run_in_executor_guarded(_kuzu_executor, _measured)+# 本进程里已打开、尚未 close 的 store。drop_graph 据此拒绝删掉一个正被+# 活着的 client 使用的图库。WeakSet:store 被回收后自动消失。+_LIVE_STORES: "weakref.WeakSet[KuzuGraphStore]" = weakref.WeakSet()+++def open_graph_paths() -> List[str]:+ """Real paths of the graph databases live stores in this process hold open."""+ import os+ return [+ os.path.realpath(str(store._db_path))+ for store in list(_LIVE_STORES)+ if store._db is not None+ ]++def kuzu_executor_snapshot() -> Dict[str, Any]:with _kuzu_stats_lock:return {@@ -350,6 +366,7 @@ class KuzuGraphStore(GraphStoreBase):)self._db = dbself._conn = conn+ _LIVE_STORES.add(self)except asyncio.TimeoutError:logger.warning(f"GraphStore init timed out after 5s (Kuzu file lock contention). "@@ -2441,6 +2458,7 @@ class KuzuGraphStore(GraphStoreBase):async def close(self) -> None:"""关闭连接"""conn, db = self._conn, self._db+ _LIVE_STORES.discard(self)self._conn = Noneself._db = Noneself._available = Falsediff --git a/hy_memory/data/vector_store_base.py b/hy_memory/data/vector_store_base.pyindex dee74bf..c0ef5d0 100644--- a/hy_memory/data/vector_store_base.py+++ b/hy_memory/data/vector_store_base.py@@ -89,6 +89,17 @@ def passes_created_after(logger = logging.getLogger(__name__)+def physical_collection_name(base_name: Optional[str], dims: Optional[int]) -> str:+ """The collection name a store opens for ``base_name`` at ``dims``.++ The dims suffix keeps two embedding widths from sharing one collection.+ `hy_memory.reembed.vector_space_names` uses this too, so callers never+ re-derive the rule.+ """+ base = base_name or "agent_memories"+ return f"{base}_{dims}" if dims else base++class VectorStoreBase(ABC):"""向量存储抽象基类@@ -98,13 +109,10 @@ class VectorStoreBase(ABC):def __init__(self, config: MemoryConfig):self.config = config- base_name = config.vector_store.collection_name or "agent_memories"- dims = config.vector_store.embedding_dims# 自动拼接维度后缀,避免不同 embedding 模型写入同一 collection 导致维度冲突- if dims:- self._collection_name = f"{base_name}_{dims}"- else:- self._collection_name = base_name+ self._collection_name = physical_collection_name(+ config.vector_store.collection_name, config.vector_store.embedding_dims,+ )_supports_retrieval_clauses: bool = False_supports_narrow_episode_queries: bool = Falsediff --git a/hy_memory/data/vector_store_chroma.py b/hy_memory/data/vector_store_chroma.pyindex 8b9a9fb..8d3565d 100644--- a/hy_memory/data/vector_store_chroma.py+++ b/hy_memory/data/vector_store_chroma.py@@ -9,10 +9,12 @@ Agent Memory V2 - VectorStore (Chroma)- 同步 API → _run_in_vdb_pool() 包装为异步"""-from typing import Optional, List, Dict, Any+from typing import Optional, List, Dict, Any, Set, Tupleimport concurrent.futuresimport functoolsimport logging+import os+import weakreffrom ..models.memory import (MemoryNode, MemoryLayer, MemoryStatus,@@ -33,6 +35,30 @@ _vdb_executor = concurrent.futures.ThreadPoolExecutor()+# 本进程里已 initialize、尚未 close 的 store。drop_vector_space 据此拒绝删掉+# 一个正被活着的 client 使用的集合。WeakSet:store 被回收后自动消失。+_LIVE_STORES: "weakref.WeakSet[ChromaVectorStore]" = weakref.WeakSet()+++def _location_key(vs_config) -> Tuple[str, str]:+ if getattr(vs_config, "host", None):+ return ("http", f"{vs_config.host}:{vs_config.port or 8000}")+ return ("local", os.path.realpath(str(vs_config.persist_directory or "")))+++def open_collection_names(location: Tuple[str, str]) -> Set[str]:+ """Collections that live stores in this process hold open at ``location``."""+ names: Set[str] = set()+ for store in list(_LIVE_STORES):+ if store._client is None or _location_key(store.config.vector_store) != location:+ continue+ names.add(store._collection_name)+ if store._entity_collection is not None:+ from ..pipelines._retrieval.config import entity_collection_name+ names.add(entity_collection_name(store._collection_name))+ return names++async def _run_in_vdb_pool(func, *args, **kwargs):"""Run a synchronous Chroma call in the dedicated VDB executor."""call = functools.partial(func, *args, **kwargs) if args or kwargs else func@@ -88,6 +114,7 @@ class ChromaVectorStore(VectorStoreBase):return client, collectionself._client, self._collection = await _run_in_vdb_pool(_init)+ _LIVE_STORES.add(self)logger.debug(f"VectorStore initialized (Chroma), collection={self._collection_name}")# ================================================================@@ -561,6 +588,7 @@ class ChromaVectorStore(VectorStoreBase):async def close(self) -> None:"""关闭客户端"""# Chroma PersistentClient 无需显式关闭+ _LIVE_STORES.discard(self)self._client = Noneself._collection = Noneself._entity_collection = Nonediff --git a/hy_memory/reembed.py b/hy_memory/reembed.pynew file mode 100644index 0000000..ab70462--- /dev/null+++ b/hy_memory/reembed.py@@ -0,0 +1,1777 @@+"""Re-embed the SDK's stores into a new embedding space.++An embedding space is the set of physical collections (and, in ultra mode, the+Kuzu graph) whose vectors were computed by one embedding model at one width.+Switching the model means computing every vector again with the new model and+writing it into a *new* space. The stored text is enough for that: each record+keeps its document and metadata, and the text that was embedded can be rebuilt+from them (`reembed_text`). The original conversations are not needed.++Contract:++- The caller pauses every writer before calling into this module and keeps+ them paused until the switch commits. The source stores are treated as frozen.+- Nothing here writes to the source. Vector sources are only read; the source+ graph is opened read-only.+- Source and target are named explicitly (`VectorSpace`, graph paths). Nothing+ is derived from the embedding provider or endpoint; the identity of a space is+ the model name plus the width (`space_identity`).+- Vector re-embeds are resumable: a target record with the same id is skipped+ (see `skip_existing`). Graph re-embeds restart from scratch: the target is+ built at a staging path that is wiped on every start and moved into place+ only after the copy verifies.++Only the Chroma vector backend and the Kuzu graph backend are supported.+"""++from __future__ import annotations++import asyncio+import concurrent.futures+import hashlib+import json+import logging+import os+import random+import re+import shutil+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import (+ Any,+ Callable,+ Dict,+ Iterable,+ List,+ Literal,+ Mapping,+ Optional,+ Sequence,+ Tuple,+ Union,+)++from .config import MemoryConfig+from .embedding import EmbeddingDimensionMismatchError, EmbeddingService+from .utils.async_executor import run_in_executor_guarded++logger = logging.getLogger(__name__)++VectorRole = Literal[+ "memory", "entities",+ "coding_keys", "coding_keys_cv",+ "working_facts", "working_facts_cv",+]+VECTOR_ROLES: Tuple[str, ...] = (+ "memory", "entities",+ "coding_keys", "coding_keys_cv",+ "working_facts", "working_facts_cv",+)++# Collection metadata keys written on target collections this module creates.+# Records themselves are copied verbatim, so the model can only live here.+COLLECTION_MODEL_KEY = "hy_embedding_model"+COLLECTION_DIMS_KEY = "hy_embedding_dims"++_SELF_RECALL_WARN_RATE = 0.9+++class ReembedCancelled(RuntimeError):+ """Raised at a batch boundary when ``cancel()`` returns True.++ Every batch written before the cancel is complete; no batch is half-written.+ ``report`` holds the progress made so far (a ``ReembedReport`` for vectors,+ ``None`` for the graph, whose staging copy is discarded).+ """++ def __init__(self, message: str = "reembed cancelled", report: Any = None) -> None:+ super().__init__(message)+ self.report = report+++# ======================================================================+# Space identity and naming+# ======================================================================++@dataclass(frozen=True)+class EmbeddingSpaceIdentity:+ """What makes two embedding spaces the same: model name and width."""++ model: str+ dims: int++ @property+ def key(self) -> str:+ return f"{self.model}#{self.dims}"++ @property+ def digest(self) -> str:+ """Short stable hash, usable inside a collection or directory name."""+ return hashlib.sha256(self.key.encode("utf-8")).hexdigest()[:12]+++def space_identity(model: str, dims: int) -> EmbeddingSpaceIdentity:+ """Identity of an embedding space. Provider and endpoint are not part of it."""+ name = " ".join(str(model or "").split())+ if not name:+ raise ValueError("space_identity needs a model name")+ width = int(dims)+ if width <= 0:+ raise ValueError(f"space_identity needs a positive width, got {dims!r}")+ return EmbeddingSpaceIdentity(model=name, dims=width)+++@dataclass(frozen=True)+class VectorSpace:+ """One embedding space in a Chroma store, named explicitly.++ ``base_name`` is the collection base without the width suffix, exactly what+ ``vector_store.collection_name`` is for a client. ``collections`` overrides+ the physical name per role when the caller manages names itself; roles left+ out of an explicit mapping are not part of the space.+ """++ base_name: str+ dims: Optional[int]+ persist_directory: Optional[str] = None+ host: Optional[str] = None+ port: Optional[int] = None+ model: Optional[str] = None+ collections: Optional[Mapping[str, str]] = None++ @classmethod+ def from_config(+ cls, config: MemoryConfig, *, base_name: Optional[str] = None,+ ) -> "VectorSpace":+ vs = config.vector_store+ provider = str(vs.provider or "chroma").lower()+ if provider != "chroma":+ raise NotImplementedError(+ f"re-embed supports the chroma vector store only, got {provider!r}"+ )+ return cls(+ base_name=base_name or vs.collection_name or "agent_memories",+ dims=vs.embedding_dims,+ persist_directory=vs.persist_directory,+ host=vs.host,+ port=vs.port,+ model=getattr(config.embedder, "model", None),+ )++ def names(self) -> Dict[str, str]:+ if self.collections is not None:+ unknown = set(self.collections) - set(VECTOR_ROLES)+ if unknown:+ raise ValueError(f"unknown vector roles: {sorted(unknown)}")+ return {role: self.collections[role] for role in VECTOR_ROLES if role in self.collections}+ return _derived_names(self.base_name, self.dims)++ def location(self) -> Tuple[str, str]:+ if self.host:+ return ("http", f"{self.host}:{self.port or 8000}")+ return ("local", os.path.realpath(str(self.persist_directory or "")))+++SpaceLike = Union[VectorSpace, MemoryConfig]+++def _derived_names(base_name: str, dims: Optional[int]) -> Dict[str, str]:+ from .coding.store import _derive_coding_collection_name, _derive_fact_collection_name+ from .data.vector_store_base import physical_collection_name+ from .pipelines._retrieval.config import entity_collection_name++ memory = physical_collection_name(base_name, dims)+ return {+ "memory": memory,+ "entities": entity_collection_name(memory),+ "coding_keys": physical_collection_name(_derive_coding_collection_name(base_name, ""), dims),+ "coding_keys_cv": physical_collection_name(_derive_coding_collection_name(base_name, "_cv"), dims),+ "working_facts": physical_collection_name(_derive_fact_collection_name(base_name, ""), dims),+ "working_facts_cv": physical_collection_name(_derive_fact_collection_name(base_name, "_cv"), dims),+ }+++def _space(value: SpaceLike) -> VectorSpace:+ if isinstance(value, VectorSpace):+ return value+ if isinstance(value, MemoryConfig):+ return VectorSpace.from_config(value)+ raise TypeError(f"expected VectorSpace or MemoryConfig, got {type(value).__name__}")+++def vector_space_names(+ config: SpaceLike, *, base_name: Optional[str] = None,+) -> Dict[str, str]:+ """Every physical collection a client with ``config`` opens for the vector roles.++ The width suffix is included. Pass ``base_name`` to name a space other than+ ``config.vector_store.collection_name``.+ """+ if isinstance(config, MemoryConfig):+ return VectorSpace.from_config(config, base_name=base_name).names()+ space = _space(config)+ if base_name is not None:+ return _derived_names(base_name, space.dims)+ return space.names()+++# ======================================================================+# The text a record was embedded from+# ======================================================================++def _custom_dict(value: Any) -> Dict[str, Any]:+ if isinstance(value, dict):+ return value+ if isinstance(value, str) and value:+ try:+ parsed = json.loads(value)+ except (TypeError, ValueError):+ return {}+ return parsed if isinstance(parsed, dict) else {}+ return {}+++def reembed_text(+ role: str, document: Optional[str], metadata: Optional[Mapping[str, Any]],+) -> str:+ """The exact text the write path embedded for one vector record.++ - memory, ``custom.memory_kind == "canonical_state"``: ``custom.state_retrieval_text``+ - memory with a non-empty ``custom.retrieval_text`` (schemas): that text+ - everything else, coding keys and working facts included: the document,+ falling back to ``metadata["content"]``+ """+ meta = metadata or {}+ if role == "memory":+ custom = _custom_dict(meta.get("custom"))+ if custom.get("memory_kind") == "canonical_state":+ text = str(custom.get("state_retrieval_text") or "").strip()+ if text:+ return text+ text = str(custom.get("retrieval_text") or "").strip()+ if text:+ return text+ if document:+ return document+ return str(meta.get("content") or "")+++def _embeddable(text: str) -> str:+ # OpenAI-compatible services reject an empty input with a 400 that would+ # fail the whole batch. A record with no text has no meaning to preserve.+ return text if text and text.strip() else " "+++def _record_digest(document: Optional[str], metadata: Optional[Mapping[str, Any]]) -> str:+ payload = json.dumps(+ [document, dict(metadata) if metadata is not None else None],+ sort_keys=True, ensure_ascii=False, default=str, separators=(",", ":"),+ )+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()+++# ======================================================================+# Chroma access (read-only on the source)+# ======================================================================++_VECTOR_PAGE = 500+++async def _vdb(func, *args, **kwargs):+ from .data.vector_store_chroma import _run_in_vdb_pool+ return await _run_in_vdb_pool(func, *args, **kwargs)+++def _client_for(space: VectorSpace):+ import chromadb++ if space.host:+ return chromadb.HttpClient(host=space.host, port=space.port or 8000)+ # Same settings as ChromaVectorStore.initialize, so both share one system.+ return chromadb.PersistentClient(path=space.persist_directory)+++def _get_collection(client, name: str):+ try:+ return client.get_collection(name=name)+ except Exception as exc: # chromadb NotFoundError / ValueError by version+ if "does not exist" in str(exc).lower() or type(exc).__name__ in (+ "NotFoundError", "InvalidCollectionException",+ ):+ return None+ raise+++def _page(collection, offset: int, limit: int, include: Sequence[str]) -> Dict[str, Any]:+ return collection.get(limit=limit, offset=offset, include=list(include))+++def _iter_pages(collection, include: Sequence[str], page: int = _VECTOR_PAGE):+ total = collection.count()+ offset = 0+ while offset < total:+ got = _page(collection, offset, page, include)+ ids = got.get("ids") or []+ if not ids:+ break+ yield got+ offset += len(ids)+++def _first_width(collection) -> Optional[int]:+ got = collection.get(limit=1, include=["embeddings"])+ embeddings = got.get("embeddings")+ if embeddings is None or len(embeddings) == 0:+ return None+ return len(embeddings[0])+++def _check_not_same(source: VectorSpace, target: VectorSpace, src_name: str, tgt_name: str) -> None:+ if source.location() == target.location() and src_name == tgt_name:+ raise ValueError(+ f"source and target are the same collection {src_name!r}; "+ "re-embed always writes a new collection"+ )+++def _roles(space: VectorSpace, roles: Optional[Iterable[str]]) -> List[str]:+ names = space.names()+ if roles is None:+ return [role for role in VECTOR_ROLES if role in names]+ wanted = list(roles)+ unknown = [role for role in wanted if role not in VECTOR_ROLES]+ if unknown:+ raise ValueError(f"unknown vector roles: {unknown}")+ return [role for role in wanted if role in names]+++# ======================================================================+# Plan+# ======================================================================++@dataclass(frozen=True)+class ReembedCollectionPlan:+ role: str+ source_name: str+ target_name: str+ source_count: int+ target_exists: bool+ target_count: int+++@dataclass(frozen=True)+class GraphReembedPlan:+ source_path: str+ node_counts: Dict[str, int]+ rel_counts: Dict[str, int]+ memory_embeddings: int+ beh_embeddings: int+ topic_embeddings: int+ hierarchy_topic_embeddings: int+ source_width: Optional[int]+ target_width: Optional[int]++ @property+ def memory_nodes(self) -> int:+ return self.node_counts.get("Memory", 0)++ @property+ def topics(self) -> int:+ return self.node_counts.get("Topic", 0)++ @property+ def hierarchy_topics(self) -> int:+ return self.node_counts.get("HierarchyTopic", 0)++ @property+ def vectors(self) -> int:+ return (+ self.memory_embeddings + self.beh_embeddings+ + self.topic_embeddings + self.hierarchy_topic_embeddings+ )+++@dataclass(frozen=True)+class ReembedPlan:+ collections: Tuple[ReembedCollectionPlan, ...]+ total_vectors: int+ source_dims: Optional[int]+ target_dims: Optional[int]+ graph: Optional[GraphReembedPlan] = None++ @property+ def estimated_vectors(self) -> int:+ """Vectors to compute: every vector record plus every graph embedding."""+ return self.total_vectors + (self.graph.vectors if self.graph else 0)++ @property+ def per_role(self) -> Dict[str, int]:+ return {item.role: item.source_count for item in self.collections}+++def _plan_sync(source: VectorSpace, target: VectorSpace) -> Tuple[List[ReembedCollectionPlan], Optional[int]]:+ src_client = _client_for(source)+ tgt_client = _client_for(target)+ src_names, tgt_names = source.names(), target.names()+ out: List[ReembedCollectionPlan] = []+ source_dims: Optional[int] = None+ for role in VECTOR_ROLES:+ if role not in src_names or role not in tgt_names:+ continue+ src = _get_collection(src_client, src_names[role])+ tgt = _get_collection(tgt_client, tgt_names[role])+ count = src.count() if src is not None else 0+ if src is not None and count and source_dims is None:+ source_dims = _first_width(src)+ out.append(ReembedCollectionPlan(+ role=role,+ source_name=src_names[role],+ target_name=tgt_names[role],+ source_count=count,+ target_exists=tgt is not None,+ target_count=tgt.count() if tgt is not None else 0,+ ))+ return out, source_dims+++async def plan_reembed(+ source: SpaceLike,+ target: SpaceLike,+ *,+ source_graph_path: Optional[Union[str, Path]] = None,+ target_graph_dims: Optional[int] = None,+) -> ReembedPlan:+ """Per-role record counts and the number of vectors a re-embed computes.++ Pass ``source_graph_path`` to include the Kuzu graph. Read-only.+ """+ src, tgt = _space(source), _space(target)+ collections, source_dims = await _vdb(_plan_sync, src, tgt)+ graph = None+ if source_graph_path is not None and Path(source_graph_path).exists():+ graph = await plan_graph_reembed(+ source_graph_path, target_dims=target_graph_dims or tgt.dims,+ )+ return ReembedPlan(+ collections=tuple(collections),+ total_vectors=sum(item.source_count for item in collections),+ source_dims=source_dims if source_dims is not None else src.dims,+ target_dims=tgt.dims,+ graph=graph,+ )+++# ======================================================================+# Vector re-embed+# ======================================================================++@dataclass(frozen=True)+class ReembedProgress:+ role: str+ done: int # source records handled so far+ total: int # source records in this role+ embedded: int # vectors computed and written+ reused: int # skipped: the target already holds an identical record+ kept: int = 0 # skipped: the target holds a different record with this id (target wins)+++@dataclass(frozen=True)+class ReembedReport:+ per_role: Dict[str, ReembedProgress]+ elapsed_s: float++ @property+ def embedded(self) -> int:+ return sum(item.embedded for item in self.per_role.values())++ @property+ def reused(self) -> int:+ return sum(item.reused for item in self.per_role.values())++ @property+ def kept(self) -> int:+ return sum(item.kept for item in self.per_role.values())+++def _default_embedding(target: SpaceLike) -> EmbeddingService:+ if isinstance(target, MemoryConfig):+ from .core.embed_service import EmbedService+ return EmbedService(target)+ raise ValueError("pass embedding= when the target is a VectorSpace")+++def _check_service_dims(embedding: Any, dims: int) -> None:+ getter = getattr(embedding, "get_embedding_dims", None)+ if not callable(getter):+ return+ try:+ declared = getter()+ except Exception:+ return+ if isinstance(declared, int) and declared and declared != dims:+ raise EmbeddingDimensionMismatchError(+ f"embedding service width {declared} != target width {dims}"+ )+++def _validate_vectors(vectors: Sequence[Any], expected: int, dims: int, where: str) -> List[List[float]]:+ if len(vectors) != expected:+ raise ValueError(f"{where}: expected {expected} vectors, got {len(vectors)}")+ out: List[List[float]] = []+ for vector in vectors:+ width = len(vector) if vector is not None else 0+ if width != dims:+ raise EmbeddingDimensionMismatchError(+ f"{where}: embedding width {width} != target width {dims}"+ )+ out.append([float(x) for x in vector])+ return out+++async def _embed(embedding: Any, texts: List[str]) -> List[Any]:+ if not texts:+ return []+ return list(await embedding.embed_batch(texts, use_cache=False))+++def _target_metadata(source_collection, target: VectorSpace) -> Dict[str, Any]:+ metadata = dict(getattr(source_collection, "metadata", None) or {})+ metadata.setdefault("hnsw:space", "cosine")+ metadata.pop(COLLECTION_MODEL_KEY, None)+ metadata.pop(COLLECTION_DIMS_KEY, None)+ if target.model:+ metadata[COLLECTION_MODEL_KEY] = str(target.model)+ if target.dims:+ metadata[COLLECTION_DIMS_KEY] = int(target.dims)+ return metadata+++async def reembed_vector_space(+ source: SpaceLike,+ target: SpaceLike,+ *,+ embedding: Optional[EmbeddingService] = None,+ batch_size: int = 64,+ max_inflight: int = 2,+ roles: Optional[Iterable[str]] = None,+ cancel: Optional[Callable[[], bool]] = None,+ on_progress: Optional[Callable[[ReembedProgress], None]] = None,+ skip_existing: bool = True,+) -> ReembedReport:+ """Re-embed every record of ``source`` into ``target`` with ``embedding``.++ All users and all layers (L1_RAW included) are read by paging the raw+ collections. For each record the text from `reembed_text` is embedded and+ the record is upserted into the target with the same id, document and+ metadata, copied verbatim.++ - Every vector's width is checked against ``target.dims`` before anything+ in its batch is written (`EmbeddingDimensionMismatchError`).+ - ``skip_existing=True`` (default): an id the target already holds is never+ touched, so a rerun resumes and a merge of a stray space never overwrites+ live data (the target wins). ``skip_existing=False``: identical records+ are still skipped, records that differ are re-embedded from the source.+ - ``cancel`` is checked before each round of up to ``max_inflight`` batches;+ a cancel raises `ReembedCancelled` with every finished batch written.+ - The source is never written.+ """+ src, tgt = _space(source), _space(target)+ if not tgt.dims:+ raise ValueError("the target space needs an explicit width (dims)")+ dims = int(tgt.dims)+ service = embedding if embedding is not None else _default_embedding(target)+ _check_service_dims(service, dims)+ batch_size = max(1, int(batch_size))+ max_inflight = max(1, int(max_inflight))+ cancelled = cancel or (lambda: False)+ src_names, tgt_names = src.names(), tgt.names()+ selected = [role for role in _roles(src, roles) if role in tgt_names]+ started = time.monotonic()+ per_role: Dict[str, ReembedProgress] = {}++ def _report() -> ReembedReport:+ return ReembedReport(per_role=dict(per_role), elapsed_s=time.monotonic() - started)++ src_client = await _vdb(_client_for, src)+ tgt_client = await _vdb(_client_for, tgt)+ for role in selected:+ src_name, tgt_name = src_names[role], tgt_names[role]+ _check_not_same(src, tgt, src_name, tgt_name)+ src_coll = await _vdb(_get_collection, src_client, src_name)+ total = await _vdb(src_coll.count) if src_coll is not None else 0+ progress = ReembedProgress(role=role, done=0, total=total, embedded=0, reused=0, kept=0)+ per_role[role] = progress+ if not total:+ if on_progress:+ on_progress(progress)+ continue+ tgt_coll = await _vdb(+ tgt_client.get_or_create_collection,+ name=tgt_name,+ metadata=_target_metadata(src_coll, tgt),+ )+ offset = 0+ while offset < total:+ if cancelled():+ raise ReembedCancelled(report=_report())+ # One round: read up to max_inflight batches, embed them together,+ # validate all of them, then write them in order.+ batches: List[Dict[str, Any]] = []+ for _ in range(max_inflight):+ if offset >= total:+ break+ got = await _vdb(_page, src_coll, offset, batch_size, ["documents", "metadatas"])+ ids = list(got.get("ids") or [])+ if not ids:+ offset = total+ break+ offset += len(ids)+ batches.append({+ "ids": ids,+ "documents": list(got.get("documents") or [None] * len(ids)),+ "metadatas": list(got.get("metadatas") or [None] * len(ids)),+ })+ if not batches:+ break+ reused = kept = 0+ for batch in batches:+ existing = await _vdb(+ tgt_coll.get, ids=batch["ids"], include=["documents", "metadatas"],+ )+ have = {+ rid: _record_digest(doc, meta)+ for rid, doc, meta in zip(+ existing.get("ids") or [],+ existing.get("documents") or [],+ existing.get("metadatas") or [],+ )+ }+ pending = []+ for i, rid in enumerate(batch["ids"]):+ doc, meta = batch["documents"][i], batch["metadatas"][i]+ if rid in have:+ if have[rid] == _record_digest(doc, meta):+ reused += 1+ continue+ if skip_existing:+ kept += 1+ continue+ pending.append(i)+ batch["pending"] = pending+ batch["texts"] = [+ _embeddable(reembed_text(role, batch["documents"][i], batch["metadatas"][i]))+ for i in pending+ ]+ results = await asyncio.gather(*(_embed(service, b["texts"]) for b in batches))+ checked = [+ _validate_vectors(vectors, len(b["pending"]), dims, f"{role}:{tgt_name}")+ for b, vectors in zip(batches, results)+ ]+ embedded = 0+ for batch, vectors in zip(batches, checked):+ if not batch["pending"]:+ continue+ idx = batch["pending"]+ await _vdb(+ tgt_coll.upsert,+ ids=[batch["ids"][i] for i in idx],+ embeddings=vectors,+ documents=[batch["documents"][i] for i in idx],+ metadatas=[batch["metadatas"][i] for i in idx],+ )+ embedded += len(idx)+ done = sum(len(b["ids"]) for b in batches)+ progress = ReembedProgress(+ role=role,+ done=progress.done + done,+ total=total,+ embedded=progress.embedded + embedded,+ reused=progress.reused + reused,+ kept=progress.kept + kept,+ )+ per_role[role] = progress+ if on_progress:+ on_progress(progress)+ report = _report()+ logger.info(+ "[reembed] vector space done: embedded=%d reused=%d kept=%d elapsed=%.1fs",+ report.embedded, report.reused, report.kept, report.elapsed_s,+ )+ return report+++# ======================================================================+# Scan and diff (MemHub startup reconcile)+# ======================================================================++_ROLE_SUFFIX = re.compile(+ r"^(?P<base>.+?)_(?P<role>coding_keys_cv|coding_keys|working_facts_cv|working_facts)"+ r"(?:_(?P<dims>\d+))?$"+)+_DIMS_SUFFIX = re.compile(r"^(?P<base>.+)_(?P<dims>\d+)$")+++def parse_collection_name(name: str) -> Optional[Tuple[str, Optional[int], str]]:+ """Split a physical collection name into (base, dims, role).++ Returns None for collections that are not vector roles (``_tag_index``).+ """+ if name.endswith("_tag_index"):+ return None+ match = _ROLE_SUFFIX.match(name)+ if match:+ dims = match.group("dims")+ return match.group("base"), int(dims) if dims else None, match.group("role")+ role = "memory"+ if name.endswith("_entities"):+ name, role = name[: -len("_entities")], "entities"+ match = _DIMS_SUFFIX.match(name)+ if match:+ return match.group("base"), int(match.group("dims")), role+ return name, None, role+++@dataclass(frozen=True)+class ScannedCollection:+ role: str+ name: str+ count: int+ width: Optional[int]+ per_user: Dict[str, int]+ models: Tuple[str, ...]+++@dataclass(frozen=True)+class ScannedSpace:+ base_name: str+ dims: Optional[int]+ roles: Dict[str, ScannedCollection]+ persist_directory: str++ @property+ def total(self) -> int:+ return sum(item.count for item in self.roles.values())++ @property+ def width(self) -> Optional[int]:+ """The stored width when every non-empty role agrees, else None."""+ widths = {item.width for item in self.roles.values() if item.width}+ return widths.pop() if len(widths) == 1 else None++ @property+ def per_user(self) -> Dict[str, int]:+ out: Dict[str, int] = {}+ for item in self.roles.values():+ for user, count in item.per_user.items():+ out[user] = out.get(user, 0) + count+ return out++ @property+ def models(self) -> Tuple[str, ...]:+ return tuple(sorted({m for item in self.roles.values() for m in item.models}))++ def space(self, *, model: Optional[str] = None) -> VectorSpace:+ """A `VectorSpace` naming exactly the collections found."""+ return VectorSpace(+ base_name=self.base_name,+ dims=self.dims if self.dims is not None else self.width,+ persist_directory=self.persist_directory,+ model=model,+ collections={role: item.name for role, item in self.roles.items()},+ )+++def _scan_sync(persist_directory: str, per_user: bool) -> List[ScannedSpace]:+ if not os.path.isdir(persist_directory):+ return []+ import chromadb++ client = chromadb.PersistentClient(path=persist_directory)+ grouped: Dict[Tuple[str, Optional[int]], Dict[str, ScannedCollection]] = {}+ for item in client.list_collections():+ name = item if isinstance(item, str) else item.name+ parsed = parse_collection_name(name)+ if parsed is None:+ continue+ base, dims, role = parsed+ collection = client.get_collection(name=name)+ count = collection.count()+ users: Dict[str, int] = {}+ models = set()+ model = (collection.metadata or {}).get(COLLECTION_MODEL_KEY)+ if model:+ models.add(str(model))+ if per_user and count:+ for got in _iter_pages(collection, ["metadatas"], page=2000):+ for meta in got.get("metadatas") or []:+ meta = meta or {}+ user = str(meta.get("user_id") or "")+ users[user] = users.get(user, 0) + 1+ record_model = meta.get(COLLECTION_MODEL_KEY) or meta.get("embedding_model")+ if record_model:+ models.add(str(record_model))+ grouped.setdefault((base, dims), {})[role] = ScannedCollection(+ role=role,+ name=name,+ count=count,+ width=_first_width(collection) if count else None,+ per_user=users,+ models=tuple(sorted(models)),+ )+ return [+ ScannedSpace(base_name=base, dims=dims, roles=roles, persist_directory=persist_directory)+ for (base, dims), roles in sorted(grouped.items(), key=lambda kv: (kv[0][0], kv[0][1] or 0))+ ]+++async def scan_vector_spaces(+ persist_directory: Union[str, Path], *, per_user: bool = True,+) -> List[ScannedSpace]:+ """Every embedding space with collections under ``persist_directory``.++ Read-only. Collections are grouped by (base, width) from their names; for+ each role found the scan reports the record count, the stored width (read+ from one embedding), the per-``user_id`` counts and the models recorded in+ collection or record metadata (often none: records do not carry the model,+ only collections created by `reembed_vector_space` do).+ """+ return await _vdb(_scan_sync, str(persist_directory), per_user)+++@dataclass(frozen=True)+class SpaceDiff:+ missing: Dict[str, Tuple[str, ...]] # per role: ids in source, absent from target++ @property+ def total_missing(self) -> int:+ return sum(len(ids) for ids in self.missing.values())++ @property+ def empty(self) -> bool:+ return self.total_missing == 0+++def _all_ids(collection) -> List[str]:+ if collection is None:+ return []+ out: List[str] = []+ for got in _iter_pages(collection, [], page=5000):+ out.extend(got.get("ids") or [])+ return out+++def _diff_sync(source: VectorSpace, target: VectorSpace, roles: Optional[Iterable[str]]) -> SpaceDiff:+ src_client, tgt_client = _client_for(source), _client_for(target)+ src_names, tgt_names = source.names(), target.names()+ missing: Dict[str, Tuple[str, ...]] = {}+ for role in _roles(source, roles):+ src_ids = _all_ids(_get_collection(src_client, src_names[role]))+ tgt_ids = set(_all_ids(+ _get_collection(tgt_client, tgt_names[role]) if role in tgt_names else None+ ))+ missing[role] = tuple(rid for rid in src_ids if rid not in tgt_ids)+ return SpaceDiff(missing=missing)+++async def diff_spaces(+ source: SpaceLike, target: SpaceLike, *, roles: Optional[Iterable[str]] = None,+) -> SpaceDiff:+ """Per role, the ids the source holds that the target does not.++ Every record counts whatever its ``status`` (deleted and superseded records+ included). An empty diff means the source holds nothing unmerged.+ """+ return await _vdb(_diff_sync, _space(source), _space(target), roles)+++# ======================================================================+# Verify and drop+# ======================================================================++@dataclass(frozen=True)+class GraphVerification:+ ok: bool+ node_counts: Dict[str, Tuple[int, int]] # table: (source, target)+ rel_counts: Dict[str, Tuple[int, int]]+ target_width: Optional[int]+ vector_index: bool+ problems: Tuple[str, ...]+++@dataclass(frozen=True)+class ReembedVerification:+ ok: bool+ per_role: Dict[str, Dict[str, int]] # source, target, missing, extra, mismatched, bad_dims+ per_user: Dict[str, int] # target record count by metadata user_id+ self_recall_hits: int+ self_recall_samples: int+ problems: Tuple[str, ...]+ warnings: Tuple[str, ...] = ()+ graph: Optional[GraphVerification] = None+++def _collect(collection, include: Sequence[str], dims: Optional[int]):+ digests: Dict[str, str] = {}+ users: Dict[str, int] = {}+ bad_dims = 0+ if collection is None:+ return digests, users, bad_dims+ for got in _iter_pages(collection, include):+ ids = got.get("ids") or []+ docs = got.get("documents") or [None] * len(ids)+ metas = got.get("metadatas") or [None] * len(ids)+ embeddings = got.get("embeddings")+ for i, rid in enumerate(ids):+ digests[rid] = _record_digest(docs[i], metas[i])+ user = str((metas[i] or {}).get("user_id") or "")+ users[user] = users.get(user, 0) + 1+ if dims is not None and embeddings is not None and len(embeddings[i]) != dims:+ bad_dims += 1+ return digests, users, bad_dims+++def _verify_sync(source: VectorSpace, target: VectorSpace, roles: Optional[Iterable[str]]):+ src_client, tgt_client = _client_for(source), _client_for(target)+ src_names, tgt_names = source.names(), target.names()+ per_role: Dict[str, Dict[str, int]] = {}+ per_user_src: Dict[str, int] = {}+ per_user_tgt: Dict[str, int] = {}+ common: List[Tuple[str, str]] = [] # (role, id)+ problems: List[str] = []+ for role in _roles(source, roles):+ if role not in tgt_names:+ continue+ src_digests, src_users, _ = _collect(+ _get_collection(src_client, src_names[role]), ["documents", "metadatas"], None,+ )+ tgt_digests, tgt_users, bad = _collect(+ _get_collection(tgt_client, tgt_names[role]),+ ["documents", "metadatas", "embeddings"], int(target.dims or 0) or None,+ )+ missing = [rid for rid in src_digests if rid not in tgt_digests]+ extra = [rid for rid in tgt_digests if rid not in src_digests]+ mismatched = [+ rid for rid, digest in src_digests.items()+ if rid in tgt_digests and tgt_digests[rid] != digest+ ]+ per_role[role] = {+ "source": len(src_digests), "target": len(tgt_digests),+ "missing": len(missing), "extra": len(extra),+ "mismatched": len(mismatched), "bad_dims": bad,+ }+ for label, values in (("missing", missing), ("extra", extra), ("mismatched", mismatched)):+ if values:+ problems.append(f"{role}: {len(values)} {label} (e.g. {values[0]})")+ if bad:+ problems.append(f"{role}: {bad} target vectors not {target.dims} wide")+ for user, count in src_users.items():+ per_user_src[user] = per_user_src.get(user, 0) + count+ for user, count in tgt_users.items():+ per_user_tgt[user] = per_user_tgt.get(user, 0) + count+ common.extend((role, rid) for rid in src_digests if rid in tgt_digests)+ for user in sorted(set(per_user_src) | set(per_user_tgt)):+ if per_user_src.get(user, 0) != per_user_tgt.get(user, 0):+ problems.append(+ f"user {user!r}: source {per_user_src.get(user, 0)} != "+ f"target {per_user_tgt.get(user, 0)}"+ )+ return per_role, per_user_tgt, common, problems+++def _sample_records(target: VectorSpace, picks: List[Tuple[str, str]]):+ client = _client_for(target)+ names = target.names()+ out = []+ by_role: Dict[str, List[str]] = {}+ for role, rid in picks:+ by_role.setdefault(role, []).append(rid)+ for role, ids in by_role.items():+ collection = _get_collection(client, names[role])+ got = collection.get(ids=ids, include=["documents", "metadatas"])+ for rid, doc, meta in zip(got["ids"], got["documents"], got["metadatas"]):+ out.append((role, rid, reembed_text(role, doc, meta)))+ return out+++def _query_sync(target: VectorSpace, role: str, vectors: List[List[float]], k: int) -> List[List[str]]:+ collection = _get_collection(_client_for(target), target.names()[role])+ got = collection.query(query_embeddings=vectors, n_results=k, include=[])+ return [list(ids) for ids in got.get("ids") or []]+++async def verify_reembed(+ source: SpaceLike,+ target: SpaceLike,+ *,+ embedding: Optional[EmbeddingService] = None,+ samples: int = 50,+ recall_k: int = 3,+ roles: Optional[Iterable[str]] = None,+ seed: Optional[int] = 0,+ source_graph_path: Optional[Union[str, Path]] = None,+ target_graph_path: Optional[Union[str, Path]] = None,+) -> ReembedVerification:+ """Check a finished re-embed.++ - Per role, the id sets are equal and every id has the same document and+ metadata on both sides (sha256 of the canonical JSON).+ - Every target vector is ``target.dims`` wide.+ - Per ``user_id``, source and target hold the same number of records.+ - Self-recall: for ``samples`` random records, querying the target with+ the re-embedded text returns the record in the top ``recall_k``. Below+ 90% this is a warning, not a failure, because near-duplicates exist.+ - With both graph paths: `verify_graph_reembed`.+ """+ src, tgt = _space(source), _space(target)+ per_role, per_user, common, problems = await _vdb(_verify_sync, src, tgt, roles)+ warnings: List[str] = []+ hits = sampled = 0+ if samples > 0 and common:+ service = embedding+ if service is None and isinstance(target, MemoryConfig):+ service = _default_embedding(target)+ if service is None:+ warnings.append("self-recall skipped: no embedding service")+ else:+ rng = random.Random(seed)+ picks = rng.sample(common, min(samples, len(common)))+ records = await _vdb(_sample_records, tgt, picks)+ by_role: Dict[str, List[Tuple[str, str]]] = {}+ for role, rid, text in records:+ by_role.setdefault(role, []).append((rid, text))+ for role, items in by_role.items():+ vectors = await _embed(service, [_embeddable(text) for _, text in items])+ vectors = _validate_vectors(vectors, len(items), int(tgt.dims), f"verify:{role}")+ found = await _vdb(_query_sync, tgt, role, vectors, recall_k)+ for (rid, _), ids in zip(items, found):+ sampled += 1+ hits += int(rid in ids)+ if sampled and hits / sampled < _SELF_RECALL_WARN_RATE:+ warnings.append(f"self-recall {hits}/{sampled} below {_SELF_RECALL_WARN_RATE:.0%}")+ graph = None+ if source_graph_path is not None and target_graph_path is not None:+ graph = await verify_graph_reembed(source_graph_path, target_graph_path, target_dims=tgt.dims)+ problems.extend(f"graph: {p}" for p in graph.problems)+ return ReembedVerification(+ ok=not problems,+ per_role=per_role,+ per_user=per_user,+ self_recall_hits=hits,+ self_recall_samples=sampled,+ problems=tuple(problems),+ warnings=tuple(warnings),+ graph=graph,+ )+++def _drop_sync(space: VectorSpace, roles: Optional[Iterable[str]]) -> List[str]:+ from .data.vector_store_chroma import open_collection_names++ names = space.names()+ selected = [names[role] for role in _roles(space, roles)]+ busy = sorted(set(selected) & open_collection_names(space.location()))+ if busy:+ raise RuntimeError(+ f"refusing to drop collections held open by a live client: {busy}"+ )+ client = _client_for(space)+ dropped = []+ for name in selected:+ if _get_collection(client, name) is None:+ continue+ client.delete_collection(name=name)+ dropped.append(name)+ return dropped+++async def drop_vector_space(+ space: SpaceLike, *, roles: Optional[Iterable[str]] = None,+) -> List[str]:+ """Delete the space's collections and return the names deleted.++ Refuses when a live store in this process holds any of them open.+ """+ return await _vdb(_drop_sync, _space(space), roles)+++# ======================================================================+# Graph (Kuzu)+# ======================================================================++# All Kuzu work of a re-embed runs on this one thread: Kuzu connections are not+# thread-safe and a QueryResult must be consumed on the thread that produced it.+_graph_executor = concurrent.futures.ThreadPoolExecutor(+ max_workers=1, thread_name_prefix="kuzu-reembed",+)+_GRAPH_PAGE = 2000+_GRAPH_WRITE_TX = 1000+_KUZU_SIDECARS = (".wal", ".shadow", ".lock", ".tmp")+_SDK_VECTOR_INDEX = (+ "CALL CREATE_VECTOR_INDEX('Memory', 'memory_content_idx', 'embedding', "+ "metric := 'cosine');"+)++# Embedding columns rebuilt from text. Everything else is copied verbatim.+_GRAPH_EMBEDDING_COLUMNS: Dict[str, Tuple[str, ...]] = {+ "Memory": ("embedding", "beh_embedding"),+ "Topic": ("embedding",),+ "HierarchyTopic": ("embedding",),+}+++async def _graph(func, *args, **kwargs):+ import functools+ return await run_in_executor_guarded(_graph_executor, functools.partial(func, *args, **kwargs))+++def hierarchy_topic_label(subject: str, path_json: Any, name: str = "") -> Tuple[str, bool]:+ """Rebuild the label a HierarchyTopic embedding was computed from.++ ``add_topic_path`` embeds ``"{subject}: kind:segment > kind:segment"`` and+ stores ``path_json`` as ``{"names": [...], "types": [...]}`` for the same+ prefix, so that form rebuilds exactly (returns ``exact=True``). A legacy+ list-form ``path_json`` has no stored types: the label is rebuilt with the+ default Topic -> Facet types the current code would use, and flagged+ inexact. With no usable path the stored ``name`` is used, also inexact.+ """+ from .structure import parse_hierarchy_path_payload++ raw = path_json+ if isinstance(raw, str):+ try:+ raw = json.loads(raw)+ except (TypeError, ValueError):+ raw = None+ parsed = parse_hierarchy_path_payload(path_json or "")+ path, types = parsed["path"], parsed["node_types"]+ if not path:+ fallback = str(name or subject or "").strip()+ return (f"{subject}: {fallback}" if subject and name else fallback), False+ exact = isinstance(raw, dict) and isinstance(raw.get("types"), list)+ label = f"{subject}: " + " > ".join(f"{kind}:{segment}" for segment, kind in zip(path, types))+ return label, exact+++def _memory_text(row: Mapping[str, Any], column: str) -> str:+ custom = _custom_dict(row.get("custom_json"))+ if column == "beh_embedding":+ return str(custom.get("beh_abstraction") or "").strip()+ text = str(custom.get("retrieval_text") or "").strip()+ return text or str(row.get("content") or "")+++def _kuzu():+ try:+ import kuzu+ except ImportError as exc: # pragma: no cover - kuzu is a core dependency+ raise ImportError("kuzu is required. Install with: pip install kuzu") from exc+ return kuzu+++def _rows(conn, query: str, params: Optional[Dict[str, Any]] = None) -> List[List[Any]]:+ result = conn.execute(query, params) if params else conn.execute(query)+ out = []+ while result.has_next():+ out.append(list(result.get_next()))+ return out+++def _tables(conn) -> Dict[str, str]:+ result = conn.execute("CALL show_tables() RETURN *;")+ columns = result.get_column_names()+ name_i, type_i = columns.index("name"), columns.index("type")+ out = {}+ while result.has_next():+ row = result.get_next()+ out[str(row[name_i])] = str(row[type_i]).upper()+ return out+++def _table_columns(conn, table: str) -> List[Tuple[str, str, bool]]:+ """(name, type, is_primary_key) per property, in declaration order."""+ out = []+ for row in _rows(conn, f"CALL table_info('{table}') RETURN *;"):+ out.append((str(row[1]), str(row[2]), row[4] is True))+ return out+++def _array_width(type_name: str) -> Optional[int]:+ match = re.match(r"^FLOAT\[(\d+)\]$", type_name.strip().upper())+ return int(match.group(1)) if match else None+++def _count(conn, query: str) -> int:+ rows = _rows(conn, query)+ return int(rows[0][0]) if rows else 0+++def _graph_counts(conn) -> Tuple[Dict[str, int], Dict[str, int]]:+ nodes, rels = {}, {}+ for name, kind in _tables(conn).items():+ if kind == "NODE":+ nodes[name] = _count(conn, f"MATCH (n:{name}) RETURN count(n);")+ elif kind == "REL":+ rels[name] = _count(conn, f"MATCH ()-[r:{name}]->() RETURN count(r);")+ return nodes, rels+++def _target_ddl(dims: int) -> List[Tuple[str, str]]:+ from .data.graph_store_kuzu import _SCHEMA_DDL_TABLES, _make_memory_ddl++ out = []+ for name, ddl in _SCHEMA_DDL_TABLES.items():+ out.append((name, _make_memory_ddl(dims) if name == "Memory" else ddl))+ return out+++def _open_readonly(path: Path):+ kuzu = _kuzu()+ db = kuzu.Database(str(path), read_only=True)+ return db, kuzu.Connection(db)+++def _close(conn, db) -> None:+ for obj in (conn, db):+ if obj is None:+ continue+ try:+ obj.close()+ except Exception:+ pass+++def _kuzu_files(path: Path) -> List[Path]:+ out = [path] if path.exists() else []+ for suffix in _KUZU_SIDECARS:+ sidecar = path.with_name(path.name + suffix)+ if sidecar.exists():+ out.append(sidecar)+ return out+++def _remove_kuzu(path: Path) -> None:+ for item in _kuzu_files(path):+ if item.is_dir() and not item.is_symlink():+ shutil.rmtree(item)+ else:+ item.unlink()+++def _source_width(conn) -> Optional[int]:+ for name, type_name, _ in _table_columns(conn, "Memory"):+ if name == "embedding":+ return _array_width(type_name)+ return None+++def _assert_not_open(path: Path, what: str) -> None:+ from .data.graph_store_kuzu import open_graph_paths++ if os.path.realpath(str(path)) in open_graph_paths():+ raise RuntimeError(f"{what} graph {path} is held open by a live client in this process")+++def _plan_graph_sync(path: Path, target_dims: Optional[int]) -> GraphReembedPlan:+ db, conn = _open_readonly(path)+ try:+ nodes, rels = _graph_counts(conn)+ tables = _tables(conn)++ def _non_null(table: str, column: str) -> int:+ if table not in tables:+ return 0+ return _count(conn, f"MATCH (n:{table}) WHERE n.{column} IS NOT NULL RETURN count(n);")++ return GraphReembedPlan(+ source_path=str(path),+ node_counts=nodes,+ rel_counts=rels,+ memory_embeddings=_non_null("Memory", "embedding"),+ beh_embeddings=_non_null("Memory", "beh_embedding"),+ topic_embeddings=_non_null("Topic", "embedding"),+ hierarchy_topic_embeddings=_non_null("HierarchyTopic", "embedding"),+ source_width=_source_width(conn) if "Memory" in tables else None,+ target_width=target_dims,+ )+ finally:+ _close(conn, db)+++async def plan_graph_reembed(+ source_graph_path: Union[str, Path], *, target_dims: Optional[int] = None,+) -> GraphReembedPlan:+ """Node, edge and embedding counts of a graph. Opens it read-only."""+ path = Path(source_graph_path)+ if not path.exists():+ raise FileNotFoundError(f"graph not found: {path}")+ _assert_not_open(path, "source")+ return await _graph(_plan_graph_sync, path, target_dims)+++@dataclass(frozen=True)+class GraphReembedProgress:+ table: str+ done: int+ total: int+ embedded: int+++@dataclass(frozen=True)+class GraphReembedReport:+ target_path: str+ source_width: Optional[int]+ target_width: int+ node_counts: Dict[str, int]+ rel_counts: Dict[str, int]+ embedded: Dict[str, int] # "Table.column": vectors computed+ cleared: Dict[str, int] # "Table.column": vectors set to NULL (no text to rebuild from)+ hierarchy_labels_exact: int+ hierarchy_labels_inexact: Tuple[str, ...] # hierarchy_topic_ids rebuilt from a fallback text+ skipped_tables: Tuple[str, ...] # source tables this SDK does not know+ dropped_columns: Tuple[str, ...] # "Table.column" in the source but not the target+ vector_indexes: Tuple[str, ...]+ elapsed_s: float+++@dataclass+class _GraphCopy:+ source: Path+ staging: Path+ dims: int+ src_db: Any = None+ src: Any = None+ dst_db: Any = None+ dst: Any = None+ node_tables: List[str] = field(default_factory=list)+ rel_tables: List[str] = field(default_factory=list)+ skipped: List[str] = field(default_factory=list)+ dropped: List[str] = field(default_factory=list)+ columns: Dict[str, List[str]] = field(default_factory=dict)+ pk: Dict[str, str] = field(default_factory=dict)+ source_width: Optional[int] = None++ def open(self) -> None:+ kuzu = _kuzu()+ self.src_db, self.src = _open_readonly(self.source)+ self.staging.parent.mkdir(parents=True, exist_ok=True)+ self.dst_db = kuzu.Database(str(self.staging))+ self.dst = kuzu.Connection(self.dst_db)+ for _, ddl in _target_ddl(self.dims):+ self.dst.execute(ddl + ";")+ src_tables, dst_tables = _tables(self.src), _tables(self.dst)+ self.skipped = sorted(name for name in src_tables if name not in dst_tables)+ for name, kind in dst_tables.items():+ if name not in src_tables:+ continue+ src_cols = _table_columns(self.src, name)+ dst_names = {col for col, _, _ in _table_columns(self.dst, name)}+ self.columns[name] = [col for col, _, _ in src_cols if col in dst_names]+ self.dropped.extend(f"{name}.{col}" for col, _, _ in src_cols if col not in dst_names)+ if kind == "NODE":+ self.node_tables.append(name)+ self.pk[name] = next(col for col, _, is_pk in src_cols if is_pk)+ else:+ self.rel_tables.append(name)+ if "Memory" in src_tables:+ self.source_width = _source_width(self.src)++ def node_total(self, table: str) -> int:+ return _count(self.src, f"MATCH (n:{table}) RETURN count(n);")++ def read_nodes(self, table: str, after: Optional[Any], limit: int) -> List[Dict[str, Any]]:+ pk = self.pk[table]+ embed_cols = _GRAPH_EMBEDDING_COLUMNS.get(table, ())+ plain = [c for c in self.columns[table] if c not in embed_cols]+ flags = [c for c in embed_cols if c in self.columns[table]]+ select = [f"n.{c}" for c in plain] + [f"n.{c} IS NOT NULL" for c in flags]+ where = f"WHERE n.{pk} > $after " if after is not None else ""+ query = (+ f"MATCH (n:{table}) {where}RETURN {', '.join(select)} "+ f"ORDER BY n.{pk} LIMIT {int(limit)};"+ )+ params = {"after": after} if after is not None else None+ out = []+ for row in _rows(self.src, query, params):+ record = dict(zip(plain, row[: len(plain)]))+ record["__has"] = {c: bool(v) for c, v in zip(flags, row[len(plain):])}+ out.append(record)+ return out++ def write_nodes(self, table: str, records: List[Dict[str, Any]]) -> None:+ if not records:+ return+ cols = self.columns[table]+ assigns = ", ".join(f"{c}: $p{i}" for i, c in enumerate(cols))+ query = f"CREATE (n:{table} {{{assigns}}});"+ self.dst.execute("BEGIN TRANSACTION;")+ try:+ for record in records:+ self.dst.execute(query, {f"p{i}": record.get(c) for i, c in enumerate(cols)})+ self.dst.execute("COMMIT;")+ except Exception:+ try:+ self.dst.execute("ROLLBACK;")+ except Exception:+ pass+ raise++ def copy_rels(self, table: str) -> int:+ props = self.columns[table]+ copied = 0+ for row in _rows(self.src, f"CALL show_connection('{table}') RETURN *;"):+ src_table, dst_table = str(row[0]), str(row[1])+ src_pk, dst_pk = self.pk.get(src_table), self.pk.get(dst_table)+ if src_pk is None or dst_pk is None:+ continue+ select = ", ".join([f"a.{src_pk}", f"b.{dst_pk}"] + [f"r.{p}" for p in props])+ result = self.src.execute(+ f"MATCH (a:{src_table})-[r:{table}]->(b:{dst_table}) RETURN {select};"+ )+ assigns = ", ".join(f"{p}: $p{i}" for i, p in enumerate(props))+ create = (+ f"MATCH (a:{src_table} {{{src_pk}: $from}}), (b:{dst_table} {{{dst_pk}: $to}}) "+ f"CREATE (a)-[:{table} {{{assigns}}}]->(b);"+ )+ pending = 0+ self.dst.execute("BEGIN TRANSACTION;")+ try:+ while result.has_next():+ values = list(result.get_next())+ params = {"from": values[0], "to": values[1]}+ params.update({f"p{i}": v for i, v in enumerate(values[2:])})+ self.dst.execute(create, params)+ copied += 1+ pending += 1+ if pending >= _GRAPH_WRITE_TX:+ self.dst.execute("COMMIT;")+ self.dst.execute("BEGIN TRANSACTION;")+ pending = 0+ self.dst.execute("COMMIT;")+ except Exception:+ try:+ self.dst.execute("ROLLBACK;")+ except Exception:+ pass+ raise+ return copied++ def create_indexes(self) -> List[str]:+ """Replay the source's vector indexes, after the data is in."""+ statements: List[str] = []+ try:+ result = self.src.execute("CALL SHOW_INDEXES() RETURN *;")+ columns = result.get_column_names()+ def_i = columns.index("index_definition")+ name_i = columns.index("index_name")+ table_i = columns.index("table_name")+ while result.has_next():+ row = result.get_next()+ definition = str(row[def_i] or "").strip()+ if (+ definition.upper().startswith("CALL CREATE_VECTOR_INDEX(")+ and str(row[table_i]) in self.node_tables+ ):+ statements.append((str(row[name_i]), definition))+ except Exception as exc:+ logger.warning("[reembed] could not read source indexes: %s", exc)+ names = [name for name, _ in statements]+ if "memory_content_idx" not in names and "Memory" in self.node_tables:+ statements.append(("memory_content_idx", _SDK_VECTOR_INDEX))+ built = []+ for name, statement in statements:+ self.dst.execute(statement if statement.endswith(";") else statement + ";")+ built.append(name)+ return built++ def counts(self) -> Tuple[Dict[str, int], Dict[str, int], Dict[str, int], Dict[str, int]]:+ src_nodes, src_rels = _graph_counts(self.src)+ dst_nodes, dst_rels = _graph_counts(self.dst)+ return src_nodes, src_rels, dst_nodes, dst_rels++ def close(self) -> None:+ _close(self.src, self.src_db)+ _close(self.dst, self.dst_db)+ self.src = self.src_db = self.dst = self.dst_db = None+++def _publish(staging: Path, target: Path, overwrite: bool) -> None:+ if _kuzu_files(target):+ if not overwrite:+ raise FileExistsError(f"graph target exists: {target}")+ _remove_kuzu(target)+ for item in _kuzu_files(staging):+ suffix = item.name[len(staging.name):]+ os.replace(item, target.with_name(target.name + suffix))+++def _staging_path(target: Path) -> Path:+ return target.with_name(target.name + ".reembed-partial")+++async def reembed_graph(+ source_graph_path: Union[str, Path],+ target_graph_path: Union[str, Path],+ *,+ embedding: EmbeddingService,+ target_dims: int,+ batch_size: int = 64,+ cancel: Optional[Callable[[], bool]] = None,+ on_progress: Optional[Callable[[GraphReembedProgress], None]] = None,+ overwrite: bool = False,+) -> GraphReembedReport:+ """Build a new Kuzu graph at ``target_graph_path`` with ``target_dims`` vectors.++ The source is opened read-only and never modified, so it stays the+ rollback copy. The new DB gets the SDK's current DDL at the new width; every+ node and every edge is copied verbatim except the embedding columns, which+ are recomputed with ``embedding``:++ - ``Memory.embedding``: ``custom.retrieval_text``, else ``content``+ - ``Memory.beh_embedding``: ``custom.beh_abstraction`` (NULL when missing)+ - ``Topic.embedding``: ``name``+ - ``HierarchyTopic.embedding``: the label from `hierarchy_topic_label`++ A NULL source vector stays NULL. Indexes are built after the data is in.+ The same path is used whether or not the width changes: Kuzu cannot SET an+ indexed column, so an in-place update is not possible anyway.++ Restart semantics: the copy is built at ``<target>.reembed-partial``, which+ is wiped at every start and on failure or cancel, then moved to the target+ only after node and edge counts match the source. A rerun starts over. An+ existing target raises `FileExistsError` unless ``overwrite=True``.+ """+ source = Path(source_graph_path)+ target = Path(target_graph_path)+ if os.path.realpath(str(source)) == os.path.realpath(str(target)):+ raise ValueError("source and target graph paths are the same")+ if not source.exists():+ raise FileNotFoundError(f"graph not found: {source}")+ if _kuzu_files(target) and not overwrite:+ raise FileExistsError(f"graph target exists: {target}")+ _assert_not_open(source, "source")+ _assert_not_open(target, "target")+ dims = int(target_dims)+ _check_service_dims(embedding, dims)+ cancelled = cancel or (lambda: False)+ page = max(1, int(batch_size))+ started = time.monotonic()+ staging = _staging_path(target)+ await _graph(_remove_kuzu, staging)+ copy = _GraphCopy(source=source, staging=staging, dims=dims)+ embedded: Dict[str, int] = {}+ cleared: Dict[str, int] = {}+ labels_exact = 0+ labels_inexact: List[str] = []+ published = False+ try:+ await _graph(copy.open)+ for table in copy.node_tables:+ embed_cols = [c for c in _GRAPH_EMBEDDING_COLUMNS.get(table, ()) if c in copy.columns[table]]+ limit = page if embed_cols else _GRAPH_PAGE+ total = await _graph(copy.node_total, table)+ done = computed = 0+ after = None+ while done < total:+ if cancelled():+ raise ReembedCancelled("graph reembed cancelled")+ records = await _graph(copy.read_nodes, table, after, limit)+ if not records:+ break+ after = records[-1][copy.pk[table]]+ for column in embed_cols:+ wanted, texts = [], []+ for record in records:+ record[column] = None+ if not record["__has"].get(column):+ continue+ if table == "Memory":+ text = _memory_text(record, column)+ elif table == "Topic":+ text = str(record.get("name") or "")+ else:+ text, exact = hierarchy_topic_label(+ str(record.get("subject") or ""),+ record.get("path_json"),+ str(record.get("name") or ""),+ )+ if exact:+ labels_exact += 1+ else:+ labels_inexact.append(str(record.get(copy.pk[table])))+ if not text.strip():+ key = f"{table}.{column}"+ cleared[key] = cleared.get(key, 0) + 1+ continue+ wanted.append(record)+ texts.append(text)+ vectors = _validate_vectors(+ await _embed(embedding, texts), len(texts), dims, f"graph:{table}.{column}",+ )+ for record, vector in zip(wanted, vectors):+ record[column] = vector+ if texts:+ key = f"{table}.{column}"+ embedded[key] = embedded.get(key, 0) + len(texts)+ computed += len(texts)+ await _graph(copy.write_nodes, table, records)+ done += len(records)+ if on_progress:+ on_progress(GraphReembedProgress(table=table, done=done, total=total, embedded=computed))+ for table in copy.rel_tables:+ if cancelled():+ raise ReembedCancelled("graph reembed cancelled")+ copied = await _graph(copy.copy_rels, table)+ if on_progress:+ on_progress(GraphReembedProgress(table=table, done=copied, total=copied, embedded=0))+ indexes = await _graph(copy.create_indexes)+ src_nodes, src_rels, dst_nodes, dst_rels = await _graph(copy.counts)+ mismatch = [+ f"{name}: {src_nodes[name]} != {dst_nodes.get(name, 0)}"+ for name in copy.node_tables if src_nodes.get(name, 0) != dst_nodes.get(name, 0)+ ] + [+ f"{name}: {src_rels[name]} != {dst_rels.get(name, 0)}"+ for name in copy.rel_tables if src_rels.get(name, 0) != dst_rels.get(name, 0)+ ]+ if mismatch:+ raise RuntimeError(f"graph copy count mismatch: {mismatch}")+ await _graph(copy.close)+ await _graph(_publish, staging, target, overwrite)+ published = True+ finally:+ await _graph(copy.close)+ if not published:+ await _graph(_remove_kuzu, staging)+ if labels_inexact:+ logger.warning(+ "[reembed] %d HierarchyTopic labels rebuilt from a fallback text (legacy path_json)",+ len(labels_inexact),+ )+ return GraphReembedReport(+ target_path=str(target),+ source_width=copy.source_width,+ target_width=dims,+ node_counts={name: dst_nodes.get(name, 0) for name in copy.node_tables},+ rel_counts={name: dst_rels.get(name, 0) for name in copy.rel_tables},+ embedded=embedded,+ cleared=cleared,+ hierarchy_labels_exact=labels_exact,+ hierarchy_labels_inexact=tuple(labels_inexact),+ skipped_tables=tuple(copy.skipped),+ dropped_columns=tuple(copy.dropped),+ vector_indexes=tuple(indexes),+ elapsed_s=time.monotonic() - started,+ )+++def _verify_graph_sync(source: Path, target: Path, dims: Optional[int]) -> GraphVerification:+ src_db, src = _open_readonly(source)+ try:+ dst_db, dst = _open_readonly(target)+ try:+ src_nodes, src_rels = _graph_counts(src)+ dst_nodes, dst_rels = _graph_counts(dst)+ width = _source_width(dst) if "Memory" in dst_nodes else None+ try:+ index_names = [str(row[1]) for row in _rows(dst, "CALL SHOW_INDEXES() RETURN *;")]+ except Exception:+ index_names = []+ problems: List[str] = []+ node_counts = {}+ for name in sorted(set(src_nodes) | set(dst_nodes)):+ pair = (src_nodes.get(name, 0), dst_nodes.get(name, 0))+ node_counts[name] = pair+ if pair[0] != pair[1]:+ problems.append(f"node table {name}: source {pair[0]} != target {pair[1]}")+ rel_counts = {}+ for name in sorted(set(src_rels) | set(dst_rels)):+ pair = (src_rels.get(name, 0), dst_rels.get(name, 0))+ rel_counts[name] = pair+ if pair[0] != pair[1]:+ problems.append(f"rel table {name}: source {pair[0]} != target {pair[1]}")+ if dims is not None and width != int(dims):+ problems.append(f"Memory.embedding width {width} != {dims}")+ has_index = "memory_content_idx" in index_names+ if "Memory" in dst_nodes and not has_index:+ problems.append("vector index memory_content_idx missing")+ for table, column in (("Memory", "embedding"), ("Topic", "embedding"), ("HierarchyTopic", "embedding")):+ if table not in src_nodes or table not in dst_nodes:+ continue+ query = f"MATCH (n:{table}) WHERE n.{column} IS NOT NULL RETURN count(n);"+ before, after = _count(src, query), _count(dst, query)+ if before != after:+ problems.append(f"{table}.{column}: {before} vectors in source, {after} in target")+ return GraphVerification(+ ok=not problems,+ node_counts=node_counts,+ rel_counts=rel_counts,+ target_width=width,+ vector_index=has_index,+ problems=tuple(problems),+ )+ finally:+ _close(dst, dst_db)+ finally:+ _close(src, src_db)+++async def verify_graph_reembed(+ source_graph_path: Union[str, Path],+ target_graph_path: Union[str, Path],+ *,+ target_dims: Optional[int] = None,+) -> GraphVerification:+ """Node and edge counts per table, target width, vector index, vector counts."""+ source, target = Path(source_graph_path), Path(target_graph_path)+ _assert_not_open(source, "source")+ _assert_not_open(target, "target")+ return await _graph(_verify_graph_sync, source, target, target_dims)+++async def drop_graph(graph_path: Union[str, Path]) -> bool:+ """Delete a Kuzu graph (and its sidecar files). Returns False if absent.++ Refuses when a live store in this process holds it open.+ """+ path = Path(graph_path)+ _assert_not_open(path, "the")+ if not _kuzu_files(path):+ return False+ await _graph(_remove_kuzu, path)+ return True+++__all__ = [+ "VECTOR_ROLES",+ "VectorRole",+ "VectorSpace",+ "EmbeddingSpaceIdentity",+ "space_identity",+ "vector_space_names",+ "reembed_text",+ "ReembedCancelled",+ "ReembedCollectionPlan",+ "ReembedPlan",+ "plan_reembed",+ "ReembedProgress",+ "ReembedReport",+ "reembed_vector_space",+ "ScannedCollection",+ "ScannedSpace",+ "scan_vector_spaces",+ "parse_collection_name",+ "SpaceDiff",+ "diff_spaces",+ "ReembedVerification",+ "verify_reembed",+ "drop_vector_space",+ "GraphReembedPlan",+ "plan_graph_reembed",+ "GraphReembedProgress",+ "GraphReembedReport",+ "reembed_graph",+ "hierarchy_topic_label",+ "GraphVerification",+ "verify_graph_reembed",+ "drop_graph",+]diff --git a/tests/test_reembed.py b/tests/test_reembed.pynew file mode 100644index 0000000..8fd77a8--- /dev/null+++ b/tests/test_reembed.py@@ -0,0 +1,642 @@+"""Re-embed into a new embedding space: vector stores, scan, diff and merge.++The switch model: writes are paused, every vector is computed again with the+new model into a new space, the source is never touched. The graph half lives+in ``test_reembed_graph.py``.+"""++import hashlib+import json+import math+import random+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from hy_memory import (+ EmbeddingDimensionMismatchError,+ MemoryConfig,+ ReembedCancelled,+ VectorSpace,+ diff_spaces,+ drop_vector_space,+ plan_reembed,+ reembed_text,+ reembed_vector_space,+ scan_vector_spaces,+ space_identity,+ vector_space_names,+ verify_reembed,+)+from hy_memory.config import EmbedderConfig+from hy_memory.core.embed_service import EmbedService+from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus+from hy_memory.reembed import VECTOR_ROLES, parse_collection_name+++class FakeEmbedder:+ """Deterministic vectors: same (model, text) -> same unit vector."""++ def __init__(self, model: str, dims: int, *, wrong_width_after: int = -1):+ self.model = model+ self.dims = dims+ self.calls = []+ self.texts = 0+ self.wrong_width_after = wrong_width_after++ def vector(self, text: str):+ seed = int(hashlib.sha256(f"{self.model}\0{text}".encode()).hexdigest()[:16], 16)+ rng = random.Random(seed)+ values = [rng.gauss(0.0, 1.0) for _ in range(self.dims)]+ norm = math.sqrt(sum(v * v for v in values)) or 1.0+ return [v / norm for v in values]++ async def embed_batch(self, texts, use_cache=True):+ self.calls.append(list(texts))+ out = []+ for text in texts:+ self.texts += 1+ vector = self.vector(text)+ if 0 <= self.wrong_width_after < self.texts:+ vector = vector + [0.0]+ out.append(vector)+ return out++ async def embed(self, text, use_cache=True):+ return self.vector(text)++ async def embed_queued(self, text, use_cache=True):+ return self.vector(text)++ def get_embedding_dims(self):+ return self.dims+++def _client(path):+ import chromadb+ return chromadb.PersistentClient(path=str(path))+++def _space(path, base, dims, model=None):+ return VectorSpace(base_name=base, dims=dims, persist_directory=str(path), model=model)+++def _custom_state():+ return {+ "memory_kind": "canonical_state",+ "state_key": "home_city",+ "state_value": "Berlin",+ "state_retrieval_text": "user lives in Berlin (home city)",+ }+++async def _seed(path, base, dims, *, model="old-model", users=("alice", "bob"), per_user=7):+ """Write a realistic source space through the SDK's own stores."""+ from hy_memory.data.vector_store_chroma import ChromaVectorStore++ old = FakeEmbedder(model, dims)+ cfg = MemoryConfig()+ cfg.vector_store.provider = "chroma"+ cfg.vector_store.persist_directory = str(path)+ cfg.vector_store.collection_name = base+ cfg.vector_store.embedding_dims = dims+ store = ChromaVectorStore(cfg)+ await store.initialize()+ nodes = []+ for user in users:+ for i in range(per_user):+ layer = [MemoryLayer.L1_RAW, MemoryLayer.L2_FACT, MemoryLayer.L3_SUMMARY][i % 3]+ node = MemoryNode(+ user_id=user, agent_id="a", session_id=f"s{i % 2}",+ layer=layer, content=f"{user} fact {i}",+ status=MemoryStatus.SUPERSEDED if i == 4 else MemoryStatus.ACTIVE,+ tags=["t1", f"t{i}"],+ custom={"k": i, "nested": {"x": [1, 2]}},+ )+ if i == 4:+ node.is_latest = False+ node.superseded_by = ["later"]+ if i == 5:+ node.custom = _custom_state()+ node.embedding = old.vector(node.content)+ nodes.append(node)+ await store.upsert_batch(nodes)+ entities = await store._get_entity_collection()+ entities.add(+ ids=["e1", "e2"],+ embeddings=[old.vector("Berlin"), old.vector("Dylan")],+ documents=["Berlin", "Dylan"],+ metadatas=[{"user_id": "alice", "label": "GPE"}, {"user_id": "bob", "label": "PERSON"}],+ )+ await store.close()+ client = _client(path)+ names = vector_space_names(_space(path, base, dims))+ for role, doc in (("coding_keys", "fix the flaky test"), ("working_facts_cv", "repo: uses uv")):+ coll = client.get_or_create_collection(names[role], metadata={"hnsw:space": "cosine"})+ coll.add(+ ids=[f"{role}:0"],+ embeddings=[old.vector(doc)],+ documents=[doc],+ metadatas=[{"user_id": "alice", "layer": "l1_raw",+ "custom": json.dumps({"coding_key_text": doc})}],+ )+ return names+++def _snapshot(path, names):+ client = _client(path)+ out = {}+ for role, name in names.items():+ try:+ coll = client.get_collection(name)+ except Exception:+ continue+ got = coll.get(include=["documents", "metadatas", "embeddings"])+ out[role] = sorted(+ (rid, doc, json.dumps(meta, sort_keys=True), [round(float(x), 6) for x in emb])+ for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])+ )+ return hashlib.sha256(json.dumps(out, sort_keys=True).encode()).hexdigest()+++def _records(path, name):+ got = _client(path).get_collection(name).get(include=["documents", "metadatas", "embeddings"])+ return {+ rid: (doc, meta, list(emb))+ for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])+ }+++# ----------------------------------------------------------------------+# naming+# ----------------------------------------------------------------------++class TestVectorSpaceNames:+ @pytest.mark.parametrize("dims", [512, None])+ async def test_names_equal_what_the_stores_open(self, tmp_path, dims):+ from hy_memory.coding.store import CodingMemoryStore+ from hy_memory.data.vector_store_chroma import ChromaVectorStore++ cfg = MemoryConfig()+ cfg.vector_store.provider = "chroma"+ cfg.vector_store.persist_directory = str(tmp_path / "v")+ cfg.vector_store.collection_name = "memhub_abc123"+ cfg.vector_store.embedding_dims = dims+ cfg.history.db_path = str(tmp_path / "history.sqlite3")+ store = ChromaVectorStore(cfg)+ await store.initialize()+ await store._get_entity_collection()+ coding = []+ for variant in ("working", "coding"):+ cs = CodingMemoryStore(cfg, FakeEmbedder("m", dims or 8), variant=variant)+ await cs.initialize()+ coding.append(cs)+ opened = {c.name for c in _client(tmp_path / "v").list_collections()}+ names = vector_space_names(cfg)+ assert set(names) == set(VECTOR_ROLES)+ assert set(names.values()) == opened+ suffix = f"_{dims}" if dims else ""+ assert names["memory"] == f"memhub_abc123{suffix}"+ assert names["coding_keys_cv"] == f"memhub_abc123_coding_keys_cv{suffix}"+ await store.close()+ for cs in coding:+ await cs.close()++ def test_explicit_base_name_and_space(self, tmp_path):+ cfg = MemoryConfig()+ cfg.vector_store.provider = "chroma"+ cfg.vector_store.embedding_dims = 1024+ names = vector_space_names(cfg, base_name="staging_x")+ assert names["memory"] == "staging_x_1024"+ assert names["entities"] == "staging_x_1024_entities"+ assert names["working_facts"] == "staging_x_working_facts_1024"+ assert vector_space_names(_space(tmp_path, "b", 512))["memory"] == "b_512"++ def test_parse_collection_name_round_trips(self):+ for dims in (512, None):+ for role, name in vector_space_names(VectorSpace("memhub_9f", dims)).items():+ assert parse_collection_name(name) == ("memhub_9f", dims, role)+ assert parse_collection_name("memhub_9f_512_tag_index") is None++ def test_space_identity_is_model_and_width_only(self):+ a = space_identity("BAAI/bge-small-zh-v1.5", 512)+ b = space_identity(" BAAI/bge-small-zh-v1.5 ", 512)+ assert a == b and a.key == "BAAI/bge-small-zh-v1.5#512"+ assert a.digest != space_identity("BAAI/bge-small-zh-v1.5", 1024).digest+ with pytest.raises(ValueError):+ space_identity("", 512)++ def test_non_chroma_provider_is_not_supported(self):+ cfg = MemoryConfig()+ cfg.vector_store.provider = "qdrant"+ with pytest.raises(NotImplementedError):+ vector_space_names(cfg)+++class TestReembedText:+ def test_canonical_state_uses_state_retrieval_text(self):+ meta = {"content": "住在柏林", "custom": json.dumps(_custom_state())}+ assert reembed_text("memory", "住在柏林", meta) == "user lives in Berlin (home city)"++ def test_retrieval_text_when_present(self):+ meta = {"custom": json.dumps({"retrieval_text": " pattern: plans ahead "})}+ assert reembed_text("memory", "画像", meta) == "pattern: plans ahead"++ def test_document_otherwise_with_content_fallback(self):+ assert reembed_text("memory", "plain fact", {"custom": "{}"}) == "plain fact"+ assert reembed_text("memory", None, {"content": "from metadata"}) == "from metadata"+ assert reembed_text("memory", "doc", {"custom": "not json"}) == "doc"++ def test_coding_key_and_working_fact_use_the_document(self):+ coding = {"custom": json.dumps({"coding_key_text": "fix flaky test",+ "retrieval_text": "ignored outside memory"})}+ assert reembed_text("coding_keys", "fix flaky test", coding) == "fix flaky test"+ assert reembed_text("working_facts", "repo: uses uv", {}) == "repo: uses uv"+++# ----------------------------------------------------------------------+# re-embed+# ----------------------------------------------------------------------++ "src_dims,tgt_dims,src_model,tgt_model",+ [(512, 512, "old-model", "new-model"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],+)+async def test_reembed_copies_everything_and_never_touches_source(+ tmp_path, src_dims, tgt_dims, src_model, tgt_model,+):+ path = tmp_path / "v"+ names = await _seed(path, "memhub_src", src_dims, model=src_model)+ before = _snapshot(path, names)+ source = _space(path, "memhub_src", src_dims)+ target = _space(path, "memhub_dst", tgt_dims, model=tgt_model)+ new = FakeEmbedder(tgt_model, tgt_dims)++ plan = await plan_reembed(source, target)+ assert plan.per_role["memory"] == 14 and plan.per_role["entities"] == 2+ assert plan.total_vectors == 18 == plan.estimated_vectors+ assert plan.source_dims == src_dims and plan.target_dims == tgt_dims++ seen = []+ report = await reembed_vector_space(+ source, target, embedding=new, batch_size=4, on_progress=seen.append,+ )+ assert report.embedded == 18 and report.reused == 0+ assert seen and seen[-1].done == seen[-1].total++ assert _snapshot(path, names) == before # source unchanged, byte for byte+ target_names = vector_space_names(target)+ for role in ("memory", "entities", "coding_keys", "working_facts_cv"):+ src = _records(path, names[role])+ dst = _records(path, target_names[role])+ assert set(src) == set(dst)+ for rid, (doc, meta, _) in src.items():+ assert dst[rid][0] == doc+ assert dst[rid][1] == meta # status, is_latest, superseded_by, timestamps, custom+ assert len(dst[rid][2]) == tgt_dims+ assert dst[rid][2] == pytest.approx(new.vector(reembed_text(role, doc, meta)), abs=1e-5)+ memory = _records(path, target_names["memory"])+ layers = {meta["layer"] for _, meta, _ in memory.values()}+ users = {meta["user_id"] for _, meta, _ in memory.values()}+ assert "l1_raw" in layers and users == {"alice", "bob"}+ statuses = {meta["status"] for _, meta, _ in memory.values()}+ assert statuses == {"active", "superseded"}+ state = [r for r in memory.values() if "canonical_state" in r[1]["custom"]]+ assert state and state[0][2] == pytest.approx(+ new.vector("user lives in Berlin (home city)"), abs=1e-5,+ )+ meta = _client(path).get_collection(target_names["memory"]).metadata+ assert meta["hnsw:space"] == "cosine" and meta["hy_embedding_model"] == tgt_model++ verified = await verify_reembed(source, target, embedding=new)+ assert verified.ok, verified.problems+ assert verified.per_user == {"alice": 10, "bob": 8}+ assert verified.self_recall_samples == 18 and verified.self_recall_hits == 18+++async def test_resume_after_kill_only_embeds_the_rest(tmp_path):+ path = tmp_path / "v"+ await _seed(path, "src", 512, per_user=20)+ source, target = _space(path, "src", 512), _space(path, "dst", 1024)++ class Killed(Exception):+ pass++ first = FakeEmbedder("new", 1024)+ batches = {"n": 0}++ def _kill():+ batches["n"] += 1+ if batches["n"] > 3:+ raise Killed()+ return False++ with pytest.raises(Killed):+ await reembed_vector_space(+ source, target, embedding=first, batch_size=5, max_inflight=1,+ roles=["memory"], cancel=_kill,+ )+ assert first.texts == 15+ second = FakeEmbedder("new", 1024)+ report = await reembed_vector_space(+ source, target, embedding=second, batch_size=5, max_inflight=1, roles=["memory"],+ )+ assert second.texts == 40 - 15+ assert report.per_role["memory"].reused == 15+ # The end state equals an uninterrupted run: every source record, verbatim,+ # with the vector of its own text.+ resumed = _records(path, "dst_1024")+ src = _records(path, "src_512")+ assert set(resumed) == set(src)+ for rid, (doc, meta, vector) in resumed.items():+ assert (doc, meta) == src[rid][:2]+ assert vector == pytest.approx(second.vector(reembed_text("memory", doc, meta)), abs=1e-5)+ assert (await verify_reembed(source, target, roles=["memory"], samples=0)).ok+++async def test_cancel_at_batch_boundary_leaves_no_partial_batch(tmp_path):+ path = tmp_path / "v"+ await _seed(path, "src", 512)+ calls = {"n": 0}++ def _cancel():+ calls["n"] += 1+ return calls["n"] > 2++ with pytest.raises(ReembedCancelled) as info:+ await reembed_vector_space(+ _space(path, "src", 512), _space(path, "dst", 512),+ embedding=FakeEmbedder("new", 512), batch_size=3, max_inflight=1,+ roles=["memory"], cancel=_cancel,+ )+ assert info.value.report.per_role["memory"].embedded == 6+ assert _client(path).get_collection("dst_512").count() == 6+++async def test_wrong_width_raises_before_any_upsert(tmp_path):+ path = tmp_path / "v"+ await _seed(path, "src", 512)+ bad = FakeEmbedder("new", 1024, wrong_width_after=2)+ with pytest.raises(EmbeddingDimensionMismatchError):+ await reembed_vector_space(+ _space(path, "src", 512), _space(path, "dst", 1024),+ embedding=bad, batch_size=4, roles=["memory"],+ )+ assert _client(path).get_collection("dst_1024").count() == 0+ declared_wrong = FakeEmbedder("new", 768)+ with pytest.raises(EmbeddingDimensionMismatchError):+ await reembed_vector_space(+ _space(path, "src", 512), _space(path, "dst", 1024), embedding=declared_wrong,+ )+++async def test_refuses_to_write_into_the_source_collection(tmp_path):+ path = tmp_path / "v"+ await _seed(path, "src", 512)+ with pytest.raises(ValueError):+ await reembed_vector_space(+ _space(path, "src", 512), _space(path, "src", 512),+ embedding=FakeEmbedder("new", 512),+ )+++async def test_verify_detects_missing_extra_drift_and_width(tmp_path):+ path = tmp_path / "v"+ await _seed(path, "src", 512)+ source, target = _space(path, "src", 512), _space(path, "dst", 512)+ new = FakeEmbedder("new", 512)+ await reembed_vector_space(source, target, embedding=new, roles=["memory"])+ coll = _client(path).get_collection("dst_512")+ got = coll.get(include=["documents", "metadatas"])+ victim, drifted = got["ids"][0], got["ids"][1]+ coll.delete(ids=[victim])+ meta = dict(got["metadatas"][1])+ meta["status"] = "archived"+ coll.update(ids=[drifted], metadatas=[meta])+ coll.add(ids=["stranger"], embeddings=[new.vector("x")], documents=["x"], metadatas=[{"user_id": "eve"}])+ result = await verify_reembed(source, target, embedding=new, roles=["memory"], samples=5)+ assert not result.ok+ counts = result.per_role["memory"]+ assert (counts["missing"], counts["extra"], counts["mismatched"]) == (1, 1, 1)+ assert any("eve" in p for p in result.problems)++ wide = _space(path, "dst", 1024)+ wide_coll = _client(path).get_or_create_collection("dst_1024", metadata={"hnsw:space": "cosine"})+ src = _records(path, "src_512")+ wide_coll.add(+ ids=list(src),+ embeddings=[[0.0] * 512 for _ in src],+ documents=[v[0] for v in src.values()],+ metadatas=[v[1] for v in src.values()],+ )+ result = await verify_reembed(source, wide, roles=["memory"], samples=0)+ assert result.per_role["memory"]["bad_dims"] == len(src) and not result.ok+++async def test_drop_vector_space_refuses_live_collections(tmp_path):+ from hy_memory.data.vector_store_chroma import ChromaVectorStore++ path = tmp_path / "v"+ await _seed(path, "src", 512)+ cfg = MemoryConfig()+ cfg.vector_store.provider = "chroma"+ cfg.vector_store.persist_directory = str(path)+ cfg.vector_store.collection_name = "src"+ cfg.vector_store.embedding_dims = 512+ live = ChromaVectorStore(cfg)+ await live.initialize()+ with pytest.raises(RuntimeError):+ await drop_vector_space(_space(path, "src", 512))+ await live.close()+ dropped = await drop_vector_space(_space(path, "src", 512))+ assert set(dropped) == {"src_512", "src_512_entities", "src_coding_keys_512", "src_working_facts_cv_512"}+ assert not {c.name for c in _client(path).list_collections()}+++# ----------------------------------------------------------------------+# scan, diff, merge+# ----------------------------------------------------------------------++async def _two_spaces(path):+ """A live 1024 space and a stray 512 space sharing some ids."""+ from hy_memory.data.vector_store_chroma import ChromaVectorStore++ async def _store(base, dims, model, contents):+ cfg = MemoryConfig()+ cfg.vector_store.provider = "chroma"+ cfg.vector_store.persist_directory = str(path)+ cfg.vector_store.collection_name = base+ cfg.vector_store.embedding_dims = dims+ store = ChromaVectorStore(cfg)+ await store.initialize()+ emb = FakeEmbedder(model, dims)+ nodes = []+ for node_id, user, content, status in contents:+ node = MemoryNode(+ node_id=node_id, user_id=user, agent_id="a", session_id="s",+ layer=MemoryLayer.L2_FACT, content=content, status=status,+ )+ node.embedding = emb.vector(content)+ nodes.append(node)+ await store.upsert_batch(nodes)+ await store.close()++ ids = [f"00000000-0000-0000-0000-00000000000{i}" for i in range(6)]+ await _store("memhub_old", 512, "bge", [+ (ids[0], "alice", "shared one", MemoryStatus.ACTIVE),+ (ids[1], "alice", "shared two (stray wording)", MemoryStatus.ACTIVE),+ (ids[2], "bob", "only in stray", MemoryStatus.ACTIVE),+ (ids[3], "bob", "only in stray, deleted", MemoryStatus.SHADOW),+ ])+ await _store("memhub_new", 1024, "e5", [+ (ids[0], "alice", "shared one", MemoryStatus.ACTIVE),+ (ids[1], "alice", "shared two (live wording)", MemoryStatus.ARCHIVED),+ (ids[4], "carol", "only in live", MemoryStatus.ACTIVE),+ ])+ return ids+++async def test_scan_diff_and_merge(tmp_path):+ path = tmp_path / "v"+ ids = await _two_spaces(path)+ spaces = await scan_vector_spaces(path)+ by_base = {(s.base_name, s.dims): s for s in spaces}+ assert set(by_base) == {("memhub_old", 512), ("memhub_new", 1024)}+ old, live = by_base[("memhub_old", 512)], by_base[("memhub_new", 1024)]+ assert old.roles["memory"].count == 4 and old.width == 512+ assert live.roles["memory"].count == 3 and live.width == 1024+ assert old.per_user == {"alice": 2, "bob": 2}+ assert live.per_user == {"alice": 2, "carol": 1}++ stray, target = old.space(), _space(path, "memhub_new", 1024, model="e5")+ diff = await diff_spaces(stray, target)+ assert set(diff.missing["memory"]) == {ids[2], ids[3]} # status ignored+ assert diff.total_missing == 2++ live_before = _records(path, "memhub_new_1024")+ stray_before = _snapshot(path, stray.names())+ merger = FakeEmbedder("e5", 1024)+ report = await reembed_vector_space(stray, target, embedding=merger)+ assert merger.texts == 2 and report.embedded == 2+ assert report.reused == 1 and report.kept == 1+ after = _records(path, "memhub_new_1024")+ assert set(after) == set(live_before) | {ids[2], ids[3]}+ for rid, record in live_before.items(): # the target wins+ assert after[rid][:2] == record[:2]+ assert after[rid][2] == pytest.approx(record[2], abs=1e-6)+ assert _snapshot(path, stray.names()) == stray_before+ assert (await diff_spaces(stray, target)).empty++ again = FakeEmbedder("e5", 1024)+ second = await reembed_vector_space(stray, target, embedding=again)+ assert again.texts == 0 and second.embedded == 0+ assert _records(path, "memhub_new_1024").keys() == after.keys()+ rescanned = {(s.base_name, s.dims): s for s in await scan_vector_spaces(path)}+ assert rescanned[("memhub_new", 1024)].roles["memory"].count == 5+++async def test_scan_of_missing_directory_is_empty(tmp_path):+ assert await scan_vector_spaces(tmp_path / "nope") == []+++# ----------------------------------------------------------------------+# EmbedService fixes+# ----------------------------------------------------------------------++def _fake_raw(n, dims):+ items = []+ for i in range(n):+ item = MagicMock()+ item.embedding = [0.1] * dims+ item.index = i+ items.append(item)+ resp = MagicMock()+ resp.model = "m"+ resp.usage = None+ resp.data = items+ raw = MagicMock()+ raw.status_code = 200+ raw.headers = {}+ raw.parse.return_value = resp+ return raw+++def _service(dims=8, **embedder):+ cfg = MemoryConfig(embedder=EmbedderConfig(+ provider="openai", model="m", api_key="sk", base_url="https://e.invalid/v1",+ embedding_dims=dims, max_retries=2, retry_delay=0.0, **embedder,+ ))+ return EmbedService(cfg)+++async def test_embed_batch_chunks_to_batch_size():+ svc = _service()+ sizes = []++ async def _create(**kwargs):+ sizes.append(len(kwargs["input"]))+ return _fake_raw(len(kwargs["input"]), 8)++ client = MagicMock()+ client.embeddings.with_raw_response.create = AsyncMock(side_effect=_create)+ with patch("openai.AsyncOpenAI", return_value=client):+ vectors = await svc.embed_batch([f"t{i}" for i in range(1000)], use_cache=False)+ assert len(vectors) == 1000+ assert len(sizes) == math.ceil(1000 / 64) and max(sizes) == 64++ small = _service(batch_size=10)+ sizes.clear()+ with patch("openai.AsyncOpenAI", return_value=client):+ _, metas = await small.embed_batch_with_meta([f"u{i}" for i in range(25)], use_cache=False)+ assert sizes == [10, 10, 5]+ assert [m["batch_index"] for m in metas] == list(range(25))+++async def test_429_honours_retry_after():+ svc = _service()++ class RateLimited(Exception):+ status_code = 429++ def __init__(self):+ super().__init__("rate limited")+ self.response = MagicMock()+ self.response.headers = {"retry-after": "2"}++ client = MagicMock()+ client.embeddings.with_raw_response.create = AsyncMock(+ side_effect=[RateLimited(), _fake_raw(1, 8)],+ )+ sleeps = []++ async def _sleep(seconds):+ sleeps.append(seconds)++ with patch("openai.AsyncOpenAI", return_value=client), \+ patch("hy_memory.core.embed_service.asyncio.sleep", _sleep):+ vectors = await svc.embed_batch(["x"], use_cache=False)+ assert len(vectors) == 1+ assert len(sleeps) == 1 and 2.0 <= sleeps[0] <= 2.5+++async def test_per_call_max_retries():+ svc = _service()++ class Down(Exception):+ status_code = 503++ client = MagicMock()+ client.embeddings.with_raw_response.create = AsyncMock(side_effect=Down("down"))+ with patch("openai.AsyncOpenAI", return_value=client), pytest.raises(Down):+ await svc.embed_batch(["x"], use_cache=False, max_retries=5)+ assert client.embeddings.with_raw_response.create.await_count == 6+++async def test_returned_width_is_validated_and_not_retried():+ svc = _service(dims=8)+ client = MagicMock()+ client.embeddings.with_raw_response.create = AsyncMock(return_value=_fake_raw(2, 7))+ with patch("openai.AsyncOpenAI", return_value=client), \+ pytest.raises(EmbeddingDimensionMismatchError):+ await svc.embed_batch(["a", "b"], use_cache=False)+ assert client.embeddings.with_raw_response.create.await_count == 1diff --git a/tests/test_reembed_graph.py b/tests/test_reembed_graph.pynew file mode 100644index 0000000..34f7549--- /dev/null+++ b/tests/test_reembed_graph.py@@ -0,0 +1,252 @@+"""Re-embed the Kuzu graph into a new embedding space.++The source graph is opened read-only and never modified. A new graph is built+at the target path with the target width: every node and edge is copied, the+embedding columns are recomputed from their text, the HNSW index is rebuilt.+Runs in CI: kuzu is a core dependency.+"""++import hashlib+import json+import os++import pytest++from hy_memory import (+ EmbeddingDimensionMismatchError,+ MemoryConfig,+ ReembedCancelled,+ drop_graph,+ reembed_graph,+ verify_graph_reembed,+)+from hy_memory.models.memory import MemoryLayer, MemoryNode+from hy_memory.reembed import hierarchy_topic_label, plan_graph_reembed++from .test_reembed import FakeEmbedder+++def _cfg(path, dims):+ cfg = MemoryConfig()+ cfg.graph_store.provider = "kuzu"+ cfg.graph_store.db_path = str(path)+ cfg.vector_store.embedding_dims = dims+ return cfg+++async def _open(path, dims):+ from hy_memory.data.graph_store_kuzu import KuzuGraphStore++ store = KuzuGraphStore(_cfg(path, dims))+ await store.initialize()+ assert store._available+ return store+++def _node(content, *, layer=MemoryLayer.L2_FACT, custom=None, user="alice"):+ return MemoryNode(+ user_id=user, agent_id="a", session_id="s1",+ layer=layer, content=content, custom=custom or {},+ )+++async def _seed_graph(path, dims, model="old"):+ """A graph with every node and edge type the SDK writes."""+ old = FakeEmbedder(model, dims)+ gs = await _open(path, dims)+ schema = _node(+ "画像:喜欢提前规划", layer=MemoryLayer.L6_SCHEMA,+ custom={"schema_type": "basic", "retrieval_text": "plans trips weeks ahead",+ "_has_beh": True, "beh_abstraction": "prepares early under uncertainty"},+ )+ schema._graph_embedding = old.vector("plans trips weeks ahead")+ core = _node("core: anticipates", layer=MemoryLayer.L6_SCHEMA, custom={"schema_type": "abstract"})+ core._graph_embedding = old.vector("core: anticipates")+ stale_beh = _node("beh without text", layer=MemoryLayer.L6_SCHEMA, custom={})+ stale_beh._graph_embedding = old.vector("beh without text")+ facts = [_node(f"fact {i}", user="bob" if i % 2 else "alice") for i in range(5)]+ for node in [schema, core, stale_beh, *facts]:+ await gs.upsert_memory_node(node)+ await gs.update_embedding(schema.node_id, beh_embedding=old.vector("prepares early under uncertainty"))+ await gs.update_embedding(stale_beh.node_id, beh_embedding=old.vector("lost text"))+ await gs.add_cross_abstracts_to(schema.node_id, core.node_id)+ ik = facts[0].get_isolation_key()+ await gs.add_edge(facts[0].node_id, facts[1].node_id, "RELATED_TO", {"relation_type": "co_occurs", "weight": 0.7})+ await gs.add_temporal_edge(facts[0].node_id, facts[2].node_id, topic_id="t")+ for fact in facts[:3]:+ await gs.add_topic_tag(fact.node_id, "travel", ik, embed_service=old)+ await gs.add_topic_tag(facts[3].node_id, "work", ik) # no embed service: NULL embedding+ await gs.add_topic_path(+ facts[0].node_id, ["Travel", "Japan trip"], ik, "alice",+ node_types=["topic", "series"], embed_service=old,+ )+ await gs.add_topic_path(facts[1].node_id, ["Work", "Standups"], ik, "alice", embed_service=old)+ await gs.add_entity_mention(facts[0].node_id, "Kyoto", "GPE", ik)+ await gs.ensure_vdbref("raw-1", "l1_raw")+ await gs.add_derived_from(schema.node_id, "raw-1")+ await gs.close()+ return old, {"schema": schema, "core": core, "stale_beh": stale_beh, "facts": facts}+++def _digest(path):+ """Hash of the Kuzu DB file and any sidecar files next to it."""+ h = hashlib.sha256()+ for name in sorted(os.listdir(path.parent)):+ if name == path.name or name.startswith(path.name + "."):+ h.update(name.encode())+ with open(path.parent / name, "rb") as handle:+ h.update(handle.read())+ return h.hexdigest()+++def _read(path, query):+ import kuzu++ db = kuzu.Database(str(path), read_only=True)+ conn = kuzu.Connection(db)+ try:+ result = conn.execute(query)+ rows = []+ while result.has_next():+ rows.append(result.get_next())+ return rows+ finally:+ conn.close()+ db.close()+++ "src_dims,tgt_dims,src_model,tgt_model",+ [(512, 512, "old", "new"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],+)+async def test_reembed_graph_copies_everything_at_the_new_width(+ tmp_path, src_dims, tgt_dims, src_model, tgt_model,+):+ source, target = tmp_path / "graph", tmp_path / "graph-new"+ _, nodes = await _seed_graph(source, src_dims, model=src_model)+ before = _digest(source)+ plan = await plan_graph_reembed(source, target_dims=tgt_dims)+ assert plan.source_width == src_dims and plan.target_width == tgt_dims+ assert plan.memory_nodes == 8 and plan.topics == 2 and plan.hierarchy_topics == 4+ assert plan.memory_embeddings == 3 and plan.beh_embeddings == 2+ assert plan.topic_embeddings == 1 and plan.hierarchy_topic_embeddings == 4++ new = FakeEmbedder(tgt_model, tgt_dims)+ seen = []+ report = await reembed_graph(+ source, target, embedding=new, target_dims=tgt_dims, batch_size=3, on_progress=seen.append,+ )+ assert _digest(source) == before # the source graph is untouched+ assert not os.path.exists(str(target) + ".reembed-partial")+ assert report.source_width == src_dims and report.target_width == tgt_dims+ assert report.node_counts == plan.node_counts+ assert report.rel_counts == plan.rel_counts+ assert report.embedded == {+ "Memory.embedding": 3, "Memory.beh_embedding": 1,+ "Topic.embedding": 1, "HierarchyTopic.embedding": 4,+ }+ assert report.cleared == {"Memory.beh_embedding": 1} # beh vector with no beh_abstraction+ assert report.hierarchy_labels_exact == 4 and report.hierarchy_labels_inexact == ()+ assert report.vector_indexes == ("memory_content_idx",)+ assert seen++ check = await verify_graph_reembed(source, target, target_dims=tgt_dims)+ assert check.ok, check.problems+ assert check.vector_index and check.target_width == tgt_dims+ assert all(src == dst for src, dst in check.node_counts.values())+ assert all(src == dst for src, dst in check.rel_counts.values())++ schema = nodes["schema"]+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{schema.node_id}'}}) RETURN m.embedding, m.beh_embedding, m.custom_json, m.content;")+ emb, beh, custom, content = rows[0]+ assert len(emb) == tgt_dims+ assert emb == pytest.approx(new.vector("plans trips weeks ahead"), abs=1e-5)+ assert beh == pytest.approx(new.vector("prepares early under uncertainty"), abs=1e-5)+ assert json.loads(custom)["retrieval_text"] == "plans trips weeks ahead" and content == schema.content+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['core'].node_id}'}}) RETURN m.embedding;")+ assert rows[0][0] == pytest.approx(new.vector("core: anticipates"), abs=1e-5)+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['facts'][0].node_id}'}}) RETURN m.embedding IS NULL;")+ assert rows[0][0] is True # a NULL source vector stays NULL+ topics = dict(_read(target, "MATCH (t:Topic) RETURN t.name, t.embedding;"))+ assert topics["travel"] == pytest.approx(new.vector("travel"), abs=1e-5)+ assert topics["work"] is None+ for subject, path_json, embedding in _read(target, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;"):+ label, exact = hierarchy_topic_label(subject, path_json)+ assert exact and embedding == pytest.approx(new.vector(label), abs=1e-5)++ # The rebuilt graph serves the SDK: vector search on the new index works.+ gs = await _open(target, tgt_dims)+ try:+ hits = await gs.vector_search(+ query_embedding=new.vector("plans trips weeks ahead"),+ isolation_key=schema.get_isolation_key(), limit=1,+ )+ assert hits and hits[0]["node_id"] == schema.node_id+ width = await gs._run("CALL table_info('Memory') RETURN *;")+ assert dict((r[1], r[2]) for r in width)["embedding"] == f"FLOAT[{tgt_dims}]"+ finally:+ await gs.close()+++async def test_source_labels_rebuild_exactly(tmp_path):+ """The stored HierarchyTopic vectors were computed from the rebuilt label."""+ source = tmp_path / "graph"+ old, _ = await _seed_graph(source, 64)+ rows = _read(source, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;")+ assert len(rows) == 4+ for subject, path_json, embedding in rows:+ label, exact = hierarchy_topic_label(subject, path_json)+ assert exact+ assert embedding == pytest.approx(old.vector(label), abs=1e-6)+++def test_legacy_path_json_is_flagged_inexact():+ label, exact = hierarchy_topic_label("alice", json.dumps(["Travel", "Japan"]), "Japan")+ assert label == "alice: topic:Travel > facet:Japan" and not exact+ label, exact = hierarchy_topic_label("alice", "", "Japan")+ assert label == "alice: Japan" and not exact+++async def test_cancel_and_failure_leave_no_target(tmp_path):+ source, target = tmp_path / "graph", tmp_path / "graph-new"+ await _seed_graph(source, 64)+ with pytest.raises(ReembedCancelled):+ await reembed_graph(+ source, target, embedding=FakeEmbedder("new", 32), target_dims=32,+ cancel=lambda: True,+ )+ with pytest.raises(EmbeddingDimensionMismatchError):+ await reembed_graph(+ source, target, embedding=FakeEmbedder("new", 32, wrong_width_after=1), target_dims=32,+ )+ assert not os.path.exists(target)+ assert not os.path.exists(str(target) + ".reembed-partial")+ # A restart starts over and succeeds.+ report = await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)+ assert report.node_counts["Memory"] == 8+ with pytest.raises(FileExistsError):+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)+ again = await reembed_graph(+ source, target, embedding=FakeEmbedder("new", 32), target_dims=32, overwrite=True,+ )+ assert again.rel_counts == report.rel_counts+++async def test_refuses_open_source_and_drops_only_closed_graphs(tmp_path):+ source, target = tmp_path / "graph", tmp_path / "graph-new"+ await _seed_graph(source, 64)+ live = await _open(source, 64)+ try:+ with pytest.raises(RuntimeError):+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)+ with pytest.raises(RuntimeError):+ await drop_graph(source)+ finally:+ await live.close()+ with pytest.raises(ValueError):+ await reembed_graph(source, source, embedding=FakeEmbedder("new", 64), target_dims=64)+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)+ assert await drop_graph(target) is True+ assert not os.path.exists(target)+ assert await drop_graph(target) is False+ assert os.path.exists(source)