Skip to content
Python

feat(sdk): re-embed vector stores and the graph into a new embedding space

alvinfei302fd30af175
diff --git a/.env.example b/.env.example
index 13ad1ed..a44ef8e 100644
--- a/.env.example
+++ b/.env.example
@@ -40,6 +40,7 @@ MEMORY_EMBEDDING_DIMS=1024
# MEMORY_EMBEDDER_TIMEOUT=15
# MEMORY_EMBEDDER_MAX_RETRIES=2
# MEMORY_EMBEDDER_RETRY_DELAY=0.3
+# MEMORY_EMBEDDER_BATCH_SIZE=64
# MEMORY_EMBEDDER_EXTRA_HEADERS={}
# MEMORY_EMBEDDER_EXTRA_BODY={}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00c9fe9..31ed08c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,46 @@ Hy Memory and MemHub are versioned and released independently.
## Unreleased
+- Hy Memory: new `hy_memory.reembed` module for switching the embedding
+ model. Writers must be paused for the whole run. Nothing is derived from
+ the provider or endpoint: source and target are named explicitly, and the
+ identity of a space is the model name plus the width (`space_identity`).
+ Only Chroma and Kuzu are supported.
+ - `vector_space_names(config, base_name=None)` returns every physical
+ collection a client opens for the six vector roles: memory, entities,
+ coding keys and working facts, each in the working and coding variants.
+ - `reembed_vector_space(source, target, embedding=...)` pages every raw
+ source collection, covering all users and all layers including L1_RAW. It
+ embeds the text the write path embedded (`reembed_text`: the canonical
+ state's `state_retrieval_text`, a schema's `retrieval_text`, otherwise the
+ document). It then writes each record into the target with the same id,
+ document and metadata. Each batch's widths are checked before anything is
+ written. A cancel stops at a batch boundary (`ReembedCancelled`). A rerun
+ skips the ids the target already holds, so it resumes, and a merge never
+ overwrites the target. The source is never written.
+ - `plan_reembed`, `verify_reembed` and `drop_vector_space` complete the
+ vector side. `verify_reembed` checks ids, metadata, per-user counts and
+ the width, and runs a self-recall sample. `drop_vector_space` refuses
+ collections that a live client in this process holds open.
+ - `scan_vector_spaces(persist_directory)` is read-only. It lists every
+ space on disk with its roles, counts, stored width, per-user counts and
+ the models recorded in metadata. `diff_spaces(source, target)` returns the
+ ids that the source holds and the target lacks, whatever their status.
+ - `reembed_graph(source_path, target_path, embedding=..., target_dims=...)`
+ builds a new Kuzu graph at the new width. It copies every node and edge
+ and recomputes `Memory.embedding`, `Memory.beh_embedding`, and the Topic
+ and HierarchyTopic embeddings, then rebuilds the HNSW index. The source
+ graph is opened read-only and kept for rollback. A run that fails or is
+ cancelled leaves no target behind, and a rerun starts over.
+ `verify_graph_reembed` and `drop_graph` complete the graph side.
+- Hy Memory: `EmbedService.embed_batch` now splits large batches into
+ requests of `embedder.batch_size` texts each (`MEMORY_EMBEDDER_BATCH_SIZE`,
+ default 64); before, it sent them all in one request. On a 429 it waits for
+ the `Retry-After` time plus jitter, capped at 60 s. It takes a per-call
+ `max_retries`. A returned vector whose width differs from the configured
+ `embedding_dims` now raises `EmbeddingDimensionMismatchError` and is not
+ retried.
+
## MemHub 1.0.5 — 2026-10-28
Build 36, on `hy-memory==2.0.8`. Linux x64/arm64 Runtime and macOS arm64
diff --git a/docs/env_reference.md b/docs/env_reference.md
index 7bd3841..699d552 100644
--- a/docs/env_reference.md
+++ b/docs/env_reference.md
@@ -48,6 +48,7 @@
| `MEMORY_EMBEDDER_TIMEOUT` | `15` | 单次超时,秒 |
| `MEMORY_EMBEDDER_MAX_RETRIES` | `2` | 最大重试次数 |
| `MEMORY_EMBEDDER_RETRY_DELAY` | `0.3` | 重试间隔,秒 |
+| `MEMORY_EMBEDDER_BATCH_SIZE` | `64` | `embed_batch` 单次请求最多发送的文本条数,超出按此切块 |
| `MEMORY_EMBEDDER_EXTRA_HEADERS` | — | JSON object,附加 HTTP headers |
| `MEMORY_EMBEDDER_EXTRA_BODY` | — | JSON object,附加请求 body |
diff --git a/hy_memory/__init__.py b/hy_memory/__init__.py
index e0a1f3a..fa03975 100644
--- a/hy_memory/__init__.py
+++ b/hy_memory/__init__.py
@@ -55,6 +55,33 @@ except Exception:
from .client import HyMemoryClient
from .inspector import MemoryInspector
from .runtime import SharedRuntime
+from .reembed import (
+ EmbeddingSpaceIdentity,
+ GraphReembedPlan,
+ GraphReembedProgress,
+ GraphReembedReport,
+ GraphVerification,
+ ReembedCancelled,
+ ReembedPlan,
+ ReembedProgress,
+ ReembedReport,
+ ReembedVerification,
+ ScannedSpace,
+ SpaceDiff,
+ VectorSpace,
+ diff_spaces,
+ drop_graph,
+ drop_vector_space,
+ plan_reembed,
+ reembed_graph,
+ reembed_text,
+ reembed_vector_space,
+ scan_vector_spaces,
+ space_identity,
+ vector_space_names,
+ verify_graph_reembed,
+ verify_reembed,
+)
from .snapshot import (
DataPlaneRestore,
DataPlaneSnapshot,
@@ -129,6 +156,32 @@ __all__ = [
"DataPlaneSnapshotRejected",
"restore_local_data_plane",
"snapshot_local_data_plane",
+ # 换 embedding 模型:把向量库和图重新 embed 进新的 embedding 空间
+ "EmbeddingSpaceIdentity",
+ "GraphReembedPlan",
+ "GraphReembedProgress",
+ "GraphReembedReport",
+ "GraphVerification",
+ "ReembedCancelled",
+ "ReembedPlan",
+ "ReembedProgress",
+ "ReembedReport",
+ "ReembedVerification",
+ "ScannedSpace",
+ "SpaceDiff",
+ "VectorSpace",
+ "diff_spaces",
+ "drop_graph",
+ "drop_vector_space",
+ "plan_reembed",
+ "reembed_graph",
+ "reembed_text",
+ "reembed_vector_space",
+ "scan_vector_spaces",
+ "space_identity",
+ "vector_space_names",
+ "verify_graph_reembed",
+ "verify_reembed",
# 配置
"MemoryConfig",
# 公共扩展协议 / 精确读取类型
diff --git a/hy_memory/config.py b/hy_memory/config.py
index 4785245..828f0e7 100644
--- a/hy_memory/config.py
+++ b/hy_memory/config.py
@@ -260,6 +260,7 @@ class EmbedderConfig:
max_retries: int = None # 最大重试次数
retry_delay: float = None # 重试间隔(秒)
timeout: int = None # 请求超时(秒)
+ batch_size: int = None # 单次 HTTP 请求最多带多少条 input(embed_batch 按此切块)
# 内部平台扩展(extra_headers / extra_body 会透传给 OpenAI client)
extra_headers: Optional[Dict[str, str]] = None
@@ -294,6 +295,10 @@ class EmbedderConfig:
# 请求切断并制造重试风暴。SDK 内层重试已关闭,15s 是每个显式 attempt
# 的真实上限;部署可通过 MEMORY_EMBEDDER_TIMEOUT 覆盖。
self.timeout = _get_env_int("MEMORY_EMBEDDER_TIMEOUT", 15)
+ if self.batch_size is None:
+ # 大批量(如重新 embed 整个库)不能一次 HTTP 发几千条:多数服务对
+ # input 数组有上限,超了直接 400。按 batch_size 切块逐块发送。
+ self.batch_size = _get_env_int("MEMORY_EMBEDDER_BATCH_SIZE", 64)
if self.extra_headers is None:
val = os.getenv("MEMORY_EMBEDDER_EXTRA_HEADERS")
if val:
diff --git a/hy_memory/core/embed_service.py b/hy_memory/core/embed_service.py
index 96fec8c..ab80cb2 100644
--- a/hy_memory/core/embed_service.py
+++ b/hy_memory/core/embed_service.py
@@ -38,10 +38,12 @@ import importlib.util
import json
from json import JSONDecodeError
import logging
+import random
import threading
import time
from ..config import MemoryConfig
+from ..embedding import EmbeddingDimensionMismatchError
from .workload import embedding_slot
from ..utils.log_setup import get_request_id
@@ -187,6 +189,41 @@ def _redact_headers(headers: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any
return out
+# Retry-After 的上限:服务端给出超长等待(或恶意值)时不让单次调用挂几十分钟。
+_RETRY_AFTER_MAX_S = 60.0
+
+
+def _retry_after_seconds(exc: Exception) -> Optional[float]:
+ """从 429 异常的响应头里取 Retry-After(秒数或 HTTP 日期),取不到返回 None。"""
+ response = getattr(exc, "response", None)
+ headers = getattr(response, "headers", None)
+ if not headers:
+ return None
+ try:
+ raw = headers.get("retry-after") or headers.get("Retry-After")
+ except Exception:
+ return None
+ if raw is None:
+ return None
+ raw = str(raw).strip()
+ try:
+ seconds = float(raw)
+ except ValueError:
+ try:
+ from email.utils import parsedate_to_datetime
+ from datetime import datetime, timezone
+
+ when = parsedate_to_datetime(raw)
+ if when.tzinfo is None:
+ when = when.replace(tzinfo=timezone.utc)
+ seconds = (when - datetime.now(timezone.utc)).total_seconds()
+ except Exception:
+ return None
+ if seconds != seconds: # NaN
+ return None
+ return min(max(0.0, seconds), _RETRY_AFTER_MAX_S)
+
+
def _embeddings_api_url(base_url: str) -> str:
"""OpenAI SDK posts to ``{base_url}/embeddings`` (base_url usually ends with /v1)."""
base = (base_url or "").rstrip("/")
@@ -205,7 +242,8 @@ class EmbedServiceConfig:
api_key: str = ""
base_url: str = ""
embedding_dims: int = 1536
- batch_size: int = 100
+ # embed_batch 单次 HTTP 最多带多少条 input;超出按块顺序发送
+ batch_size: int = 64
enable_cache: bool = True
cache_size: int = 10000
max_retries: int = 5
@@ -287,6 +325,7 @@ class EmbedService:
max_retries=embedder.max_retries if embedder.max_retries is not None else 5,
retry_delay=embedder.retry_delay if embedder.retry_delay is not None else 3.0,
timeout=embedder.timeout if embedder.timeout is not None else 60,
+ batch_size=max(1, int(getattr(embedder, "batch_size", None) or 64)),
queue_batch_size=int(os.getenv("MEMORY_EMBED_BATCH_SIZE", "8")),
queue_batch_window_ms=float(os.getenv("MEMORY_EMBED_BATCH_WINDOW_MS", "100")),
extra_headers=embedder.extra_headers,
@@ -640,6 +679,8 @@ class EmbedService:
self,
texts: List[str],
use_cache: bool = True,
+ *,
+ max_retries: Optional[int] = None,
) -> List[List[float]]:
"""
批量向量化
@@ -647,17 +688,23 @@ class EmbedService:
Args:
texts: 文本列表
use_cache: 是否使用缓存
+ max_retries: 本次调用的重试次数;None 用配置值。长批量任务(重新
+ embed 整个库)可以给更宽的重试预算,不影响在线路径。
Returns:
向量列表
"""
- vectors, _metas = await self.embed_batch_with_meta(texts, use_cache=use_cache)
+ vectors, _metas = await self.embed_batch_with_meta(
+ texts, use_cache=use_cache, max_retries=max_retries,
+ )
return vectors
async def embed_batch_with_meta(
self,
texts: List[str],
use_cache: bool = True,
+ *,
+ max_retries: Optional[int] = None,
) -> Tuple[List[List[float]], List[Dict[str, Any]]]:
"""批量向量化,同时返回每条的 provider metadata(非 vector)。"""
if not texts:
@@ -682,7 +729,12 @@ class EmbedService:
indices_to_embed.append(i)
if texts_to_embed:
- vectors, batch_metas = await self._embed_texts_with_meta(texts_to_embed)
+ # max_retries 只在显式给出时透传:保持 `_embed_texts_with_meta(texts)`
+ # 这个既有调用形状不变(测试与下游替身按它打桩)。
+ retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}
+ vectors, batch_metas = await self._embed_texts_with_meta(
+ texts_to_embed, **retry_kwargs,
+ )
for j, vector in enumerate(vectors):
idx = indices_to_embed[j]
@@ -700,9 +752,9 @@ class EmbedService:
return results, metas
async def _embed_texts_with_meta(
- self, texts: List[str]
+ self, texts: List[str], *, max_retries: Optional[int] = None,
) -> Tuple[List[List[float]], List[Dict[str, Any]]]:
- """实际的向量化调用:**一次 HTTP 带走整批**;返回 (vectors, metas)。
+ """实际的向量化调用:**一次 HTTP 带走一块**;返回 (vectors, metas)。
早期实现是 `for text in texts: await _embed_openai([text])` 逐条串行,
原意是"避免 batch size 限制",但代价极大:线上一批 8 条 × ~5.6s 串行
@@ -719,21 +771,34 @@ class EmbedService:
截断在这里做(发送前逐条过 `_truncate_for_embedding`):所有 embedding
路径(embed / embed_batch / embed_queued / *_with_meta)最终都落到这个
方法 —— 放在各入口会漏。理由与阈值见模块头部 `_EMBED_MAX_TOKENS`。
+
+ 切块:一次带走整批只适用于小批。几千条的 input 数组会被多数服务直接
+ 400(input 上限),所以按 `batch_size`(默认 64)切块,块之间顺序发送。
+ 不超过一块时行为与切块前逐字相同(`batch_mode="single_request"`)。
"""
if not texts:
return [], []
safe_texts = [_truncate_for_embedding(t) for t in texts]
- vectors, meta = await self._embed_openai(safe_texts)
- base_meta = dict(meta or {})
- base_meta["batch_mode"] = "single_request"
- base_meta["batch_input_count"] = len(safe_texts)
-
+ chunk = max(1, int(self._embed_config.batch_size or 64))
+ vectors: List[List[float]] = []
all_metas: List[Dict[str, Any]] = []
- for idx in range(len(safe_texts)):
- m = dict(base_meta)
- m["batch_index"] = idx
- all_metas.append(m)
+ chunk_count = (len(safe_texts) + chunk - 1) // chunk
+ for start in range(0, len(safe_texts), chunk):
+ part = safe_texts[start:start + chunk]
+ retry_kwargs = {} if max_retries is None else {"max_retries": max_retries}
+ part_vectors, meta = await self._embed_openai(part, **retry_kwargs)
+ base_meta = dict(meta or {})
+ base_meta["batch_mode"] = "single_request" if chunk_count == 1 else "chunked"
+ base_meta["batch_input_count"] = len(part)
+ if chunk_count > 1:
+ base_meta["batch_chunk_index"] = start // chunk
+ base_meta["batch_chunk_count"] = chunk_count
+ for idx in range(len(part)):
+ m = dict(base_meta)
+ m["batch_index"] = start + idx
+ all_metas.append(m)
+ vectors.extend(part_vectors)
self._last_embed_meta = all_metas[-1] if all_metas else {}
return vectors, all_metas
@@ -783,7 +848,7 @@ class EmbedService:
)
async def _embed_openai(
- self, texts: List[str]
+ self, texts: List[str], *, max_retries: Optional[int] = None,
) -> Tuple[List[List[float]], Dict[str, Any]]:
"""
使用 OpenAI 兼容接口向量化
@@ -873,7 +938,9 @@ class EmbedService:
"endpoint_overridden": bool(_ov),
}
- _max_retries = max(0, int(self._embed_config.max_retries or 0))
+ _max_retries = max(0, int(
+ (self._embed_config.max_retries or 0) if max_retries is None else max_retries
+ ))
_retry_delay = max(0.0, float(self._embed_config.retry_delay or 0.0))
last_exc: Optional[Exception] = None
@@ -919,6 +986,17 @@ class EmbedService:
f"got={len(vectors)}"
)
dims = [len(v) for v in vectors]
+ expected_dims = self._embed_config.embedding_dims
+ if expected_dims:
+ bad = [d for d in dims if d != int(expected_dims)]
+ if bad:
+ # 宽度不对的向量写进集合会把集合的维度钉死在错误值上(Chroma
+ # 首次 add 定维),或在已有集合上整批失败。确定性错误,不重试。
+ raise EmbeddingDimensionMismatchError(
+ "embedding dimension mismatch: "
+ f"expected={int(expected_dims)} got={bad[0]} "
+ f"model={self._embed_config.model}"
+ )
meta = {
"http_status": raw.status_code,
"response_headers": resp_headers,
@@ -954,6 +1032,8 @@ class EmbedService:
},
)
return vectors, meta
+ except EmbeddingDimensionMismatchError:
+ raise
except Exception as e:
last_exc = e
_err_name = type(e).__name__
@@ -987,12 +1067,18 @@ class EmbedService:
if _retryable and attempt < _max_retries:
# 逐出可能卡死的 client,下个 attempt 重建干净连接(恢复原防御语义)
self._evict_client(_eff_base_url, _eff_api_key, _eff_timeout)
+ _delay = _retry_delay
+ _retry_after = _retry_after_seconds(e) if _status == 429 else None
+ if _retry_after is not None:
+ # 限流时服务端明确告诉了要等多久:按它等,再加一点抖动,
+ # 避免多个 worker 在同一时刻一起醒来再次撞限流。
+ _delay = max(_delay, _retry_after) + random.uniform(0, 0.25 * _retry_after)
logger.warning(
f"[embed] attempt {attempt + 1}/{_max_retries + 1} failed "
- f"({_err_name}): {_err_msg[:200]}; retry in {_retry_delay}s"
+ f"({_err_name}): {_err_msg[:200]}; retry in {_delay:.2f}s"
)
- if _retry_delay > 0:
- await asyncio.sleep(_retry_delay)
+ if _delay > 0:
+ await asyncio.sleep(_delay)
continue
self._log_embed_call(
status="error",
diff --git a/hy_memory/data/graph_store_kuzu.py b/hy_memory/data/graph_store_kuzu.py
index e11eeb4..7c4cddd 100644
--- a/hy_memory/data/graph_store_kuzu.py
+++ b/hy_memory/data/graph_store_kuzu.py
@@ -22,6 +22,7 @@ import logging
import json
import threading
import time
+import weakref
from ..models.memory import MemoryNode, MemoryLayer, MemoryStatus
from ..config import MemoryConfig
@@ -77,6 +78,21 @@ async def _run_in_kuzu_pool(func, *args, **kwargs):
return await run_in_executor_guarded(_kuzu_executor, _measured)
+# 本进程里已打开、尚未 close 的 store。drop_graph 据此拒绝删掉一个正被
+# 活着的 client 使用的图库。WeakSet:store 被回收后自动消失。
+_LIVE_STORES: "weakref.WeakSet[KuzuGraphStore]" = weakref.WeakSet()
+
+
+def open_graph_paths() -> List[str]:
+ """Real paths of the graph databases live stores in this process hold open."""
+ import os
+ return [
+ os.path.realpath(str(store._db_path))
+ for store in list(_LIVE_STORES)
+ if store._db is not None
+ ]
+
+
def kuzu_executor_snapshot() -> Dict[str, Any]:
with _kuzu_stats_lock:
return {
@@ -350,6 +366,7 @@ class KuzuGraphStore(GraphStoreBase):
)
self._db = db
self._conn = conn
+ _LIVE_STORES.add(self)
except asyncio.TimeoutError:
logger.warning(
f"GraphStore init timed out after 5s (Kuzu file lock contention). "
@@ -2441,6 +2458,7 @@ class KuzuGraphStore(GraphStoreBase):
async def close(self) -> None:
"""关闭连接"""
conn, db = self._conn, self._db
+ _LIVE_STORES.discard(self)
self._conn = None
self._db = None
self._available = False
diff --git a/hy_memory/data/vector_store_base.py b/hy_memory/data/vector_store_base.py
index dee74bf..c0ef5d0 100644
--- a/hy_memory/data/vector_store_base.py
+++ b/hy_memory/data/vector_store_base.py
@@ -89,6 +89,17 @@ def passes_created_after(
logger = logging.getLogger(__name__)
+def physical_collection_name(base_name: Optional[str], dims: Optional[int]) -> str:
+ """The collection name a store opens for ``base_name`` at ``dims``.
+
+ The dims suffix keeps two embedding widths from sharing one collection.
+ `hy_memory.reembed.vector_space_names` uses this too, so callers never
+ re-derive the rule.
+ """
+ base = base_name or "agent_memories"
+ return f"{base}_{dims}" if dims else base
+
+
class VectorStoreBase(ABC):
"""
向量存储抽象基类
@@ -98,13 +109,10 @@ class VectorStoreBase(ABC):
def __init__(self, config: MemoryConfig):
self.config = config
- base_name = config.vector_store.collection_name or "agent_memories"
- dims = config.vector_store.embedding_dims
# 自动拼接维度后缀,避免不同 embedding 模型写入同一 collection 导致维度冲突
- if dims:
- self._collection_name = f"{base_name}_{dims}"
- else:
- self._collection_name = base_name
+ self._collection_name = physical_collection_name(
+ config.vector_store.collection_name, config.vector_store.embedding_dims,
+ )
_supports_retrieval_clauses: bool = False
_supports_narrow_episode_queries: bool = False
diff --git a/hy_memory/data/vector_store_chroma.py b/hy_memory/data/vector_store_chroma.py
index 8b9a9fb..8d3565d 100644
--- a/hy_memory/data/vector_store_chroma.py
+++ b/hy_memory/data/vector_store_chroma.py
@@ -9,10 +9,12 @@ Agent Memory V2 - VectorStore (Chroma)
- 同步 API → _run_in_vdb_pool() 包装为异步
"""
-from typing import Optional, List, Dict, Any
+from typing import Optional, List, Dict, Any, Set, Tuple
import concurrent.futures
import functools
import logging
+import os
+import weakref
from ..models.memory import (
MemoryNode, MemoryLayer, MemoryStatus,
@@ -33,6 +35,30 @@ _vdb_executor = concurrent.futures.ThreadPoolExecutor(
)
+# 本进程里已 initialize、尚未 close 的 store。drop_vector_space 据此拒绝删掉
+# 一个正被活着的 client 使用的集合。WeakSet:store 被回收后自动消失。
+_LIVE_STORES: "weakref.WeakSet[ChromaVectorStore]" = weakref.WeakSet()
+
+
+def _location_key(vs_config) -> Tuple[str, str]:
+ if getattr(vs_config, "host", None):
+ return ("http", f"{vs_config.host}:{vs_config.port or 8000}")
+ return ("local", os.path.realpath(str(vs_config.persist_directory or "")))
+
+
+def open_collection_names(location: Tuple[str, str]) -> Set[str]:
+ """Collections that live stores in this process hold open at ``location``."""
+ names: Set[str] = set()
+ for store in list(_LIVE_STORES):
+ if store._client is None or _location_key(store.config.vector_store) != location:
+ continue
+ names.add(store._collection_name)
+ if store._entity_collection is not None:
+ from ..pipelines._retrieval.config import entity_collection_name
+ names.add(entity_collection_name(store._collection_name))
+ return names
+
+
async def _run_in_vdb_pool(func, *args, **kwargs):
"""Run a synchronous Chroma call in the dedicated VDB executor."""
call = functools.partial(func, *args, **kwargs) if args or kwargs else func
@@ -88,6 +114,7 @@ class ChromaVectorStore(VectorStoreBase):
return client, collection
self._client, self._collection = await _run_in_vdb_pool(_init)
+ _LIVE_STORES.add(self)
logger.debug(f"VectorStore initialized (Chroma), collection={self._collection_name}")
# ================================================================
@@ -561,6 +588,7 @@ class ChromaVectorStore(VectorStoreBase):
async def close(self) -> None:
"""关闭客户端"""
# Chroma PersistentClient 无需显式关闭
+ _LIVE_STORES.discard(self)
self._client = None
self._collection = None
self._entity_collection = None
diff --git a/hy_memory/reembed.py b/hy_memory/reembed.py
new file mode 100644
index 0000000..ab70462
--- /dev/null
+++ b/hy_memory/reembed.py
@@ -0,0 +1,1777 @@
+"""Re-embed the SDK's stores into a new embedding space.
+
+An embedding space is the set of physical collections (and, in ultra mode, the
+Kuzu graph) whose vectors were computed by one embedding model at one width.
+Switching the model means computing every vector again with the new model and
+writing it into a *new* space. The stored text is enough for that: each record
+keeps its document and metadata, and the text that was embedded can be rebuilt
+from them (`reembed_text`). The original conversations are not needed.
+
+Contract:
+
+- The caller pauses every writer before calling into this module and keeps
+ them paused until the switch commits. The source stores are treated as frozen.
+- Nothing here writes to the source. Vector sources are only read; the source
+ graph is opened read-only.
+- Source and target are named explicitly (`VectorSpace`, graph paths). Nothing
+ is derived from the embedding provider or endpoint; the identity of a space is
+ the model name plus the width (`space_identity`).
+- Vector re-embeds are resumable: a target record with the same id is skipped
+ (see `skip_existing`). Graph re-embeds restart from scratch: the target is
+ built at a staging path that is wiped on every start and moved into place
+ only after the copy verifies.
+
+Only the Chroma vector backend and the Kuzu graph backend are supported.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import hashlib
+import json
+import logging
+import os
+import random
+import re
+import shutil
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterable,
+ List,
+ Literal,
+ Mapping,
+ Optional,
+ Sequence,
+ Tuple,
+ Union,
+)
+
+from .config import MemoryConfig
+from .embedding import EmbeddingDimensionMismatchError, EmbeddingService
+from .utils.async_executor import run_in_executor_guarded
+
+logger = logging.getLogger(__name__)
+
+VectorRole = Literal[
+ "memory", "entities",
+ "coding_keys", "coding_keys_cv",
+ "working_facts", "working_facts_cv",
+]
+VECTOR_ROLES: Tuple[str, ...] = (
+ "memory", "entities",
+ "coding_keys", "coding_keys_cv",
+ "working_facts", "working_facts_cv",
+)
+
+# Collection metadata keys written on target collections this module creates.
+# Records themselves are copied verbatim, so the model can only live here.
+COLLECTION_MODEL_KEY = "hy_embedding_model"
+COLLECTION_DIMS_KEY = "hy_embedding_dims"
+
+_SELF_RECALL_WARN_RATE = 0.9
+
+
+class ReembedCancelled(RuntimeError):
+ """Raised at a batch boundary when ``cancel()`` returns True.
+
+ Every batch written before the cancel is complete; no batch is half-written.
+ ``report`` holds the progress made so far (a ``ReembedReport`` for vectors,
+ ``None`` for the graph, whose staging copy is discarded).
+ """
+
+ def __init__(self, message: str = "reembed cancelled", report: Any = None) -> None:
+ super().__init__(message)
+ self.report = report
+
+
+# ======================================================================
+# Space identity and naming
+# ======================================================================
+
+@dataclass(frozen=True)
+class EmbeddingSpaceIdentity:
+ """What makes two embedding spaces the same: model name and width."""
+
+ model: str
+ dims: int
+
+ @property
+ def key(self) -> str:
+ return f"{self.model}#{self.dims}"
+
+ @property
+ def digest(self) -> str:
+ """Short stable hash, usable inside a collection or directory name."""
+ return hashlib.sha256(self.key.encode("utf-8")).hexdigest()[:12]
+
+
+def space_identity(model: str, dims: int) -> EmbeddingSpaceIdentity:
+ """Identity of an embedding space. Provider and endpoint are not part of it."""
+ name = " ".join(str(model or "").split())
+ if not name:
+ raise ValueError("space_identity needs a model name")
+ width = int(dims)
+ if width <= 0:
+ raise ValueError(f"space_identity needs a positive width, got {dims!r}")
+ return EmbeddingSpaceIdentity(model=name, dims=width)
+
+
+@dataclass(frozen=True)
+class VectorSpace:
+ """One embedding space in a Chroma store, named explicitly.
+
+ ``base_name`` is the collection base without the width suffix, exactly what
+ ``vector_store.collection_name`` is for a client. ``collections`` overrides
+ the physical name per role when the caller manages names itself; roles left
+ out of an explicit mapping are not part of the space.
+ """
+
+ base_name: str
+ dims: Optional[int]
+ persist_directory: Optional[str] = None
+ host: Optional[str] = None
+ port: Optional[int] = None
+ model: Optional[str] = None
+ collections: Optional[Mapping[str, str]] = None
+
+ @classmethod
+ def from_config(
+ cls, config: MemoryConfig, *, base_name: Optional[str] = None,
+ ) -> "VectorSpace":
+ vs = config.vector_store
+ provider = str(vs.provider or "chroma").lower()
+ if provider != "chroma":
+ raise NotImplementedError(
+ f"re-embed supports the chroma vector store only, got {provider!r}"
+ )
+ return cls(
+ base_name=base_name or vs.collection_name or "agent_memories",
+ dims=vs.embedding_dims,
+ persist_directory=vs.persist_directory,
+ host=vs.host,
+ port=vs.port,
+ model=getattr(config.embedder, "model", None),
+ )
+
+ def names(self) -> Dict[str, str]:
+ if self.collections is not None:
+ unknown = set(self.collections) - set(VECTOR_ROLES)
+ if unknown:
+ raise ValueError(f"unknown vector roles: {sorted(unknown)}")
+ return {role: self.collections[role] for role in VECTOR_ROLES if role in self.collections}
+ return _derived_names(self.base_name, self.dims)
+
+ def location(self) -> Tuple[str, str]:
+ if self.host:
+ return ("http", f"{self.host}:{self.port or 8000}")
+ return ("local", os.path.realpath(str(self.persist_directory or "")))
+
+
+SpaceLike = Union[VectorSpace, MemoryConfig]
+
+
+def _derived_names(base_name: str, dims: Optional[int]) -> Dict[str, str]:
+ from .coding.store import _derive_coding_collection_name, _derive_fact_collection_name
+ from .data.vector_store_base import physical_collection_name
+ from .pipelines._retrieval.config import entity_collection_name
+
+ memory = physical_collection_name(base_name, dims)
+ return {
+ "memory": memory,
+ "entities": entity_collection_name(memory),
+ "coding_keys": physical_collection_name(_derive_coding_collection_name(base_name, ""), dims),
+ "coding_keys_cv": physical_collection_name(_derive_coding_collection_name(base_name, "_cv"), dims),
+ "working_facts": physical_collection_name(_derive_fact_collection_name(base_name, ""), dims),
+ "working_facts_cv": physical_collection_name(_derive_fact_collection_name(base_name, "_cv"), dims),
+ }
+
+
+def _space(value: SpaceLike) -> VectorSpace:
+ if isinstance(value, VectorSpace):
+ return value
+ if isinstance(value, MemoryConfig):
+ return VectorSpace.from_config(value)
+ raise TypeError(f"expected VectorSpace or MemoryConfig, got {type(value).__name__}")
+
+
+def vector_space_names(
+ config: SpaceLike, *, base_name: Optional[str] = None,
+) -> Dict[str, str]:
+ """Every physical collection a client with ``config`` opens for the vector roles.
+
+ The width suffix is included. Pass ``base_name`` to name a space other than
+ ``config.vector_store.collection_name``.
+ """
+ if isinstance(config, MemoryConfig):
+ return VectorSpace.from_config(config, base_name=base_name).names()
+ space = _space(config)
+ if base_name is not None:
+ return _derived_names(base_name, space.dims)
+ return space.names()
+
+
+# ======================================================================
+# The text a record was embedded from
+# ======================================================================
+
+def _custom_dict(value: Any) -> Dict[str, Any]:
+ if isinstance(value, dict):
+ return value
+ if isinstance(value, str) and value:
+ try:
+ parsed = json.loads(value)
+ except (TypeError, ValueError):
+ return {}
+ return parsed if isinstance(parsed, dict) else {}
+ return {}
+
+
+def reembed_text(
+ role: str, document: Optional[str], metadata: Optional[Mapping[str, Any]],
+) -> str:
+ """The exact text the write path embedded for one vector record.
+
+ - memory, ``custom.memory_kind == "canonical_state"``: ``custom.state_retrieval_text``
+ - memory with a non-empty ``custom.retrieval_text`` (schemas): that text
+ - everything else, coding keys and working facts included: the document,
+ falling back to ``metadata["content"]``
+ """
+ meta = metadata or {}
+ if role == "memory":
+ custom = _custom_dict(meta.get("custom"))
+ if custom.get("memory_kind") == "canonical_state":
+ text = str(custom.get("state_retrieval_text") or "").strip()
+ if text:
+ return text
+ text = str(custom.get("retrieval_text") or "").strip()
+ if text:
+ return text
+ if document:
+ return document
+ return str(meta.get("content") or "")
+
+
+def _embeddable(text: str) -> str:
+ # OpenAI-compatible services reject an empty input with a 400 that would
+ # fail the whole batch. A record with no text has no meaning to preserve.
+ return text if text and text.strip() else " "
+
+
+def _record_digest(document: Optional[str], metadata: Optional[Mapping[str, Any]]) -> str:
+ payload = json.dumps(
+ [document, dict(metadata) if metadata is not None else None],
+ sort_keys=True, ensure_ascii=False, default=str, separators=(",", ":"),
+ )
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+# ======================================================================
+# Chroma access (read-only on the source)
+# ======================================================================
+
+_VECTOR_PAGE = 500
+
+
+async def _vdb(func, *args, **kwargs):
+ from .data.vector_store_chroma import _run_in_vdb_pool
+ return await _run_in_vdb_pool(func, *args, **kwargs)
+
+
+def _client_for(space: VectorSpace):
+ import chromadb
+
+ if space.host:
+ return chromadb.HttpClient(host=space.host, port=space.port or 8000)
+ # Same settings as ChromaVectorStore.initialize, so both share one system.
+ return chromadb.PersistentClient(path=space.persist_directory)
+
+
+def _get_collection(client, name: str):
+ try:
+ return client.get_collection(name=name)
+ except Exception as exc: # chromadb NotFoundError / ValueError by version
+ if "does not exist" in str(exc).lower() or type(exc).__name__ in (
+ "NotFoundError", "InvalidCollectionException",
+ ):
+ return None
+ raise
+
+
+def _page(collection, offset: int, limit: int, include: Sequence[str]) -> Dict[str, Any]:
+ return collection.get(limit=limit, offset=offset, include=list(include))
+
+
+def _iter_pages(collection, include: Sequence[str], page: int = _VECTOR_PAGE):
+ total = collection.count()
+ offset = 0
+ while offset < total:
+ got = _page(collection, offset, page, include)
+ ids = got.get("ids") or []
+ if not ids:
+ break
+ yield got
+ offset += len(ids)
+
+
+def _first_width(collection) -> Optional[int]:
+ got = collection.get(limit=1, include=["embeddings"])
+ embeddings = got.get("embeddings")
+ if embeddings is None or len(embeddings) == 0:
+ return None
+ return len(embeddings[0])
+
+
+def _check_not_same(source: VectorSpace, target: VectorSpace, src_name: str, tgt_name: str) -> None:
+ if source.location() == target.location() and src_name == tgt_name:
+ raise ValueError(
+ f"source and target are the same collection {src_name!r}; "
+ "re-embed always writes a new collection"
+ )
+
+
+def _roles(space: VectorSpace, roles: Optional[Iterable[str]]) -> List[str]:
+ names = space.names()
+ if roles is None:
+ return [role for role in VECTOR_ROLES if role in names]
+ wanted = list(roles)
+ unknown = [role for role in wanted if role not in VECTOR_ROLES]
+ if unknown:
+ raise ValueError(f"unknown vector roles: {unknown}")
+ return [role for role in wanted if role in names]
+
+
+# ======================================================================
+# Plan
+# ======================================================================
+
+@dataclass(frozen=True)
+class ReembedCollectionPlan:
+ role: str
+ source_name: str
+ target_name: str
+ source_count: int
+ target_exists: bool
+ target_count: int
+
+
+@dataclass(frozen=True)
+class GraphReembedPlan:
+ source_path: str
+ node_counts: Dict[str, int]
+ rel_counts: Dict[str, int]
+ memory_embeddings: int
+ beh_embeddings: int
+ topic_embeddings: int
+ hierarchy_topic_embeddings: int
+ source_width: Optional[int]
+ target_width: Optional[int]
+
+ @property
+ def memory_nodes(self) -> int:
+ return self.node_counts.get("Memory", 0)
+
+ @property
+ def topics(self) -> int:
+ return self.node_counts.get("Topic", 0)
+
+ @property
+ def hierarchy_topics(self) -> int:
+ return self.node_counts.get("HierarchyTopic", 0)
+
+ @property
+ def vectors(self) -> int:
+ return (
+ self.memory_embeddings + self.beh_embeddings
+ + self.topic_embeddings + self.hierarchy_topic_embeddings
+ )
+
+
+@dataclass(frozen=True)
+class ReembedPlan:
+ collections: Tuple[ReembedCollectionPlan, ...]
+ total_vectors: int
+ source_dims: Optional[int]
+ target_dims: Optional[int]
+ graph: Optional[GraphReembedPlan] = None
+
+ @property
+ def estimated_vectors(self) -> int:
+ """Vectors to compute: every vector record plus every graph embedding."""
+ return self.total_vectors + (self.graph.vectors if self.graph else 0)
+
+ @property
+ def per_role(self) -> Dict[str, int]:
+ return {item.role: item.source_count for item in self.collections}
+
+
+def _plan_sync(source: VectorSpace, target: VectorSpace) -> Tuple[List[ReembedCollectionPlan], Optional[int]]:
+ src_client = _client_for(source)
+ tgt_client = _client_for(target)
+ src_names, tgt_names = source.names(), target.names()
+ out: List[ReembedCollectionPlan] = []
+ source_dims: Optional[int] = None
+ for role in VECTOR_ROLES:
+ if role not in src_names or role not in tgt_names:
+ continue
+ src = _get_collection(src_client, src_names[role])
+ tgt = _get_collection(tgt_client, tgt_names[role])
+ count = src.count() if src is not None else 0
+ if src is not None and count and source_dims is None:
+ source_dims = _first_width(src)
+ out.append(ReembedCollectionPlan(
+ role=role,
+ source_name=src_names[role],
+ target_name=tgt_names[role],
+ source_count=count,
+ target_exists=tgt is not None,
+ target_count=tgt.count() if tgt is not None else 0,
+ ))
+ return out, source_dims
+
+
+async def plan_reembed(
+ source: SpaceLike,
+ target: SpaceLike,
+ *,
+ source_graph_path: Optional[Union[str, Path]] = None,
+ target_graph_dims: Optional[int] = None,
+) -> ReembedPlan:
+ """Per-role record counts and the number of vectors a re-embed computes.
+
+ Pass ``source_graph_path`` to include the Kuzu graph. Read-only.
+ """
+ src, tgt = _space(source), _space(target)
+ collections, source_dims = await _vdb(_plan_sync, src, tgt)
+ graph = None
+ if source_graph_path is not None and Path(source_graph_path).exists():
+ graph = await plan_graph_reembed(
+ source_graph_path, target_dims=target_graph_dims or tgt.dims,
+ )
+ return ReembedPlan(
+ collections=tuple(collections),
+ total_vectors=sum(item.source_count for item in collections),
+ source_dims=source_dims if source_dims is not None else src.dims,
+ target_dims=tgt.dims,
+ graph=graph,
+ )
+
+
+# ======================================================================
+# Vector re-embed
+# ======================================================================
+
+@dataclass(frozen=True)
+class ReembedProgress:
+ role: str
+ done: int # source records handled so far
+ total: int # source records in this role
+ embedded: int # vectors computed and written
+ reused: int # skipped: the target already holds an identical record
+ kept: int = 0 # skipped: the target holds a different record with this id (target wins)
+
+
+@dataclass(frozen=True)
+class ReembedReport:
+ per_role: Dict[str, ReembedProgress]
+ elapsed_s: float
+
+ @property
+ def embedded(self) -> int:
+ return sum(item.embedded for item in self.per_role.values())
+
+ @property
+ def reused(self) -> int:
+ return sum(item.reused for item in self.per_role.values())
+
+ @property
+ def kept(self) -> int:
+ return sum(item.kept for item in self.per_role.values())
+
+
+def _default_embedding(target: SpaceLike) -> EmbeddingService:
+ if isinstance(target, MemoryConfig):
+ from .core.embed_service import EmbedService
+ return EmbedService(target)
+ raise ValueError("pass embedding= when the target is a VectorSpace")
+
+
+def _check_service_dims(embedding: Any, dims: int) -> None:
+ getter = getattr(embedding, "get_embedding_dims", None)
+ if not callable(getter):
+ return
+ try:
+ declared = getter()
+ except Exception:
+ return
+ if isinstance(declared, int) and declared and declared != dims:
+ raise EmbeddingDimensionMismatchError(
+ f"embedding service width {declared} != target width {dims}"
+ )
+
+
+def _validate_vectors(vectors: Sequence[Any], expected: int, dims: int, where: str) -> List[List[float]]:
+ if len(vectors) != expected:
+ raise ValueError(f"{where}: expected {expected} vectors, got {len(vectors)}")
+ out: List[List[float]] = []
+ for vector in vectors:
+ width = len(vector) if vector is not None else 0
+ if width != dims:
+ raise EmbeddingDimensionMismatchError(
+ f"{where}: embedding width {width} != target width {dims}"
+ )
+ out.append([float(x) for x in vector])
+ return out
+
+
+async def _embed(embedding: Any, texts: List[str]) -> List[Any]:
+ if not texts:
+ return []
+ return list(await embedding.embed_batch(texts, use_cache=False))
+
+
+def _target_metadata(source_collection, target: VectorSpace) -> Dict[str, Any]:
+ metadata = dict(getattr(source_collection, "metadata", None) or {})
+ metadata.setdefault("hnsw:space", "cosine")
+ metadata.pop(COLLECTION_MODEL_KEY, None)
+ metadata.pop(COLLECTION_DIMS_KEY, None)
+ if target.model:
+ metadata[COLLECTION_MODEL_KEY] = str(target.model)
+ if target.dims:
+ metadata[COLLECTION_DIMS_KEY] = int(target.dims)
+ return metadata
+
+
+async def reembed_vector_space(
+ source: SpaceLike,
+ target: SpaceLike,
+ *,
+ embedding: Optional[EmbeddingService] = None,
+ batch_size: int = 64,
+ max_inflight: int = 2,
+ roles: Optional[Iterable[str]] = None,
+ cancel: Optional[Callable[[], bool]] = None,
+ on_progress: Optional[Callable[[ReembedProgress], None]] = None,
+ skip_existing: bool = True,
+) -> ReembedReport:
+ """Re-embed every record of ``source`` into ``target`` with ``embedding``.
+
+ All users and all layers (L1_RAW included) are read by paging the raw
+ collections. For each record the text from `reembed_text` is embedded and
+ the record is upserted into the target with the same id, document and
+ metadata, copied verbatim.
+
+ - Every vector's width is checked against ``target.dims`` before anything
+ in its batch is written (`EmbeddingDimensionMismatchError`).
+ - ``skip_existing=True`` (default): an id the target already holds is never
+ touched, so a rerun resumes and a merge of a stray space never overwrites
+ live data (the target wins). ``skip_existing=False``: identical records
+ are still skipped, records that differ are re-embedded from the source.
+ - ``cancel`` is checked before each round of up to ``max_inflight`` batches;
+ a cancel raises `ReembedCancelled` with every finished batch written.
+ - The source is never written.
+ """
+ src, tgt = _space(source), _space(target)
+ if not tgt.dims:
+ raise ValueError("the target space needs an explicit width (dims)")
+ dims = int(tgt.dims)
+ service = embedding if embedding is not None else _default_embedding(target)
+ _check_service_dims(service, dims)
+ batch_size = max(1, int(batch_size))
+ max_inflight = max(1, int(max_inflight))
+ cancelled = cancel or (lambda: False)
+ src_names, tgt_names = src.names(), tgt.names()
+ selected = [role for role in _roles(src, roles) if role in tgt_names]
+ started = time.monotonic()
+ per_role: Dict[str, ReembedProgress] = {}
+
+ def _report() -> ReembedReport:
+ return ReembedReport(per_role=dict(per_role), elapsed_s=time.monotonic() - started)
+
+ src_client = await _vdb(_client_for, src)
+ tgt_client = await _vdb(_client_for, tgt)
+ for role in selected:
+ src_name, tgt_name = src_names[role], tgt_names[role]
+ _check_not_same(src, tgt, src_name, tgt_name)
+ src_coll = await _vdb(_get_collection, src_client, src_name)
+ total = await _vdb(src_coll.count) if src_coll is not None else 0
+ progress = ReembedProgress(role=role, done=0, total=total, embedded=0, reused=0, kept=0)
+ per_role[role] = progress
+ if not total:
+ if on_progress:
+ on_progress(progress)
+ continue
+ tgt_coll = await _vdb(
+ tgt_client.get_or_create_collection,
+ name=tgt_name,
+ metadata=_target_metadata(src_coll, tgt),
+ )
+ offset = 0
+ while offset < total:
+ if cancelled():
+ raise ReembedCancelled(report=_report())
+ # One round: read up to max_inflight batches, embed them together,
+ # validate all of them, then write them in order.
+ batches: List[Dict[str, Any]] = []
+ for _ in range(max_inflight):
+ if offset >= total:
+ break
+ got = await _vdb(_page, src_coll, offset, batch_size, ["documents", "metadatas"])
+ ids = list(got.get("ids") or [])
+ if not ids:
+ offset = total
+ break
+ offset += len(ids)
+ batches.append({
+ "ids": ids,
+ "documents": list(got.get("documents") or [None] * len(ids)),
+ "metadatas": list(got.get("metadatas") or [None] * len(ids)),
+ })
+ if not batches:
+ break
+ reused = kept = 0
+ for batch in batches:
+ existing = await _vdb(
+ tgt_coll.get, ids=batch["ids"], include=["documents", "metadatas"],
+ )
+ have = {
+ rid: _record_digest(doc, meta)
+ for rid, doc, meta in zip(
+ existing.get("ids") or [],
+ existing.get("documents") or [],
+ existing.get("metadatas") or [],
+ )
+ }
+ pending = []
+ for i, rid in enumerate(batch["ids"]):
+ doc, meta = batch["documents"][i], batch["metadatas"][i]
+ if rid in have:
+ if have[rid] == _record_digest(doc, meta):
+ reused += 1
+ continue
+ if skip_existing:
+ kept += 1
+ continue
+ pending.append(i)
+ batch["pending"] = pending
+ batch["texts"] = [
+ _embeddable(reembed_text(role, batch["documents"][i], batch["metadatas"][i]))
+ for i in pending
+ ]
+ results = await asyncio.gather(*(_embed(service, b["texts"]) for b in batches))
+ checked = [
+ _validate_vectors(vectors, len(b["pending"]), dims, f"{role}:{tgt_name}")
+ for b, vectors in zip(batches, results)
+ ]
+ embedded = 0
+ for batch, vectors in zip(batches, checked):
+ if not batch["pending"]:
+ continue
+ idx = batch["pending"]
+ await _vdb(
+ tgt_coll.upsert,
+ ids=[batch["ids"][i] for i in idx],
+ embeddings=vectors,
+ documents=[batch["documents"][i] for i in idx],
+ metadatas=[batch["metadatas"][i] for i in idx],
+ )
+ embedded += len(idx)
+ done = sum(len(b["ids"]) for b in batches)
+ progress = ReembedProgress(
+ role=role,
+ done=progress.done + done,
+ total=total,
+ embedded=progress.embedded + embedded,
+ reused=progress.reused + reused,
+ kept=progress.kept + kept,
+ )
+ per_role[role] = progress
+ if on_progress:
+ on_progress(progress)
+ report = _report()
+ logger.info(
+ "[reembed] vector space done: embedded=%d reused=%d kept=%d elapsed=%.1fs",
+ report.embedded, report.reused, report.kept, report.elapsed_s,
+ )
+ return report
+
+
+# ======================================================================
+# Scan and diff (MemHub startup reconcile)
+# ======================================================================
+
+_ROLE_SUFFIX = re.compile(
+ r"^(?P<base>.+?)_(?P<role>coding_keys_cv|coding_keys|working_facts_cv|working_facts)"
+ r"(?:_(?P<dims>\d+))?$"
+)
+_DIMS_SUFFIX = re.compile(r"^(?P<base>.+)_(?P<dims>\d+)$")
+
+
+def parse_collection_name(name: str) -> Optional[Tuple[str, Optional[int], str]]:
+ """Split a physical collection name into (base, dims, role).
+
+ Returns None for collections that are not vector roles (``_tag_index``).
+ """
+ if name.endswith("_tag_index"):
+ return None
+ match = _ROLE_SUFFIX.match(name)
+ if match:
+ dims = match.group("dims")
+ return match.group("base"), int(dims) if dims else None, match.group("role")
+ role = "memory"
+ if name.endswith("_entities"):
+ name, role = name[: -len("_entities")], "entities"
+ match = _DIMS_SUFFIX.match(name)
+ if match:
+ return match.group("base"), int(match.group("dims")), role
+ return name, None, role
+
+
+@dataclass(frozen=True)
+class ScannedCollection:
+ role: str
+ name: str
+ count: int
+ width: Optional[int]
+ per_user: Dict[str, int]
+ models: Tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ScannedSpace:
+ base_name: str
+ dims: Optional[int]
+ roles: Dict[str, ScannedCollection]
+ persist_directory: str
+
+ @property
+ def total(self) -> int:
+ return sum(item.count for item in self.roles.values())
+
+ @property
+ def width(self) -> Optional[int]:
+ """The stored width when every non-empty role agrees, else None."""
+ widths = {item.width for item in self.roles.values() if item.width}
+ return widths.pop() if len(widths) == 1 else None
+
+ @property
+ def per_user(self) -> Dict[str, int]:
+ out: Dict[str, int] = {}
+ for item in self.roles.values():
+ for user, count in item.per_user.items():
+ out[user] = out.get(user, 0) + count
+ return out
+
+ @property
+ def models(self) -> Tuple[str, ...]:
+ return tuple(sorted({m for item in self.roles.values() for m in item.models}))
+
+ def space(self, *, model: Optional[str] = None) -> VectorSpace:
+ """A `VectorSpace` naming exactly the collections found."""
+ return VectorSpace(
+ base_name=self.base_name,
+ dims=self.dims if self.dims is not None else self.width,
+ persist_directory=self.persist_directory,
+ model=model,
+ collections={role: item.name for role, item in self.roles.items()},
+ )
+
+
+def _scan_sync(persist_directory: str, per_user: bool) -> List[ScannedSpace]:
+ if not os.path.isdir(persist_directory):
+ return []
+ import chromadb
+
+ client = chromadb.PersistentClient(path=persist_directory)
+ grouped: Dict[Tuple[str, Optional[int]], Dict[str, ScannedCollection]] = {}
+ for item in client.list_collections():
+ name = item if isinstance(item, str) else item.name
+ parsed = parse_collection_name(name)
+ if parsed is None:
+ continue
+ base, dims, role = parsed
+ collection = client.get_collection(name=name)
+ count = collection.count()
+ users: Dict[str, int] = {}
+ models = set()
+ model = (collection.metadata or {}).get(COLLECTION_MODEL_KEY)
+ if model:
+ models.add(str(model))
+ if per_user and count:
+ for got in _iter_pages(collection, ["metadatas"], page=2000):
+ for meta in got.get("metadatas") or []:
+ meta = meta or {}
+ user = str(meta.get("user_id") or "")
+ users[user] = users.get(user, 0) + 1
+ record_model = meta.get(COLLECTION_MODEL_KEY) or meta.get("embedding_model")
+ if record_model:
+ models.add(str(record_model))
+ grouped.setdefault((base, dims), {})[role] = ScannedCollection(
+ role=role,
+ name=name,
+ count=count,
+ width=_first_width(collection) if count else None,
+ per_user=users,
+ models=tuple(sorted(models)),
+ )
+ return [
+ ScannedSpace(base_name=base, dims=dims, roles=roles, persist_directory=persist_directory)
+ for (base, dims), roles in sorted(grouped.items(), key=lambda kv: (kv[0][0], kv[0][1] or 0))
+ ]
+
+
+async def scan_vector_spaces(
+ persist_directory: Union[str, Path], *, per_user: bool = True,
+) -> List[ScannedSpace]:
+ """Every embedding space with collections under ``persist_directory``.
+
+ Read-only. Collections are grouped by (base, width) from their names; for
+ each role found the scan reports the record count, the stored width (read
+ from one embedding), the per-``user_id`` counts and the models recorded in
+ collection or record metadata (often none: records do not carry the model,
+ only collections created by `reembed_vector_space` do).
+ """
+ return await _vdb(_scan_sync, str(persist_directory), per_user)
+
+
+@dataclass(frozen=True)
+class SpaceDiff:
+ missing: Dict[str, Tuple[str, ...]] # per role: ids in source, absent from target
+
+ @property
+ def total_missing(self) -> int:
+ return sum(len(ids) for ids in self.missing.values())
+
+ @property
+ def empty(self) -> bool:
+ return self.total_missing == 0
+
+
+def _all_ids(collection) -> List[str]:
+ if collection is None:
+ return []
+ out: List[str] = []
+ for got in _iter_pages(collection, [], page=5000):
+ out.extend(got.get("ids") or [])
+ return out
+
+
+def _diff_sync(source: VectorSpace, target: VectorSpace, roles: Optional[Iterable[str]]) -> SpaceDiff:
+ src_client, tgt_client = _client_for(source), _client_for(target)
+ src_names, tgt_names = source.names(), target.names()
+ missing: Dict[str, Tuple[str, ...]] = {}
+ for role in _roles(source, roles):
+ src_ids = _all_ids(_get_collection(src_client, src_names[role]))
+ tgt_ids = set(_all_ids(
+ _get_collection(tgt_client, tgt_names[role]) if role in tgt_names else None
+ ))
+ missing[role] = tuple(rid for rid in src_ids if rid not in tgt_ids)
+ return SpaceDiff(missing=missing)
+
+
+async def diff_spaces(
+ source: SpaceLike, target: SpaceLike, *, roles: Optional[Iterable[str]] = None,
+) -> SpaceDiff:
+ """Per role, the ids the source holds that the target does not.
+
+ Every record counts whatever its ``status`` (deleted and superseded records
+ included). An empty diff means the source holds nothing unmerged.
+ """
+ return await _vdb(_diff_sync, _space(source), _space(target), roles)
+
+
+# ======================================================================
+# Verify and drop
+# ======================================================================
+
+@dataclass(frozen=True)
+class GraphVerification:
+ ok: bool
+ node_counts: Dict[str, Tuple[int, int]] # table: (source, target)
+ rel_counts: Dict[str, Tuple[int, int]]
+ target_width: Optional[int]
+ vector_index: bool
+ problems: Tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ReembedVerification:
+ ok: bool
+ per_role: Dict[str, Dict[str, int]] # source, target, missing, extra, mismatched, bad_dims
+ per_user: Dict[str, int] # target record count by metadata user_id
+ self_recall_hits: int
+ self_recall_samples: int
+ problems: Tuple[str, ...]
+ warnings: Tuple[str, ...] = ()
+ graph: Optional[GraphVerification] = None
+
+
+def _collect(collection, include: Sequence[str], dims: Optional[int]):
+ digests: Dict[str, str] = {}
+ users: Dict[str, int] = {}
+ bad_dims = 0
+ if collection is None:
+ return digests, users, bad_dims
+ for got in _iter_pages(collection, include):
+ ids = got.get("ids") or []
+ docs = got.get("documents") or [None] * len(ids)
+ metas = got.get("metadatas") or [None] * len(ids)
+ embeddings = got.get("embeddings")
+ for i, rid in enumerate(ids):
+ digests[rid] = _record_digest(docs[i], metas[i])
+ user = str((metas[i] or {}).get("user_id") or "")
+ users[user] = users.get(user, 0) + 1
+ if dims is not None and embeddings is not None and len(embeddings[i]) != dims:
+ bad_dims += 1
+ return digests, users, bad_dims
+
+
+def _verify_sync(source: VectorSpace, target: VectorSpace, roles: Optional[Iterable[str]]):
+ src_client, tgt_client = _client_for(source), _client_for(target)
+ src_names, tgt_names = source.names(), target.names()
+ per_role: Dict[str, Dict[str, int]] = {}
+ per_user_src: Dict[str, int] = {}
+ per_user_tgt: Dict[str, int] = {}
+ common: List[Tuple[str, str]] = [] # (role, id)
+ problems: List[str] = []
+ for role in _roles(source, roles):
+ if role not in tgt_names:
+ continue
+ src_digests, src_users, _ = _collect(
+ _get_collection(src_client, src_names[role]), ["documents", "metadatas"], None,
+ )
+ tgt_digests, tgt_users, bad = _collect(
+ _get_collection(tgt_client, tgt_names[role]),
+ ["documents", "metadatas", "embeddings"], int(target.dims or 0) or None,
+ )
+ missing = [rid for rid in src_digests if rid not in tgt_digests]
+ extra = [rid for rid in tgt_digests if rid not in src_digests]
+ mismatched = [
+ rid for rid, digest in src_digests.items()
+ if rid in tgt_digests and tgt_digests[rid] != digest
+ ]
+ per_role[role] = {
+ "source": len(src_digests), "target": len(tgt_digests),
+ "missing": len(missing), "extra": len(extra),
+ "mismatched": len(mismatched), "bad_dims": bad,
+ }
+ for label, values in (("missing", missing), ("extra", extra), ("mismatched", mismatched)):
+ if values:
+ problems.append(f"{role}: {len(values)} {label} (e.g. {values[0]})")
+ if bad:
+ problems.append(f"{role}: {bad} target vectors not {target.dims} wide")
+ for user, count in src_users.items():
+ per_user_src[user] = per_user_src.get(user, 0) + count
+ for user, count in tgt_users.items():
+ per_user_tgt[user] = per_user_tgt.get(user, 0) + count
+ common.extend((role, rid) for rid in src_digests if rid in tgt_digests)
+ for user in sorted(set(per_user_src) | set(per_user_tgt)):
+ if per_user_src.get(user, 0) != per_user_tgt.get(user, 0):
+ problems.append(
+ f"user {user!r}: source {per_user_src.get(user, 0)} != "
+ f"target {per_user_tgt.get(user, 0)}"
+ )
+ return per_role, per_user_tgt, common, problems
+
+
+def _sample_records(target: VectorSpace, picks: List[Tuple[str, str]]):
+ client = _client_for(target)
+ names = target.names()
+ out = []
+ by_role: Dict[str, List[str]] = {}
+ for role, rid in picks:
+ by_role.setdefault(role, []).append(rid)
+ for role, ids in by_role.items():
+ collection = _get_collection(client, names[role])
+ got = collection.get(ids=ids, include=["documents", "metadatas"])
+ for rid, doc, meta in zip(got["ids"], got["documents"], got["metadatas"]):
+ out.append((role, rid, reembed_text(role, doc, meta)))
+ return out
+
+
+def _query_sync(target: VectorSpace, role: str, vectors: List[List[float]], k: int) -> List[List[str]]:
+ collection = _get_collection(_client_for(target), target.names()[role])
+ got = collection.query(query_embeddings=vectors, n_results=k, include=[])
+ return [list(ids) for ids in got.get("ids") or []]
+
+
+async def verify_reembed(
+ source: SpaceLike,
+ target: SpaceLike,
+ *,
+ embedding: Optional[EmbeddingService] = None,
+ samples: int = 50,
+ recall_k: int = 3,
+ roles: Optional[Iterable[str]] = None,
+ seed: Optional[int] = 0,
+ source_graph_path: Optional[Union[str, Path]] = None,
+ target_graph_path: Optional[Union[str, Path]] = None,
+) -> ReembedVerification:
+ """Check a finished re-embed.
+
+ - Per role, the id sets are equal and every id has the same document and
+ metadata on both sides (sha256 of the canonical JSON).
+ - Every target vector is ``target.dims`` wide.
+ - Per ``user_id``, source and target hold the same number of records.
+ - Self-recall: for ``samples`` random records, querying the target with
+ the re-embedded text returns the record in the top ``recall_k``. Below
+ 90% this is a warning, not a failure, because near-duplicates exist.
+ - With both graph paths: `verify_graph_reembed`.
+ """
+ src, tgt = _space(source), _space(target)
+ per_role, per_user, common, problems = await _vdb(_verify_sync, src, tgt, roles)
+ warnings: List[str] = []
+ hits = sampled = 0
+ if samples > 0 and common:
+ service = embedding
+ if service is None and isinstance(target, MemoryConfig):
+ service = _default_embedding(target)
+ if service is None:
+ warnings.append("self-recall skipped: no embedding service")
+ else:
+ rng = random.Random(seed)
+ picks = rng.sample(common, min(samples, len(common)))
+ records = await _vdb(_sample_records, tgt, picks)
+ by_role: Dict[str, List[Tuple[str, str]]] = {}
+ for role, rid, text in records:
+ by_role.setdefault(role, []).append((rid, text))
+ for role, items in by_role.items():
+ vectors = await _embed(service, [_embeddable(text) for _, text in items])
+ vectors = _validate_vectors(vectors, len(items), int(tgt.dims), f"verify:{role}")
+ found = await _vdb(_query_sync, tgt, role, vectors, recall_k)
+ for (rid, _), ids in zip(items, found):
+ sampled += 1
+ hits += int(rid in ids)
+ if sampled and hits / sampled < _SELF_RECALL_WARN_RATE:
+ warnings.append(f"self-recall {hits}/{sampled} below {_SELF_RECALL_WARN_RATE:.0%}")
+ graph = None
+ if source_graph_path is not None and target_graph_path is not None:
+ graph = await verify_graph_reembed(source_graph_path, target_graph_path, target_dims=tgt.dims)
+ problems.extend(f"graph: {p}" for p in graph.problems)
+ return ReembedVerification(
+ ok=not problems,
+ per_role=per_role,
+ per_user=per_user,
+ self_recall_hits=hits,
+ self_recall_samples=sampled,
+ problems=tuple(problems),
+ warnings=tuple(warnings),
+ graph=graph,
+ )
+
+
+def _drop_sync(space: VectorSpace, roles: Optional[Iterable[str]]) -> List[str]:
+ from .data.vector_store_chroma import open_collection_names
+
+ names = space.names()
+ selected = [names[role] for role in _roles(space, roles)]
+ busy = sorted(set(selected) & open_collection_names(space.location()))
+ if busy:
+ raise RuntimeError(
+ f"refusing to drop collections held open by a live client: {busy}"
+ )
+ client = _client_for(space)
+ dropped = []
+ for name in selected:
+ if _get_collection(client, name) is None:
+ continue
+ client.delete_collection(name=name)
+ dropped.append(name)
+ return dropped
+
+
+async def drop_vector_space(
+ space: SpaceLike, *, roles: Optional[Iterable[str]] = None,
+) -> List[str]:
+ """Delete the space's collections and return the names deleted.
+
+ Refuses when a live store in this process holds any of them open.
+ """
+ return await _vdb(_drop_sync, _space(space), roles)
+
+
+# ======================================================================
+# Graph (Kuzu)
+# ======================================================================
+
+# All Kuzu work of a re-embed runs on this one thread: Kuzu connections are not
+# thread-safe and a QueryResult must be consumed on the thread that produced it.
+_graph_executor = concurrent.futures.ThreadPoolExecutor(
+ max_workers=1, thread_name_prefix="kuzu-reembed",
+)
+_GRAPH_PAGE = 2000
+_GRAPH_WRITE_TX = 1000
+_KUZU_SIDECARS = (".wal", ".shadow", ".lock", ".tmp")
+_SDK_VECTOR_INDEX = (
+ "CALL CREATE_VECTOR_INDEX('Memory', 'memory_content_idx', 'embedding', "
+ "metric := 'cosine');"
+)
+
+# Embedding columns rebuilt from text. Everything else is copied verbatim.
+_GRAPH_EMBEDDING_COLUMNS: Dict[str, Tuple[str, ...]] = {
+ "Memory": ("embedding", "beh_embedding"),
+ "Topic": ("embedding",),
+ "HierarchyTopic": ("embedding",),
+}
+
+
+async def _graph(func, *args, **kwargs):
+ import functools
+ return await run_in_executor_guarded(_graph_executor, functools.partial(func, *args, **kwargs))
+
+
+def hierarchy_topic_label(subject: str, path_json: Any, name: str = "") -> Tuple[str, bool]:
+ """Rebuild the label a HierarchyTopic embedding was computed from.
+
+ ``add_topic_path`` embeds ``"{subject}: kind:segment > kind:segment"`` and
+ stores ``path_json`` as ``{"names": [...], "types": [...]}`` for the same
+ prefix, so that form rebuilds exactly (returns ``exact=True``). A legacy
+ list-form ``path_json`` has no stored types: the label is rebuilt with the
+ default Topic -> Facet types the current code would use, and flagged
+ inexact. With no usable path the stored ``name`` is used, also inexact.
+ """
+ from .structure import parse_hierarchy_path_payload
+
+ raw = path_json
+ if isinstance(raw, str):
+ try:
+ raw = json.loads(raw)
+ except (TypeError, ValueError):
+ raw = None
+ parsed = parse_hierarchy_path_payload(path_json or "")
+ path, types = parsed["path"], parsed["node_types"]
+ if not path:
+ fallback = str(name or subject or "").strip()
+ return (f"{subject}: {fallback}" if subject and name else fallback), False
+ exact = isinstance(raw, dict) and isinstance(raw.get("types"), list)
+ label = f"{subject}: " + " > ".join(f"{kind}:{segment}" for segment, kind in zip(path, types))
+ return label, exact
+
+
+def _memory_text(row: Mapping[str, Any], column: str) -> str:
+ custom = _custom_dict(row.get("custom_json"))
+ if column == "beh_embedding":
+ return str(custom.get("beh_abstraction") or "").strip()
+ text = str(custom.get("retrieval_text") or "").strip()
+ return text or str(row.get("content") or "")
+
+
+def _kuzu():
+ try:
+ import kuzu
+ except ImportError as exc: # pragma: no cover - kuzu is a core dependency
+ raise ImportError("kuzu is required. Install with: pip install kuzu") from exc
+ return kuzu
+
+
+def _rows(conn, query: str, params: Optional[Dict[str, Any]] = None) -> List[List[Any]]:
+ result = conn.execute(query, params) if params else conn.execute(query)
+ out = []
+ while result.has_next():
+ out.append(list(result.get_next()))
+ return out
+
+
+def _tables(conn) -> Dict[str, str]:
+ result = conn.execute("CALL show_tables() RETURN *;")
+ columns = result.get_column_names()
+ name_i, type_i = columns.index("name"), columns.index("type")
+ out = {}
+ while result.has_next():
+ row = result.get_next()
+ out[str(row[name_i])] = str(row[type_i]).upper()
+ return out
+
+
+def _table_columns(conn, table: str) -> List[Tuple[str, str, bool]]:
+ """(name, type, is_primary_key) per property, in declaration order."""
+ out = []
+ for row in _rows(conn, f"CALL table_info('{table}') RETURN *;"):
+ out.append((str(row[1]), str(row[2]), row[4] is True))
+ return out
+
+
+def _array_width(type_name: str) -> Optional[int]:
+ match = re.match(r"^FLOAT\[(\d+)\]$", type_name.strip().upper())
+ return int(match.group(1)) if match else None
+
+
+def _count(conn, query: str) -> int:
+ rows = _rows(conn, query)
+ return int(rows[0][0]) if rows else 0
+
+
+def _graph_counts(conn) -> Tuple[Dict[str, int], Dict[str, int]]:
+ nodes, rels = {}, {}
+ for name, kind in _tables(conn).items():
+ if kind == "NODE":
+ nodes[name] = _count(conn, f"MATCH (n:{name}) RETURN count(n);")
+ elif kind == "REL":
+ rels[name] = _count(conn, f"MATCH ()-[r:{name}]->() RETURN count(r);")
+ return nodes, rels
+
+
+def _target_ddl(dims: int) -> List[Tuple[str, str]]:
+ from .data.graph_store_kuzu import _SCHEMA_DDL_TABLES, _make_memory_ddl
+
+ out = []
+ for name, ddl in _SCHEMA_DDL_TABLES.items():
+ out.append((name, _make_memory_ddl(dims) if name == "Memory" else ddl))
+ return out
+
+
+def _open_readonly(path: Path):
+ kuzu = _kuzu()
+ db = kuzu.Database(str(path), read_only=True)
+ return db, kuzu.Connection(db)
+
+
+def _close(conn, db) -> None:
+ for obj in (conn, db):
+ if obj is None:
+ continue
+ try:
+ obj.close()
+ except Exception:
+ pass
+
+
+def _kuzu_files(path: Path) -> List[Path]:
+ out = [path] if path.exists() else []
+ for suffix in _KUZU_SIDECARS:
+ sidecar = path.with_name(path.name + suffix)
+ if sidecar.exists():
+ out.append(sidecar)
+ return out
+
+
+def _remove_kuzu(path: Path) -> None:
+ for item in _kuzu_files(path):
+ if item.is_dir() and not item.is_symlink():
+ shutil.rmtree(item)
+ else:
+ item.unlink()
+
+
+def _source_width(conn) -> Optional[int]:
+ for name, type_name, _ in _table_columns(conn, "Memory"):
+ if name == "embedding":
+ return _array_width(type_name)
+ return None
+
+
+def _assert_not_open(path: Path, what: str) -> None:
+ from .data.graph_store_kuzu import open_graph_paths
+
+ if os.path.realpath(str(path)) in open_graph_paths():
+ raise RuntimeError(f"{what} graph {path} is held open by a live client in this process")
+
+
+def _plan_graph_sync(path: Path, target_dims: Optional[int]) -> GraphReembedPlan:
+ db, conn = _open_readonly(path)
+ try:
+ nodes, rels = _graph_counts(conn)
+ tables = _tables(conn)
+
+ def _non_null(table: str, column: str) -> int:
+ if table not in tables:
+ return 0
+ return _count(conn, f"MATCH (n:{table}) WHERE n.{column} IS NOT NULL RETURN count(n);")
+
+ return GraphReembedPlan(
+ source_path=str(path),
+ node_counts=nodes,
+ rel_counts=rels,
+ memory_embeddings=_non_null("Memory", "embedding"),
+ beh_embeddings=_non_null("Memory", "beh_embedding"),
+ topic_embeddings=_non_null("Topic", "embedding"),
+ hierarchy_topic_embeddings=_non_null("HierarchyTopic", "embedding"),
+ source_width=_source_width(conn) if "Memory" in tables else None,
+ target_width=target_dims,
+ )
+ finally:
+ _close(conn, db)
+
+
+async def plan_graph_reembed(
+ source_graph_path: Union[str, Path], *, target_dims: Optional[int] = None,
+) -> GraphReembedPlan:
+ """Node, edge and embedding counts of a graph. Opens it read-only."""
+ path = Path(source_graph_path)
+ if not path.exists():
+ raise FileNotFoundError(f"graph not found: {path}")
+ _assert_not_open(path, "source")
+ return await _graph(_plan_graph_sync, path, target_dims)
+
+
+@dataclass(frozen=True)
+class GraphReembedProgress:
+ table: str
+ done: int
+ total: int
+ embedded: int
+
+
+@dataclass(frozen=True)
+class GraphReembedReport:
+ target_path: str
+ source_width: Optional[int]
+ target_width: int
+ node_counts: Dict[str, int]
+ rel_counts: Dict[str, int]
+ embedded: Dict[str, int] # "Table.column": vectors computed
+ cleared: Dict[str, int] # "Table.column": vectors set to NULL (no text to rebuild from)
+ hierarchy_labels_exact: int
+ hierarchy_labels_inexact: Tuple[str, ...] # hierarchy_topic_ids rebuilt from a fallback text
+ skipped_tables: Tuple[str, ...] # source tables this SDK does not know
+ dropped_columns: Tuple[str, ...] # "Table.column" in the source but not the target
+ vector_indexes: Tuple[str, ...]
+ elapsed_s: float
+
+
+@dataclass
+class _GraphCopy:
+ source: Path
+ staging: Path
+ dims: int
+ src_db: Any = None
+ src: Any = None
+ dst_db: Any = None
+ dst: Any = None
+ node_tables: List[str] = field(default_factory=list)
+ rel_tables: List[str] = field(default_factory=list)
+ skipped: List[str] = field(default_factory=list)
+ dropped: List[str] = field(default_factory=list)
+ columns: Dict[str, List[str]] = field(default_factory=dict)
+ pk: Dict[str, str] = field(default_factory=dict)
+ source_width: Optional[int] = None
+
+ def open(self) -> None:
+ kuzu = _kuzu()
+ self.src_db, self.src = _open_readonly(self.source)
+ self.staging.parent.mkdir(parents=True, exist_ok=True)
+ self.dst_db = kuzu.Database(str(self.staging))
+ self.dst = kuzu.Connection(self.dst_db)
+ for _, ddl in _target_ddl(self.dims):
+ self.dst.execute(ddl + ";")
+ src_tables, dst_tables = _tables(self.src), _tables(self.dst)
+ self.skipped = sorted(name for name in src_tables if name not in dst_tables)
+ for name, kind in dst_tables.items():
+ if name not in src_tables:
+ continue
+ src_cols = _table_columns(self.src, name)
+ dst_names = {col for col, _, _ in _table_columns(self.dst, name)}
+ self.columns[name] = [col for col, _, _ in src_cols if col in dst_names]
+ self.dropped.extend(f"{name}.{col}" for col, _, _ in src_cols if col not in dst_names)
+ if kind == "NODE":
+ self.node_tables.append(name)
+ self.pk[name] = next(col for col, _, is_pk in src_cols if is_pk)
+ else:
+ self.rel_tables.append(name)
+ if "Memory" in src_tables:
+ self.source_width = _source_width(self.src)
+
+ def node_total(self, table: str) -> int:
+ return _count(self.src, f"MATCH (n:{table}) RETURN count(n);")
+
+ def read_nodes(self, table: str, after: Optional[Any], limit: int) -> List[Dict[str, Any]]:
+ pk = self.pk[table]
+ embed_cols = _GRAPH_EMBEDDING_COLUMNS.get(table, ())
+ plain = [c for c in self.columns[table] if c not in embed_cols]
+ flags = [c for c in embed_cols if c in self.columns[table]]
+ select = [f"n.{c}" for c in plain] + [f"n.{c} IS NOT NULL" for c in flags]
+ where = f"WHERE n.{pk} > $after " if after is not None else ""
+ query = (
+ f"MATCH (n:{table}) {where}RETURN {', '.join(select)} "
+ f"ORDER BY n.{pk} LIMIT {int(limit)};"
+ )
+ params = {"after": after} if after is not None else None
+ out = []
+ for row in _rows(self.src, query, params):
+ record = dict(zip(plain, row[: len(plain)]))
+ record["__has"] = {c: bool(v) for c, v in zip(flags, row[len(plain):])}
+ out.append(record)
+ return out
+
+ def write_nodes(self, table: str, records: List[Dict[str, Any]]) -> None:
+ if not records:
+ return
+ cols = self.columns[table]
+ assigns = ", ".join(f"{c}: $p{i}" for i, c in enumerate(cols))
+ query = f"CREATE (n:{table} {{{assigns}}});"
+ self.dst.execute("BEGIN TRANSACTION;")
+ try:
+ for record in records:
+ self.dst.execute(query, {f"p{i}": record.get(c) for i, c in enumerate(cols)})
+ self.dst.execute("COMMIT;")
+ except Exception:
+ try:
+ self.dst.execute("ROLLBACK;")
+ except Exception:
+ pass
+ raise
+
+ def copy_rels(self, table: str) -> int:
+ props = self.columns[table]
+ copied = 0
+ for row in _rows(self.src, f"CALL show_connection('{table}') RETURN *;"):
+ src_table, dst_table = str(row[0]), str(row[1])
+ src_pk, dst_pk = self.pk.get(src_table), self.pk.get(dst_table)
+ if src_pk is None or dst_pk is None:
+ continue
+ select = ", ".join([f"a.{src_pk}", f"b.{dst_pk}"] + [f"r.{p}" for p in props])
+ result = self.src.execute(
+ f"MATCH (a:{src_table})-[r:{table}]->(b:{dst_table}) RETURN {select};"
+ )
+ assigns = ", ".join(f"{p}: $p{i}" for i, p in enumerate(props))
+ create = (
+ f"MATCH (a:{src_table} {{{src_pk}: $from}}), (b:{dst_table} {{{dst_pk}: $to}}) "
+ f"CREATE (a)-[:{table} {{{assigns}}}]->(b);"
+ )
+ pending = 0
+ self.dst.execute("BEGIN TRANSACTION;")
+ try:
+ while result.has_next():
+ values = list(result.get_next())
+ params = {"from": values[0], "to": values[1]}
+ params.update({f"p{i}": v for i, v in enumerate(values[2:])})
+ self.dst.execute(create, params)
+ copied += 1
+ pending += 1
+ if pending >= _GRAPH_WRITE_TX:
+ self.dst.execute("COMMIT;")
+ self.dst.execute("BEGIN TRANSACTION;")
+ pending = 0
+ self.dst.execute("COMMIT;")
+ except Exception:
+ try:
+ self.dst.execute("ROLLBACK;")
+ except Exception:
+ pass
+ raise
+ return copied
+
+ def create_indexes(self) -> List[str]:
+ """Replay the source's vector indexes, after the data is in."""
+ statements: List[str] = []
+ try:
+ result = self.src.execute("CALL SHOW_INDEXES() RETURN *;")
+ columns = result.get_column_names()
+ def_i = columns.index("index_definition")
+ name_i = columns.index("index_name")
+ table_i = columns.index("table_name")
+ while result.has_next():
+ row = result.get_next()
+ definition = str(row[def_i] or "").strip()
+ if (
+ definition.upper().startswith("CALL CREATE_VECTOR_INDEX(")
+ and str(row[table_i]) in self.node_tables
+ ):
+ statements.append((str(row[name_i]), definition))
+ except Exception as exc:
+ logger.warning("[reembed] could not read source indexes: %s", exc)
+ names = [name for name, _ in statements]
+ if "memory_content_idx" not in names and "Memory" in self.node_tables:
+ statements.append(("memory_content_idx", _SDK_VECTOR_INDEX))
+ built = []
+ for name, statement in statements:
+ self.dst.execute(statement if statement.endswith(";") else statement + ";")
+ built.append(name)
+ return built
+
+ def counts(self) -> Tuple[Dict[str, int], Dict[str, int], Dict[str, int], Dict[str, int]]:
+ src_nodes, src_rels = _graph_counts(self.src)
+ dst_nodes, dst_rels = _graph_counts(self.dst)
+ return src_nodes, src_rels, dst_nodes, dst_rels
+
+ def close(self) -> None:
+ _close(self.src, self.src_db)
+ _close(self.dst, self.dst_db)
+ self.src = self.src_db = self.dst = self.dst_db = None
+
+
+def _publish(staging: Path, target: Path, overwrite: bool) -> None:
+ if _kuzu_files(target):
+ if not overwrite:
+ raise FileExistsError(f"graph target exists: {target}")
+ _remove_kuzu(target)
+ for item in _kuzu_files(staging):
+ suffix = item.name[len(staging.name):]
+ os.replace(item, target.with_name(target.name + suffix))
+
+
+def _staging_path(target: Path) -> Path:
+ return target.with_name(target.name + ".reembed-partial")
+
+
+async def reembed_graph(
+ source_graph_path: Union[str, Path],
+ target_graph_path: Union[str, Path],
+ *,
+ embedding: EmbeddingService,
+ target_dims: int,
+ batch_size: int = 64,
+ cancel: Optional[Callable[[], bool]] = None,
+ on_progress: Optional[Callable[[GraphReembedProgress], None]] = None,
+ overwrite: bool = False,
+) -> GraphReembedReport:
+ """Build a new Kuzu graph at ``target_graph_path`` with ``target_dims`` vectors.
+
+ The source is opened read-only and never modified, so it stays the
+ rollback copy. The new DB gets the SDK's current DDL at the new width; every
+ node and every edge is copied verbatim except the embedding columns, which
+ are recomputed with ``embedding``:
+
+ - ``Memory.embedding``: ``custom.retrieval_text``, else ``content``
+ - ``Memory.beh_embedding``: ``custom.beh_abstraction`` (NULL when missing)
+ - ``Topic.embedding``: ``name``
+ - ``HierarchyTopic.embedding``: the label from `hierarchy_topic_label`
+
+ A NULL source vector stays NULL. Indexes are built after the data is in.
+ The same path is used whether or not the width changes: Kuzu cannot SET an
+ indexed column, so an in-place update is not possible anyway.
+
+ Restart semantics: the copy is built at ``<target>.reembed-partial``, which
+ is wiped at every start and on failure or cancel, then moved to the target
+ only after node and edge counts match the source. A rerun starts over. An
+ existing target raises `FileExistsError` unless ``overwrite=True``.
+ """
+ source = Path(source_graph_path)
+ target = Path(target_graph_path)
+ if os.path.realpath(str(source)) == os.path.realpath(str(target)):
+ raise ValueError("source and target graph paths are the same")
+ if not source.exists():
+ raise FileNotFoundError(f"graph not found: {source}")
+ if _kuzu_files(target) and not overwrite:
+ raise FileExistsError(f"graph target exists: {target}")
+ _assert_not_open(source, "source")
+ _assert_not_open(target, "target")
+ dims = int(target_dims)
+ _check_service_dims(embedding, dims)
+ cancelled = cancel or (lambda: False)
+ page = max(1, int(batch_size))
+ started = time.monotonic()
+ staging = _staging_path(target)
+ await _graph(_remove_kuzu, staging)
+ copy = _GraphCopy(source=source, staging=staging, dims=dims)
+ embedded: Dict[str, int] = {}
+ cleared: Dict[str, int] = {}
+ labels_exact = 0
+ labels_inexact: List[str] = []
+ published = False
+ try:
+ await _graph(copy.open)
+ for table in copy.node_tables:
+ embed_cols = [c for c in _GRAPH_EMBEDDING_COLUMNS.get(table, ()) if c in copy.columns[table]]
+ limit = page if embed_cols else _GRAPH_PAGE
+ total = await _graph(copy.node_total, table)
+ done = computed = 0
+ after = None
+ while done < total:
+ if cancelled():
+ raise ReembedCancelled("graph reembed cancelled")
+ records = await _graph(copy.read_nodes, table, after, limit)
+ if not records:
+ break
+ after = records[-1][copy.pk[table]]
+ for column in embed_cols:
+ wanted, texts = [], []
+ for record in records:
+ record[column] = None
+ if not record["__has"].get(column):
+ continue
+ if table == "Memory":
+ text = _memory_text(record, column)
+ elif table == "Topic":
+ text = str(record.get("name") or "")
+ else:
+ text, exact = hierarchy_topic_label(
+ str(record.get("subject") or ""),
+ record.get("path_json"),
+ str(record.get("name") or ""),
+ )
+ if exact:
+ labels_exact += 1
+ else:
+ labels_inexact.append(str(record.get(copy.pk[table])))
+ if not text.strip():
+ key = f"{table}.{column}"
+ cleared[key] = cleared.get(key, 0) + 1
+ continue
+ wanted.append(record)
+ texts.append(text)
+ vectors = _validate_vectors(
+ await _embed(embedding, texts), len(texts), dims, f"graph:{table}.{column}",
+ )
+ for record, vector in zip(wanted, vectors):
+ record[column] = vector
+ if texts:
+ key = f"{table}.{column}"
+ embedded[key] = embedded.get(key, 0) + len(texts)
+ computed += len(texts)
+ await _graph(copy.write_nodes, table, records)
+ done += len(records)
+ if on_progress:
+ on_progress(GraphReembedProgress(table=table, done=done, total=total, embedded=computed))
+ for table in copy.rel_tables:
+ if cancelled():
+ raise ReembedCancelled("graph reembed cancelled")
+ copied = await _graph(copy.copy_rels, table)
+ if on_progress:
+ on_progress(GraphReembedProgress(table=table, done=copied, total=copied, embedded=0))
+ indexes = await _graph(copy.create_indexes)
+ src_nodes, src_rels, dst_nodes, dst_rels = await _graph(copy.counts)
+ mismatch = [
+ f"{name}: {src_nodes[name]} != {dst_nodes.get(name, 0)}"
+ for name in copy.node_tables if src_nodes.get(name, 0) != dst_nodes.get(name, 0)
+ ] + [
+ f"{name}: {src_rels[name]} != {dst_rels.get(name, 0)}"
+ for name in copy.rel_tables if src_rels.get(name, 0) != dst_rels.get(name, 0)
+ ]
+ if mismatch:
+ raise RuntimeError(f"graph copy count mismatch: {mismatch}")
+ await _graph(copy.close)
+ await _graph(_publish, staging, target, overwrite)
+ published = True
+ finally:
+ await _graph(copy.close)
+ if not published:
+ await _graph(_remove_kuzu, staging)
+ if labels_inexact:
+ logger.warning(
+ "[reembed] %d HierarchyTopic labels rebuilt from a fallback text (legacy path_json)",
+ len(labels_inexact),
+ )
+ return GraphReembedReport(
+ target_path=str(target),
+ source_width=copy.source_width,
+ target_width=dims,
+ node_counts={name: dst_nodes.get(name, 0) for name in copy.node_tables},
+ rel_counts={name: dst_rels.get(name, 0) for name in copy.rel_tables},
+ embedded=embedded,
+ cleared=cleared,
+ hierarchy_labels_exact=labels_exact,
+ hierarchy_labels_inexact=tuple(labels_inexact),
+ skipped_tables=tuple(copy.skipped),
+ dropped_columns=tuple(copy.dropped),
+ vector_indexes=tuple(indexes),
+ elapsed_s=time.monotonic() - started,
+ )
+
+
+def _verify_graph_sync(source: Path, target: Path, dims: Optional[int]) -> GraphVerification:
+ src_db, src = _open_readonly(source)
+ try:
+ dst_db, dst = _open_readonly(target)
+ try:
+ src_nodes, src_rels = _graph_counts(src)
+ dst_nodes, dst_rels = _graph_counts(dst)
+ width = _source_width(dst) if "Memory" in dst_nodes else None
+ try:
+ index_names = [str(row[1]) for row in _rows(dst, "CALL SHOW_INDEXES() RETURN *;")]
+ except Exception:
+ index_names = []
+ problems: List[str] = []
+ node_counts = {}
+ for name in sorted(set(src_nodes) | set(dst_nodes)):
+ pair = (src_nodes.get(name, 0), dst_nodes.get(name, 0))
+ node_counts[name] = pair
+ if pair[0] != pair[1]:
+ problems.append(f"node table {name}: source {pair[0]} != target {pair[1]}")
+ rel_counts = {}
+ for name in sorted(set(src_rels) | set(dst_rels)):
+ pair = (src_rels.get(name, 0), dst_rels.get(name, 0))
+ rel_counts[name] = pair
+ if pair[0] != pair[1]:
+ problems.append(f"rel table {name}: source {pair[0]} != target {pair[1]}")
+ if dims is not None and width != int(dims):
+ problems.append(f"Memory.embedding width {width} != {dims}")
+ has_index = "memory_content_idx" in index_names
+ if "Memory" in dst_nodes and not has_index:
+ problems.append("vector index memory_content_idx missing")
+ for table, column in (("Memory", "embedding"), ("Topic", "embedding"), ("HierarchyTopic", "embedding")):
+ if table not in src_nodes or table not in dst_nodes:
+ continue
+ query = f"MATCH (n:{table}) WHERE n.{column} IS NOT NULL RETURN count(n);"
+ before, after = _count(src, query), _count(dst, query)
+ if before != after:
+ problems.append(f"{table}.{column}: {before} vectors in source, {after} in target")
+ return GraphVerification(
+ ok=not problems,
+ node_counts=node_counts,
+ rel_counts=rel_counts,
+ target_width=width,
+ vector_index=has_index,
+ problems=tuple(problems),
+ )
+ finally:
+ _close(dst, dst_db)
+ finally:
+ _close(src, src_db)
+
+
+async def verify_graph_reembed(
+ source_graph_path: Union[str, Path],
+ target_graph_path: Union[str, Path],
+ *,
+ target_dims: Optional[int] = None,
+) -> GraphVerification:
+ """Node and edge counts per table, target width, vector index, vector counts."""
+ source, target = Path(source_graph_path), Path(target_graph_path)
+ _assert_not_open(source, "source")
+ _assert_not_open(target, "target")
+ return await _graph(_verify_graph_sync, source, target, target_dims)
+
+
+async def drop_graph(graph_path: Union[str, Path]) -> bool:
+ """Delete a Kuzu graph (and its sidecar files). Returns False if absent.
+
+ Refuses when a live store in this process holds it open.
+ """
+ path = Path(graph_path)
+ _assert_not_open(path, "the")
+ if not _kuzu_files(path):
+ return False
+ await _graph(_remove_kuzu, path)
+ return True
+
+
+__all__ = [
+ "VECTOR_ROLES",
+ "VectorRole",
+ "VectorSpace",
+ "EmbeddingSpaceIdentity",
+ "space_identity",
+ "vector_space_names",
+ "reembed_text",
+ "ReembedCancelled",
+ "ReembedCollectionPlan",
+ "ReembedPlan",
+ "plan_reembed",
+ "ReembedProgress",
+ "ReembedReport",
+ "reembed_vector_space",
+ "ScannedCollection",
+ "ScannedSpace",
+ "scan_vector_spaces",
+ "parse_collection_name",
+ "SpaceDiff",
+ "diff_spaces",
+ "ReembedVerification",
+ "verify_reembed",
+ "drop_vector_space",
+ "GraphReembedPlan",
+ "plan_graph_reembed",
+ "GraphReembedProgress",
+ "GraphReembedReport",
+ "reembed_graph",
+ "hierarchy_topic_label",
+ "GraphVerification",
+ "verify_graph_reembed",
+ "drop_graph",
+]
diff --git a/tests/test_reembed.py b/tests/test_reembed.py
new file mode 100644
index 0000000..8fd77a8
--- /dev/null
+++ b/tests/test_reembed.py
@@ -0,0 +1,642 @@
+"""Re-embed into a new embedding space: vector stores, scan, diff and merge.
+
+The switch model: writes are paused, every vector is computed again with the
+new model into a new space, the source is never touched. The graph half lives
+in ``test_reembed_graph.py``.
+"""
+
+import hashlib
+import json
+import math
+import random
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from hy_memory import (
+ EmbeddingDimensionMismatchError,
+ MemoryConfig,
+ ReembedCancelled,
+ VectorSpace,
+ diff_spaces,
+ drop_vector_space,
+ plan_reembed,
+ reembed_text,
+ reembed_vector_space,
+ scan_vector_spaces,
+ space_identity,
+ vector_space_names,
+ verify_reembed,
+)
+from hy_memory.config import EmbedderConfig
+from hy_memory.core.embed_service import EmbedService
+from hy_memory.models.memory import MemoryLayer, MemoryNode, MemoryStatus
+from hy_memory.reembed import VECTOR_ROLES, parse_collection_name
+
+
+class FakeEmbedder:
+ """Deterministic vectors: same (model, text) -> same unit vector."""
+
+ def __init__(self, model: str, dims: int, *, wrong_width_after: int = -1):
+ self.model = model
+ self.dims = dims
+ self.calls = []
+ self.texts = 0
+ self.wrong_width_after = wrong_width_after
+
+ def vector(self, text: str):
+ seed = int(hashlib.sha256(f"{self.model}\0{text}".encode()).hexdigest()[:16], 16)
+ rng = random.Random(seed)
+ values = [rng.gauss(0.0, 1.0) for _ in range(self.dims)]
+ norm = math.sqrt(sum(v * v for v in values)) or 1.0
+ return [v / norm for v in values]
+
+ async def embed_batch(self, texts, use_cache=True):
+ self.calls.append(list(texts))
+ out = []
+ for text in texts:
+ self.texts += 1
+ vector = self.vector(text)
+ if 0 <= self.wrong_width_after < self.texts:
+ vector = vector + [0.0]
+ out.append(vector)
+ return out
+
+ async def embed(self, text, use_cache=True):
+ return self.vector(text)
+
+ async def embed_queued(self, text, use_cache=True):
+ return self.vector(text)
+
+ def get_embedding_dims(self):
+ return self.dims
+
+
+def _client(path):
+ import chromadb
+ return chromadb.PersistentClient(path=str(path))
+
+
+def _space(path, base, dims, model=None):
+ return VectorSpace(base_name=base, dims=dims, persist_directory=str(path), model=model)
+
+
+def _custom_state():
+ return {
+ "memory_kind": "canonical_state",
+ "state_key": "home_city",
+ "state_value": "Berlin",
+ "state_retrieval_text": "user lives in Berlin (home city)",
+ }
+
+
+async def _seed(path, base, dims, *, model="old-model", users=("alice", "bob"), per_user=7):
+ """Write a realistic source space through the SDK's own stores."""
+ from hy_memory.data.vector_store_chroma import ChromaVectorStore
+
+ old = FakeEmbedder(model, dims)
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "chroma"
+ cfg.vector_store.persist_directory = str(path)
+ cfg.vector_store.collection_name = base
+ cfg.vector_store.embedding_dims = dims
+ store = ChromaVectorStore(cfg)
+ await store.initialize()
+ nodes = []
+ for user in users:
+ for i in range(per_user):
+ layer = [MemoryLayer.L1_RAW, MemoryLayer.L2_FACT, MemoryLayer.L3_SUMMARY][i % 3]
+ node = MemoryNode(
+ user_id=user, agent_id="a", session_id=f"s{i % 2}",
+ layer=layer, content=f"{user} fact {i}",
+ status=MemoryStatus.SUPERSEDED if i == 4 else MemoryStatus.ACTIVE,
+ tags=["t1", f"t{i}"],
+ custom={"k": i, "nested": {"x": [1, 2]}},
+ )
+ if i == 4:
+ node.is_latest = False
+ node.superseded_by = ["later"]
+ if i == 5:
+ node.custom = _custom_state()
+ node.embedding = old.vector(node.content)
+ nodes.append(node)
+ await store.upsert_batch(nodes)
+ entities = await store._get_entity_collection()
+ entities.add(
+ ids=["e1", "e2"],
+ embeddings=[old.vector("Berlin"), old.vector("Dylan")],
+ documents=["Berlin", "Dylan"],
+ metadatas=[{"user_id": "alice", "label": "GPE"}, {"user_id": "bob", "label": "PERSON"}],
+ )
+ await store.close()
+ client = _client(path)
+ names = vector_space_names(_space(path, base, dims))
+ for role, doc in (("coding_keys", "fix the flaky test"), ("working_facts_cv", "repo: uses uv")):
+ coll = client.get_or_create_collection(names[role], metadata={"hnsw:space": "cosine"})
+ coll.add(
+ ids=[f"{role}:0"],
+ embeddings=[old.vector(doc)],
+ documents=[doc],
+ metadatas=[{"user_id": "alice", "layer": "l1_raw",
+ "custom": json.dumps({"coding_key_text": doc})}],
+ )
+ return names
+
+
+def _snapshot(path, names):
+ client = _client(path)
+ out = {}
+ for role, name in names.items():
+ try:
+ coll = client.get_collection(name)
+ except Exception:
+ continue
+ got = coll.get(include=["documents", "metadatas", "embeddings"])
+ out[role] = sorted(
+ (rid, doc, json.dumps(meta, sort_keys=True), [round(float(x), 6) for x in emb])
+ for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])
+ )
+ return hashlib.sha256(json.dumps(out, sort_keys=True).encode()).hexdigest()
+
+
+def _records(path, name):
+ got = _client(path).get_collection(name).get(include=["documents", "metadatas", "embeddings"])
+ return {
+ rid: (doc, meta, list(emb))
+ for rid, doc, meta, emb in zip(got["ids"], got["documents"], got["metadatas"], got["embeddings"])
+ }
+
+
+# ----------------------------------------------------------------------
+# naming
+# ----------------------------------------------------------------------
+
+class TestVectorSpaceNames:
+ @pytest.mark.parametrize("dims", [512, None])
+ async def test_names_equal_what_the_stores_open(self, tmp_path, dims):
+ from hy_memory.coding.store import CodingMemoryStore
+ from hy_memory.data.vector_store_chroma import ChromaVectorStore
+
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "chroma"
+ cfg.vector_store.persist_directory = str(tmp_path / "v")
+ cfg.vector_store.collection_name = "memhub_abc123"
+ cfg.vector_store.embedding_dims = dims
+ cfg.history.db_path = str(tmp_path / "history.sqlite3")
+ store = ChromaVectorStore(cfg)
+ await store.initialize()
+ await store._get_entity_collection()
+ coding = []
+ for variant in ("working", "coding"):
+ cs = CodingMemoryStore(cfg, FakeEmbedder("m", dims or 8), variant=variant)
+ await cs.initialize()
+ coding.append(cs)
+ opened = {c.name for c in _client(tmp_path / "v").list_collections()}
+ names = vector_space_names(cfg)
+ assert set(names) == set(VECTOR_ROLES)
+ assert set(names.values()) == opened
+ suffix = f"_{dims}" if dims else ""
+ assert names["memory"] == f"memhub_abc123{suffix}"
+ assert names["coding_keys_cv"] == f"memhub_abc123_coding_keys_cv{suffix}"
+ await store.close()
+ for cs in coding:
+ await cs.close()
+
+ def test_explicit_base_name_and_space(self, tmp_path):
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "chroma"
+ cfg.vector_store.embedding_dims = 1024
+ names = vector_space_names(cfg, base_name="staging_x")
+ assert names["memory"] == "staging_x_1024"
+ assert names["entities"] == "staging_x_1024_entities"
+ assert names["working_facts"] == "staging_x_working_facts_1024"
+ assert vector_space_names(_space(tmp_path, "b", 512))["memory"] == "b_512"
+
+ def test_parse_collection_name_round_trips(self):
+ for dims in (512, None):
+ for role, name in vector_space_names(VectorSpace("memhub_9f", dims)).items():
+ assert parse_collection_name(name) == ("memhub_9f", dims, role)
+ assert parse_collection_name("memhub_9f_512_tag_index") is None
+
+ def test_space_identity_is_model_and_width_only(self):
+ a = space_identity("BAAI/bge-small-zh-v1.5", 512)
+ b = space_identity(" BAAI/bge-small-zh-v1.5 ", 512)
+ assert a == b and a.key == "BAAI/bge-small-zh-v1.5#512"
+ assert a.digest != space_identity("BAAI/bge-small-zh-v1.5", 1024).digest
+ with pytest.raises(ValueError):
+ space_identity("", 512)
+
+ def test_non_chroma_provider_is_not_supported(self):
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "qdrant"
+ with pytest.raises(NotImplementedError):
+ vector_space_names(cfg)
+
+
+class TestReembedText:
+ def test_canonical_state_uses_state_retrieval_text(self):
+ meta = {"content": "住在柏林", "custom": json.dumps(_custom_state())}
+ assert reembed_text("memory", "住在柏林", meta) == "user lives in Berlin (home city)"
+
+ def test_retrieval_text_when_present(self):
+ meta = {"custom": json.dumps({"retrieval_text": " pattern: plans ahead "})}
+ assert reembed_text("memory", "画像", meta) == "pattern: plans ahead"
+
+ def test_document_otherwise_with_content_fallback(self):
+ assert reembed_text("memory", "plain fact", {"custom": "{}"}) == "plain fact"
+ assert reembed_text("memory", None, {"content": "from metadata"}) == "from metadata"
+ assert reembed_text("memory", "doc", {"custom": "not json"}) == "doc"
+
+ def test_coding_key_and_working_fact_use_the_document(self):
+ coding = {"custom": json.dumps({"coding_key_text": "fix flaky test",
+ "retrieval_text": "ignored outside memory"})}
+ assert reembed_text("coding_keys", "fix flaky test", coding) == "fix flaky test"
+ assert reembed_text("working_facts", "repo: uses uv", {}) == "repo: uses uv"
+
+
+# ----------------------------------------------------------------------
+# re-embed
+# ----------------------------------------------------------------------
+
+ "src_dims,tgt_dims,src_model,tgt_model",
+ [(512, 512, "old-model", "new-model"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],
+)
+async def test_reembed_copies_everything_and_never_touches_source(
+ tmp_path, src_dims, tgt_dims, src_model, tgt_model,
+):
+ path = tmp_path / "v"
+ names = await _seed(path, "memhub_src", src_dims, model=src_model)
+ before = _snapshot(path, names)
+ source = _space(path, "memhub_src", src_dims)
+ target = _space(path, "memhub_dst", tgt_dims, model=tgt_model)
+ new = FakeEmbedder(tgt_model, tgt_dims)
+
+ plan = await plan_reembed(source, target)
+ assert plan.per_role["memory"] == 14 and plan.per_role["entities"] == 2
+ assert plan.total_vectors == 18 == plan.estimated_vectors
+ assert plan.source_dims == src_dims and plan.target_dims == tgt_dims
+
+ seen = []
+ report = await reembed_vector_space(
+ source, target, embedding=new, batch_size=4, on_progress=seen.append,
+ )
+ assert report.embedded == 18 and report.reused == 0
+ assert seen and seen[-1].done == seen[-1].total
+
+ assert _snapshot(path, names) == before # source unchanged, byte for byte
+ target_names = vector_space_names(target)
+ for role in ("memory", "entities", "coding_keys", "working_facts_cv"):
+ src = _records(path, names[role])
+ dst = _records(path, target_names[role])
+ assert set(src) == set(dst)
+ for rid, (doc, meta, _) in src.items():
+ assert dst[rid][0] == doc
+ assert dst[rid][1] == meta # status, is_latest, superseded_by, timestamps, custom
+ assert len(dst[rid][2]) == tgt_dims
+ assert dst[rid][2] == pytest.approx(new.vector(reembed_text(role, doc, meta)), abs=1e-5)
+ memory = _records(path, target_names["memory"])
+ layers = {meta["layer"] for _, meta, _ in memory.values()}
+ users = {meta["user_id"] for _, meta, _ in memory.values()}
+ assert "l1_raw" in layers and users == {"alice", "bob"}
+ statuses = {meta["status"] for _, meta, _ in memory.values()}
+ assert statuses == {"active", "superseded"}
+ state = [r for r in memory.values() if "canonical_state" in r[1]["custom"]]
+ assert state and state[0][2] == pytest.approx(
+ new.vector("user lives in Berlin (home city)"), abs=1e-5,
+ )
+ meta = _client(path).get_collection(target_names["memory"]).metadata
+ assert meta["hnsw:space"] == "cosine" and meta["hy_embedding_model"] == tgt_model
+
+ verified = await verify_reembed(source, target, embedding=new)
+ assert verified.ok, verified.problems
+ assert verified.per_user == {"alice": 10, "bob": 8}
+ assert verified.self_recall_samples == 18 and verified.self_recall_hits == 18
+
+
+async def test_resume_after_kill_only_embeds_the_rest(tmp_path):
+ path = tmp_path / "v"
+ await _seed(path, "src", 512, per_user=20)
+ source, target = _space(path, "src", 512), _space(path, "dst", 1024)
+
+ class Killed(Exception):
+ pass
+
+ first = FakeEmbedder("new", 1024)
+ batches = {"n": 0}
+
+ def _kill():
+ batches["n"] += 1
+ if batches["n"] > 3:
+ raise Killed()
+ return False
+
+ with pytest.raises(Killed):
+ await reembed_vector_space(
+ source, target, embedding=first, batch_size=5, max_inflight=1,
+ roles=["memory"], cancel=_kill,
+ )
+ assert first.texts == 15
+ second = FakeEmbedder("new", 1024)
+ report = await reembed_vector_space(
+ source, target, embedding=second, batch_size=5, max_inflight=1, roles=["memory"],
+ )
+ assert second.texts == 40 - 15
+ assert report.per_role["memory"].reused == 15
+ # The end state equals an uninterrupted run: every source record, verbatim,
+ # with the vector of its own text.
+ resumed = _records(path, "dst_1024")
+ src = _records(path, "src_512")
+ assert set(resumed) == set(src)
+ for rid, (doc, meta, vector) in resumed.items():
+ assert (doc, meta) == src[rid][:2]
+ assert vector == pytest.approx(second.vector(reembed_text("memory", doc, meta)), abs=1e-5)
+ assert (await verify_reembed(source, target, roles=["memory"], samples=0)).ok
+
+
+async def test_cancel_at_batch_boundary_leaves_no_partial_batch(tmp_path):
+ path = tmp_path / "v"
+ await _seed(path, "src", 512)
+ calls = {"n": 0}
+
+ def _cancel():
+ calls["n"] += 1
+ return calls["n"] > 2
+
+ with pytest.raises(ReembedCancelled) as info:
+ await reembed_vector_space(
+ _space(path, "src", 512), _space(path, "dst", 512),
+ embedding=FakeEmbedder("new", 512), batch_size=3, max_inflight=1,
+ roles=["memory"], cancel=_cancel,
+ )
+ assert info.value.report.per_role["memory"].embedded == 6
+ assert _client(path).get_collection("dst_512").count() == 6
+
+
+async def test_wrong_width_raises_before_any_upsert(tmp_path):
+ path = tmp_path / "v"
+ await _seed(path, "src", 512)
+ bad = FakeEmbedder("new", 1024, wrong_width_after=2)
+ with pytest.raises(EmbeddingDimensionMismatchError):
+ await reembed_vector_space(
+ _space(path, "src", 512), _space(path, "dst", 1024),
+ embedding=bad, batch_size=4, roles=["memory"],
+ )
+ assert _client(path).get_collection("dst_1024").count() == 0
+ declared_wrong = FakeEmbedder("new", 768)
+ with pytest.raises(EmbeddingDimensionMismatchError):
+ await reembed_vector_space(
+ _space(path, "src", 512), _space(path, "dst", 1024), embedding=declared_wrong,
+ )
+
+
+async def test_refuses_to_write_into_the_source_collection(tmp_path):
+ path = tmp_path / "v"
+ await _seed(path, "src", 512)
+ with pytest.raises(ValueError):
+ await reembed_vector_space(
+ _space(path, "src", 512), _space(path, "src", 512),
+ embedding=FakeEmbedder("new", 512),
+ )
+
+
+async def test_verify_detects_missing_extra_drift_and_width(tmp_path):
+ path = tmp_path / "v"
+ await _seed(path, "src", 512)
+ source, target = _space(path, "src", 512), _space(path, "dst", 512)
+ new = FakeEmbedder("new", 512)
+ await reembed_vector_space(source, target, embedding=new, roles=["memory"])
+ coll = _client(path).get_collection("dst_512")
+ got = coll.get(include=["documents", "metadatas"])
+ victim, drifted = got["ids"][0], got["ids"][1]
+ coll.delete(ids=[victim])
+ meta = dict(got["metadatas"][1])
+ meta["status"] = "archived"
+ coll.update(ids=[drifted], metadatas=[meta])
+ coll.add(ids=["stranger"], embeddings=[new.vector("x")], documents=["x"], metadatas=[{"user_id": "eve"}])
+ result = await verify_reembed(source, target, embedding=new, roles=["memory"], samples=5)
+ assert not result.ok
+ counts = result.per_role["memory"]
+ assert (counts["missing"], counts["extra"], counts["mismatched"]) == (1, 1, 1)
+ assert any("eve" in p for p in result.problems)
+
+ wide = _space(path, "dst", 1024)
+ wide_coll = _client(path).get_or_create_collection("dst_1024", metadata={"hnsw:space": "cosine"})
+ src = _records(path, "src_512")
+ wide_coll.add(
+ ids=list(src),
+ embeddings=[[0.0] * 512 for _ in src],
+ documents=[v[0] for v in src.values()],
+ metadatas=[v[1] for v in src.values()],
+ )
+ result = await verify_reembed(source, wide, roles=["memory"], samples=0)
+ assert result.per_role["memory"]["bad_dims"] == len(src) and not result.ok
+
+
+async def test_drop_vector_space_refuses_live_collections(tmp_path):
+ from hy_memory.data.vector_store_chroma import ChromaVectorStore
+
+ path = tmp_path / "v"
+ await _seed(path, "src", 512)
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "chroma"
+ cfg.vector_store.persist_directory = str(path)
+ cfg.vector_store.collection_name = "src"
+ cfg.vector_store.embedding_dims = 512
+ live = ChromaVectorStore(cfg)
+ await live.initialize()
+ with pytest.raises(RuntimeError):
+ await drop_vector_space(_space(path, "src", 512))
+ await live.close()
+ dropped = await drop_vector_space(_space(path, "src", 512))
+ assert set(dropped) == {"src_512", "src_512_entities", "src_coding_keys_512", "src_working_facts_cv_512"}
+ assert not {c.name for c in _client(path).list_collections()}
+
+
+# ----------------------------------------------------------------------
+# scan, diff, merge
+# ----------------------------------------------------------------------
+
+async def _two_spaces(path):
+ """A live 1024 space and a stray 512 space sharing some ids."""
+ from hy_memory.data.vector_store_chroma import ChromaVectorStore
+
+ async def _store(base, dims, model, contents):
+ cfg = MemoryConfig()
+ cfg.vector_store.provider = "chroma"
+ cfg.vector_store.persist_directory = str(path)
+ cfg.vector_store.collection_name = base
+ cfg.vector_store.embedding_dims = dims
+ store = ChromaVectorStore(cfg)
+ await store.initialize()
+ emb = FakeEmbedder(model, dims)
+ nodes = []
+ for node_id, user, content, status in contents:
+ node = MemoryNode(
+ node_id=node_id, user_id=user, agent_id="a", session_id="s",
+ layer=MemoryLayer.L2_FACT, content=content, status=status,
+ )
+ node.embedding = emb.vector(content)
+ nodes.append(node)
+ await store.upsert_batch(nodes)
+ await store.close()
+
+ ids = [f"00000000-0000-0000-0000-00000000000{i}" for i in range(6)]
+ await _store("memhub_old", 512, "bge", [
+ (ids[0], "alice", "shared one", MemoryStatus.ACTIVE),
+ (ids[1], "alice", "shared two (stray wording)", MemoryStatus.ACTIVE),
+ (ids[2], "bob", "only in stray", MemoryStatus.ACTIVE),
+ (ids[3], "bob", "only in stray, deleted", MemoryStatus.SHADOW),
+ ])
+ await _store("memhub_new", 1024, "e5", [
+ (ids[0], "alice", "shared one", MemoryStatus.ACTIVE),
+ (ids[1], "alice", "shared two (live wording)", MemoryStatus.ARCHIVED),
+ (ids[4], "carol", "only in live", MemoryStatus.ACTIVE),
+ ])
+ return ids
+
+
+async def test_scan_diff_and_merge(tmp_path):
+ path = tmp_path / "v"
+ ids = await _two_spaces(path)
+ spaces = await scan_vector_spaces(path)
+ by_base = {(s.base_name, s.dims): s for s in spaces}
+ assert set(by_base) == {("memhub_old", 512), ("memhub_new", 1024)}
+ old, live = by_base[("memhub_old", 512)], by_base[("memhub_new", 1024)]
+ assert old.roles["memory"].count == 4 and old.width == 512
+ assert live.roles["memory"].count == 3 and live.width == 1024
+ assert old.per_user == {"alice": 2, "bob": 2}
+ assert live.per_user == {"alice": 2, "carol": 1}
+
+ stray, target = old.space(), _space(path, "memhub_new", 1024, model="e5")
+ diff = await diff_spaces(stray, target)
+ assert set(diff.missing["memory"]) == {ids[2], ids[3]} # status ignored
+ assert diff.total_missing == 2
+
+ live_before = _records(path, "memhub_new_1024")
+ stray_before = _snapshot(path, stray.names())
+ merger = FakeEmbedder("e5", 1024)
+ report = await reembed_vector_space(stray, target, embedding=merger)
+ assert merger.texts == 2 and report.embedded == 2
+ assert report.reused == 1 and report.kept == 1
+ after = _records(path, "memhub_new_1024")
+ assert set(after) == set(live_before) | {ids[2], ids[3]}
+ for rid, record in live_before.items(): # the target wins
+ assert after[rid][:2] == record[:2]
+ assert after[rid][2] == pytest.approx(record[2], abs=1e-6)
+ assert _snapshot(path, stray.names()) == stray_before
+ assert (await diff_spaces(stray, target)).empty
+
+ again = FakeEmbedder("e5", 1024)
+ second = await reembed_vector_space(stray, target, embedding=again)
+ assert again.texts == 0 and second.embedded == 0
+ assert _records(path, "memhub_new_1024").keys() == after.keys()
+ rescanned = {(s.base_name, s.dims): s for s in await scan_vector_spaces(path)}
+ assert rescanned[("memhub_new", 1024)].roles["memory"].count == 5
+
+
+async def test_scan_of_missing_directory_is_empty(tmp_path):
+ assert await scan_vector_spaces(tmp_path / "nope") == []
+
+
+# ----------------------------------------------------------------------
+# EmbedService fixes
+# ----------------------------------------------------------------------
+
+def _fake_raw(n, dims):
+ items = []
+ for i in range(n):
+ item = MagicMock()
+ item.embedding = [0.1] * dims
+ item.index = i
+ items.append(item)
+ resp = MagicMock()
+ resp.model = "m"
+ resp.usage = None
+ resp.data = items
+ raw = MagicMock()
+ raw.status_code = 200
+ raw.headers = {}
+ raw.parse.return_value = resp
+ return raw
+
+
+def _service(dims=8, **embedder):
+ cfg = MemoryConfig(embedder=EmbedderConfig(
+ provider="openai", model="m", api_key="sk", base_url="https://e.invalid/v1",
+ embedding_dims=dims, max_retries=2, retry_delay=0.0, **embedder,
+ ))
+ return EmbedService(cfg)
+
+
+async def test_embed_batch_chunks_to_batch_size():
+ svc = _service()
+ sizes = []
+
+ async def _create(**kwargs):
+ sizes.append(len(kwargs["input"]))
+ return _fake_raw(len(kwargs["input"]), 8)
+
+ client = MagicMock()
+ client.embeddings.with_raw_response.create = AsyncMock(side_effect=_create)
+ with patch("openai.AsyncOpenAI", return_value=client):
+ vectors = await svc.embed_batch([f"t{i}" for i in range(1000)], use_cache=False)
+ assert len(vectors) == 1000
+ assert len(sizes) == math.ceil(1000 / 64) and max(sizes) == 64
+
+ small = _service(batch_size=10)
+ sizes.clear()
+ with patch("openai.AsyncOpenAI", return_value=client):
+ _, metas = await small.embed_batch_with_meta([f"u{i}" for i in range(25)], use_cache=False)
+ assert sizes == [10, 10, 5]
+ assert [m["batch_index"] for m in metas] == list(range(25))
+
+
+async def test_429_honours_retry_after():
+ svc = _service()
+
+ class RateLimited(Exception):
+ status_code = 429
+
+ def __init__(self):
+ super().__init__("rate limited")
+ self.response = MagicMock()
+ self.response.headers = {"retry-after": "2"}
+
+ client = MagicMock()
+ client.embeddings.with_raw_response.create = AsyncMock(
+ side_effect=[RateLimited(), _fake_raw(1, 8)],
+ )
+ sleeps = []
+
+ async def _sleep(seconds):
+ sleeps.append(seconds)
+
+ with patch("openai.AsyncOpenAI", return_value=client), \
+ patch("hy_memory.core.embed_service.asyncio.sleep", _sleep):
+ vectors = await svc.embed_batch(["x"], use_cache=False)
+ assert len(vectors) == 1
+ assert len(sleeps) == 1 and 2.0 <= sleeps[0] <= 2.5
+
+
+async def test_per_call_max_retries():
+ svc = _service()
+
+ class Down(Exception):
+ status_code = 503
+
+ client = MagicMock()
+ client.embeddings.with_raw_response.create = AsyncMock(side_effect=Down("down"))
+ with patch("openai.AsyncOpenAI", return_value=client), pytest.raises(Down):
+ await svc.embed_batch(["x"], use_cache=False, max_retries=5)
+ assert client.embeddings.with_raw_response.create.await_count == 6
+
+
+async def test_returned_width_is_validated_and_not_retried():
+ svc = _service(dims=8)
+ client = MagicMock()
+ client.embeddings.with_raw_response.create = AsyncMock(return_value=_fake_raw(2, 7))
+ with patch("openai.AsyncOpenAI", return_value=client), \
+ pytest.raises(EmbeddingDimensionMismatchError):
+ await svc.embed_batch(["a", "b"], use_cache=False)
+ assert client.embeddings.with_raw_response.create.await_count == 1
diff --git a/tests/test_reembed_graph.py b/tests/test_reembed_graph.py
new file mode 100644
index 0000000..34f7549
--- /dev/null
+++ b/tests/test_reembed_graph.py
@@ -0,0 +1,252 @@
+"""Re-embed the Kuzu graph into a new embedding space.
+
+The source graph is opened read-only and never modified. A new graph is built
+at the target path with the target width: every node and edge is copied, the
+embedding columns are recomputed from their text, the HNSW index is rebuilt.
+Runs in CI: kuzu is a core dependency.
+"""
+
+import hashlib
+import json
+import os
+
+import pytest
+
+from hy_memory import (
+ EmbeddingDimensionMismatchError,
+ MemoryConfig,
+ ReembedCancelled,
+ drop_graph,
+ reembed_graph,
+ verify_graph_reembed,
+)
+from hy_memory.models.memory import MemoryLayer, MemoryNode
+from hy_memory.reembed import hierarchy_topic_label, plan_graph_reembed
+
+from .test_reembed import FakeEmbedder
+
+
+def _cfg(path, dims):
+ cfg = MemoryConfig()
+ cfg.graph_store.provider = "kuzu"
+ cfg.graph_store.db_path = str(path)
+ cfg.vector_store.embedding_dims = dims
+ return cfg
+
+
+async def _open(path, dims):
+ from hy_memory.data.graph_store_kuzu import KuzuGraphStore
+
+ store = KuzuGraphStore(_cfg(path, dims))
+ await store.initialize()
+ assert store._available
+ return store
+
+
+def _node(content, *, layer=MemoryLayer.L2_FACT, custom=None, user="alice"):
+ return MemoryNode(
+ user_id=user, agent_id="a", session_id="s1",
+ layer=layer, content=content, custom=custom or {},
+ )
+
+
+async def _seed_graph(path, dims, model="old"):
+ """A graph with every node and edge type the SDK writes."""
+ old = FakeEmbedder(model, dims)
+ gs = await _open(path, dims)
+ schema = _node(
+ "画像:喜欢提前规划", layer=MemoryLayer.L6_SCHEMA,
+ custom={"schema_type": "basic", "retrieval_text": "plans trips weeks ahead",
+ "_has_beh": True, "beh_abstraction": "prepares early under uncertainty"},
+ )
+ schema._graph_embedding = old.vector("plans trips weeks ahead")
+ core = _node("core: anticipates", layer=MemoryLayer.L6_SCHEMA, custom={"schema_type": "abstract"})
+ core._graph_embedding = old.vector("core: anticipates")
+ stale_beh = _node("beh without text", layer=MemoryLayer.L6_SCHEMA, custom={})
+ stale_beh._graph_embedding = old.vector("beh without text")
+ facts = [_node(f"fact {i}", user="bob" if i % 2 else "alice") for i in range(5)]
+ for node in [schema, core, stale_beh, *facts]:
+ await gs.upsert_memory_node(node)
+ await gs.update_embedding(schema.node_id, beh_embedding=old.vector("prepares early under uncertainty"))
+ await gs.update_embedding(stale_beh.node_id, beh_embedding=old.vector("lost text"))
+ await gs.add_cross_abstracts_to(schema.node_id, core.node_id)
+ ik = facts[0].get_isolation_key()
+ await gs.add_edge(facts[0].node_id, facts[1].node_id, "RELATED_TO", {"relation_type": "co_occurs", "weight": 0.7})
+ await gs.add_temporal_edge(facts[0].node_id, facts[2].node_id, topic_id="t")
+ for fact in facts[:3]:
+ await gs.add_topic_tag(fact.node_id, "travel", ik, embed_service=old)
+ await gs.add_topic_tag(facts[3].node_id, "work", ik) # no embed service: NULL embedding
+ await gs.add_topic_path(
+ facts[0].node_id, ["Travel", "Japan trip"], ik, "alice",
+ node_types=["topic", "series"], embed_service=old,
+ )
+ await gs.add_topic_path(facts[1].node_id, ["Work", "Standups"], ik, "alice", embed_service=old)
+ await gs.add_entity_mention(facts[0].node_id, "Kyoto", "GPE", ik)
+ await gs.ensure_vdbref("raw-1", "l1_raw")
+ await gs.add_derived_from(schema.node_id, "raw-1")
+ await gs.close()
+ return old, {"schema": schema, "core": core, "stale_beh": stale_beh, "facts": facts}
+
+
+def _digest(path):
+ """Hash of the Kuzu DB file and any sidecar files next to it."""
+ h = hashlib.sha256()
+ for name in sorted(os.listdir(path.parent)):
+ if name == path.name or name.startswith(path.name + "."):
+ h.update(name.encode())
+ with open(path.parent / name, "rb") as handle:
+ h.update(handle.read())
+ return h.hexdigest()
+
+
+def _read(path, query):
+ import kuzu
+
+ db = kuzu.Database(str(path), read_only=True)
+ conn = kuzu.Connection(db)
+ try:
+ result = conn.execute(query)
+ rows = []
+ while result.has_next():
+ rows.append(result.get_next())
+ return rows
+ finally:
+ conn.close()
+ db.close()
+
+
+ "src_dims,tgt_dims,src_model,tgt_model",
+ [(512, 512, "old", "new"), (512, 1024, "old", "new"), (1024, 512, "old", "new")],
+)
+async def test_reembed_graph_copies_everything_at_the_new_width(
+ tmp_path, src_dims, tgt_dims, src_model, tgt_model,
+):
+ source, target = tmp_path / "graph", tmp_path / "graph-new"
+ _, nodes = await _seed_graph(source, src_dims, model=src_model)
+ before = _digest(source)
+ plan = await plan_graph_reembed(source, target_dims=tgt_dims)
+ assert plan.source_width == src_dims and plan.target_width == tgt_dims
+ assert plan.memory_nodes == 8 and plan.topics == 2 and plan.hierarchy_topics == 4
+ assert plan.memory_embeddings == 3 and plan.beh_embeddings == 2
+ assert plan.topic_embeddings == 1 and plan.hierarchy_topic_embeddings == 4
+
+ new = FakeEmbedder(tgt_model, tgt_dims)
+ seen = []
+ report = await reembed_graph(
+ source, target, embedding=new, target_dims=tgt_dims, batch_size=3, on_progress=seen.append,
+ )
+ assert _digest(source) == before # the source graph is untouched
+ assert not os.path.exists(str(target) + ".reembed-partial")
+ assert report.source_width == src_dims and report.target_width == tgt_dims
+ assert report.node_counts == plan.node_counts
+ assert report.rel_counts == plan.rel_counts
+ assert report.embedded == {
+ "Memory.embedding": 3, "Memory.beh_embedding": 1,
+ "Topic.embedding": 1, "HierarchyTopic.embedding": 4,
+ }
+ assert report.cleared == {"Memory.beh_embedding": 1} # beh vector with no beh_abstraction
+ assert report.hierarchy_labels_exact == 4 and report.hierarchy_labels_inexact == ()
+ assert report.vector_indexes == ("memory_content_idx",)
+ assert seen
+
+ check = await verify_graph_reembed(source, target, target_dims=tgt_dims)
+ assert check.ok, check.problems
+ assert check.vector_index and check.target_width == tgt_dims
+ assert all(src == dst for src, dst in check.node_counts.values())
+ assert all(src == dst for src, dst in check.rel_counts.values())
+
+ schema = nodes["schema"]
+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{schema.node_id}'}}) RETURN m.embedding, m.beh_embedding, m.custom_json, m.content;")
+ emb, beh, custom, content = rows[0]
+ assert len(emb) == tgt_dims
+ assert emb == pytest.approx(new.vector("plans trips weeks ahead"), abs=1e-5)
+ assert beh == pytest.approx(new.vector("prepares early under uncertainty"), abs=1e-5)
+ assert json.loads(custom)["retrieval_text"] == "plans trips weeks ahead" and content == schema.content
+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['core'].node_id}'}}) RETURN m.embedding;")
+ assert rows[0][0] == pytest.approx(new.vector("core: anticipates"), abs=1e-5)
+ rows = _read(target, f"MATCH (m:Memory {{node_id: '{nodes['facts'][0].node_id}'}}) RETURN m.embedding IS NULL;")
+ assert rows[0][0] is True # a NULL source vector stays NULL
+ topics = dict(_read(target, "MATCH (t:Topic) RETURN t.name, t.embedding;"))
+ assert topics["travel"] == pytest.approx(new.vector("travel"), abs=1e-5)
+ assert topics["work"] is None
+ for subject, path_json, embedding in _read(target, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;"):
+ label, exact = hierarchy_topic_label(subject, path_json)
+ assert exact and embedding == pytest.approx(new.vector(label), abs=1e-5)
+
+ # The rebuilt graph serves the SDK: vector search on the new index works.
+ gs = await _open(target, tgt_dims)
+ try:
+ hits = await gs.vector_search(
+ query_embedding=new.vector("plans trips weeks ahead"),
+ isolation_key=schema.get_isolation_key(), limit=1,
+ )
+ assert hits and hits[0]["node_id"] == schema.node_id
+ width = await gs._run("CALL table_info('Memory') RETURN *;")
+ assert dict((r[1], r[2]) for r in width)["embedding"] == f"FLOAT[{tgt_dims}]"
+ finally:
+ await gs.close()
+
+
+async def test_source_labels_rebuild_exactly(tmp_path):
+ """The stored HierarchyTopic vectors were computed from the rebuilt label."""
+ source = tmp_path / "graph"
+ old, _ = await _seed_graph(source, 64)
+ rows = _read(source, "MATCH (t:HierarchyTopic) RETURN t.subject, t.path_json, t.embedding;")
+ assert len(rows) == 4
+ for subject, path_json, embedding in rows:
+ label, exact = hierarchy_topic_label(subject, path_json)
+ assert exact
+ assert embedding == pytest.approx(old.vector(label), abs=1e-6)
+
+
+def test_legacy_path_json_is_flagged_inexact():
+ label, exact = hierarchy_topic_label("alice", json.dumps(["Travel", "Japan"]), "Japan")
+ assert label == "alice: topic:Travel > facet:Japan" and not exact
+ label, exact = hierarchy_topic_label("alice", "", "Japan")
+ assert label == "alice: Japan" and not exact
+
+
+async def test_cancel_and_failure_leave_no_target(tmp_path):
+ source, target = tmp_path / "graph", tmp_path / "graph-new"
+ await _seed_graph(source, 64)
+ with pytest.raises(ReembedCancelled):
+ await reembed_graph(
+ source, target, embedding=FakeEmbedder("new", 32), target_dims=32,
+ cancel=lambda: True,
+ )
+ with pytest.raises(EmbeddingDimensionMismatchError):
+ await reembed_graph(
+ source, target, embedding=FakeEmbedder("new", 32, wrong_width_after=1), target_dims=32,
+ )
+ assert not os.path.exists(target)
+ assert not os.path.exists(str(target) + ".reembed-partial")
+ # A restart starts over and succeeds.
+ report = await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)
+ assert report.node_counts["Memory"] == 8
+ with pytest.raises(FileExistsError):
+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 32), target_dims=32)
+ again = await reembed_graph(
+ source, target, embedding=FakeEmbedder("new", 32), target_dims=32, overwrite=True,
+ )
+ assert again.rel_counts == report.rel_counts
+
+
+async def test_refuses_open_source_and_drops_only_closed_graphs(tmp_path):
+ source, target = tmp_path / "graph", tmp_path / "graph-new"
+ await _seed_graph(source, 64)
+ live = await _open(source, 64)
+ try:
+ with pytest.raises(RuntimeError):
+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)
+ with pytest.raises(RuntimeError):
+ await drop_graph(source)
+ finally:
+ await live.close()
+ with pytest.raises(ValueError):
+ await reembed_graph(source, source, embedding=FakeEmbedder("new", 64), target_dims=64)
+ await reembed_graph(source, target, embedding=FakeEmbedder("new", 64), target_dims=64)
+ assert await drop_graph(target) is True
+ assert not os.path.exists(target)
+ assert await drop_graph(target) is False
+ assert os.path.exists(source)