Skip to content
Python
11.5 KB
Raw Blame
"""An in-memory `ReembedBackend` for embedding switch tests.

It keeps collections as plain dicts named exactly as the SDK names them, so a
test can lay out a home -- a live space, a stray 512 space left by an older
build, a retired space -- and check what a switch or merge did to each. The
vectors themselves are not computed: a record remembers the model and width
it was "embedded" with, which is what the assertions need.
"""

from __future__ import annotations

import asyncio
import re
import shutil
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path

from one_memory.embedding_backend import (
    EmbeddingTarget,
    EmbeddingWidthError,
    GraphCheck,
    PhysicalSpace,
    ReembedCancelledError,
    ReembedStep,
    ScannedSpace,
    VectorCheck,
)
from one_memory.embedding_spaces import VECTOR_ROLES, space_collection_names

_ROLE_SUFFIX = re.compile(
    r"^(?P<base>.+?)_(?P<role>coding_keys_cv|coding_keys|working_facts_cv|working_facts)"
    r"(?:_(?P<dims>\d+))?$"
)
_DIMS_SUFFIX = re.compile(r"^(?P<base>.+)_(?P<dims>\d+)$")


class Crash(BaseException):
    """Stands in for the process dying: nothing in the worker catches it."""


@dataclass
class Record:
    document: str
    metadata: dict[str, object]
    width: int
    model: str


@dataclass
class FakeVectorStore:
    collections: dict[str, dict[str, Record]] = field(default_factory=dict)

    def put(
        self,
        base: str,
        name_dims: int | None,
        records: dict[str, tuple[str, str]],
        *,
        role: str = "memory",
        width: int | None = None,
        model: str = "old-model",
    ) -> None:
        """`records` maps id -> (document, user_id)."""
        name = space_collection_names(base, name_dims)[role]
        target = self.collections.setdefault(name, {})
        for record_id, (document, user) in records.items():
            target[record_id] = Record(
                document=document,
                metadata={"user_id": user, "content": document},
                width=width or name_dims or 0,
                model=model,
            )

    def space(self, base: str, name_dims: int | None) -> dict[str, dict[str, Record]]:
        return {
            role: dict(self.collections.get(name, {}))
            for role, name in space_collection_names(base, name_dims).items()
            if name in self.collections
        }

    def ids(self, base: str, name_dims: int | None, role: str = "memory") -> set[str]:
        return set(self.collections.get(space_collection_names(base, name_dims)[role], {}))


def parse(name: str) -> tuple[str, int | None, str] | None:
    if name.endswith("_tag_index"):
        return None
    match = _ROLE_SUFFIX.match(name)
    if match:
        dims = match.group("dims")
        return match.group("base"), int(dims) if dims else None, match.group("role")
    role = "memory"
    if name.endswith("_entities"):
        name, role = name[: -len("_entities")], "entities"
    match = _DIMS_SUFFIX.match(name)
    if match:
        return match.group("base"), int(match.group("dims")), role
    return name, None, role


class FakeReembedBackend:
    """The backend contract, over `FakeVectorStore`."""

    def __init__(self, store: FakeVectorStore | None = None) -> None:
        self.store = store or FakeVectorStore()
        self.unavailable: str | None = None
        #: Called before every batch; a test raises `Crash` or flips a cancel here.
        self.before_batch: Callable[[str, int], None] = lambda _role, _done: None
        #: Width the "embedding service" returns; None returns the target's.
        self.returned_width: int | None = None
        #: Ids the re-embed silently loses, to prove verification catches it.
        self.lose_ids: set[str] = set()
        self.batch_size = 2
        self.reembed_calls: list[tuple[str, str]] = []
        self.dropped: list[str] = []
        self.graphs_built: list[Path] = []
        #: When set, every batch waits for it: a switch a test holds mid-way.
        self.pause: asyncio.Event | None = None

    def unavailable_reason(self) -> str | None:
        return self.unavailable

    async def scan(self, *, per_user: bool = False) -> tuple[ScannedSpace, ...]:
        grouped: dict[tuple[str, int | None], dict[str, str]] = {}
        for name in self.store.collections:
            parsed = parse(name)
            if parsed is None:
                continue
            base, dims, role = parsed
            grouped.setdefault((base, dims), {})[role] = name
        out = []
        for (base, dims), roles in sorted(
            grouped.items(), key=lambda kv: (kv[0][0], kv[0][1] or 0)
        ):
            counts = {role: len(self.store.collections[name]) for role, name in roles.items()}
            widths = {
                record.width
                for name in roles.values()
                for record in self.store.collections[name].values()
            }
            users: dict[str, int] = {}
            if per_user:
                for name in roles.values():
                    for record in self.store.collections[name].values():
                        user = str(record.metadata.get("user_id") or "")
                        users[user] = users.get(user, 0) + 1
            out.append(
                ScannedSpace(
                    collection_base=base,
                    name_dims=dims,
                    width=widths.pop() if len(widths) == 1 else None,
                    counts=counts,
                    collections=roles,
                    per_user=users,
                )
            )
        return tuple(out)

    async def diff(self, source: PhysicalSpace, target: PhysicalSpace) -> dict[str, int]:
        target_names = target.names()
        out = {}
        for role, name in source.names().items():
            have = set(self.store.collections.get(target_names.get(role, ""), {}))
            out[role] = len(set(self.store.collections.get(name, {})) - have)
        return out

    async def reembed(
        self,
        source: PhysicalSpace,
        target: PhysicalSpace,
        *,
        embedding: EmbeddingTarget,
        cancel: Callable[[], bool],
        on_step: Callable[[ReembedStep], None],
        batch_size: int = 64,
    ) -> None:
        del batch_size
        if target.dims is None:
            raise ValueError("the target space needs an explicit width")
        self.reembed_calls.append((source.collection_base, target.collection_base))
        target_names = target.names()
        model = embedding.space.model or "?"
        for role in VECTOR_ROLES:
            name = source.names().get(role)
            if name is None or name not in self.store.collections:
                continue
            records = sorted(self.store.collections[name].items())
            destination = self.store.collections.setdefault(target_names[role], {})
            done = embedded = kept = 0
            for start in range(0, len(records), self.batch_size):
                if cancel():
                    raise ReembedCancelledError("embedding_switch_cancelled")
                self.before_batch(role, done)
                if self.pause is not None:
                    await self.pause.wait()
                batch = records[start : start + self.batch_size]
                width = self.returned_width or target.dims
                if width != target.dims:
                    raise EmbeddingWidthError("embedding_dimensions_mismatch")
                for record_id, record in batch:
                    if record_id in destination:
                        kept += 1
                        continue
                    if record_id in self.lose_ids:
                        continue
                    destination[record_id] = Record(
                        document=record.document,
                        metadata=dict(record.metadata),
                        width=width,
                        model=model,
                    )
                    embedded += 1
                done += len(batch)
                on_step(
                    ReembedStep(
                        phase="vectors",
                        role=role,
                        done=done,
                        total=len(records),
                        embedded=embedded,
                        kept=kept,
                    )
                )

    async def verify(
        self, source: PhysicalSpace, target: PhysicalSpace, *, embedding: EmbeddingTarget
    ) -> VectorCheck:
        del embedding
        problems: list[str] = []
        per_role: dict[str, dict[str, int]] = {}
        users_source: dict[str, int] = {}
        users_target: dict[str, int] = {}
        target_names = target.names()
        for role, name in source.names().items():
            src = self.store.collections.get(name, {})
            tgt = self.store.collections.get(target_names.get(role, ""), {})
            missing = set(src) - set(tgt)
            extra = set(tgt) - set(src)
            mismatched = {
                key for key in set(src) & set(tgt) if src[key].document != tgt[key].document
            }
            bad = sum(1 for record in tgt.values() if record.width != target.dims)
            per_role[role] = {
                "source": len(src),
                "target": len(tgt),
                "missing": len(missing),
                "extra": len(extra),
                "mismatched": len(mismatched),
                "bad_dims": bad,
            }
            for label, value in (
                ("missing", missing),
                ("extra", extra),
                ("mismatched", mismatched),
            ):
                if value:
                    problems.append(f"{role}: {len(value)} {label}")
            if bad:
                problems.append(f"{role}: {bad} bad dims")
            for record in src.values():
                user = str(record.metadata.get("user_id") or "")
                users_source[user] = users_source.get(user, 0) + 1
            for record in tgt.values():
                user = str(record.metadata.get("user_id") or "")
                users_target[user] = users_target.get(user, 0) + 1
        for user in sorted(set(users_source) | set(users_target)):
            if users_source.get(user, 0) != users_target.get(user, 0):
                problems.append(f"user {user!r}: counts differ")
        return VectorCheck(per_role=per_role, per_user=users_target, problems=tuple(problems))

    async def graph_vectors(self, path: Path) -> int | None:
        return 3 if path.exists() else None

    async def reembed_graph(
        self,
        source: Path,
        target: Path,
        *,
        embedding: EmbeddingTarget,
        target_dims: int,
        cancel: Callable[[], bool],
        on_step: Callable[[ReembedStep], None],
    ) -> None:
        del embedding, cancel
        target.mkdir(parents=True, exist_ok=True)
        (target / "width").write_text(str(target_dims))
        self.graphs_built.append(target)
        on_step(ReembedStep(phase="graph", role="Memory", done=3, total=3, embedded=3))

    async def verify_graph(self, source: Path, target: Path, *, target_dims: int) -> GraphCheck:
        ok = (target / "width").is_file() and (target / "width").read_text() == str(target_dims)
        return GraphCheck(ok=ok, problems=() if ok else ("graph width",))

    async def drop(self, space: PhysicalSpace) -> None:
        for name in space.names().values():
            if self.store.collections.pop(name, None) is not None:
                self.dropped.append(name)

    async def drop_graph(self, path: Path) -> None:
        shutil.rmtree(path, ignore_errors=True)