Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

24.2 KB
Raw Blame
"""Offline re-embedding migration of the LanceDB memory table."""

from __future__ import annotations

import json
import sqlite3

import lancedb  # type: ignore
import pyarrow as pa  # type: ignore
import pytest

import xagent.core.memory.embedding_migration as migration
from xagent.core.memory.core import MemoryNote
from xagent.core.memory.embedding_migration import (
    MemoryReembedStatus,
    backup_table_name,
    reembed_memory_table,
    staging_table_name,
)
from xagent.core.memory.lancedb import LanceDBMemoryStore
from xagent.core.memory.scope_columns import SCOPE_DIMS_COLUMN, USER_ID_COLUMN
from xagent.core.memory.storage_admission import (
    DormantLanceDBMemoryHandle,
    admit_lancedb_memory_storage,
)
from xagent.core.memory.vector_compatibility import (
    VECTOR_IDENTITY_METADATA_KEY,
    EmbeddingIdentity,
    VectorCompatibility,
    _is_fully_admitted,
    classify_vector_compatibility,
    create_or_recreate_vector_capable_table,
)
from xagent.core.model.embedding import BaseEmbedding
from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import _safe_close_table
from xagent.providers.vector_store.lancedb import clear_connection_cache

OLD = EmbeddingIdentity(
    "openai", "text-embedding-3-small", "https://api.openai.com/v1/embeddings", 4, None
)
# Same dimension as OLD on purpose: nothing but the declared space can tell them
# apart, which is the case a config change alone silently mixes.
NEW = EmbeddingIdentity(
    "bailian", "qwen-embedding-v4", "https://example.test/v1", 4, None
)


class Encoder:
    """Records every batch it is asked to embed and returns a fixed value."""

    def __init__(self, value: float = 0.9, dimension: int = 4):
        self.value = value
        self.dimension = dimension
        self.calls: list[list[str]] = []

    def __call__(self, texts):
        self.calls.append(list(texts))
        return [[self.value] * self.dimension for _ in texts]


def _row(note_id, text, user_id, value):
    return {
        "id": note_id,
        "text": text,
        "metadata": json.dumps({"content": text, "user_id": user_id}),
        "vector": [value] * 4,
        USER_ID_COLUMN: user_id,
        SCOPE_DIMS_COLUMN: [],
    }


def _live(connection, rows=None, identity=OLD):
    create_or_recreate_vector_capable_table(connection, "memories", identity)
    if rows:
        _append(connection, rows)
    return connection.open_table("memories")


def _append(connection, rows):
    table = connection.open_table("memories")
    try:
        table.add(pa.Table.from_pylist(rows, schema=table.schema))
    finally:
        _safe_close_table(table)


def _vectors(connection, name="memories"):
    table = connection.open_table(name)
    try:
        return table.to_arrow().to_pylist()
    finally:
        _safe_close_table(table)


def _names(connection):
    return {str(name) for name in connection.table_names()}


@pytest.fixture
def connection(tmp_path):
    clear_connection_cache()
    yield lancedb.connect(str(tmp_path))
    clear_connection_cache()


def _three_notes():
    return [
        _row("a", "alpha", 1, 0.1),
        _row("b", "beta", 1, 0.1),
        _row("c", "gamma", 2, 0.1),
    ]


def test_same_dimension_swap_replaces_the_declared_space_and_every_vector(connection):
    _live(connection, _three_notes())
    encoder = Encoder(0.9)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=encoder)

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert (outcome.source_rows, outcome.reembedded_rows) == (3, 3)
    rows = _vectors(connection)
    assert [row["id"] for row in rows] == ["a", "b", "c"]
    assert all(row["vector"] == pytest.approx([0.9] * 4) for row in rows)
    # Owners and scope survive the rebuild; they still come from the metadata.
    assert [(row[USER_ID_COLUMN], row[SCOPE_DIMS_COLUMN]) for row in rows] == [
        (1, []),
        (1, []),
        (2, []),
    ]
    live = connection.open_table("memories")
    try:
        assert (
            classify_vector_compatibility(live.schema, NEW)
            is VectorCompatibility.MATCHING
        )
        assert (
            classify_vector_compatibility(live.schema, OLD)
            is VectorCompatibility.MISMATCHING
        )
        assert json.loads(live.schema.metadata[VECTOR_IDENTITY_METADATA_KEY]) == (
            NEW.as_dict()
        )
        assert _is_fully_admitted(live)
    finally:
        _safe_close_table(live)

    backup = _vectors(connection, backup_table_name("memories", NEW))
    assert all(row["vector"] == pytest.approx([0.1] * 4) for row in backup)
    assert backup_table_name("memories", NEW) in _names(connection)
    assert staging_table_name("memories", NEW) not in _names(connection)


def test_a_second_run_is_a_no_op(connection):
    _live(connection, _three_notes())
    assert (
        reembed_memory_table(connection, "memories", NEW, encode=Encoder()).status
        is MemoryReembedStatus.COMPLETE
    )
    encoder = Encoder()

    outcome = reembed_memory_table(connection, "memories", NEW, encode=encoder)

    assert outcome.status is MemoryReembedStatus.ALREADY_TARGET
    assert encoder.calls == []


def test_a_failed_staging_table_aborts_before_the_live_table_is_touched(
    connection, monkeypatch
):
    _live(connection, _three_notes())

    def drop_one(stage, batch=None):
        if stage == "batch_filled" and batch == 1:
            staging = connection.open_table(staging_table_name("memories", NEW))
            try:
                staging.delete("id = 'b'")
            finally:
                _safe_close_table(staging)

    monkeypatch.setattr(migration, "_checkpoint", drop_one)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.VERIFICATION_FAILED
    assert "missing" in outcome.detail
    live = connection.open_table("memories")
    try:
        assert live.count_rows() == 3
        assert (
            classify_vector_compatibility(live.schema, OLD)
            is VectorCompatibility.MATCHING
        )
    finally:
        _safe_close_table(live)
    assert backup_table_name("memories", NEW) not in _names(connection)


def test_staging_ids_must_match_and_not_merely_add_up(connection, monkeypatch):
    _live(connection, _three_notes())

    def swap_ids(stage, batch=None):
        if stage == "batch_filled" and batch == 1:
            staging = connection.open_table(staging_table_name("memories", NEW))
            try:
                staging.delete("id = 'b'")
                staging.add(
                    pa.Table.from_pylist(
                        [_row("ghost", "beta", 1, 0.9)], schema=staging.schema
                    )
                )
            finally:
                _safe_close_table(staging)

    monkeypatch.setattr(migration, "_checkpoint", swap_ids)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.VERIFICATION_FAILED
    assert "missing" in outcome.detail
    assert [row["id"] for row in _vectors(connection)] == ["a", "b", "c"]
    assert all(
        row["vector"] == pytest.approx([0.1] * 4) for row in _vectors(connection)
    )


def test_a_metadata_only_change_between_passes_is_carried(connection, monkeypatch):
    _live(connection, _three_notes())
    changed = []

    def change_metadata(stage, batch=None):
        if stage == "batch_filled" and not changed:
            changed.append(True)
            table = connection.open_table("memories")
            try:
                table.delete("id = 'b'")
            finally:
                _safe_close_table(table)
            _append(
                connection,
                [
                    {
                        **_row("b", "beta", 9, 0.1),
                        "metadata": json.dumps(
                            {"content": "beta", "user_id": 9, "tags": ["carried"]}
                        ),
                    }
                ],
            )

    monkeypatch.setattr(migration, "_checkpoint", change_metadata)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert outcome.passes == 2
    rows = {row["id"]: row for row in _vectors(connection)}
    assert rows["b"][USER_ID_COLUMN] == 9
    assert json.loads(rows["b"]["metadata"])["tags"] == ["carried"]


def _staging_table(connection):
    live = connection.open_table("memories")
    try:
        return migration._staging_table(
            connection, staging_table_name("memories", NEW), live.schema, NEW
        )
    finally:
        _safe_close_table(live)


def _staged_row(note_id, text, user_id, value):
    row = _row(note_id, text, user_id, 0.9)
    row["vector"] = None if value is None else [value] * 4
    return row


def test_a_staged_row_without_a_vector_fails_verification(connection):
    _live(connection, _three_notes())
    staging = _staging_table(connection)
    staging.add(
        pa.Table.from_pylist(
            [
                _staged_row("a", "alpha", 1, 0.9),
                _staged_row("b", "beta", 1, None),
                _staged_row("c", "gamma", 2, 0.9),
            ],
            schema=staging.schema,
        )
    )
    seen = sqlite3.connect(":memory:")
    seen.execute("CREATE TABLE source_ids (id TEXT PRIMARY KEY) WITHOUT ROWID")
    for note_id in ("a", "b", "c"):
        seen.execute("INSERT INTO source_ids VALUES (?)", (note_id,))

    detail = migration._verify(
        connection,
        staging_table_name("memories", NEW),
        identity=NEW,
        source_rows=3,
        seen=seen,
        batch_size=512,
    )

    _safe_close_table(staging)
    seen.close()
    assert "carry no vector" in detail
    assert len(_vectors(connection)) == 3


def test_a_staged_row_without_a_vector_is_embedded_by_the_next_run(connection):
    _live(connection, _three_notes())
    staging = _staging_table(connection)
    staging.add(
        pa.Table.from_pylist(
            [
                _staged_row("a", "alpha", 1, 0.9),
                _staged_row("b", "beta", 1, None),
                _staged_row("c", "gamma", 2, 0.9),
            ],
            schema=staging.schema,
        )
    )
    _safe_close_table(staging)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert all(
        row["vector"] == pytest.approx([0.9] * 4) for row in _vectors(connection)
    )


def test_an_aborted_swap_leaves_no_backup_behind(connection, monkeypatch):
    _live(connection, _three_notes())

    def stop(stage, batch=None):
        if stage == "before_swap":
            raise RuntimeError("operator stopped the run")

    monkeypatch.setattr(migration, "_checkpoint", stop)

    with pytest.raises(RuntimeError, match="operator stopped"):
        reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert backup_table_name("memories", NEW) not in _names(connection)
    assert all(
        row["vector"] == pytest.approx([0.1] * 4) for row in _vectors(connection)
    )


def test_a_failure_after_the_swap_keeps_the_backup(connection, monkeypatch):
    _live(connection, _three_notes())

    def stop(stage, batch=None):
        if stage == "after_swap":
            raise RuntimeError("process killed after the swap")

    monkeypatch.setattr(migration, "_checkpoint", stop)

    with pytest.raises(RuntimeError, match="after the swap"):
        reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert all(
        row["vector"] == pytest.approx([0.9] * 4) for row in _vectors(connection)
    )
    backup = _vectors(connection, backup_table_name("memories", NEW))
    assert len(backup) == 3
    assert all(row["vector"] == pytest.approx([0.1] * 4) for row in backup)


def test_a_failure_once_the_swap_committed_keeps_the_backup(connection, monkeypatch):
    _live(connection, _three_notes())
    replace_from = migration._replace_from

    def fail_after_the_commit(connection_, target_name, spilled, schema, **kwargs):
        replace_from(connection_, target_name, spilled, schema, **kwargs)
        if target_name == "memories":
            raise OSError("the spilled file could not be removed")

    monkeypatch.setattr(migration, "_replace_from", fail_after_the_commit)

    with pytest.raises(OSError, match="could not be removed"):
        reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert all(
        row["vector"] == pytest.approx([0.9] * 4) for row in _vectors(connection)
    )
    backup = _vectors(connection, backup_table_name("memories", NEW))
    assert all(row["vector"] == pytest.approx([0.1] * 4) for row in backup)


def test_a_write_overwritten_by_the_swap_is_reported(connection, monkeypatch):
    _live(connection, _three_notes())
    replace_from = migration._replace_from

    def write_first(connection_, target_name, spilled, schema, **kwargs):
        if target_name == "memories":
            _append(connection, [_row("late", "delta", 3, 0.1)])
        return replace_from(connection_, target_name, spilled, schema, **kwargs)

    monkeypatch.setattr(migration, "_replace_from", write_first)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.SWAPPED_OVER_A_WRITE
    assert outcome.backup_table == backup_table_name("memories", NEW)
    assert outcome.backup_table in _names(connection)
    assert "late" not in {row["id"] for row in _vectors(connection)}
    live = connection.open_table("memories")
    try:
        live.checkout(live.version - 1)
        assert "late" in {row["id"] for row in live.to_arrow().to_pylist()}
    finally:
        _safe_close_table(live)


def test_blank_notes_stay_without_a_vector(connection):
    blank = _row("b", "", 1, 0.1)
    blank["vector"] = None
    spaces = _row("c", " \n\t", 2, 0.1)
    spaces["vector"] = None
    _live(connection, [_row("a", "alpha", 1, 0.1), blank, spaces])
    encoder = Encoder()

    outcome = reembed_memory_table(connection, "memories", NEW, encode=encoder)

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert outcome.reembedded_rows == 1
    assert encoder.calls == [["alpha"]]
    rows = {row["id"]: row["vector"] for row in _vectors(connection)}
    assert rows["a"] == pytest.approx([0.9] * 4)
    assert (rows["b"], rows["c"]) == (None, None)


def test_a_staged_blank_note_takes_the_source_text_verbatim(connection):
    _live(connection, [_row("a", "", 1, 0.1)])
    staging = _staging_table(connection)
    staged = _staged_row("a", "", 1, None)
    staged["text"] = None
    staging.add(pa.Table.from_pylist([staged], schema=staging.schema))
    _safe_close_table(staging)
    encoder = Encoder()

    outcome = reembed_memory_table(connection, "memories", NEW, encode=encoder)

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert encoder.calls == []
    row = _vectors(connection)[0]
    assert (row["text"], row["vector"]) == ("", None)


def _target_table_with_a_vectorless_note(connection):
    missing = _row("b", "beta", 1, 0.1)
    missing["vector"] = None
    blank = _row("c", "", 2, 0.1)
    blank["vector"] = None
    _live(connection, [_row("a", "alpha", 1, 0.1), missing, blank], identity=NEW)


def test_a_table_in_the_target_space_has_vectorless_notes_backfilled(connection):
    _target_table_with_a_vectorless_note(connection)
    encoder = Encoder()

    outcome = reembed_memory_table(connection, "memories", NEW, encode=encoder)

    assert outcome.status is MemoryReembedStatus.ALREADY_TARGET
    assert outcome.reembedded_rows == 1
    assert encoder.calls == [["beta"]]
    rows = {row["id"]: row["vector"] for row in _vectors(connection)}
    assert rows["a"] == pytest.approx([0.1] * 4)
    assert rows["b"] == pytest.approx([0.9] * 4)
    assert rows["c"] is None


def test_a_backfill_never_overwrites_a_note_edited_meanwhile(connection):
    _target_table_with_a_vectorless_note(connection)

    def edit_then_encode(texts):
        table = connection.open_table("memories")
        try:
            table.update(where="id = 'b'", values={"text": "edited"})
        finally:
            _safe_close_table(table)
        return [[0.9] * 4 for _ in texts]

    outcome = reembed_memory_table(connection, "memories", NEW, encode=edit_then_encode)

    assert outcome.reembedded_rows == 0
    row = next(row for row in _vectors(connection) if row["id"] == "b")
    assert (row["text"], row["vector"]) == ("edited", None)


def test_a_write_before_the_swap_joins_the_new_table_with_a_new_vector(
    connection, monkeypatch
):
    _live(connection, _three_notes())
    added = []

    def add_note(stage, batch=None):
        if stage == "batch_filled" and not added:
            added.append("later")
            _append(connection, [_row("later", "delta", 3, 0.1)])

    monkeypatch.setattr(migration, "_checkpoint", add_note)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert outcome.passes == 2
    rows = {row["id"]: row for row in _vectors(connection)}
    assert set(rows) == {"a", "b", "c", "later"}
    assert all(row["vector"] == pytest.approx([0.9] * 4) for row in rows.values())
    assert rows["later"][USER_ID_COLUMN] == 3


def test_notes_deleted_before_the_swap_do_not_survive(connection, monkeypatch):
    _live(connection, _three_notes())

    def delete_note(stage, batch=None):
        if stage == "batch_filled" and batch == 1:
            table = connection.open_table("memories")
            try:
                table.delete("id = 'c'")
            finally:
                _safe_close_table(table)

    monkeypatch.setattr(migration, "_checkpoint", delete_note)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert [row["id"] for row in _vectors(connection)] == ["a", "b"]


def test_writers_that_never_stop_leave_the_old_table_in_place(connection, monkeypatch):
    _live(connection, _three_notes())
    counter = []

    def keep_writing(stage, batch=None):
        if stage == "batch_filled":
            counter.append(1)
            _append(connection, [_row(f"w{len(counter)}", "delta", 4, 0.1)])

    monkeypatch.setattr(migration, "_checkpoint", keep_writing)

    outcome = reembed_memory_table(
        connection, "memories", NEW, encode=Encoder(), max_passes=3
    )

    assert outcome.status is MemoryReembedStatus.NOT_CONVERGED
    assert outcome.passes == 3
    live = connection.open_table("memories")
    try:
        assert (
            classify_vector_compatibility(live.schema, OLD)
            is VectorCompatibility.MATCHING
        )
        assert live.count_rows() == 3 + len(counter)
    finally:
        _safe_close_table(live)
    assert backup_table_name("memories", NEW) not in _names(connection)


def test_interrupted_run_resumes_and_only_embeds_what_is_missing(
    connection, monkeypatch
):
    _live(connection, _three_notes() + [_row("d", "delta", 2, 0.1)])

    def interrupt(stage, batch=None):
        if stage == "batch_filled" and batch == 1:
            raise RuntimeError("interrupted")

    monkeypatch.setattr(migration, "_checkpoint", interrupt)
    with pytest.raises(RuntimeError, match="interrupted"):
        reembed_memory_table(
            connection, "memories", NEW, encode=Encoder(), batch_size=2
        )
    assert staging_table_name("memories", NEW) in _names(connection)
    assert len(_vectors(connection)) == 4

    monkeypatch.setattr(migration, "_checkpoint", lambda *args: None)
    encoder = Encoder()
    outcome = reembed_memory_table(
        connection, "memories", NEW, encode=encoder, batch_size=2
    )

    assert outcome.status is MemoryReembedStatus.COMPLETE
    assert sorted(text for call in encoder.calls for text in call) == ["delta", "gamma"]
    assert all(
        row["vector"] == pytest.approx([0.9] * 4) for row in _vectors(connection)
    )


def test_embedding_failure_keeps_the_live_table(connection):
    _live(connection, _three_notes())

    def broken(_texts):
        raise RuntimeError("provider unavailable")

    outcome = reembed_memory_table(connection, "memories", NEW, encode=broken)

    assert outcome.status is MemoryReembedStatus.EMBEDDING_FAILED
    assert "provider unavailable" in outcome.detail
    assert all(
        row["vector"] == pytest.approx([0.1] * 4) for row in _vectors(connection)
    )
    assert backup_table_name("memories", NEW) not in _names(connection)


def test_a_vector_of_the_wrong_width_is_refused(connection):
    _live(connection, _three_notes())

    outcome = reembed_memory_table(
        connection, "memories", NEW, encode=Encoder(dimension=8)
    )

    assert outcome.status is MemoryReembedStatus.EMBEDDING_FAILED
    live = connection.open_table("memories")
    try:
        assert (
            classify_vector_compatibility(live.schema, OLD)
            is VectorCompatibility.MATCHING
        )
    finally:
        _safe_close_table(live)


def test_duplicate_source_ids_abort_the_migration(connection):
    _live(connection, _three_notes() + [_row("a", "alpha again", 1, 0.1)])

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.INVALID_SOURCE_DATA
    assert "unique" in outcome.detail
    assert len(_vectors(connection)) == 4


def test_a_vectorless_table_is_rebuilt_with_vectors(connection):
    table = connection.create_table(
        "memories",
        pa.table(
            {
                "id": ["a", "b"],
                "text": ["alpha", "beta"],
                "metadata": [json.dumps({"user_id": 1})] * 2,
            }
        ),
    )
    _safe_close_table(table)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.COMPLETE
    rows = _vectors(connection)
    assert [row["vector"] for row in rows] == [
        pytest.approx([0.9] * 4),
        pytest.approx([0.9] * 4),
    ]
    assert [row[USER_ID_COLUMN] for row in rows] == [1, 1]


def test_an_absent_table_is_reported_not_created(connection):
    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.ABSENT
    assert "memories" not in _names(connection)


def test_a_staging_table_from_another_space_is_refused(connection):
    _live(connection, _three_notes())
    staging = connection.create_table(
        staging_table_name("memories", NEW), data=[_row("x", "x", 1, 0.9)]
    )
    _safe_close_table(staging)

    outcome = reembed_memory_table(connection, "memories", NEW, encode=Encoder())

    assert outcome.status is MemoryReembedStatus.INCOMPATIBLE_SCHEMA
    assert "does not hold the target vector space" in outcome.detail
    assert len(_vectors(connection)) == 3


class ConstantEmbedding(BaseEmbedding):
    def __init__(self, value: float = 0.5):
        self.value = value

    def encode(self, text, dimension=None, instruct=None):
        vector = [self.value] * 4
        return vector if isinstance(text, str) else [vector for _ in text]

    def get_dimension(self):
        return 4

    @property
    def abilities(self):
        return ["embed"]


def _admitted_store(connection, tmp_path):
    outcome = admit_lancedb_memory_storage(
        DormantLanceDBMemoryHandle(connection, "memories"),
        OLD,
        writers_quiesced=True,
    )
    assert outcome.admitted is not None
    return LanceDBMemoryStore.open_admitted(
        outcome.admitted,
        OLD,
        db_dir=str(tmp_path),
        embedding_model=ConstantEmbedding(),
    )


def test_a_store_that_lost_its_table_to_a_reembed_writes_no_old_vectors(
    connection, tmp_path
):
    _live(connection)
    store = _admitted_store(connection, tmp_path)
    assert store.add(MemoryNote(id="before", content="alpha")).success

    assert (
        reembed_memory_table(connection, "memories", NEW, encode=Encoder()).status
        is MemoryReembedStatus.COMPLETE
    )

    assert store.add(MemoryNote(id="after", content="beta")).success
    rows = {row["id"]: row for row in _vectors(connection)}
    assert rows["before"]["vector"] == pytest.approx([0.9] * 4)
    assert rows["after"]["vector"] is None
    # The store's old adapter never ranked the new table either.
    assert [note.id for note in store.search("beta", k=1)] == ["after"]
    assert all(
        row["vector"] in (None, pytest.approx([0.9] * 4)) for row in rows.values()
    )