Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

18.2 KB
Raw Blame
"""Real-LanceDB coverage for resumable scope-column maintenance."""

import json
import multiprocessing
import socket
import time
from datetime import timedelta

import lancedb  # type: ignore
import pyarrow as pa  # type: ignore
import pytest
from filelock import FileLock

from xagent.core.memory import lancedb_maintenance as maintenance
from xagent.core.memory.lancedb_maintenance import (
    MAINTENANCE_METADATA_KEY,
    MAINTENANCE_TABLE_VERSION_KEY,
    MAINTENANCE_VERSION,
    MaintenanceLockTimeout,
    MaintenanceStatus,
    maintain_lancedb_memory_table,
)
from xagent.core.memory.scope_columns import SCOPE_DIMS_COLUMN, USER_ID_COLUMN
from xagent.core.memory.vector_compatibility import VECTOR_IDENTITY_METADATA_KEY
from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import _safe_close_table


def _metadata(index):
    return json.dumps(
        {
            "user_id": index,
            "execution_scope_agent": f"agent-{index % 2}",
        }
    )


def _connection(
    tmp_path,
    count=2,
    *,
    ids=None,
    metadata=None,
    scoped=False,
    connection_kwargs=None,
):
    connection = lancedb.connect(tmp_path, **(connection_kwargs or {}))
    ids = ids if ids is not None else [f"note-{index}" for index in range(count)]
    metadata = (
        metadata if metadata is not None else [_metadata(i) for i in range(count)]
    )
    columns = {
        "id": ids,
        "text": [f"text-{i}" for i in range(count)],
        "metadata": pa.array(metadata, pa.string()),
        "vector": pa.array(
            [[float(i), 1.0] for i in range(count)], pa.list_(pa.float32(), 2)
        ),
    }
    if scoped:
        columns[USER_ID_COLUMN] = pa.array([None] * count, pa.int64())
        columns[SCOPE_DIMS_COLUMN] = pa.array([None] * count, pa.list_(pa.string()))
    data = pa.table(columns).replace_schema_metadata(
        {VECTOR_IDENTITY_METADATA_KEY: b"preserved"}
    )
    table = connection.create_table("memories", data)
    _safe_close_table(table)
    return connection


def _table(connection):
    return connection.open_table("memories")


def _marker(table):
    if USER_ID_COLUMN not in table.schema.names:
        return None
    return (table.schema.field(USER_ID_COLUMN).metadata or {}).get(
        MAINTENANCE_METADATA_KEY
    )


def _marker_table_version(table):
    if USER_ID_COLUMN not in table.schema.names:
        return None
    return (table.schema.field(USER_ID_COLUMN).metadata or {}).get(
        MAINTENANCE_TABLE_VERSION_KEY
    )


def _hold_file_lock(lock_path, ready, release):
    with FileLock(lock_path):
        ready.set()
        release.wait(30)


def test_complete_migration_and_second_run_is_schema_only(tmp_path, monkeypatch):
    connection = _connection(tmp_path)
    first = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert first.status is MaintenanceStatus.COMPLETE
    assert table.to_arrow().select([USER_ID_COLUMN, SCOPE_DIMS_COLUMN]).to_pylist() == [
        {USER_ID_COLUMN: 0, SCOPE_DIMS_COLUMN: ["agent=agent-0"]},
        {USER_ID_COLUMN: 1, SCOPE_DIMS_COLUMN: ["agent=agent-1"]},
    ]
    assert _marker(table) == MAINTENANCE_VERSION
    assert _marker_table_version(table) == str(table.version).encode()
    assert table.schema.metadata[VECTOR_IDENTITY_METADATA_KEY] == b"preserved"
    _safe_close_table(table)

    monkeypatch.setattr(
        maintenance, "_read_rows", lambda _table: pytest.fail("fast path scanned")
    )
    second = maintain_lancedb_memory_table(connection, "memories")
    assert second == maintenance.MaintenanceOutcome(MaintenanceStatus.COMPLETE)


@pytest.mark.parametrize(("count", "batches"), [(512, 1), (513, 2)])
def test_512_boundary_uses_one_commit_per_batch(tmp_path, count, batches):
    connection = _connection(tmp_path, count=count)
    table = _table(connection)
    before = table.version
    _safe_close_table(table)
    outcome = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert outcome.batches_committed == batches
    assert outcome.updated_rows == count
    assert table.version - before == batches + 2  # add columns + batches + marker
    _safe_close_table(table)


def test_sql_null_metadata_projects_to_empty_scope_without_network(
    tmp_path, monkeypatch
):
    connection = _connection(tmp_path, count=1, metadata=[None])

    def forbidden(*_args, **_kwargs):
        raise AssertionError("maintenance must not embed or use the network")

    monkeypatch.setattr(socket.socket, "connect", forbidden)
    outcome = maintain_lancedb_memory_table(connection, "memories")
    row = _table(connection).to_arrow().to_pylist()[0]
    assert outcome.status is MaintenanceStatus.COMPLETE
    assert row[USER_ID_COLUMN] is None
    assert row[SCOPE_DIMS_COLUMN] == []


@pytest.mark.parametrize(
    "ids",
    [[None, "ok"], [1, 2], ["same", "same"], ["", "ok"]],
)
def test_invalid_ids_leave_table_unchanged(tmp_path, ids):
    connection = _connection(tmp_path, ids=ids)
    table = _table(connection)
    before = (table.version, table.schema, table.to_arrow().to_pylist())
    _safe_close_table(table)
    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.INVALID_LEGACY_DATA
    assert (table.version, table.schema, table.to_arrow().to_pylist()) == before
    _safe_close_table(table)


@pytest.mark.parametrize(
    ("metadata", "detail"),
    [
        (pa.array([7], pa.int64()), "must be a string or SQL NULL"),
        (pa.array([json.dumps({"user_id": 2**63})], pa.string()), "signed int64"),
        (
            pa.array([json.dumps({"user_id": -(2**63) - 1})], pa.string()),
            "signed int64",
        ),
    ],
)
def test_invalid_metadata_or_user_id_range_leaves_table_unchanged(
    tmp_path, metadata, detail
):
    connection = lancedb.connect(tmp_path)
    table = connection.create_table(
        "memories",
        pa.table({"id": ["note-0"], "text": ["text-0"], "metadata": metadata}),
    )
    before = (table.version, table.schema, table.to_arrow().to_pylist())
    _safe_close_table(table)

    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.INVALID_LEGACY_DATA
    assert detail in result.detail
    assert (table.version, table.schema, table.to_arrow().to_pylist()) == before
    _safe_close_table(table)


@pytest.mark.parametrize("stage", ["columns_added", "batch_committed"])
def test_partial_failure_resumes_without_repeating_completed_rows(
    tmp_path, monkeypatch, stage
):
    connection = _connection(tmp_path, count=513)
    original = maintenance._checkpoint

    def fail(selected, batch=None):
        if selected == stage and (batch is None or batch == 1):
            raise RuntimeError("injected failure")
        original(selected, batch)

    monkeypatch.setattr(maintenance, "_checkpoint", fail)
    with pytest.raises(RuntimeError, match="injected failure"):
        maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert {USER_ID_COLUMN, SCOPE_DIMS_COLUMN} <= set(table.schema.names)
    assert _marker(table) is None
    _safe_close_table(table)

    monkeypatch.setattr(maintenance, "_checkpoint", original)
    resumed = maintain_lancedb_memory_table(connection, "memories")
    assert resumed.status is MaintenanceStatus.COMPLETE
    assert resumed.updated_rows == (513 if stage == "columns_added" else 1)


@pytest.mark.parametrize("change_metadata", [False, True])
def test_concurrent_changes_are_preserved_and_metadata_change_is_reported(
    tmp_path, monkeypatch, change_metadata
):
    connection = _connection(tmp_path, count=1, scoped=True)
    original = maintenance._backfill_batch
    changed_metadata = json.dumps({"user_id": 99})

    def concurrent_update(table, rows):
        writer = _table(connection)
        values = {"text": "writer-text", "vector": [9.0, 9.0]}
        if change_metadata:
            values["metadata"] = changed_metadata
        writer.update("id = 'note-0'", values=values)
        _safe_close_table(writer)
        return original(table, rows)

    monkeypatch.setattr(maintenance, "_backfill_batch", concurrent_update)
    result = maintain_lancedb_memory_table(connection, "memories")
    row = _table(connection).to_arrow().to_pylist()[0]
    assert (row["text"], row["vector"]) == ("writer-text", [9.0, 9.0])
    if change_metadata:
        assert result.status is MaintenanceStatus.INCOMPLETE
        assert result.cas_skipped_rows
        assert row["metadata"] == changed_metadata
        assert _marker(_table(connection)) is None
    else:
        assert result.status is MaintenanceStatus.COMPLETE
        assert row[USER_ID_COLUMN] == 0


def test_late_metadata_writer_invalidates_marker_and_resumes(tmp_path, monkeypatch):
    connection = _connection(tmp_path, count=1, scoped=True)
    changed_metadata = json.dumps({"user_id": 99})

    def late_update(stage, _batch=None):
        if stage == "before_completion":
            writer = _table(connection)
            writer.update("id = 'note-0'", values={"metadata": changed_metadata})
            _safe_close_table(writer)

    monkeypatch.setattr(maintenance, "_checkpoint", late_update)
    first = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    row = table.to_arrow().to_pylist()[0]
    assert first.status is MaintenanceStatus.INCOMPLETE
    assert row["metadata"] == changed_metadata
    assert row[USER_ID_COLUMN] == 0
    assert _marker_table_version(table) != str(table.version).encode()
    _safe_close_table(table)

    monkeypatch.setattr(maintenance, "_checkpoint", lambda *_args: None)
    resumed = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert resumed.status is MaintenanceStatus.COMPLETE
    assert table.to_arrow().to_pylist()[0][USER_ID_COLUMN] == 99
    assert _marker_table_version(table) == str(table.version).encode()
    _safe_close_table(table)


def test_later_write_keeps_the_completion_fast_path(tmp_path, monkeypatch):
    """Completion is migration state: an ordinary write does not undo it."""
    connection = _connection(tmp_path, count=1)
    assert (
        maintain_lancedb_memory_table(connection, "memories").status
        is MaintenanceStatus.COMPLETE
    )
    writer = _table(connection)
    writer.update("id = 'note-0'", values={"text": "new text"})
    version = writer.version
    _safe_close_table(writer)

    monkeypatch.setattr(
        maintenance, "_read_rows", lambda _table: pytest.fail("fast path scanned")
    )
    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result == maintenance.MaintenanceOutcome(MaintenanceStatus.COMPLETE)
    assert table.version == version
    assert table.to_arrow().to_pylist()[0]["text"] == "new text"
    _safe_close_table(table)


def test_strong_consistency_detects_version_change_during_final_read(
    tmp_path, monkeypatch
):
    connection = _connection(
        tmp_path,
        count=1,
        scoped=True,
        connection_kwargs={"read_consistency_interval": timedelta(0)},
    )
    original = maintenance._read_rows
    calls = 0

    def update_after_final_read(table):
        nonlocal calls
        rows = original(table)
        calls += 1
        if calls == 2:
            writer = _table(connection)
            writer.update("id = 'note-0'", values={"text": "concurrent text"})
            _safe_close_table(writer)
        return rows

    monkeypatch.setattr(maintenance, "_read_rows", update_after_final_read)
    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.INCOMPLETE
    assert table.to_arrow().to_pylist()[0]["text"] == "concurrent text"
    assert _marker(table) is None
    _safe_close_table(table)


def test_final_invalid_id_returns_invalid_and_preserves_concurrent_row(
    tmp_path, monkeypatch
):
    connection = _connection(
        tmp_path,
        count=1,
        connection_kwargs={"read_consistency_interval": timedelta(0)},
    )

    def append_invalid_id(stage, _batch=None):
        if stage == "batch_committed":
            writer = _table(connection)
            writer.add(
                [
                    {
                        "id": None,
                        "text": "concurrent text",
                        "metadata": "{}",
                        "vector": [3.0, 4.0],
                        USER_ID_COLUMN: None,
                        SCOPE_DIMS_COLUMN: [],
                    }
                ]
            )
            _safe_close_table(writer)

    monkeypatch.setattr(maintenance, "_checkpoint", append_invalid_id)
    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.INVALID_LEGACY_DATA
    assert any(row["id"] is None for row in table.to_arrow().to_pylist())
    assert _marker(table) is None
    _safe_close_table(table)


def test_exact_scope_schema_preserves_existing_field_metadata(tmp_path):
    connection = _connection(tmp_path, count=1, scoped=True)
    table = _table(connection)
    metadata = {"preserved": "yes"}
    if hasattr(table, "update_field_metadata"):
        table.update_field_metadata({"path": USER_ID_COLUMN, "metadata": metadata})
    else:
        table.replace_field_metadata(USER_ID_COLUMN, metadata)
    _safe_close_table(table)

    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.COMPLETE
    assert table.schema.field(USER_ID_COLUMN).metadata[b"preserved"] == b"yes"
    assert _marker_table_version(table) == str(table.version).encode()
    _safe_close_table(table)


@pytest.mark.parametrize(
    ("partial", "old_marker"),
    [(False, False), (True, False), (False, True)],
)
def test_incompatible_scope_schema_is_reported_without_mutation(
    tmp_path, partial, old_marker
):
    connection = lancedb.connect(tmp_path)
    columns = {
        "id": ["note-0"],
        "text": ["text-0"],
        "metadata": pa.array([json.dumps({"user_id": 0})], pa.string()),
        USER_ID_COLUMN: pa.array([0], pa.int32()),
    }
    if not partial:
        columns[SCOPE_DIMS_COLUMN] = pa.array([[]], pa.list_(pa.int32()))
    table = connection.create_table("memories", pa.table(columns))
    if old_marker:
        marker = {MAINTENANCE_METADATA_KEY.decode(): MAINTENANCE_VERSION.decode()}
        if hasattr(table, "update_field_metadata"):
            table.update_field_metadata({"path": USER_ID_COLUMN, "metadata": marker})
        else:
            table.replace_field_metadata(USER_ID_COLUMN, marker)
    before = (table.version, table.schema, table.to_arrow().to_pylist())
    _safe_close_table(table)

    result = maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert result.status is MaintenanceStatus.INCOMPATIBLE_SCHEMA
    assert USER_ID_COLUMN in result.detail
    assert (table.version, table.schema, table.to_arrow().to_pylist()) == before
    _safe_close_table(table)


def test_completion_marker_is_last_and_failure_before_it_resumes(tmp_path, monkeypatch):
    connection = _connection(tmp_path)

    def fail(stage, _batch=None):
        if stage == "before_completion":
            raise RuntimeError("before marker")

    monkeypatch.setattr(maintenance, "_checkpoint", fail)
    with pytest.raises(RuntimeError, match="before marker"):
        maintain_lancedb_memory_table(connection, "memories")
    table = _table(connection)
    assert _marker(table) is None
    version = table.version
    _safe_close_table(table)
    monkeypatch.setattr(maintenance, "_checkpoint", lambda *_args: None)
    assert (
        maintain_lancedb_memory_table(connection, "memories").status
        is MaintenanceStatus.COMPLETE
    )
    table = _table(connection)
    assert table.version == version + 1
    assert _marker(table) == MAINTENANCE_VERSION
    _safe_close_table(table)


def test_lock_timeout_is_bounded_and_success_releases_lock(tmp_path):
    connection = _connection(tmp_path)
    lock_path = maintenance._lock_path(connection, "memories")
    with FileLock(lock_path):
        started = time.monotonic()
        with pytest.raises(MaintenanceLockTimeout, match="stop the other"):
            maintain_lancedb_memory_table(connection, "memories", lock_timeout=0.05)
        assert time.monotonic() - started < 1

    maintain_lancedb_memory_table(connection, "memories", lock_timeout=0.05)
    with FileLock(lock_path, timeout=0.05):
        pass


def test_lock_timeout_against_second_process(tmp_path):
    connection = _connection(tmp_path)
    lock_path = maintenance._lock_path(connection, "memories")
    context = multiprocessing.get_context("spawn")
    ready = context.Event()
    release = context.Event()
    process = context.Process(target=_hold_file_lock, args=(lock_path, ready, release))
    process.start()
    try:
        assert ready.wait(20)
        with pytest.raises(MaintenanceLockTimeout, match="stop the other"):
            maintain_lancedb_memory_table(connection, "memories", lock_timeout=0.05)
    finally:
        release.set()
        process.join(10)
    assert process.exitcode == 0


def test_non_local_uri_is_rejected():
    class RemoteConnection:
        uri = "s3://bucket/database"

    with pytest.raises(ValueError, match="writable local database URI"):
        maintain_lancedb_memory_table(RemoteConnection(), "memories")


@pytest.mark.parametrize("timeout", [0, float("inf")])
def test_nonpositive_lock_timeout_is_rejected(tmp_path, timeout):
    connection = _connection(tmp_path)
    with pytest.raises(ValueError, match="finite positive"):
        maintain_lancedb_memory_table(connection, "memories", lock_timeout=timeout)


@pytest.mark.parametrize(
    "kwargs",
    [
        {"batch_size": True},
        {"batch_size": 1.5},
        {"batch_size": "1"},
        {"lock_timeout": True},
        {"lock_timeout": "1"},
        {"lock_timeout": float("nan")},
    ],
)
def test_invalid_parameter_types_do_not_access_connection(kwargs):
    class ForbiddenConnection:
        @property
        def uri(self):
            raise AssertionError(
                "invalid parameters must fail before connection access"
            )

    with pytest.raises(ValueError):
        maintain_lancedb_memory_table(ForbiddenConnection(), "memories", **kwargs)