Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

8.9 KB
Raw Blame
"""Real-LanceDB coverage for dormant streaming retrieval compatibility (#2346)."""

import json

import pyarrow as pa  # type: ignore
import pytest

from xagent.core.memory import retrieval_compatibility
from xagent.core.memory.core import MemoryNote
from xagent.core.memory.lancedb import LanceDBMemoryStore
from xagent.core.memory.retrieval_compatibility import (
    stream_lexical_top_k,
    stream_list_all,
    stream_stats,
)
from xagent.core.memory.scope_columns import SCOPE_DIMS_COLUMN, USER_ID_COLUMN
from xagent.core.memory.storage_admission import DormantLanceDBMemoryHandle
from xagent.core.tools.core.RAG_tools.LanceDB.schema_manager import _safe_close_table
from xagent.providers.vector_store.lancedb import clear_connection_cache


@pytest.fixture
def store(tmp_path):
    """A real table whose rows all carry NULL vectors (no embedding model)."""
    clear_connection_cache()
    result = LanceDBMemoryStore(str(tmp_path), collection_name="memories")
    yield result
    clear_connection_cache()


@pytest.fixture
def observed(monkeypatch):
    """Rows materialised per scan batch and per retained set, in the style of
    the ``scanned`` counter in ``test_storage_admission_matrix.py``."""
    seen: dict[str, list[int]] = {"scan_batch": [], "retained": []}
    monkeypatch.setattr(
        retrieval_compatibility,
        "_checkpoint",
        lambda stage, batch=None: seen.setdefault(stage, []).append(batch),
    )
    return seen


def _add(store, note_id, text, *, user_id=7, metadata=None, **fields):
    assert store.add(
        MemoryNote(
            id=note_id,
            content=text,
            metadata={"user_id": user_id, **(metadata or {})},
            **fields,
        )
    ).success


def _add_raw(store, rows):
    """Insert rows the store's own writer would reject (malformed metadata)."""
    table = store._vector_store.get_raw_connection().open_table("memories")
    table.add(pa.Table.from_pylist(rows, schema=table.schema))
    _safe_close_table(table)


def _raw(note_id, text, metadata, *, user_id=7):
    return {
        "id": note_id,
        "text": text,
        "metadata": json.dumps(metadata),
        "vector": None,
        USER_ID_COLUMN: user_id,
        SCOPE_DIMS_COLUMN: [],
    }


def _handle(store):
    return DormantLanceDBMemoryHandle(
        store._vector_store.get_raw_connection(), "memories"
    )


def _stream_kwargs(store):
    return dict(
        row_to_note=store._dict_to_memory_note,
        note_filter_factory=store._residual_note_filter,
    )


def _top_k(store, query, k, **kwargs):
    return [
        note.id
        for note in stream_lexical_top_k(
            _handle(store),
            query,
            k,
            **_stream_kwargs(store),
            **kwargs,
        )
    ]


def test_tail_exact_winner_in_the_last_batch_beats_earlier_partial_matches(store):
    _add(store, "p1", "alpha beta")
    _add(store, "p2", "alpha gamma")
    _add(store, "tail", "alpha")
    # batch_size=1 puts the exact match alone in the final batch: it still wins,
    # so nothing capped the raw rows before ranking.
    assert _top_k(store, "alpha", 2, batch_size=1) == ["tail", "p1"]
    assert _top_k(store, "alpha", 1, batch_size=1) == ["tail"]


def test_residual_filter_rejection_does_not_consume_a_slot(store):
    _add(store, "drop", "alpha", metadata={"priority": "drop"})
    _add(store, "keep", "alpha alpha", metadata={"priority": "keep"})
    # "drop" is the better lexical match and is read first, but fails a residual
    # filter, so the single slot must still go to "keep".
    assert _top_k(
        store,
        "alpha",
        1,
        filters={"metadata": {"user_id": 7}, "priority": "keep"},
        batch_size=1,
    ) == ["keep"]


def test_cross_batch_ordering_is_identical_for_every_batch_size(store):
    for index, text in enumerate(
        ["alpha", "alpha alpha", "beta alpha", "alpha beta", "alpha alpha alpha"]
    ):
        _add(store, f"n{index}", text)
    orderings = {
        batch_size: tuple(_top_k(store, "alpha", 4, batch_size=batch_size))
        for batch_size in (1, 2, 3, 64)
    }
    assert len(set(orderings.values())) == 1
    assert orderings[1] == ("n0", "n4", "n1", "n3")


def test_malformed_row_is_skipped_without_aborting_the_scan(store):
    _add_raw(store, [_raw("bad", "alpha", {"user_id": 7, "timestamp": "not-a-date"})])
    _add(store, "good", "alpha")
    assert _top_k(store, "alpha", 5, batch_size=1) == ["good"]


def test_ann_duplicate_does_not_consume_lexical_quota(store):
    _add(store, "dup", "alpha")
    _add(store, "win", "alpha beta")
    # "dup" outranks "win", so without the ANN-id skip it would take the slot.
    assert _top_k(store, "alpha", 1, exclude_ids={"dup"}, batch_size=1) == ["win"]
    assert _top_k(store, "alpha", 1, batch_size=1) == ["dup"]


def test_user_and_scope_isolation_hold_on_a_real_two_principal_table(store, observed):
    _add(store, "mine", "alpha", metadata={"execution_scope_agent": "x"})
    _add(
        store, "other-user", "alpha", user_id=8, metadata={"execution_scope_agent": "x"}
    )
    _add(store, "other-scope", "alpha", metadata={"execution_scope_agent": "y"})
    filters = {"metadata": {"user_id": 7, "execution_scope_agent": "x"}}
    assert _top_k(store, "alpha", 5, filters=filters, batch_size=1) == ["mine"]
    # Both dimensions were pushed into `where`: the foreign rows were never read.
    assert sum(observed["scan_batch"]) == 1


def test_streaming_list_and_stats_match_the_existing_paths(store, monkeypatch):
    _add(store, "a", "alpha", category="c1", tags=["t1"])
    _add(store, "b", "beta", category="c2", tags=["t1", "t2"])
    _add(store, "c", "gamma", category="c1")
    expected_ids = [note.id for note in store.list_all()]
    expected_stats = store.get_stats()

    def _no_ranking(*args, **kwargs):
        raise AssertionError("streaming paths must not route through search()")

    monkeypatch.setattr(store, "search", _no_ranking)
    listed = stream_list_all(_handle(store), batch_size=2, **_stream_kwargs(store))
    assert [note.id for note in listed] == expected_ids
    assert (
        stream_stats(_handle(store), batch_size=2, **_stream_kwargs(store))
        == expected_stats
    )


def test_scan_and_selection_residency_stay_bounded(store, observed):
    for index in range(5):
        _add(store, f"n{index}", "alpha")
    assert _top_k(store, "alpha", 2, batch_size=2) == ["n0", "n1"]
    # Never more than batch_size rows resident, nothing skipped, and the
    # selection buffer never grows past k.
    assert max(observed["scan_batch"]) <= 2
    assert sum(observed["scan_batch"]) == 5
    assert max(observed["retained"]) <= 2


def _old_full_scan_search(store, query, k, filters=None):
    """The full-table text fallback ``search()`` used before it streamed: every
    row materialised in scan order, then case-insensitive substring match, the
    full filters in Python, and the first ``k`` survivors."""
    table = store._vector_store.get_raw_connection().open_table("memories")
    try:
        rows = table.to_arrow().select(["id", "text", "metadata"]).to_pylist()
    finally:
        _safe_close_table(table)
    other_filters = store._flat_other_filters(filters)
    results = []
    for row in rows:
        if query and query.lower() not in row["text"].lower():
            continue
        note = store._dict_to_memory_note(row)
        if filters and not store._matches_filters(note, filters, other_filters):
            continue
        results.append(note.id)
        if len(results) >= k:
            break
    return results[:k]


def _populate_two_principals(store):
    for index in range(12):
        _add(
            store,
            f"m{index:02d}",
            "Alpha" if index % 4 == 0 else f"note alpha {index}",
            user_id=7 if index % 3 else 8,
            category="c1" if index % 2 else "c2",
        )
    _add(store, "other", "beta only", user_id=7)


@pytest.mark.parametrize(
    ("query", "k", "filters"),
    [
        ("alpha", 3, None),
        ("ALPHA", 20, None),
        ("", 4, None),
        ("alpha", 2, {"metadata": {"user_id": 7}}),
        ("alpha", 20, {"metadata": {"user_id": 8}}),
        ("alpha", 3, {"category": "c1", "metadata": {"user_id": 7}}),
        ("missing", 5, None),
    ],
)
def test_text_only_search_matches_the_old_full_scan(store, query, k, filters):
    _populate_two_principals(store)
    expected = _old_full_scan_search(store, query, k, filters)
    assert [note.id for note in store.search(query, k=k, filters=filters)] == expected


def test_text_only_search_reads_bounded_batches_not_the_table(store, observed):
    store._stream_batch_size = 2
    for index in range(10):
        _add(store, f"n{index}", "alpha")

    assert [note.id for note in store.search("alpha", k=3)] == ["n0", "n1", "n2"]
    # Read in batches of at most 2, stopped once k were retained, and never
    # held more than k.
    assert observed["scan_batch"]
    assert max(observed["scan_batch"]) <= 2
    assert sum(observed["scan_batch"]) < 10
    assert max(observed["retained"]) <= 3